mirror of
https://github.com/freebsd/freebsd-src.git
synced 2026-11-07 04:36:52 -10:00
openssl: import missing files from 3.5.0
This change completes the import of OpenSSL 3.5.0 from upstream. The source archive has been verified via PGP and SHA256: 1. https://github.com/openssl/openssl/releases/download/openssl-3.5.0/openssl-3.5.0.tar.gz 2. https://github.com/openssl/openssl/releases/download/openssl-3.5.0/openssl-3.5.0.tar.gz.asc 3. https://github.com/openssl/openssl/releases/download/openssl-3.5.0/openssl-3.5.0.tar.gz.sha256 4. https://github.com/openssl/openssl/blob/openssl-3.5.0/NEWS.md Reviewed by: ngie, philip Approved by: philip (mentor) Sponsored by: The FreeBSD Foundation
This commit is contained in:
Pierre Pronchery committed
1 parent
29536654cc
commit
09a2519227
1095 files changed
+321722
No files matched your search
@@ -0,0 +1,6 @@
|
||||
Code of Conduct
|
||||
===============
|
||||
|
||||
The OpenSSL [Code of Conduct] is published on the project's website.
|
||||
|
||||
[Code of Conduct]: https://www.openssl.org/community/conduct.html
|
||||
@@ -0,0 +1,36 @@
|
||||
## -*- mode: perl; -*-
|
||||
# Windows on Arm clang-cl targets.
|
||||
#
|
||||
|
||||
my %targets = (
|
||||
"VC-WIN64-CLANGASM-ARM" => {
|
||||
inherit_from => [ "VC-noCE-common" ],
|
||||
defines => add("_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE",
|
||||
"OPENSSL_SYS_WIN_CORE"),
|
||||
bn_ops => "SIXTY_FOUR_BIT RC4_CHAR",
|
||||
multilib => "-arm64",
|
||||
asm_arch => "aarch64",
|
||||
AS => "clang-cl.exe",
|
||||
ASFLAGS => "/nologo /Zi --target=arm64-pc-windows-msvc",
|
||||
asflags => "/c",
|
||||
asoutflag => "/Fo",
|
||||
perlasm_scheme => "win64",
|
||||
uplink_arch => 'armv8',
|
||||
},
|
||||
"VC-CLANG-WIN64-CLANGASM-ARM" => {
|
||||
CC => "clang-cl",
|
||||
inherit_from => [ "VC-noCE-common" ],
|
||||
defines => add("_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE",
|
||||
"OPENSSL_SYS_WIN_CORE"),
|
||||
bn_ops => "SIXTY_FOUR_BIT RC4_CHAR",
|
||||
multilib => "-arm64",
|
||||
asm_arch => "aarch64",
|
||||
CFLAGS => add("--target=arm64-pc-windows-msvc"),
|
||||
AS => "clang-cl.exe",
|
||||
ASFLAGS => "/nologo /Zi --target=arm64-pc-windows-msvc",
|
||||
asflags => "/c",
|
||||
asoutflag => "/Fo",
|
||||
perlasm_scheme => "win64",
|
||||
uplink_arch => 'armv8',
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,37 @@
|
||||
## -*- mode: perl; -*-
|
||||
# Windows HybridCRT targets.
|
||||
#
|
||||
# https://github.com/microsoft/WindowsAppSDK/blob/77761e244289fda6b3d5f14c7bded189fed4fb89/docs/Coding-Guidelines/HybridCRT.md
|
||||
# Link statically against the runtime and STL, but link dynamically against the CRT by ignoring the static CRT
|
||||
# lib and instead linking against the Universal CRT DLL import library. This "Hybrid" linking mechanism is
|
||||
# supported according to the CRT maintainer. Dynamic linking against the CRT makes the binaries a bit smaller
|
||||
# than they would otherwise be if the CRT, runtime, and STL were all statically linked in.
|
||||
|
||||
|
||||
sub remove_from_flags {
|
||||
my ($toRemove, $flags) = @_;
|
||||
|
||||
$flags =~ s/$toRemove//;
|
||||
return $flags;
|
||||
}
|
||||
|
||||
my %targets = (
|
||||
"VC-WIN32-HYBRIDCRT" => {
|
||||
inherit_from => [ "VC-WIN32" ],
|
||||
cflags => sub {
|
||||
remove_from_flags(qr/\/MDd?\s/, add(picker(debug => "/MTd",
|
||||
release => "/MT"))->(@_))
|
||||
},
|
||||
lflags => add(picker(debug => "/NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib",
|
||||
release => "/NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")),
|
||||
},
|
||||
"VC-WIN64A-HYBRIDCRT" => {
|
||||
inherit_from => [ "VC-WIN64A" ],
|
||||
cflags => sub {
|
||||
remove_from_flags(qr/\/MDd?\s/, add(picker(debug => "/MTd",
|
||||
release => "/MT"))->(@_))
|
||||
},
|
||||
lflags => add(picker(debug => "/NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib",
|
||||
release => "/NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")),
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,33 @@
|
||||
Notes on ANSI C
|
||||
===============
|
||||
|
||||
When building for pure ANSI C (C89/C90), you must configure with at least
|
||||
the following configuration settings:
|
||||
|
||||
- `no-asm`
|
||||
|
||||
There are cases of `asm()` calls in our C source, which isn't supported
|
||||
in pure ANSI C.
|
||||
|
||||
- `no-secure-memory`
|
||||
|
||||
The secure memory calls aren't supported with ANSI C.
|
||||
|
||||
- `-D_XOPEN_SOURCE=1`
|
||||
|
||||
This macro enables the use of the following types, functions and global
|
||||
variables:
|
||||
|
||||
- `timezone`
|
||||
|
||||
- `-D_POSIX_C_SOURCE=200809L`
|
||||
|
||||
This macro enables the use of the following types, functions and global
|
||||
variables:
|
||||
|
||||
- `ssize_t`
|
||||
- `strdup()`
|
||||
|
||||
It's arguable that with gcc and clang, all of these issues are removed when
|
||||
defining the macro `_DEFAULT_SOURCE`. However, that effectively sets the C
|
||||
language level to C99, which isn't ANSI C.
|
||||
@@ -0,0 +1,20 @@
|
||||
Notes on POSIX
|
||||
==============
|
||||
|
||||
There are few instances where OpenSSL requires a POSIX C library, at least
|
||||
version 1-2008, or compatible enough functionality.
|
||||
|
||||
There are exceptions, though, for platforms that do not have a POSIX
|
||||
library, or where there are quirks that need working around. A notable
|
||||
platform is Windows, where POSIX functionality may be available, but where
|
||||
the function names are prefixed with an underscore, and where some POSIX
|
||||
types are not present (such as `ssize_t`).
|
||||
|
||||
Platforms that do have a POSIX library may still not have them accessible
|
||||
unless the following macros are defined:
|
||||
|
||||
_POSIX_C_SOURCE=200809L
|
||||
_XOPEN_SOURCE=1
|
||||
|
||||
This is, for example, the case when building with gcc or clang and using the
|
||||
flag `-ansi`.
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
Using OpenSSL with QUIC
|
||||
=======================
|
||||
|
||||
From OpenSSL 3.2, OpenSSL features support for making QUIC connections as a
|
||||
client. Starting with OpenSSL 3.5, server-side QUIC support has also been added.
|
||||
|
||||
Users interested in using the new QUIC functionality are encouraged to look at
|
||||
some of the following resources:
|
||||
|
||||
- The new [OpenSSL Guide], which provides introductory guides on the use of TLS,
|
||||
QUIC, and other OpenSSL functionality.
|
||||
- The [OpenSSL Guide] incorporates various code samples. The complete source
|
||||
for these can be [found in the source tree under `demos/guide`](./demos/guide/).
|
||||
- The [openssl-quic(7) manual page], which provides a basic reference overview
|
||||
of QUIC functionality and how use of QUIC differs from use of TLS with regard
|
||||
to our API.
|
||||
- The [Demo-Driven Design (DDD)][DDD] demos, which demonstrate the use of QUIC
|
||||
using simple examples. These can be [found in the source tree under
|
||||
`doc/designs/ddd`].
|
||||
- The [demo found in `demos/http3`], which provides an HTTP/3 client example
|
||||
using the nghttp3 HTTP/3 library.
|
||||
|
||||
FAQ
|
||||
---
|
||||
|
||||
### Why would I want to use QUIC, and what functionality does QUIC offer relative to TLS or DTLS?
|
||||
|
||||
QUIC is a state-of-the-art secure transport protocol carried over UDP. It can
|
||||
serve many of the use cases of SSL/TLS as well as those of DTLS.
|
||||
|
||||
QUIC delivers a number of advantages such as support for multiple streams of
|
||||
communication; it is the basis for HTTP/3 [RFC 9114]; fast connection
|
||||
initiation; and connection migration (enabling a connection to survive IP
|
||||
address changes). For a more complete description of what QUIC is and its
|
||||
advantages see the [QUIC Introduction] in the [OpenSSL Guide].
|
||||
|
||||
For a comprehensive overview of OpenSSL's QUIC implementation, see the
|
||||
[openssl-quic(7) manual page].
|
||||
|
||||
### How can I use HTTP/3 with OpenSSL?
|
||||
|
||||
There are many HTTP/3 implementations in C available. The use of one such HTTP/3
|
||||
library with OpenSSL QUIC is demonstrated via the [demo found in `demos/http3`].
|
||||
|
||||
### How can I use OpenSSL QUIC in my own application for a different protocol?
|
||||
|
||||
The [OpenSSL Guide] provides introductory examples for how to make use of
|
||||
OpenSSL QUIC.
|
||||
|
||||
The [openssl-quic(7) manual page] and the [Demo-Driven Design (DDD)][DDD] demos
|
||||
may also be helpful to illustrate the changes needed if you are trying to adapt
|
||||
an existing application.
|
||||
|
||||
### How can I test QUIC using `openssl s_client`?
|
||||
|
||||
There is basic support for single-stream QUIC using `openssl s_client`:
|
||||
|
||||
```shell
|
||||
$ openssl s_client -quic -alpn myalpn -connect host:port
|
||||
```
|
||||
|
||||
In the above example replace `host` with the hostname of the server (e.g.
|
||||
`www.example.com`) and `port` with the port for the server (e.g. `443`). Replace
|
||||
`myalpn` with the Application Layer Protocol to use (e.g.`h3` represents
|
||||
HTTP/3). IANA maintains a standard list of [ALPN ids] that can be used.
|
||||
|
||||
This example connects to a QUIC server and opens a single bidirectional stream.
|
||||
Data can be passed via stdin/stdout as usual. This allows test usage of QUIC
|
||||
using simple TCP/TLS-like usage. Note that OpenSSL has no direct support for
|
||||
HTTP/3 so connecting to an HTTP/3 server should be possible but sending an
|
||||
HTTP/3 request or receiving any response data is not.
|
||||
|
||||
### How can I create a QUIC server with OpenSSL?
|
||||
|
||||
Starting with OpenSSL 3.5, you can create a QUIC server. OpenSSL provides a server
|
||||
implementation example that you can use as a reference:
|
||||
|
||||
The example QUIC server implementation can be found in the source tree under
|
||||
[`demos/quic/server`](./demos/quic/server/). This demonstrates how to implement a
|
||||
basic QUIC server using the OpenSSL API.
|
||||
|
||||
To run the example QUIC server:
|
||||
|
||||
```shell
|
||||
$ ./demos/quic/server/server <port-number> <certificate-file> <key-file>
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```shell
|
||||
$ ./demos/quic/server/server 4433 server.pem server.key
|
||||
```
|
||||
|
||||
Replace `server.pem` and `server.key` with your certificate and private key files.
|
||||
Note that the standard `openssl s_server` command does NOT support QUIC - you must
|
||||
use this dedicated server example instead.
|
||||
|
||||
For more information about implementing QUIC servers with OpenSSL, refer to the
|
||||
[OpenSSL Guide] and the [openssl-quic(7) manual page].
|
||||
|
||||
[openssl-quic(7) manual page]: https://www.openssl.org/docs/manmaster/man7/openssl-quic.html
|
||||
[OpenSSL Guide]: https://www.openssl.org/docs/manmaster/man7/ossl-guide-introduction.html
|
||||
[DDD]: https://github.com/openssl/openssl/tree/master/doc/designs/ddd
|
||||
[found in the source tree under `doc/designs/ddd`]: ./doc/designs/ddd/
|
||||
[demo found in `demos/http3`]: ./demos/http3/
|
||||
[QUIC Introduction]: https://www.openssl.org/docs/manmaster/man7/ossl-guide-quic-introduction.html
|
||||
[RFC 9114]: https://tools.ietf.org/html/rfc9114
|
||||
[ALPN ids]: https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#ifndef OSSL_APPS_LOG_H
|
||||
# define OSSL_APPS_LOG_H
|
||||
|
||||
# include <openssl/bio.h>
|
||||
# if !defined(OPENSSL_SYS_VMS) && !defined(OPENSSL_SYS_WINDOWS) \
|
||||
&& !defined(OPENSSL_NO_SOCK) && !defined(OPENSSL_NO_POSIX_IO)
|
||||
# include <syslog.h>
|
||||
# else
|
||||
# define LOG_EMERG 0
|
||||
# define LOG_ALERT 1
|
||||
# define LOG_CRIT 2
|
||||
# define LOG_ERR 3
|
||||
# define LOG_WARNING 4
|
||||
# define LOG_NOTICE 5
|
||||
# define LOG_INFO 6
|
||||
# define LOG_DEBUG 7
|
||||
# endif
|
||||
|
||||
# undef LOG_TRACE
|
||||
# define LOG_TRACE (LOG_DEBUG + 1)
|
||||
|
||||
int log_set_verbosity(const char *prog, int level);
|
||||
int log_get_verbosity(void);
|
||||
|
||||
/*-
|
||||
* Output a message using the trace API with the given category
|
||||
* if the category is >= 0 and tracing is enabled.
|
||||
* Log the message to syslog if multi-threaded HTTP_DAEMON, else to bio_err
|
||||
* if the verbosity is sufficient for the given level of severity.
|
||||
* Yet cannot do both types of output in strict ANSI mode.
|
||||
* category: trace category as defined in trace.h, or -1
|
||||
* prog: the name of the current app, or NULL
|
||||
* level: the severity of the message, e.g., LOG_ERR
|
||||
* fmt: message format, which should not include a trailing newline
|
||||
* ...: potential extra parameters like with printf()
|
||||
* returns nothing
|
||||
*/
|
||||
void trace_log_message(int category,
|
||||
const char *prog, int level, const char *fmt, ...);
|
||||
|
||||
#endif /* OSSL_APPS_LOG_H */
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "opt.h"
|
||||
#include <openssl/ui.h>
|
||||
#include "apps_ui.h"
|
||||
|
||||
/* This function is defined here due to visibility of bio_err */
|
||||
int opt_printf_stderr(const char *fmt, ...)
|
||||
{
|
||||
va_list ap;
|
||||
int ret;
|
||||
|
||||
va_start(ap, fmt);
|
||||
ret = BIO_vprintf(bio_err, fmt, ap);
|
||||
va_end(ap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/trace.h>
|
||||
#include "apps.h"
|
||||
#include "log.h"
|
||||
|
||||
static int verbosity = LOG_INFO;
|
||||
|
||||
int log_set_verbosity(const char *prog, int level)
|
||||
{
|
||||
if (level < LOG_EMERG || level > LOG_TRACE) {
|
||||
trace_log_message(-1, prog, LOG_ERR,
|
||||
"Invalid verbosity level %d", level);
|
||||
return 0;
|
||||
}
|
||||
verbosity = level;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int log_get_verbosity(void)
|
||||
{
|
||||
return verbosity;
|
||||
}
|
||||
|
||||
#ifdef HTTP_DAEMON
|
||||
static int print_syslog(const char *str, size_t len, void *levPtr)
|
||||
{
|
||||
int level = *(int *)levPtr;
|
||||
int ilen = len > MAXERRLEN ? MAXERRLEN : len;
|
||||
|
||||
syslog(level, "%.*s", ilen, str);
|
||||
|
||||
return ilen;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void log_with_prefix(const char *prog, const char *fmt, va_list ap)
|
||||
{
|
||||
char prefix[80];
|
||||
BIO *bio, *pre = BIO_new(BIO_f_prefix());
|
||||
|
||||
(void)BIO_snprintf(prefix, sizeof(prefix), "%s: ", prog);
|
||||
(void)BIO_set_prefix(pre, prefix);
|
||||
bio = BIO_push(pre, bio_err);
|
||||
(void)BIO_vprintf(bio, fmt, ap);
|
||||
(void)BIO_printf(bio, "\n");
|
||||
(void)BIO_flush(bio);
|
||||
(void)BIO_pop(pre);
|
||||
BIO_free(pre);
|
||||
}
|
||||
|
||||
/*
|
||||
* Unfortunately, C before C99 does not define va_copy, so we must
|
||||
* check if it can be assumed to be present. We do that with an internal
|
||||
* antifeature macro.
|
||||
* C versions since C94 define __STDC_VERSION__, so it's enough to
|
||||
* check its existence and value.
|
||||
*/
|
||||
#undef OSSL_NO_C99
|
||||
#if !defined(__STDC_VERSION__) || __STDC_VERSION__ + 0 < 199900L
|
||||
# define OSSL_NO_C99
|
||||
#endif
|
||||
|
||||
void trace_log_message(int category,
|
||||
const char *prog, int level, const char *fmt, ...)
|
||||
{
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
|
||||
#ifdef OSSL_NO_C99
|
||||
if (verbosity >= level)
|
||||
category = -1; /* disabling trace output in addition to logging */
|
||||
#endif
|
||||
if (category >= 0 && OSSL_trace_enabled(category)) {
|
||||
BIO *out = OSSL_trace_begin(category);
|
||||
#ifndef OSSL_NO_C99
|
||||
va_list ap_copy;
|
||||
|
||||
va_copy(ap_copy, ap);
|
||||
(void)BIO_vprintf(out, fmt, ap_copy);
|
||||
va_end(ap_copy);
|
||||
#else
|
||||
(void)BIO_vprintf(out, fmt, ap);
|
||||
#endif
|
||||
(void)BIO_printf(out, "\n");
|
||||
OSSL_trace_end(category, out);
|
||||
}
|
||||
if (verbosity < level) {
|
||||
va_end(ap);
|
||||
return;
|
||||
}
|
||||
#ifdef HTTP_DAEMON
|
||||
if (n_responders != 0) {
|
||||
vsyslog(level, fmt, ap);
|
||||
if (level <= LOG_ERR)
|
||||
ERR_print_errors_cb(print_syslog, &level);
|
||||
} else
|
||||
#endif
|
||||
log_with_prefix(prog, fmt, ap);
|
||||
va_end(ap);
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
#include "apps.h"
|
||||
#include "progs.h"
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/err.h>
|
||||
#include <openssl/evp.h>
|
||||
|
||||
typedef enum OPTION_choice {
|
||||
OPT_COMMON,
|
||||
OPT_PROV_ENUM,
|
||||
OPT_CIPHER,
|
||||
OPT_SKEYOPT, OPT_SKEYMGMT, OPT_GENKEY
|
||||
} OPTION_CHOICE;
|
||||
|
||||
const OPTIONS skeyutl_options[] = {
|
||||
OPT_SECTION("General"),
|
||||
{"help", OPT_HELP, '-', "Display this summary"},
|
||||
{"skeyopt", OPT_SKEYOPT, 's', "Key options as opt:value for opaque keys handling"},
|
||||
{"skeymgmt", OPT_SKEYMGMT, 's', "Symmetric key management name for opaque keys handling"},
|
||||
{"genkey", OPT_GENKEY, '-', "Generate an opaque symmetric key"},
|
||||
{"cipher", OPT_CIPHER, 's', "The cipher to generate key for"},
|
||||
OPT_PROV_OPTIONS,
|
||||
{NULL}
|
||||
};
|
||||
|
||||
int skeyutl_main(int argc, char **argv)
|
||||
{
|
||||
EVP_CIPHER *cipher = NULL;
|
||||
int ret = 1;
|
||||
OPTION_CHOICE o;
|
||||
int genkey = 0;
|
||||
char *prog, *ciphername = NULL;
|
||||
STACK_OF(OPENSSL_STRING) *skeyopts = NULL;
|
||||
const char *skeymgmt = NULL;
|
||||
EVP_SKEY *skey = NULL;
|
||||
EVP_SKEYMGMT *mgmt = NULL;
|
||||
|
||||
prog = opt_init(argc, argv, skeyutl_options);
|
||||
while ((o = opt_next()) != OPT_EOF) {
|
||||
switch (o) {
|
||||
case OPT_EOF:
|
||||
case OPT_ERR:
|
||||
opthelp:
|
||||
BIO_printf(bio_err, "%s: Use -help for summary.\n", prog);
|
||||
goto end;
|
||||
case OPT_HELP:
|
||||
opt_help(skeyutl_options);
|
||||
ret = 0;
|
||||
goto end;
|
||||
case OPT_GENKEY:
|
||||
genkey = 1;
|
||||
break;
|
||||
case OPT_CIPHER:
|
||||
ciphername = opt_arg();
|
||||
break;
|
||||
case OPT_SKEYOPT:
|
||||
if ((skeyopts == NULL &&
|
||||
(skeyopts = sk_OPENSSL_STRING_new_null()) == NULL) ||
|
||||
sk_OPENSSL_STRING_push(skeyopts, opt_arg()) == 0) {
|
||||
BIO_printf(bio_err, "%s: out of memory\n", prog);
|
||||
goto end;
|
||||
}
|
||||
break;
|
||||
case OPT_SKEYMGMT:
|
||||
skeymgmt = opt_arg();
|
||||
break;
|
||||
case OPT_PROV_CASES:
|
||||
if (!opt_provider(o))
|
||||
goto end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Get the cipher name, either from progname (if set) or flag. */
|
||||
if (!opt_cipher_any(ciphername, &cipher))
|
||||
goto opthelp;
|
||||
|
||||
if (cipher == NULL && skeymgmt == NULL) {
|
||||
BIO_printf(bio_err, "Either -skeymgmt -or -cipher option should be specified\n");
|
||||
goto end;
|
||||
}
|
||||
|
||||
if (genkey) {
|
||||
OSSL_PARAM *params = NULL;
|
||||
|
||||
mgmt = EVP_SKEYMGMT_fetch(app_get0_libctx(),
|
||||
skeymgmt ? skeymgmt : EVP_CIPHER_name(cipher),
|
||||
app_get0_propq());
|
||||
if (mgmt == NULL)
|
||||
goto end;
|
||||
params = app_params_new_from_opts(skeyopts,
|
||||
EVP_SKEYMGMT_get0_gen_settable_params(mgmt));
|
||||
|
||||
skey = EVP_SKEY_generate(app_get0_libctx(),
|
||||
skeymgmt ? skeymgmt : EVP_CIPHER_name(cipher),
|
||||
app_get0_propq(), params);
|
||||
OSSL_PARAM_free(params);
|
||||
if (skey == NULL) {
|
||||
BIO_printf(bio_err, "Error creating opaque key for skeymgmt %s\n",
|
||||
skeymgmt ? skeymgmt : EVP_CIPHER_name(cipher));
|
||||
ERR_print_errors(bio_err);
|
||||
} else {
|
||||
const char *key_name = EVP_SKEY_get0_key_id(skey);
|
||||
|
||||
BIO_printf(bio_out, "An opaque key identified by %s is created\n",
|
||||
key_name ? key_name : "<unknown>");
|
||||
BIO_printf(bio_out, "Provider: %s\n", EVP_SKEY_get0_provider_name(skey));
|
||||
BIO_printf(bio_out, "Key management: %s\n", EVP_SKEY_get0_skeymgmt_name(skey));
|
||||
ret = 0;
|
||||
}
|
||||
goto end;
|
||||
} else {
|
||||
BIO_printf(bio_err, "Key generation is the only supported operation as of now\n");
|
||||
}
|
||||
|
||||
end:
|
||||
ERR_print_errors(bio_err);
|
||||
sk_OPENSSL_STRING_free(skeyopts);
|
||||
EVP_SKEYMGMT_free(mgmt);
|
||||
EVP_SKEY_free(skey);
|
||||
EVP_CIPHER_free(cipher);
|
||||
return ret;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,594 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2022, Hongren (Zenithal) Zheng <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
################################################################################
|
||||
# Utility functions to help with keeping track of which registers to stack/
|
||||
# unstack when entering / exiting routines.
|
||||
################################################################################
|
||||
{
|
||||
# Callee-saved registers
|
||||
my @callee_saved = map("x$_",(2,8,9,18..27));
|
||||
# Caller-saved registers
|
||||
my @caller_saved = map("x$_",(1,5..7,10..17,28..31));
|
||||
my @must_save;
|
||||
sub use_reg {
|
||||
my $reg = shift;
|
||||
if (grep(/^$reg$/, @callee_saved)) {
|
||||
push(@must_save, $reg);
|
||||
} elsif (!grep(/^$reg$/, @caller_saved)) {
|
||||
# Register is not usable!
|
||||
die("Unusable register ".$reg);
|
||||
}
|
||||
return $reg;
|
||||
}
|
||||
sub use_regs {
|
||||
return map(use_reg("x$_"), @_);
|
||||
}
|
||||
sub save_regs {
|
||||
my $ret = '';
|
||||
my $stack_reservation = ($#must_save + 1) * 8;
|
||||
my $stack_offset = $stack_reservation;
|
||||
if ($stack_reservation % 16) {
|
||||
$stack_reservation += 8;
|
||||
}
|
||||
$ret.=" addi sp,sp,-$stack_reservation\n";
|
||||
foreach (@must_save) {
|
||||
$stack_offset -= 8;
|
||||
$ret.=" sd $_,$stack_offset(sp)\n";
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
sub load_regs {
|
||||
my $ret = '';
|
||||
my $stack_reservation = ($#must_save + 1) * 8;
|
||||
my $stack_offset = $stack_reservation;
|
||||
if ($stack_reservation % 16) {
|
||||
$stack_reservation += 8;
|
||||
}
|
||||
foreach (@must_save) {
|
||||
$stack_offset -= 8;
|
||||
$ret.=" ld $_,$stack_offset(sp)\n";
|
||||
}
|
||||
$ret.=" addi sp,sp,$stack_reservation\n";
|
||||
return $ret;
|
||||
}
|
||||
sub clear_regs {
|
||||
@must_save = ();
|
||||
}
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Register assignment for rv64i_zkne_encrypt and rv64i_zknd_decrypt
|
||||
################################################################################
|
||||
|
||||
# Registers to hold AES state (called s0-s3 or y0-y3 elsewhere)
|
||||
my ($Q0,$Q1,$Q2,$Q3) = use_regs(6..9);
|
||||
|
||||
# Function arguments (x10-x12 are a0-a2 in the ABI)
|
||||
# Input block pointer, output block pointer, key pointer
|
||||
my ($INP,$OUTP,$KEYP) = use_regs(10..12);
|
||||
|
||||
# Temporaries
|
||||
my ($T0,$T1) = use_regs(13..14);
|
||||
|
||||
# Loop counter
|
||||
my ($loopcntr) = use_regs(30);
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zkne_encrypt(const unsigned char *in, unsigned char *out,
|
||||
# const AES_KEY *key);
|
||||
################################################################################
|
||||
my $code .= <<___;
|
||||
.text
|
||||
.balign 16
|
||||
.globl rv64i_zkne_encrypt
|
||||
.type rv64i_zkne_encrypt,\@function
|
||||
rv64i_zkne_encrypt:
|
||||
___
|
||||
|
||||
$code .= save_regs();
|
||||
|
||||
$code .= <<___;
|
||||
|
||||
# Load input to block cipher
|
||||
ld $Q0,0($INP)
|
||||
ld $Q1,8($INP)
|
||||
|
||||
# Load key
|
||||
ld $T0,0($KEYP)
|
||||
ld $T1,8($KEYP)
|
||||
|
||||
# Load number of rounds
|
||||
lwu $loopcntr,240($KEYP)
|
||||
|
||||
# initial transformation
|
||||
xor $Q0,$Q0,$T0
|
||||
xor $Q1,$Q1,$T1
|
||||
|
||||
# The main loop only executes the first N-1 rounds.
|
||||
add $loopcntr,$loopcntr,-1
|
||||
|
||||
# Do Nr - 1 rounds (final round is special)
|
||||
1:
|
||||
@{[aes64esm $Q2,$Q0,$Q1]}
|
||||
@{[aes64esm $Q3,$Q1,$Q0]}
|
||||
|
||||
# Update key ptr to point to next key in schedule
|
||||
add $KEYP,$KEYP,16
|
||||
|
||||
# Grab next key in schedule
|
||||
ld $T0,0($KEYP)
|
||||
ld $T1,8($KEYP)
|
||||
xor $Q0,$Q2,$T0
|
||||
xor $Q1,$Q3,$T1
|
||||
|
||||
add $loopcntr,$loopcntr,-1
|
||||
bgtz $loopcntr,1b
|
||||
|
||||
# final round
|
||||
@{[aes64es $Q2,$Q0,$Q1]}
|
||||
@{[aes64es $Q3,$Q1,$Q0]}
|
||||
|
||||
# since not added 16 before
|
||||
ld $T0,16($KEYP)
|
||||
ld $T1,24($KEYP)
|
||||
xor $Q0,$Q2,$T0
|
||||
xor $Q1,$Q3,$T1
|
||||
|
||||
sd $Q0,0($OUTP)
|
||||
sd $Q1,8($OUTP)
|
||||
|
||||
# Pop registers and return
|
||||
___
|
||||
|
||||
$code .= load_regs();
|
||||
|
||||
$code .= <<___;
|
||||
ret
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zknd_decrypt(const unsigned char *in, unsigned char *out,
|
||||
# const AES_KEY *key);
|
||||
################################################################################
|
||||
$code .= <<___;
|
||||
.text
|
||||
.balign 16
|
||||
.globl rv64i_zknd_decrypt
|
||||
.type rv64i_zknd_decrypt,\@function
|
||||
rv64i_zknd_decrypt:
|
||||
___
|
||||
|
||||
$code .= save_regs();
|
||||
|
||||
$code .= <<___;
|
||||
|
||||
# Load input to block cipher
|
||||
ld $Q0,0($INP)
|
||||
ld $Q1,8($INP)
|
||||
|
||||
# Load number of rounds
|
||||
lwu $loopcntr,240($KEYP)
|
||||
|
||||
# Load the last key
|
||||
slli $T0,$loopcntr,4
|
||||
add $KEYP,$KEYP,$T0
|
||||
ld $T0,0($KEYP)
|
||||
ld $T1,8($KEYP)
|
||||
|
||||
xor $Q0,$Q0,$T0
|
||||
xor $Q1,$Q1,$T1
|
||||
|
||||
# The main loop only executes the first N-1 rounds.
|
||||
add $loopcntr,$loopcntr,-1
|
||||
|
||||
# Do Nr - 1 rounds (final round is special)
|
||||
1:
|
||||
@{[aes64dsm $Q2,$Q0,$Q1]}
|
||||
@{[aes64dsm $Q3,$Q1,$Q0]}
|
||||
|
||||
# Update key ptr to point to next key in schedule
|
||||
add $KEYP,$KEYP,-16
|
||||
|
||||
# Grab next key in schedule
|
||||
ld $T0,0($KEYP)
|
||||
ld $T1,8($KEYP)
|
||||
xor $Q0,$Q2,$T0
|
||||
xor $Q1,$Q3,$T1
|
||||
|
||||
add $loopcntr,$loopcntr,-1
|
||||
bgtz $loopcntr,1b
|
||||
|
||||
# final round
|
||||
@{[aes64ds $Q2,$Q0,$Q1]}
|
||||
@{[aes64ds $Q3,$Q1,$Q0]}
|
||||
|
||||
add $KEYP,$KEYP,-16
|
||||
ld $T0,0($KEYP)
|
||||
ld $T1,8($KEYP)
|
||||
xor $Q0,$Q2,$T0
|
||||
xor $Q1,$Q3,$T1
|
||||
|
||||
sd $Q0,0($OUTP)
|
||||
sd $Q1,8($OUTP)
|
||||
# Pop registers and return
|
||||
___
|
||||
|
||||
$code .= load_regs();
|
||||
|
||||
$code .= <<___;
|
||||
ret
|
||||
___
|
||||
|
||||
clear_regs();
|
||||
|
||||
################################################################################
|
||||
# Register assignment for rv64i_zkn[e/d]_set_[en/de]crypt_key
|
||||
################################################################################
|
||||
|
||||
# Function arguments (x10-x12 are a0-a2 in the ABI)
|
||||
# Pointer to user key, number of bits in key, key pointer
|
||||
my ($UKEY,$BITS,$KEYP) = use_regs(10..12);
|
||||
|
||||
# Temporaries
|
||||
my ($T0,$T1,$T2,$T3,$T4) = use_regs(6..8,13..14);
|
||||
|
||||
################################################################################
|
||||
# utility functions for rv64i_zkne_set_encrypt_key
|
||||
################################################################################
|
||||
sub ke128enc {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
while($rnum < 10) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T2,$T1,$rnum]}
|
||||
@{[aes64ks2 $T0,$T2,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
add $KEYP,$KEYP,16
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
sub ke192enc {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
ld $T2,16($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
sd $T2,16($KEYP)
|
||||
___
|
||||
while($rnum < 8) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T3,$T2,$rnum]}
|
||||
@{[aes64ks2 $T0,$T3,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
___
|
||||
if ($rnum != 7) {
|
||||
# note that (8+1)*24 = 216, (12+1)*16 = 208
|
||||
# thus the last 8 bytes can be dropped
|
||||
$ret .= <<___;
|
||||
@{[aes64ks2 $T2,$T1,$T2]}
|
||||
___
|
||||
}
|
||||
$ret .= <<___;
|
||||
add $KEYP,$KEYP,24
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
if ($rnum != 7) {
|
||||
$ret .= <<___;
|
||||
sd $T2,16($KEYP)
|
||||
___
|
||||
}
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
sub ke256enc {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
ld $T2,16($UKEY)
|
||||
ld $T3,24($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
sd $T2,16($KEYP)
|
||||
sd $T3,24($KEYP)
|
||||
___
|
||||
while($rnum < 7) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T4,$T3,$rnum]}
|
||||
@{[aes64ks2 $T0,$T4,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
add $KEYP,$KEYP,32
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
if ($rnum != 6) {
|
||||
# note that (7+1)*32 = 256, (14+1)*16 = 240
|
||||
# thus the last 16 bytes can be dropped
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T4,$T1,0xA]}
|
||||
@{[aes64ks2 $T2,$T4,$T2]}
|
||||
@{[aes64ks2 $T3,$T2,$T3]}
|
||||
sd $T2,16($KEYP)
|
||||
sd $T3,24($KEYP)
|
||||
___
|
||||
}
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
|
||||
# AES_KEY *key)
|
||||
################################################################################
|
||||
sub AES_set_common {
|
||||
my ($ke128, $ke192, $ke256) = @_;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
bnez $UKEY,1f # if (!userKey || !key) return -1;
|
||||
bnez $KEYP,1f
|
||||
li a0,-1
|
||||
ret
|
||||
1:
|
||||
# Determine number of rounds from key size in bits
|
||||
li $T0,128
|
||||
bne $BITS,$T0,1f
|
||||
li $T1,10 # key->rounds = 10 if bits == 128
|
||||
sw $T1,240($KEYP) # store key->rounds
|
||||
$ke128
|
||||
j 4f
|
||||
1:
|
||||
li $T0,192
|
||||
bne $BITS,$T0,2f
|
||||
li $T1,12 # key->rounds = 12 if bits == 192
|
||||
sw $T1,240($KEYP) # store key->rounds
|
||||
$ke192
|
||||
j 4f
|
||||
2:
|
||||
li $T1,14 # key->rounds = 14 if bits == 256
|
||||
li $T0,256
|
||||
beq $BITS,$T0,3f
|
||||
li a0,-2 # If bits != 128, 192, or 256, return -2
|
||||
j 5f
|
||||
3:
|
||||
sw $T1,240($KEYP) # store key->rounds
|
||||
$ke256
|
||||
4: # return 0
|
||||
li a0,0
|
||||
5: # return a0
|
||||
___
|
||||
return $ret;
|
||||
}
|
||||
$code .= <<___;
|
||||
.text
|
||||
.balign 16
|
||||
.globl rv64i_zkne_set_encrypt_key
|
||||
.type rv64i_zkne_set_encrypt_key,\@function
|
||||
rv64i_zkne_set_encrypt_key:
|
||||
___
|
||||
$code .= save_regs();
|
||||
$code .= AES_set_common(ke128enc(), ke192enc(),ke256enc());
|
||||
$code .= load_regs();
|
||||
$code .= <<___;
|
||||
ret
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# utility functions for rv64i_zknd_set_decrypt_key
|
||||
################################################################################
|
||||
sub ke128dec {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
while($rnum < 10) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T2,$T1,$rnum]}
|
||||
@{[aes64ks2 $T0,$T2,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
add $KEYP,$KEYP,16
|
||||
___
|
||||
# need to aes64im for [1:N-1] round keys
|
||||
# this is from the fact that aes64dsm subwords first then mix column
|
||||
# intuitively decryption needs to first mix column then subwords
|
||||
# however, for merging datapaths (encryption first subwords then mix column)
|
||||
# aes64dsm chooses to inverse the order of them, thus
|
||||
# transform should then be done on the round key
|
||||
if ($rnum < 9) {
|
||||
$ret .= <<___;
|
||||
@{[aes64im $T2,$T0]}
|
||||
sd $T2,0($KEYP)
|
||||
@{[aes64im $T2,$T1]}
|
||||
sd $T2,8($KEYP)
|
||||
___
|
||||
} else {
|
||||
$ret .= <<___;
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
}
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
sub ke192dec {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
ld $T2,16($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
@{[aes64im $T3,$T2]}
|
||||
sd $T3,16($KEYP)
|
||||
___
|
||||
while($rnum < 8) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T3,$T2,$rnum]}
|
||||
@{[aes64ks2 $T0,$T3,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
add $KEYP,$KEYP,24
|
||||
___
|
||||
if ($rnum < 7) {
|
||||
$ret .= <<___;
|
||||
@{[aes64im $T3,$T0]}
|
||||
sd $T3,0($KEYP)
|
||||
@{[aes64im $T3,$T1]}
|
||||
sd $T3,8($KEYP)
|
||||
# the reason is in ke192enc
|
||||
@{[aes64ks2 $T2,$T1,$T2]}
|
||||
@{[aes64im $T3,$T2]}
|
||||
sd $T3,16($KEYP)
|
||||
___
|
||||
} else { # rnum == 7
|
||||
$ret .= <<___;
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
___
|
||||
}
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
sub ke256dec {
|
||||
my $rnum = 0;
|
||||
my $ret = '';
|
||||
$ret .= <<___;
|
||||
ld $T0,0($UKEY)
|
||||
ld $T1,8($UKEY)
|
||||
ld $T2,16($UKEY)
|
||||
ld $T3,24($UKEY)
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
@{[aes64im $T4,$T2]}
|
||||
sd $T4,16($KEYP)
|
||||
@{[aes64im $T4,$T3]}
|
||||
sd $T4,24($KEYP)
|
||||
___
|
||||
while($rnum < 7) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T4,$T3,$rnum]}
|
||||
@{[aes64ks2 $T0,$T4,$T0]}
|
||||
@{[aes64ks2 $T1,$T0,$T1]}
|
||||
add $KEYP,$KEYP,32
|
||||
___
|
||||
if ($rnum < 6) {
|
||||
$ret .= <<___;
|
||||
@{[aes64ks1i $T4,$T1,0xA]}
|
||||
@{[aes64ks2 $T2,$T4,$T2]}
|
||||
@{[aes64ks2 $T3,$T2,$T3]}
|
||||
@{[aes64im $T4,$T0]}
|
||||
sd $T4,0($KEYP)
|
||||
@{[aes64im $T4,$T1]}
|
||||
sd $T4,8($KEYP)
|
||||
@{[aes64im $T4,$T2]}
|
||||
sd $T4,16($KEYP)
|
||||
@{[aes64im $T4,$T3]}
|
||||
sd $T4,24($KEYP)
|
||||
___
|
||||
} else {
|
||||
$ret .= <<___;
|
||||
sd $T0,0($KEYP)
|
||||
sd $T1,8($KEYP)
|
||||
# last two one dropped
|
||||
___
|
||||
}
|
||||
$rnum++;
|
||||
}
|
||||
return $ret;
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zknd_set_decrypt_key(const unsigned char *userKey, const int bits,
|
||||
# AES_KEY *key)
|
||||
################################################################################
|
||||
$code .= <<___;
|
||||
.text
|
||||
.balign 16
|
||||
.globl rv64i_zknd_set_decrypt_key
|
||||
.type rv64i_zknd_set_decrypt_key,\@function
|
||||
rv64i_zknd_set_decrypt_key:
|
||||
___
|
||||
$code .= save_regs();
|
||||
$code .= AES_set_common(ke128dec(), ke192dec(),ke256dec());
|
||||
$code .= load_regs();
|
||||
$code .= <<___;
|
||||
ret
|
||||
___
|
||||
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,710 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Jerry Shih <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Bit-manipulation extension ('Zvbb')
|
||||
# - RISC-V Vector GCM/GMAC extension ('Zvkg')
|
||||
# - RISC-V Vector AES block cipher extension ('Zvkned')
|
||||
# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
{
|
||||
################################################################################
|
||||
# void rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt(const unsigned char *in,
|
||||
# unsigned char *out, size_t length,
|
||||
# const AES_KEY *key1,
|
||||
# const AES_KEY *key2,
|
||||
# const unsigned char iv[16])
|
||||
my ($INPUT, $OUTPUT, $LENGTH, $KEY1, $KEY2, $IV) = ("a0", "a1", "a2", "a3", "a4", "a5");
|
||||
my ($TAIL_LENGTH) = ("a6");
|
||||
my ($VL) = ("a7");
|
||||
my ($T0, $T1, $T2) = ("t0", "t1", "t2");
|
||||
my ($STORE_LEN32) = ("t3");
|
||||
my ($LEN32) = ("t4");
|
||||
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
|
||||
$V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
|
||||
$V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
|
||||
$V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map("v$_",(0..31));
|
||||
|
||||
sub compute_xts_iv0 {
|
||||
my $code=<<___;
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEY2)
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V28, $IV]}
|
||||
@{[vle32_v $V29, $KEY2]}
|
||||
@{[vaesz_vs $V28, $V29]}
|
||||
addi $T0, $T0, -1
|
||||
addi $KEY2, $KEY2, 16
|
||||
1:
|
||||
@{[vle32_v $V29, $KEY2]}
|
||||
@{[vaesem_vs $V28, $V29]}
|
||||
addi $T0, $T0, -1
|
||||
addi $KEY2, $KEY2, 16
|
||||
bnez $T0, 1b
|
||||
@{[vle32_v $V29, $KEY2]}
|
||||
@{[vaesef_vs $V28, $V29]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# prepare input data(v24), iv(v28), bit-reversed-iv(v16), bit-reversed-iv-multiplier(v20)
|
||||
sub init_first_round {
|
||||
my $code=<<___;
|
||||
# load input
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
|
||||
li $T0, 5
|
||||
# We could simplify the initialization steps if we have `block<=1`.
|
||||
blt $LEN32, $T0, 1f
|
||||
|
||||
# Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses
|
||||
# different order of coefficients. We should use`vbrev8` to reverse the
|
||||
# data when we use `vgmul`.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vbrev8_v $V0, $V28]}
|
||||
@{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V16, 0]}
|
||||
# v16: [r-IV0, r-IV0, ...]
|
||||
@{[vaesz_vs $V16, $V0]}
|
||||
|
||||
# Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8.
|
||||
slli $T0, $LEN32, 2
|
||||
@{[vsetvli "zero", $T0, "e32", "m1", "ta", "ma"]}
|
||||
# v2: [`1`, `1`, `1`, `1`, ...]
|
||||
@{[vmv_v_i $V2, 1]}
|
||||
# v3: [`0`, `1`, `2`, `3`, ...]
|
||||
@{[vid_v $V3]}
|
||||
@{[vsetvli "zero", $T0, "e64", "m2", "ta", "ma"]}
|
||||
# v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...]
|
||||
@{[vzext_vf2 $V4, $V2]}
|
||||
# v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...]
|
||||
@{[vzext_vf2 $V6, $V3]}
|
||||
slli $T0, $LEN32, 1
|
||||
@{[vsetvli "zero", $T0, "e32", "m2", "ta", "ma"]}
|
||||
# v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...]
|
||||
@{[vwsll_vv $V8, $V4, $V6]}
|
||||
|
||||
# Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16
|
||||
@{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vbrev8_v $V8, $V8]}
|
||||
@{[vgmul_vv $V16, $V8]}
|
||||
|
||||
# Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28.
|
||||
# Reverse the bits order back.
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
|
||||
# Prepare the x^n multiplier in v20. The `n` is the aes-xts block number
|
||||
# in a LMUL=4 register group.
|
||||
# n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4))
|
||||
# = (VLEN/32)
|
||||
# We could use vsetvli with `e32, m1` to compute the `n` number.
|
||||
@{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]}
|
||||
li $T1, 1
|
||||
sll $T0, $T1, $T0
|
||||
@{[vsetivli "zero", 2, "e64", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_i $V0, 0]}
|
||||
@{[vsetivli "zero", 1, "e64", "m1", "tu", "ma"]}
|
||||
@{[vmv_v_x $V0, $T0]}
|
||||
@{[vsetivli "zero", 2, "e64", "m1", "ta", "ma"]}
|
||||
@{[vbrev8_v $V0, $V0]}
|
||||
@{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V20, 0]}
|
||||
@{[vaesz_vs $V20, $V0]}
|
||||
|
||||
j 2f
|
||||
1:
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vbrev8_v $V16, $V28]}
|
||||
2:
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# prepare xts enc last block's input(v24) and iv(v28)
|
||||
sub handle_xts_enc_last_block {
|
||||
my $code=<<___;
|
||||
bnez $TAIL_LENGTH, 1f
|
||||
ret
|
||||
1:
|
||||
# slidedown second to last block
|
||||
addi $VL, $VL, -4
|
||||
@{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
|
||||
# ciphertext
|
||||
@{[vslidedown_vx $V24, $V24, $VL]}
|
||||
# multiplier
|
||||
@{[vslidedown_vx $V16, $V16, $VL]}
|
||||
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_v $V25, $V24]}
|
||||
|
||||
# load last block into v24
|
||||
# note: We should load the last block before store the second to last block
|
||||
# for in-place operation.
|
||||
@{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]}
|
||||
@{[vle8_v $V24, $INPUT]}
|
||||
|
||||
# setup `x` multiplier with byte-reversed order
|
||||
# 0b00000010 => 0b01000000 (0x40)
|
||||
li $T0, 0x40
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_i $V28, 0]}
|
||||
@{[vsetivli "zero", 1, "e8", "m1", "tu", "ma"]}
|
||||
@{[vmv_v_x $V28, $T0]}
|
||||
|
||||
# compute IV for last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vgmul_vv $V16, $V28]}
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
|
||||
# store second to last block
|
||||
@{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "ta", "ma"]}
|
||||
@{[vse8_v $V25, $OUTPUT]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# prepare xts dec second to last block's input(v24) and iv(v29) and
|
||||
# last block's and iv(v28)
|
||||
sub handle_xts_dec_last_block {
|
||||
my $code=<<___;
|
||||
bnez $TAIL_LENGTH, 1f
|
||||
ret
|
||||
1:
|
||||
# load second to last block's ciphertext
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
addi $INPUT, $INPUT, 16
|
||||
|
||||
# setup `x` multiplier with byte-reversed order
|
||||
# 0b00000010 => 0b01000000 (0x40)
|
||||
li $T0, 0x40
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_i $V20, 0]}
|
||||
@{[vsetivli "zero", 1, "e8", "m1", "tu", "ma"]}
|
||||
@{[vmv_v_x $V20, $T0]}
|
||||
|
||||
beqz $LENGTH, 1f
|
||||
# slidedown third to last block
|
||||
addi $VL, $VL, -4
|
||||
@{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
|
||||
# multiplier
|
||||
@{[vslidedown_vx $V16, $V16, $VL]}
|
||||
|
||||
# compute IV for last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
|
||||
# compute IV for second to last block
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
@{[vbrev8_v $V29, $V16]}
|
||||
j 2f
|
||||
1:
|
||||
# compute IV for second to last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
@{[vbrev8_v $V29, $V16]}
|
||||
2:
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Load all 11 round keys to v1-v11 registers.
|
||||
sub aes_128_load_key {
|
||||
my $code=<<___;
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V2, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V3, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V4, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V5, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V6, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V7, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V8, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V9, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V10, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V11, $KEY1]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Load all 15 round keys to v1-v15 registers.
|
||||
sub aes_256_load_key {
|
||||
my $code=<<___;
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V2, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V3, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V4, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V5, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V6, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V7, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V8, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V9, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V10, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V11, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V12, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V13, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V14, $KEY1]}
|
||||
addi $KEY1, $KEY1, 16
|
||||
@{[vle32_v $V15, $KEY1]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# aes-128 enc with round keys v1-v11
|
||||
sub aes_128_enc {
|
||||
my $code=<<___;
|
||||
@{[vaesz_vs $V24, $V1]}
|
||||
@{[vaesem_vs $V24, $V2]}
|
||||
@{[vaesem_vs $V24, $V3]}
|
||||
@{[vaesem_vs $V24, $V4]}
|
||||
@{[vaesem_vs $V24, $V5]}
|
||||
@{[vaesem_vs $V24, $V6]}
|
||||
@{[vaesem_vs $V24, $V7]}
|
||||
@{[vaesem_vs $V24, $V8]}
|
||||
@{[vaesem_vs $V24, $V9]}
|
||||
@{[vaesem_vs $V24, $V10]}
|
||||
@{[vaesef_vs $V24, $V11]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# aes-128 dec with round keys v1-v11
|
||||
sub aes_128_dec {
|
||||
my $code=<<___;
|
||||
@{[vaesz_vs $V24, $V11]}
|
||||
@{[vaesdm_vs $V24, $V10]}
|
||||
@{[vaesdm_vs $V24, $V9]}
|
||||
@{[vaesdm_vs $V24, $V8]}
|
||||
@{[vaesdm_vs $V24, $V7]}
|
||||
@{[vaesdm_vs $V24, $V6]}
|
||||
@{[vaesdm_vs $V24, $V5]}
|
||||
@{[vaesdm_vs $V24, $V4]}
|
||||
@{[vaesdm_vs $V24, $V3]}
|
||||
@{[vaesdm_vs $V24, $V2]}
|
||||
@{[vaesdf_vs $V24, $V1]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# aes-256 enc with round keys v1-v15
|
||||
sub aes_256_enc {
|
||||
my $code=<<___;
|
||||
@{[vaesz_vs $V24, $V1]}
|
||||
@{[vaesem_vs $V24, $V2]}
|
||||
@{[vaesem_vs $V24, $V3]}
|
||||
@{[vaesem_vs $V24, $V4]}
|
||||
@{[vaesem_vs $V24, $V5]}
|
||||
@{[vaesem_vs $V24, $V6]}
|
||||
@{[vaesem_vs $V24, $V7]}
|
||||
@{[vaesem_vs $V24, $V8]}
|
||||
@{[vaesem_vs $V24, $V9]}
|
||||
@{[vaesem_vs $V24, $V10]}
|
||||
@{[vaesem_vs $V24, $V11]}
|
||||
@{[vaesem_vs $V24, $V12]}
|
||||
@{[vaesem_vs $V24, $V13]}
|
||||
@{[vaesem_vs $V24, $V14]}
|
||||
@{[vaesef_vs $V24, $V15]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# aes-256 dec with round keys v1-v15
|
||||
sub aes_256_dec {
|
||||
my $code=<<___;
|
||||
@{[vaesz_vs $V24, $V15]}
|
||||
@{[vaesdm_vs $V24, $V14]}
|
||||
@{[vaesdm_vs $V24, $V13]}
|
||||
@{[vaesdm_vs $V24, $V12]}
|
||||
@{[vaesdm_vs $V24, $V11]}
|
||||
@{[vaesdm_vs $V24, $V10]}
|
||||
@{[vaesdm_vs $V24, $V9]}
|
||||
@{[vaesdm_vs $V24, $V8]}
|
||||
@{[vaesdm_vs $V24, $V7]}
|
||||
@{[vaesdm_vs $V24, $V6]}
|
||||
@{[vaesdm_vs $V24, $V5]}
|
||||
@{[vaesdm_vs $V24, $V4]}
|
||||
@{[vaesdm_vs $V24, $V3]}
|
||||
@{[vaesdm_vs $V24, $V2]}
|
||||
@{[vaesdf_vs $V24, $V1]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt
|
||||
.type rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,\@function
|
||||
rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt:
|
||||
@{[compute_xts_iv0]}
|
||||
|
||||
# aes block size is 16
|
||||
andi $TAIL_LENGTH, $LENGTH, 15
|
||||
mv $STORE_LEN32, $LENGTH
|
||||
beqz $TAIL_LENGTH, 1f
|
||||
sub $LENGTH, $LENGTH, $TAIL_LENGTH
|
||||
addi $STORE_LEN32, $LENGTH, -16
|
||||
1:
|
||||
# We make the `LENGTH` become e32 length here.
|
||||
srli $LEN32, $LENGTH, 2
|
||||
srli $STORE_LEN32, $STORE_LEN32, 2
|
||||
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEY1)
|
||||
li $T1, 14
|
||||
li $T2, 10
|
||||
beq $T0, $T1, aes_xts_enc_256
|
||||
beq $T0, $T2, aes_xts_enc_128
|
||||
.size rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_xts_enc_128:
|
||||
@{[init_first_round]}
|
||||
@{[aes_128_load_key]}
|
||||
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
j 1f
|
||||
|
||||
.Lenc_blocks_128:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
# load plaintext into v24
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
# update iv
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
# reverse the iv's bits order back
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
1:
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
slli $T0, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INPUT, $INPUT, $T0
|
||||
@{[aes_128_enc]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store ciphertext
|
||||
@{[vsetvli "zero", $STORE_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
add $OUTPUT, $OUTPUT, $T0
|
||||
sub $STORE_LEN32, $STORE_LEN32, $VL
|
||||
|
||||
bnez $LEN32, .Lenc_blocks_128
|
||||
|
||||
@{[handle_xts_enc_last_block]}
|
||||
|
||||
# xts last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
@{[aes_128_enc]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store last block ciphertext
|
||||
addi $OUTPUT, $OUTPUT, -16
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
|
||||
ret
|
||||
.size aes_xts_enc_128,.-aes_xts_enc_128
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_xts_enc_256:
|
||||
@{[init_first_round]}
|
||||
@{[aes_256_load_key]}
|
||||
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
j 1f
|
||||
|
||||
.Lenc_blocks_256:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
# load plaintext into v24
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
# update iv
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
# reverse the iv's bits order back
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
1:
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
slli $T0, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INPUT, $INPUT, $T0
|
||||
@{[aes_256_enc]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store ciphertext
|
||||
@{[vsetvli "zero", $STORE_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
add $OUTPUT, $OUTPUT, $T0
|
||||
sub $STORE_LEN32, $STORE_LEN32, $VL
|
||||
|
||||
bnez $LEN32, .Lenc_blocks_256
|
||||
|
||||
@{[handle_xts_enc_last_block]}
|
||||
|
||||
# xts last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
@{[aes_256_enc]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store last block ciphertext
|
||||
addi $OUTPUT, $OUTPUT, -16
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
|
||||
ret
|
||||
.size aes_xts_enc_256,.-aes_xts_enc_256
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt(const unsigned char *in,
|
||||
# unsigned char *out, size_t length,
|
||||
# const AES_KEY *key1,
|
||||
# const AES_KEY *key2,
|
||||
# const unsigned char iv[16])
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt
|
||||
.type rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,\@function
|
||||
rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt:
|
||||
@{[compute_xts_iv0]}
|
||||
|
||||
# aes block size is 16
|
||||
andi $TAIL_LENGTH, $LENGTH, 15
|
||||
beqz $TAIL_LENGTH, 1f
|
||||
sub $LENGTH, $LENGTH, $TAIL_LENGTH
|
||||
addi $LENGTH, $LENGTH, -16
|
||||
1:
|
||||
# We make the `LENGTH` become e32 length here.
|
||||
srli $LEN32, $LENGTH, 2
|
||||
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEY1)
|
||||
li $T1, 14
|
||||
li $T2, 10
|
||||
beq $T0, $T1, aes_xts_dec_256
|
||||
beq $T0, $T2, aes_xts_dec_128
|
||||
.size rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_xts_dec_128:
|
||||
@{[init_first_round]}
|
||||
@{[aes_128_load_key]}
|
||||
|
||||
beqz $LEN32, 2f
|
||||
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
j 1f
|
||||
|
||||
.Ldec_blocks_128:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
# load ciphertext into v24
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
# update iv
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
# reverse the iv's bits order back
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
1:
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
slli $T0, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INPUT, $INPUT, $T0
|
||||
@{[aes_128_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store plaintext
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
add $OUTPUT, $OUTPUT, $T0
|
||||
|
||||
bnez $LEN32, .Ldec_blocks_128
|
||||
|
||||
2:
|
||||
@{[handle_xts_dec_last_block]}
|
||||
|
||||
## xts second to last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V29]}
|
||||
@{[aes_128_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V29]}
|
||||
@{[vmv_v_v $V25, $V24]}
|
||||
|
||||
# load last block ciphertext
|
||||
@{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]}
|
||||
@{[vle8_v $V24, $INPUT]}
|
||||
|
||||
# store second to last block plaintext
|
||||
addi $T0, $OUTPUT, 16
|
||||
@{[vse8_v $V25, $T0]}
|
||||
|
||||
## xts last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
@{[aes_128_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store second to last block plaintext
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
|
||||
ret
|
||||
.size aes_xts_dec_128,.-aes_xts_dec_128
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_xts_dec_256:
|
||||
@{[init_first_round]}
|
||||
@{[aes_256_load_key]}
|
||||
|
||||
beqz $LEN32, 2f
|
||||
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
j 1f
|
||||
|
||||
.Ldec_blocks_256:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
# load ciphertext into v24
|
||||
@{[vle32_v $V24, $INPUT]}
|
||||
# update iv
|
||||
@{[vgmul_vv $V16, $V20]}
|
||||
# reverse the iv's bits order back
|
||||
@{[vbrev8_v $V28, $V16]}
|
||||
1:
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
slli $T0, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INPUT, $INPUT, $T0
|
||||
@{[aes_256_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store plaintext
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
add $OUTPUT, $OUTPUT, $T0
|
||||
|
||||
bnez $LEN32, .Ldec_blocks_256
|
||||
|
||||
2:
|
||||
@{[handle_xts_dec_last_block]}
|
||||
|
||||
## xts second to last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V29]}
|
||||
@{[aes_256_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V29]}
|
||||
@{[vmv_v_v $V25, $V24]}
|
||||
|
||||
# load last block ciphertext
|
||||
@{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]}
|
||||
@{[vle8_v $V24, $INPUT]}
|
||||
|
||||
# store second to last block plaintext
|
||||
addi $T0, $OUTPUT, 16
|
||||
@{[vse8_v $V25, $T0]}
|
||||
|
||||
## xts last block
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
@{[aes_256_dec]}
|
||||
@{[vxor_vv $V24, $V24, $V28]}
|
||||
|
||||
# store second to last block plaintext
|
||||
@{[vse32_v $V24, $OUTPUT]}
|
||||
|
||||
ret
|
||||
.size aes_xts_dec_256,.-aes_xts_dec_256
|
||||
___
|
||||
}
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,376 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Jerry Shih <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
# - RISC-V Vector AES block cipher extension ('Zvkned')
|
||||
# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in,
|
||||
# unsigned char *out, size_t blocks,
|
||||
# const void *key,
|
||||
# const unsigned char ivec[16]);
|
||||
{
|
||||
my ($INP, $OUTP, $BLOCK_NUM, $KEYP, $IVP) = ("a0", "a1", "a2", "a3", "a4");
|
||||
my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3");
|
||||
my ($VL) = ("t4");
|
||||
my ($LEN32) = ("t5");
|
||||
my ($CTR) = ("t6");
|
||||
my ($MASK) = ("v0");
|
||||
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
|
||||
$V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
|
||||
$V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
|
||||
$V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map("v$_",(0..31));
|
||||
|
||||
# Prepare the AES ctr input data into v16.
|
||||
sub init_aes_ctr_input {
|
||||
my $code=<<___;
|
||||
# Setup mask into v0
|
||||
# The mask pattern for 4*N-th elements
|
||||
# mask v0: [000100010001....]
|
||||
# Note:
|
||||
# We could setup the mask just for the maximum element length instead of
|
||||
# the VLMAX.
|
||||
li $T0, 0b10001000
|
||||
@{[vsetvli $T2, "zero", "e8", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_x $MASK, $T0]}
|
||||
# Load IV.
|
||||
# v31:[IV0, IV1, IV2, big-endian count]
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V31, $IVP]}
|
||||
# Convert the big-endian counter into little-endian.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]}
|
||||
@{[vrev8_v $V31, $V31, $MASK]}
|
||||
# Splat the IV to v16
|
||||
@{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V16, 0]}
|
||||
@{[vaesz_vs $V16, $V31]}
|
||||
# Prepare the ctr pattern into v20
|
||||
# v20: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...]
|
||||
@{[viota_m $V20, $MASK, $MASK]}
|
||||
# v16:[IV0, IV1, IV2, count+0, IV0, IV1, IV2, count+1, ...]
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]}
|
||||
@{[vadd_vv $V16, $V16, $V20, $MASK]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl rv64i_zvkb_zvkned_ctr32_encrypt_blocks
|
||||
.type rv64i_zvkb_zvkned_ctr32_encrypt_blocks,\@function
|
||||
rv64i_zvkb_zvkned_ctr32_encrypt_blocks:
|
||||
beqz $BLOCK_NUM, 1f
|
||||
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEYP)
|
||||
li $T1, 14
|
||||
li $T2, 12
|
||||
li $T3, 10
|
||||
|
||||
slli $LEN32, $BLOCK_NUM, 2
|
||||
|
||||
beq $T0, $T1, ctr32_encrypt_blocks_256
|
||||
beq $T0, $T2, ctr32_encrypt_blocks_192
|
||||
beq $T0, $T3, ctr32_encrypt_blocks_128
|
||||
|
||||
1:
|
||||
ret
|
||||
|
||||
.size rv64i_zvkb_zvkned_ctr32_encrypt_blocks,.-rv64i_zvkb_zvkned_ctr32_encrypt_blocks
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
ctr32_encrypt_blocks_128:
|
||||
# Load all 11 round keys to v1-v11 registers.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V2, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V3, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V4, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V5, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V6, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V7, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V8, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V9, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V10, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V11, $KEYP]}
|
||||
|
||||
@{[init_aes_ctr_input]}
|
||||
|
||||
##### AES body
|
||||
j 2f
|
||||
1:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]}
|
||||
# Increase ctr in v16.
|
||||
@{[vadd_vx $V16, $V16, $CTR, $MASK]}
|
||||
2:
|
||||
# Load plaintext into v20
|
||||
@{[vle32_v $V20, $INP]}
|
||||
slli $T0, $VL, 2
|
||||
srli $CTR, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INP, $INP, $T0
|
||||
# Prepare the AES ctr input into v24.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vmv_v_v $V24, $V16]}
|
||||
@{[vrev8_v $V24, $V24, $MASK]}
|
||||
|
||||
@{[vaesz_vs $V24, $V1]}
|
||||
@{[vaesem_vs $V24, $V2]}
|
||||
@{[vaesem_vs $V24, $V3]}
|
||||
@{[vaesem_vs $V24, $V4]}
|
||||
@{[vaesem_vs $V24, $V5]}
|
||||
@{[vaesem_vs $V24, $V6]}
|
||||
@{[vaesem_vs $V24, $V7]}
|
||||
@{[vaesem_vs $V24, $V8]}
|
||||
@{[vaesem_vs $V24, $V9]}
|
||||
@{[vaesem_vs $V24, $V10]}
|
||||
@{[vaesef_vs $V24, $V11]}
|
||||
|
||||
# ciphertext
|
||||
@{[vxor_vv $V24, $V24, $V20]}
|
||||
|
||||
# Store the ciphertext.
|
||||
@{[vse32_v $V24, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
bnez $LEN32, 1b
|
||||
|
||||
ret
|
||||
.size ctr32_encrypt_blocks_128,.-ctr32_encrypt_blocks_128
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
ctr32_encrypt_blocks_192:
|
||||
# Load all 13 round keys to v1-v13 registers.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V2, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V3, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V4, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V5, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V6, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V7, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V8, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V9, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V10, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V11, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V12, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V13, $KEYP]}
|
||||
|
||||
@{[init_aes_ctr_input]}
|
||||
|
||||
##### AES body
|
||||
j 2f
|
||||
1:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]}
|
||||
# Increase ctr in v16.
|
||||
@{[vadd_vx $V16, $V16, $CTR, $MASK]}
|
||||
2:
|
||||
# Load plaintext into v20
|
||||
@{[vle32_v $V20, $INP]}
|
||||
slli $T0, $VL, 2
|
||||
srli $CTR, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INP, $INP, $T0
|
||||
# Prepare the AES ctr input into v24.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vmv_v_v $V24, $V16]}
|
||||
@{[vrev8_v $V24, $V24, $MASK]}
|
||||
|
||||
@{[vaesz_vs $V24, $V1]}
|
||||
@{[vaesem_vs $V24, $V2]}
|
||||
@{[vaesem_vs $V24, $V3]}
|
||||
@{[vaesem_vs $V24, $V4]}
|
||||
@{[vaesem_vs $V24, $V5]}
|
||||
@{[vaesem_vs $V24, $V6]}
|
||||
@{[vaesem_vs $V24, $V7]}
|
||||
@{[vaesem_vs $V24, $V8]}
|
||||
@{[vaesem_vs $V24, $V9]}
|
||||
@{[vaesem_vs $V24, $V10]}
|
||||
@{[vaesem_vs $V24, $V11]}
|
||||
@{[vaesem_vs $V24, $V12]}
|
||||
@{[vaesef_vs $V24, $V13]}
|
||||
|
||||
# ciphertext
|
||||
@{[vxor_vv $V24, $V24, $V20]}
|
||||
|
||||
# Store the ciphertext.
|
||||
@{[vse32_v $V24, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
bnez $LEN32, 1b
|
||||
|
||||
ret
|
||||
.size ctr32_encrypt_blocks_192,.-ctr32_encrypt_blocks_192
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
ctr32_encrypt_blocks_256:
|
||||
# Load all 15 round keys to v1-v15 registers.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V2, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V3, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V4, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V5, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V6, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V7, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V8, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V9, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V10, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V11, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V12, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V13, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V14, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V15, $KEYP]}
|
||||
|
||||
@{[init_aes_ctr_input]}
|
||||
|
||||
##### AES body
|
||||
j 2f
|
||||
1:
|
||||
@{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]}
|
||||
# Increase ctr in v16.
|
||||
@{[vadd_vx $V16, $V16, $CTR, $MASK]}
|
||||
2:
|
||||
# Load plaintext into v20
|
||||
@{[vle32_v $V20, $INP]}
|
||||
slli $T0, $VL, 2
|
||||
srli $CTR, $VL, 2
|
||||
sub $LEN32, $LEN32, $VL
|
||||
add $INP, $INP, $T0
|
||||
# Prepare the AES ctr input into v24.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vmv_v_v $V24, $V16]}
|
||||
@{[vrev8_v $V24, $V24, $MASK]}
|
||||
|
||||
@{[vaesz_vs $V24, $V1]}
|
||||
@{[vaesem_vs $V24, $V2]}
|
||||
@{[vaesem_vs $V24, $V3]}
|
||||
@{[vaesem_vs $V24, $V4]}
|
||||
@{[vaesem_vs $V24, $V5]}
|
||||
@{[vaesem_vs $V24, $V6]}
|
||||
@{[vaesem_vs $V24, $V7]}
|
||||
@{[vaesem_vs $V24, $V8]}
|
||||
@{[vaesem_vs $V24, $V9]}
|
||||
@{[vaesem_vs $V24, $V10]}
|
||||
@{[vaesem_vs $V24, $V11]}
|
||||
@{[vaesem_vs $V24, $V12]}
|
||||
@{[vaesem_vs $V24, $V13]}
|
||||
@{[vaesem_vs $V24, $V14]}
|
||||
@{[vaesef_vs $V24, $V15]}
|
||||
|
||||
# ciphertext
|
||||
@{[vxor_vv $V24, $V24, $V20]}
|
||||
|
||||
# Store the ciphertext.
|
||||
@{[vse32_v $V24, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
bnez $LEN32, 1b
|
||||
|
||||
ret
|
||||
.size ctr32_encrypt_blocks_256,.-ctr32_encrypt_blocks_256
|
||||
___
|
||||
}
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,744 @@
|
||||
# Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
# Copyright (c) 2020, Intel Corporation. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
#
|
||||
# Originally written by Sergey Kirillov and Andrey Matyukov.
|
||||
# Special thanks to Ilya Albrekht for his valuable hints.
|
||||
# Intel Corporation
|
||||
#
|
||||
# December 2020
|
||||
#
|
||||
# Initial release.
|
||||
#
|
||||
# Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues.
|
||||
#
|
||||
# IceLake-Client @ 1.3GHz
|
||||
# |---------+----------------------+--------------+-------------|
|
||||
# | | OpenSSL 3.0.0-alpha9 | this | Unit |
|
||||
# |---------+----------------------+--------------+-------------|
|
||||
# | rsa2048 | 2 127 659 | 1 015 625 | cycles/sign |
|
||||
# | | 611 | 1280 / +109% | sign/s |
|
||||
# |---------+----------------------+--------------+-------------|
|
||||
#
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
||||
$avx512ifma=0;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||||
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
||||
die "can't locate x86_64-xlate.pl";
|
||||
|
||||
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
||||
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
|
||||
$avx512ifma = ($1>=2.26);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
|
||||
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
|
||||
$avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && `$ENV{CC} -v 2>&1`
|
||||
=~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
|
||||
my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
|
||||
if ($1) {
|
||||
# Apple conditions, they use a different version series, see
|
||||
# https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2
|
||||
# clang 7.0.0 is Apple clang 10.0.1
|
||||
$avx512ifma = ($ver>=10.0001)
|
||||
} else {
|
||||
$avx512ifma = ($ver>=7.0);
|
||||
}
|
||||
}
|
||||
|
||||
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
|
||||
or die "can't call $xlate: $!";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
if ($avx512ifma>0) {{{
|
||||
@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
|
||||
|
||||
$code.=<<___;
|
||||
.extern OPENSSL_ia32cap_P
|
||||
.globl ossl_rsaz_avx512ifma_eligible
|
||||
.type ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent
|
||||
.align 32
|
||||
ossl_rsaz_avx512ifma_eligible:
|
||||
mov OPENSSL_ia32cap_P+8(%rip), %ecx
|
||||
xor %eax,%eax
|
||||
and \$`1<<31|1<<21|1<<17|1<<16`, %ecx # avx512vl + avx512ifma + avx512dq + avx512f
|
||||
cmp \$`1<<31|1<<21|1<<17|1<<16`, %ecx
|
||||
cmove %ecx,%eax
|
||||
ret
|
||||
.size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52.
|
||||
#
|
||||
# AMM is defined as presented in the paper [1].
|
||||
#
|
||||
# The input and output are presented in 2^52 radix domain, i.e.
|
||||
# |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed.
|
||||
# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
|
||||
#
|
||||
# NB: the AMM implementation does not perform "conditional" subtraction step
|
||||
# specified in the original algorithm as according to the Lemma 1 from the paper
|
||||
# [2], the result will be always < 2*m and can be used as a direct input to
|
||||
# the next AMM iteration. This post-condition is true, provided the correct
|
||||
# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k,
|
||||
# which matches our case: 1040 > 1024 + 2 * 1.
|
||||
#
|
||||
# [1] Gueron, S. Efficient software implementations of modular exponentiation.
|
||||
# DOI: 10.1007/s13389-012-0031-5
|
||||
# [2] Gueron, S. Enhanced Montgomery Multiplication.
|
||||
# DOI: 10.1007/3-540-36400-5_5
|
||||
#
|
||||
# void ossl_rsaz_amm52x20_x1_ifma256(BN_ULONG *res,
|
||||
# const BN_ULONG *a,
|
||||
# const BN_ULONG *b,
|
||||
# const BN_ULONG *m,
|
||||
# BN_ULONG k0);
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
|
||||
my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
|
||||
|
||||
my $mask52 = "%rax";
|
||||
my $acc0_0 = "%r9";
|
||||
my $acc0_0_low = "%r9d";
|
||||
my $acc0_1 = "%r15";
|
||||
my $acc0_1_low = "%r15d";
|
||||
my $b_ptr = "%r11";
|
||||
|
||||
my $iter = "%ebx";
|
||||
|
||||
my $zero = "%ymm0";
|
||||
my $Bi = "%ymm1";
|
||||
my $Yi = "%ymm2";
|
||||
my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(16..19)));
|
||||
my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(20..23)));
|
||||
|
||||
# Registers mapping for normalization.
|
||||
my ($T0,$T0h,$T1,$T1h,$T2) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (25..26)));
|
||||
|
||||
sub amm52x20_x1() {
|
||||
# _data_offset - offset in the |a| or |m| arrays pointing to the beginning
|
||||
# of data for corresponding AMM operation;
|
||||
# _b_offset - offset in the |b| array pointing to the next qword digit;
|
||||
my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_;
|
||||
my $_R0_xmm = $_R0;
|
||||
$_R0_xmm =~ s/%y/%x/;
|
||||
$code.=<<___;
|
||||
movq $_b_offset($b_ptr), %r13 # b[i]
|
||||
|
||||
vpbroadcastq %r13, $Bi # broadcast b[i]
|
||||
movq $_data_offset($a), %rdx
|
||||
mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2)
|
||||
addq %r13, $_acc # acc += t0
|
||||
movq %r12, %r10
|
||||
adcq \$0, %r10 # t2 += CF
|
||||
|
||||
movq $_k0, %r13
|
||||
imulq $_acc, %r13 # acc * k0
|
||||
andq $mask52, %r13 # yi = (acc * k0) & mask52
|
||||
|
||||
vpbroadcastq %r13, $Yi # broadcast y[i]
|
||||
movq $_data_offset($m), %rdx
|
||||
mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1)
|
||||
addq %r13, $_acc # acc += t0
|
||||
adcq %r12, %r10 # t2 += (t1 + CF)
|
||||
|
||||
shrq \$52, $_acc
|
||||
salq \$12, %r10
|
||||
or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12))
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
|
||||
# Shift accumulators right by 1 qword, zero extending the highest one
|
||||
valignq \$1, $_R0, $_R0h, $_R0
|
||||
valignq \$1, $_R0h, $_R1, $_R0h
|
||||
valignq \$1, $_R1, $_R1h, $_R1
|
||||
valignq \$1, $_R1h, $_R2, $_R1h
|
||||
valignq \$1, $_R2, $zero, $_R2
|
||||
|
||||
vmovq $_R0_xmm, %r13
|
||||
addq %r13, $_acc # acc += R0[0]
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
___
|
||||
}
|
||||
|
||||
# Normalization routine: handles carry bits and gets bignum qwords to normalized
|
||||
# 2^52 representation.
|
||||
#
|
||||
# Uses %r8-14,%e[bcd]x
|
||||
sub amm52x20_x1_norm {
|
||||
my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_;
|
||||
$code.=<<___;
|
||||
# Put accumulator to low qword in R0
|
||||
vpbroadcastq $_acc, $T0
|
||||
vpblendd \$3, $T0, $_R0, $_R0
|
||||
|
||||
# Extract "carries" (12 high bits) from each QW of R0..R2
|
||||
# Save them to LSB of QWs in T0..T2
|
||||
vpsrlq \$52, $_R0, $T0
|
||||
vpsrlq \$52, $_R0h, $T0h
|
||||
vpsrlq \$52, $_R1, $T1
|
||||
vpsrlq \$52, $_R1h, $T1h
|
||||
vpsrlq \$52, $_R2, $T2
|
||||
|
||||
# "Shift left" T0..T2 by 1 QW
|
||||
valignq \$3, $T1h, $T2, $T2
|
||||
valignq \$3, $T1, $T1h, $T1h
|
||||
valignq \$3, $T0h, $T1, $T1
|
||||
valignq \$3, $T0, $T0h, $T0h
|
||||
valignq \$3, .Lzeros(%rip), $T0, $T0
|
||||
|
||||
# Drop "carries" from R0..R2 QWs
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
|
||||
# Sum R0..R2 with corresponding adjusted carries
|
||||
vpaddq $T0, $_R0, $_R0
|
||||
vpaddq $T0h, $_R0h, $_R0h
|
||||
vpaddq $T1, $_R1, $_R1
|
||||
vpaddq $T1h, $_R1h, $_R1h
|
||||
vpaddq $T2, $_R2, $_R2
|
||||
|
||||
# Now handle carry bits from this addition
|
||||
# Get mask of QWs which 52-bit parts overflow...
|
||||
vpcmpuq \$6, .Lmask52x4(%rip), $_R0, %k1 # OP=nle (i.e. gt)
|
||||
vpcmpuq \$6, .Lmask52x4(%rip), $_R0h, %k2
|
||||
vpcmpuq \$6, .Lmask52x4(%rip), $_R1, %k3
|
||||
vpcmpuq \$6, .Lmask52x4(%rip), $_R1h, %k4
|
||||
vpcmpuq \$6, .Lmask52x4(%rip), $_R2, %k5
|
||||
kmovb %k1, %r14d # k1
|
||||
kmovb %k2, %r13d # k1h
|
||||
kmovb %k3, %r12d # k2
|
||||
kmovb %k4, %r11d # k2h
|
||||
kmovb %k5, %r10d # k3
|
||||
|
||||
# ...or saturated
|
||||
vpcmpuq \$0, .Lmask52x4(%rip), $_R0, %k1 # OP=eq
|
||||
vpcmpuq \$0, .Lmask52x4(%rip), $_R0h, %k2
|
||||
vpcmpuq \$0, .Lmask52x4(%rip), $_R1, %k3
|
||||
vpcmpuq \$0, .Lmask52x4(%rip), $_R1h, %k4
|
||||
vpcmpuq \$0, .Lmask52x4(%rip), $_R2, %k5
|
||||
kmovb %k1, %r9d # k4
|
||||
kmovb %k2, %r8d # k4h
|
||||
kmovb %k3, %ebx # k5
|
||||
kmovb %k4, %ecx # k5h
|
||||
kmovb %k5, %edx # k6
|
||||
|
||||
# Get mask of QWs where carries shall be propagated to.
|
||||
# Merge 4-bit masks to 8-bit values to use add with carry.
|
||||
shl \$4, %r13b
|
||||
or %r13b, %r14b
|
||||
shl \$4, %r11b
|
||||
or %r11b, %r12b
|
||||
|
||||
add %r14b, %r14b
|
||||
adc %r12b, %r12b
|
||||
adc %r10b, %r10b
|
||||
|
||||
shl \$4, %r8b
|
||||
or %r8b,%r9b
|
||||
shl \$4, %cl
|
||||
or %cl, %bl
|
||||
|
||||
add %r9b, %r14b
|
||||
adc %bl, %r12b
|
||||
adc %dl, %r10b
|
||||
|
||||
xor %r9b, %r14b
|
||||
xor %bl, %r12b
|
||||
xor %dl, %r10b
|
||||
|
||||
kmovb %r14d, %k1
|
||||
shr \$4, %r14b
|
||||
kmovb %r14d, %k2
|
||||
kmovb %r12d, %k3
|
||||
shr \$4, %r12b
|
||||
kmovb %r12d, %k4
|
||||
kmovb %r10d, %k5
|
||||
|
||||
# Add carries according to the obtained mask
|
||||
vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1}
|
||||
vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2}
|
||||
vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3}
|
||||
vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4}
|
||||
vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5}
|
||||
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
___
|
||||
}
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x1_ifma256
|
||||
.type ossl_rsaz_amm52x20_x1_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x20_x1_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
.Lossl_rsaz_amm52x20_x1_ifma256_body:
|
||||
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
# Loop over 20 digits unrolled by 4
|
||||
mov \$5, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop5:
|
||||
___
|
||||
foreach my $idx (0..3) {
|
||||
&amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0);
|
||||
}
|
||||
$code.=<<___;
|
||||
lea `4*8`($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop5
|
||||
___
|
||||
&amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
|
||||
vzeroupper
|
||||
mov 0(%rsp),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rsp),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rsp),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rsp),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rsp),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rsp),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rsp),%rsp
|
||||
.cfi_adjust_cfa_offset -48
|
||||
.Lossl_rsaz_amm52x20_x1_ifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256
|
||||
___
|
||||
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lmask52x4:
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52
|
||||
#
|
||||
# See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost
|
||||
# Montgomery Multiplication algorithm and function input parameters description.
|
||||
#
|
||||
# This function does two AMMs for two independent inputs, hence dual.
|
||||
#
|
||||
# void ossl_rsaz_amm52x20_x2_ifma256(BN_ULONG out[2][20],
|
||||
# const BN_ULONG a[2][20],
|
||||
# const BN_ULONG b[2][20],
|
||||
# const BN_ULONG m[2][20],
|
||||
# const BN_ULONG k0[2]);
|
||||
###############################################################################
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x2_ifma256
|
||||
.type ossl_rsaz_amm52x20_x2_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x20_x2_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
.Lossl_rsaz_amm52x20_x2_ifma256_body:
|
||||
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
vmovdqa64 $zero, $R0_1
|
||||
vmovdqa64 $zero, $R0_1h
|
||||
vmovdqa64 $zero, $R1_1
|
||||
vmovdqa64 $zero, $R1_1h
|
||||
vmovdqa64 $zero, $R2_1
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
xorl $acc0_1_low, $acc0_1_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
mov \$20, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop20:
|
||||
___
|
||||
&amm52x20_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)");
|
||||
# 20*8 = offset of the next dimension in two-dimension array
|
||||
&amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)");
|
||||
$code.=<<___;
|
||||
lea 8($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop20
|
||||
___
|
||||
&amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
|
||||
&amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
|
||||
vmovdqu64 $R0_1, `5*32`($res)
|
||||
vmovdqu64 $R0_1h, `6*32`($res)
|
||||
vmovdqu64 $R1_1, `7*32`($res)
|
||||
vmovdqu64 $R1_1h, `8*32`($res)
|
||||
vmovdqu64 $R2_1, `9*32`($res)
|
||||
|
||||
vzeroupper
|
||||
mov 0(%rsp),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rsp),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rsp),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rsp),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rsp),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rsp),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rsp),%rsp
|
||||
.cfi_adjust_cfa_offset -48
|
||||
.Lossl_rsaz_amm52x20_x2_ifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x20_x2_ifma256, .-ossl_rsaz_amm52x20_x2_ifma256
|
||||
___
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Constant time extraction from the precomputed table of powers base^i, where
|
||||
# i = 0..2^EXP_WIN_SIZE-1
|
||||
#
|
||||
# The input |red_table| contains precomputations for two independent base values.
|
||||
# |red_table_idx1| and |red_table_idx2| are corresponding power indexes.
|
||||
#
|
||||
# Extracted value (output) is 2 20 digit numbers in 2^52 radix.
|
||||
#
|
||||
# void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y,
|
||||
# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20],
|
||||
# int red_table_idx1, int red_table_idx2);
|
||||
#
|
||||
# EXP_WIN_SIZE = 5
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters
|
||||
my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
||||
("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
||||
|
||||
my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5));
|
||||
my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19));
|
||||
my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24));
|
||||
|
||||
my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9);
|
||||
my $t0xmm = $t0;
|
||||
$t0xmm =~ s/%y/%x/;
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.align 32
|
||||
.globl ossl_extract_multiplier_2x20_win5
|
||||
.type ossl_extract_multiplier_2x20_win5,\@abi-omnipotent
|
||||
ossl_extract_multiplier_2x20_win5:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
vmovdqa64 .Lones(%rip), $ones # broadcast ones
|
||||
vpbroadcastq $red_tbl_idx1, $idx1
|
||||
vpbroadcastq $red_tbl_idx2, $idx2
|
||||
leaq `(1<<5)*2*20*8`($red_tbl), %rax # holds end of the tbl
|
||||
|
||||
# zeroing t0..n, cur_idx
|
||||
vpxor $t0xmm, $t0xmm, $t0xmm
|
||||
vmovdqa64 $t0, $cur_idx
|
||||
___
|
||||
foreach (1..9) {
|
||||
$code.="vmovdqa64 $t0, $t[$_] \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
|
||||
.align 32
|
||||
.Lloop:
|
||||
vpcmpq \$0, $cur_idx, $idx1, %k1 # mask of (idx1 == cur_idx)
|
||||
vpcmpq \$0, $cur_idx, $idx2, %k2 # mask of (idx2 == cur_idx)
|
||||
___
|
||||
foreach (0..9) {
|
||||
my $mask = $_<5?"%k1":"%k2";
|
||||
$code.=<<___;
|
||||
vmovdqu64 `${_}*32`($red_tbl), $tmp # load data from red_tbl
|
||||
vpblendmq $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx
|
||||
addq \$`2*20*8`, $red_tbl
|
||||
cmpq $red_tbl, %rax
|
||||
jne .Lloop
|
||||
___
|
||||
# store t0..n
|
||||
foreach (0..9) {
|
||||
$code.="vmovdqu64 $t[$_], `${_}*32`($out) \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5
|
||||
___
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lones:
|
||||
.quad 1,1,1,1
|
||||
.Lzeros:
|
||||
.quad 0,0,0,0
|
||||
___
|
||||
}
|
||||
|
||||
if ($win64) {
|
||||
$rec="%rcx";
|
||||
$frame="%rdx";
|
||||
$context="%r8";
|
||||
$disp="%r9";
|
||||
|
||||
$code.=<<___;
|
||||
.extern __imp_RtlVirtualUnwind
|
||||
.type rsaz_def_handler,\@abi-omnipotent
|
||||
.align 16
|
||||
rsaz_def_handler:
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %rbx
|
||||
push %rbp
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
pushfq
|
||||
sub \$64,%rsp
|
||||
|
||||
mov 120($context),%rax # pull context->Rax
|
||||
mov 248($context),%rbx # pull context->Rip
|
||||
|
||||
mov 8($disp),%rsi # disp->ImageBase
|
||||
mov 56($disp),%r11 # disp->HandlerData
|
||||
|
||||
mov 0(%r11),%r10d # HandlerData[0]
|
||||
lea (%rsi,%r10),%r10 # prologue label
|
||||
cmp %r10,%rbx # context->Rip<.Lprologue
|
||||
jb .Lcommon_seh_tail
|
||||
|
||||
mov 152($context),%rax # pull context->Rsp
|
||||
|
||||
mov 4(%r11),%r10d # HandlerData[1]
|
||||
lea (%rsi,%r10),%r10 # epilogue label
|
||||
cmp %r10,%rbx # context->Rip>=.Lepilogue
|
||||
jae .Lcommon_seh_tail
|
||||
|
||||
lea 48(%rax),%rax
|
||||
|
||||
mov -8(%rax),%rbx
|
||||
mov -16(%rax),%rbp
|
||||
mov -24(%rax),%r12
|
||||
mov -32(%rax),%r13
|
||||
mov -40(%rax),%r14
|
||||
mov -48(%rax),%r15
|
||||
mov %rbx,144($context) # restore context->Rbx
|
||||
mov %rbp,160($context) # restore context->Rbp
|
||||
mov %r12,216($context) # restore context->R12
|
||||
mov %r13,224($context) # restore context->R13
|
||||
mov %r14,232($context) # restore context->R14
|
||||
mov %r15,240($context) # restore context->R14
|
||||
|
||||
.Lcommon_seh_tail:
|
||||
mov 8(%rax),%rdi
|
||||
mov 16(%rax),%rsi
|
||||
mov %rax,152($context) # restore context->Rsp
|
||||
mov %rsi,168($context) # restore context->Rsi
|
||||
mov %rdi,176($context) # restore context->Rdi
|
||||
|
||||
mov 40($disp),%rdi # disp->ContextRecord
|
||||
mov $context,%rsi # context
|
||||
mov \$154,%ecx # sizeof(CONTEXT)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
mov $disp,%rsi
|
||||
xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
|
||||
mov 8(%rsi),%rdx # arg2, disp->ImageBase
|
||||
mov 0(%rsi),%r8 # arg3, disp->ControlPc
|
||||
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
|
||||
mov 40(%rsi),%r10 # disp->ContextRecord
|
||||
lea 56(%rsi),%r11 # &disp->HandlerData
|
||||
lea 24(%rsi),%r12 # &disp->EstablisherFrame
|
||||
mov %r10,32(%rsp) # arg5
|
||||
mov %r11,40(%rsp) # arg6
|
||||
mov %r12,48(%rsp) # arg7
|
||||
mov %rcx,56(%rsp) # arg8, (NULL)
|
||||
call *__imp_RtlVirtualUnwind(%rip)
|
||||
|
||||
mov \$1,%eax # ExceptionContinueSearch
|
||||
add \$64,%rsp
|
||||
popfq
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %rbp
|
||||
pop %rbx
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
ret
|
||||
.size rsaz_def_handler,.-rsaz_def_handler
|
||||
|
||||
.section .pdata
|
||||
.align 4
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x20_x1_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x20_x1_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256
|
||||
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x20_x2_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x20_x2_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256
|
||||
|
||||
.section .xdata
|
||||
.align 8
|
||||
.LSEH_info_ossl_rsaz_amm52x20_x1_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_def_handler
|
||||
.rva .Lossl_rsaz_amm52x20_x1_ifma256_body,.Lossl_rsaz_amm52x20_x1_ifma256_epilogue
|
||||
.LSEH_info_ossl_rsaz_amm52x20_x2_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_def_handler
|
||||
.rva .Lossl_rsaz_amm52x20_x2_ifma256_body,.Lossl_rsaz_amm52x20_x2_ifma256_epilogue
|
||||
___
|
||||
}
|
||||
}}} else {{{ # fallback for old assembler
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_avx512ifma_eligible
|
||||
.type ossl_rsaz_avx512ifma_eligible,\@abi-omnipotent
|
||||
ossl_rsaz_avx512ifma_eligible:
|
||||
xor %eax,%eax
|
||||
ret
|
||||
.size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x1_ifma256
|
||||
.globl ossl_rsaz_amm52x20_x2_ifma256
|
||||
.globl ossl_extract_multiplier_2x20_win5
|
||||
.type ossl_rsaz_amm52x20_x1_ifma256,\@abi-omnipotent
|
||||
ossl_rsaz_amm52x20_x1_ifma256:
|
||||
ossl_rsaz_amm52x20_x2_ifma256:
|
||||
ossl_extract_multiplier_2x20_win5:
|
||||
.byte 0x0f,0x0b # ud2
|
||||
ret
|
||||
.size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256
|
||||
___
|
||||
}}}
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,863 @@
|
||||
# Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
# Copyright (c) 2024, Intel Corporation. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# Written by Zhiguo Zhou <[email protected]> and Wangyang Guo <[email protected]>.
|
||||
# Special thanks to Tomasz Kantecki <[email protected]> for his valuable suggestions.
|
||||
#
|
||||
# October 2024
|
||||
#
|
||||
# Initial release.
|
||||
#
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
||||
$avxifma=0;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||||
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
||||
die "can't locate x86_64-xlate.pl";
|
||||
|
||||
# TODO: Find out the version of NASM that supports VEX-encoded AVX-IFMA instructions
|
||||
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
||||
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
|
||||
$avxifma = ($1>=2.40);
|
||||
}
|
||||
|
||||
if (!$avxifma && `$ENV{CC} -v 2>&1`
|
||||
=~ /\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
|
||||
my $ver = $2 + $3/100.0 + $4/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
|
||||
$avxifma = ($ver>=16.0);
|
||||
}
|
||||
|
||||
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
|
||||
or die "can't call $xlate: $!";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
if ($avxifma>0) {{{
|
||||
@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
.extern OPENSSL_ia32cap_P
|
||||
.globl ossl_rsaz_avxifma_eligible
|
||||
.type ossl_rsaz_avxifma_eligible,\@abi-omnipotent
|
||||
.align 32
|
||||
ossl_rsaz_avxifma_eligible:
|
||||
mov OPENSSL_ia32cap_P+20(%rip), %ecx
|
||||
xor %eax,%eax
|
||||
and \$`1<<23`, %ecx # avxifma
|
||||
cmp \$`1<<23`, %ecx
|
||||
cmove %ecx,%eax
|
||||
ret
|
||||
.size ossl_rsaz_avxifma_eligible, .-ossl_rsaz_avxifma_eligible
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52.
|
||||
#
|
||||
# AMM is defined as presented in the paper [1].
|
||||
#
|
||||
# The input and output are presented in 2^52 radix domain, i.e.
|
||||
# |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed.
|
||||
# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
|
||||
#
|
||||
# NB: the AMM implementation does not perform "conditional" subtraction step
|
||||
# specified in the original algorithm as according to the Lemma 1 from the paper
|
||||
# [2], the result will be always < 2*m and can be used as a direct input to
|
||||
# the next AMM iteration. This post-condition is true, provided the correct
|
||||
# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k,
|
||||
# which matches our case: 1040 > 1024 + 2 * 1.
|
||||
#
|
||||
# [1] Gueron, S. Efficient software implementations of modular exponentiation.
|
||||
# DOI: 10.1007/s13389-012-0031-5
|
||||
# [2] Gueron, S. Enhanced Montgomery Multiplication.
|
||||
# DOI: 10.1007/3-540-36400-5_5
|
||||
#
|
||||
# void ossl_rsaz_amm52x20_x1_avxifma256(BN_ULONG *res,
|
||||
# const BN_ULONG *a,
|
||||
# const BN_ULONG *b,
|
||||
# const BN_ULONG *m,
|
||||
# BN_ULONG k0);
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
|
||||
my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
|
||||
|
||||
my $mask52 = "%rax";
|
||||
my $acc0_0 = "%r9";
|
||||
my $acc0_0_low = "%r9d";
|
||||
my $acc0_1 = "%r15";
|
||||
my $acc0_1_low = "%r15d";
|
||||
my $b_ptr = "%r11";
|
||||
|
||||
my $iter = "%ebx";
|
||||
|
||||
my $zero = "%ymm0";
|
||||
my $Bi = "%ymm1";
|
||||
my $Yi = "%ymm2";
|
||||
my $Yi_xmm = "%xmm2";
|
||||
my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(5..8)));
|
||||
my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(9..12)));
|
||||
|
||||
# Registers mapping for normalization.
|
||||
my ($T0,$T0h,$T1,$T1h,$T2,$tmp) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (13..15)));
|
||||
my $T0_xmm = "%xmm0";
|
||||
|
||||
sub amm52x20_x1() {
|
||||
# _data_offset - offset in the |a| or |m| arrays pointing to the beginning
|
||||
# of data for corresponding AMM operation;
|
||||
# _b_offset - offset in the |b| array pointing to the next qword digit;
|
||||
my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_;
|
||||
$code.=<<___;
|
||||
movq $_b_offset($b_ptr), %r13 # b[i]
|
||||
|
||||
vpbroadcastq $_b_offset($b_ptr), $Bi # broadcast b[i]
|
||||
movq $_data_offset($a), %rdx
|
||||
mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2)
|
||||
addq %r13, $_acc # acc += t0
|
||||
movq %r12, %r10
|
||||
adcq \$0, %r10 # t2 += CF
|
||||
|
||||
movq $_k0, %r13
|
||||
imulq $_acc, %r13 # acc * k0
|
||||
andq $mask52, %r13 # yi = (acc * k0) & mask52
|
||||
|
||||
vmovq %r13, $Yi_xmm
|
||||
vpbroadcastq $Yi_xmm, $Yi # broadcast y[i]
|
||||
movq $_data_offset($m), %rdx
|
||||
mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1)
|
||||
addq %r13, $_acc # acc += t0
|
||||
adcq %r12, %r10 # t2 += (t1 + CF)
|
||||
|
||||
shrq \$52, $_acc
|
||||
salq \$12, %r10
|
||||
or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12))
|
||||
|
||||
lea -168(%rsp), %rsp
|
||||
{vex} vpmadd52luq `$_data_offset`($a), $Bi, $_R0
|
||||
{vex} vpmadd52luq `$_data_offset+32`($a), $Bi, $_R0h
|
||||
{vex} vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
{vex} vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
{vex} vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
|
||||
{vex} vpmadd52luq `$_data_offset`($m), $Yi, $_R0
|
||||
{vex} vpmadd52luq `$_data_offset+32`($m), $Yi, $_R0h
|
||||
{vex} vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
{vex} vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
{vex} vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
|
||||
# Shift accumulators right by 1 qword, zero extending the highest one
|
||||
vmovdqu $_R0, `32*0`(%rsp)
|
||||
vmovdqu $_R0h, `32*1`(%rsp)
|
||||
vmovdqu $_R1, `32*2`(%rsp)
|
||||
vmovdqu $_R1h, `32*3`(%rsp)
|
||||
vmovdqu $_R2, `32*4`(%rsp)
|
||||
movq \$0, `32*5`(%rsp)
|
||||
|
||||
vmovdqu `32*0 + 8`(%rsp), $_R0
|
||||
vmovdqu `32*1 + 8`(%rsp), $_R0h
|
||||
vmovdqu `32*2 + 8`(%rsp), $_R1
|
||||
vmovdqu `32*3 + 8`(%rsp), $_R1h
|
||||
vmovdqu `32*4 + 8`(%rsp), $_R2
|
||||
|
||||
addq 8(%rsp), $_acc # acc += R0[0]
|
||||
|
||||
{vex} vpmadd52huq `$_data_offset`($a), $Bi, $_R0
|
||||
{vex} vpmadd52huq `$_data_offset+32`($a), $Bi, $_R0h
|
||||
{vex} vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
{vex} vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
{vex} vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
|
||||
{vex} vpmadd52huq `$_data_offset`($m), $Yi, $_R0
|
||||
{vex} vpmadd52huq `$_data_offset+32`($m), $Yi, $_R0h
|
||||
{vex} vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
{vex} vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
{vex} vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
lea 168(%rsp),%rsp
|
||||
___
|
||||
}
|
||||
|
||||
# Normalization routine: handles carry bits and gets bignum qwords to normalized
|
||||
# 2^52 representation.
|
||||
#
|
||||
# Uses %r8-14,%e[bcd]x
|
||||
sub amm52x20_x1_norm {
|
||||
my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_;
|
||||
$code.=<<___;
|
||||
# Put accumulator to low qword in R0
|
||||
vmovq $_acc, $T0_xmm
|
||||
vpbroadcastq $T0_xmm, $T0
|
||||
vpblendd \$3, $T0, $_R0, $_R0
|
||||
|
||||
# Extract "carries" (12 high bits) from each QW of R0..R2
|
||||
# Save them to LSB of QWs in T0..T2
|
||||
vpsrlq \$52, $_R0, $T0
|
||||
vpsrlq \$52, $_R0h, $T0h
|
||||
vpsrlq \$52, $_R1, $T1
|
||||
vpsrlq \$52, $_R1h, $T1h
|
||||
vpsrlq \$52, $_R2, $T2
|
||||
|
||||
# "Shift left" T0..T2 by 1 QW
|
||||
vpermq \$144, $T2, $T2
|
||||
vpermq \$3, $T1h, $tmp
|
||||
vblendpd \$1, $tmp, $T2, $T2
|
||||
|
||||
vpermq \$144, $T1h, $T1h
|
||||
vpermq \$3, $T1, $tmp
|
||||
vblendpd \$1, $tmp, $T1h, $T1h
|
||||
|
||||
vpermq \$144, $T1, $T1
|
||||
vpermq \$3, $T0h, $tmp
|
||||
vblendpd \$1, $tmp, $T1, $T1
|
||||
|
||||
vpermq \$144, $T0h, $T0h
|
||||
vpermq \$3, $T0, $tmp
|
||||
vblendpd \$1, $tmp, $T0h, $T0h
|
||||
|
||||
vpermq \$144, $T0, $T0
|
||||
vpand .Lhigh64x3(%rip), $T0, $T0
|
||||
|
||||
# Drop "carries" from R0..R2 QWs
|
||||
vpand .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpand .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpand .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpand .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpand .Lmask52x4(%rip), $_R2, $_R2
|
||||
|
||||
# Sum R0..R2 with corresponding adjusted carries
|
||||
vpaddq $T0, $_R0, $_R0
|
||||
vpaddq $T0h, $_R0h, $_R0h
|
||||
vpaddq $T1, $_R1, $_R1
|
||||
vpaddq $T1h, $_R1h, $_R1h
|
||||
vpaddq $T2, $_R2, $_R2
|
||||
|
||||
# Now handle carry bits from this addition
|
||||
# Get mask of QWs which 52-bit parts overflow...
|
||||
vpcmpgtq .Lmask52x4(%rip), $_R0, $T0
|
||||
vpcmpgtq .Lmask52x4(%rip), $_R0h, $T0h
|
||||
vpcmpgtq .Lmask52x4(%rip), $_R1, $T1
|
||||
vpcmpgtq .Lmask52x4(%rip), $_R1h, $T1h
|
||||
vpcmpgtq .Lmask52x4(%rip), $_R2, $T2
|
||||
vmovmskpd $T0, %r14d
|
||||
vmovmskpd $T0h, %r13d
|
||||
vmovmskpd $T1, %r12d
|
||||
vmovmskpd $T1h, %r11d
|
||||
vmovmskpd $T2, %r10d
|
||||
|
||||
# ...or saturated
|
||||
vpcmpeqq .Lmask52x4(%rip), $_R0, $T0
|
||||
vpcmpeqq .Lmask52x4(%rip), $_R0h, $T0h
|
||||
vpcmpeqq .Lmask52x4(%rip), $_R1, $T1
|
||||
vpcmpeqq .Lmask52x4(%rip), $_R1h, $T1h
|
||||
vpcmpeqq .Lmask52x4(%rip), $_R2, $T2
|
||||
vmovmskpd $T0, %r9d
|
||||
vmovmskpd $T0h, %r8d
|
||||
vmovmskpd $T1, %ebx
|
||||
vmovmskpd $T1h, %ecx
|
||||
vmovmskpd $T2, %edx
|
||||
|
||||
# Get mask of QWs where carries shall be propagated to.
|
||||
# Merge 4-bit masks to 8-bit values to use add with carry.
|
||||
shl \$4, %r13b
|
||||
or %r13b, %r14b
|
||||
shl \$4, %r11b
|
||||
or %r11b, %r12b
|
||||
|
||||
add %r14b, %r14b
|
||||
adc %r12b, %r12b
|
||||
adc %r10b, %r10b
|
||||
|
||||
shl \$4, %r8b
|
||||
or %r8b,%r9b
|
||||
shl \$4, %cl
|
||||
or %cl, %bl
|
||||
|
||||
add %r9b, %r14b
|
||||
adc %bl, %r12b
|
||||
adc %dl, %r10b
|
||||
|
||||
xor %r9b, %r14b
|
||||
xor %bl, %r12b
|
||||
xor %dl, %r10b
|
||||
|
||||
lea .Lkmasklut(%rip), %rdx
|
||||
|
||||
mov %r14b, %r13b
|
||||
and \$0xf, %r14
|
||||
vpsubq .Lmask52x4(%rip), $_R0, $T0
|
||||
shl \$5, %r14
|
||||
vmovapd (%rdx, %r14), $T1
|
||||
vblendvpd $T1, $T0, $_R0, $_R0
|
||||
|
||||
shr \$4, %r13b
|
||||
and \$0xf, %r13
|
||||
vpsubq .Lmask52x4(%rip), $_R0h, $T0
|
||||
shl \$5, %r13
|
||||
vmovapd (%rdx, %r13), $T1
|
||||
vblendvpd $T1, $T0, $_R0h, $_R0h
|
||||
|
||||
mov %r12b, %r11b
|
||||
and \$0xf, %r12
|
||||
vpsubq .Lmask52x4(%rip), $_R1, $T0
|
||||
shl \$5, %r12
|
||||
vmovapd (%rdx, %r12), $T1
|
||||
vblendvpd $T1, $T0, $_R1, $_R1
|
||||
|
||||
shr \$4, %r11b
|
||||
and \$0xf, %r11
|
||||
vpsubq .Lmask52x4(%rip), $_R1h, $T0
|
||||
shl \$5, %r11
|
||||
vmovapd (%rdx, %r11), $T1
|
||||
vblendvpd $T1, $T0, $_R1h, $_R1h
|
||||
|
||||
and \$0xf, %r10
|
||||
vpsubq .Lmask52x4(%rip), $_R2, $T0
|
||||
shl \$5, %r10
|
||||
vmovapd (%rdx, %r10), $T1
|
||||
vblendvpd $T1, $T0, $_R2, $_R2
|
||||
|
||||
# Add carries according to the obtained mask
|
||||
vpand .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpand .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpand .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpand .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpand .Lmask52x4(%rip), $_R2, $_R2
|
||||
___
|
||||
}
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x1_avxifma256
|
||||
.type ossl_rsaz_amm52x20_x1_avxifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x20_x1_avxifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
.Lossl_rsaz_amm52x20_x1_avxifma256_body:
|
||||
|
||||
# Zeroing accumulators
|
||||
vpxor $zero, $zero, $zero
|
||||
vmovapd $zero, $R0_0
|
||||
vmovapd $zero, $R0_0h
|
||||
vmovapd $zero, $R1_0
|
||||
vmovapd $zero, $R1_0h
|
||||
vmovapd $zero, $R2_0
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
# Loop over 20 digits unrolled by 4
|
||||
mov \$5, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop5:
|
||||
___
|
||||
foreach my $idx (0..3) {
|
||||
&amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0);
|
||||
}
|
||||
$code.=<<___;
|
||||
lea `4*8`($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop5
|
||||
___
|
||||
&amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu $R0_0, `0*32`($res)
|
||||
vmovdqu $R0_0h, `1*32`($res)
|
||||
vmovdqu $R1_0, `2*32`($res)
|
||||
vmovdqu $R1_0h, `3*32`($res)
|
||||
vmovdqu $R2_0, `4*32`($res)
|
||||
|
||||
vzeroupper
|
||||
mov 0(%rsp),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rsp),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rsp),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rsp),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rsp),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rsp),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rsp),%rsp
|
||||
.cfi_adjust_cfa_offset -48
|
||||
.Lossl_rsaz_amm52x20_x1_avxifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x20_x1_avxifma256, .-ossl_rsaz_amm52x20_x1_avxifma256
|
||||
___
|
||||
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lmask52x4:
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.Lhigh64x3:
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.Lkmasklut:
|
||||
#0000
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
#0001
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
#0010
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
#0011
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
#0100
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
#0101
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
#0110
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
#0111
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
#1000
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
#1001
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
#1010
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
#1011
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
#1100
|
||||
.quad 0x0
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
#1101
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
#1110
|
||||
.quad 0x0
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
#1111
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
.quad 0xffffffffffffffff
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52
|
||||
#
|
||||
# See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost
|
||||
# Montgomery Multiplication algorithm and function input parameters description.
|
||||
#
|
||||
# This function does two AMMs for two independent inputs, hence dual.
|
||||
#
|
||||
# void ossl_rsaz_amm52x20_x2_avxifma256(BN_ULONG out[2][20],
|
||||
# const BN_ULONG a[2][20],
|
||||
# const BN_ULONG b[2][20],
|
||||
# const BN_ULONG m[2][20],
|
||||
# const BN_ULONG k0[2]);
|
||||
###############################################################################
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x2_avxifma256
|
||||
.type ossl_rsaz_amm52x20_x2_avxifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x20_x2_avxifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
.Lossl_rsaz_amm52x20_x2_avxifma256_body:
|
||||
|
||||
# Zeroing accumulators
|
||||
vpxor $zero, $zero, $zero
|
||||
vmovapd $zero, $R0_0
|
||||
vmovapd $zero, $R0_0h
|
||||
vmovapd $zero, $R1_0
|
||||
vmovapd $zero, $R1_0h
|
||||
vmovapd $zero, $R2_0
|
||||
vmovapd $zero, $R0_1
|
||||
vmovapd $zero, $R0_1h
|
||||
vmovapd $zero, $R1_1
|
||||
vmovapd $zero, $R1_1h
|
||||
vmovapd $zero, $R2_1
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
xorl $acc0_1_low, $acc0_1_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
mov \$20, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop20:
|
||||
___
|
||||
&amm52x20_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)");
|
||||
# 20*8 = offset of the next dimension in two-dimension array
|
||||
&amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)");
|
||||
$code.=<<___;
|
||||
lea 8($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop20
|
||||
___
|
||||
&amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0);
|
||||
&amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu $R0_0, `0*32`($res)
|
||||
vmovdqu $R0_0h, `1*32`($res)
|
||||
vmovdqu $R1_0, `2*32`($res)
|
||||
vmovdqu $R1_0h, `3*32`($res)
|
||||
vmovdqu $R2_0, `4*32`($res)
|
||||
|
||||
vmovdqu $R0_1, `5*32`($res)
|
||||
vmovdqu $R0_1h, `6*32`($res)
|
||||
vmovdqu $R1_1, `7*32`($res)
|
||||
vmovdqu $R1_1h, `8*32`($res)
|
||||
vmovdqu $R2_1, `9*32`($res)
|
||||
|
||||
vzeroupper
|
||||
mov 0(%rsp),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rsp),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rsp),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rsp),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rsp),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rsp),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rsp),%rsp
|
||||
.cfi_adjust_cfa_offset -48
|
||||
.Lossl_rsaz_amm52x20_x2_avxifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x20_x2_avxifma256, .-ossl_rsaz_amm52x20_x2_avxifma256
|
||||
___
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Constant time extraction from the precomputed table of powers base^i, where
|
||||
# i = 0..2^EXP_WIN_SIZE-1
|
||||
#
|
||||
# The input |red_table| contains precomputations for two independent base values.
|
||||
# |red_table_idx1| and |red_table_idx2| are corresponding power indexes.
|
||||
#
|
||||
# Extracted value (output) is 2 20 digit numbers in 2^52 radix.
|
||||
#
|
||||
# void ossl_extract_multiplier_2x20_win5_avx(BN_ULONG *red_Y,
|
||||
# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20],
|
||||
# int red_table_idx1, int red_table_idx2);
|
||||
#
|
||||
# EXP_WIN_SIZE = 5
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters
|
||||
my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
||||
("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
||||
|
||||
my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5));
|
||||
my ($t6,$t7,$t8,$t9) = map("%ymm$_", (6..9));
|
||||
my ($tmp,$cur_idx,$idx1,$idx2,$ones,$mask) = map("%ymm$_", (10..15));
|
||||
|
||||
my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9);
|
||||
my $t0xmm = $t0;
|
||||
my $tmp_xmm = "%xmm10";
|
||||
$t0xmm =~ s/%y/%x/;
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.align 32
|
||||
.globl ossl_extract_multiplier_2x20_win5_avx
|
||||
.type ossl_extract_multiplier_2x20_win5_avx,\@abi-omnipotent
|
||||
ossl_extract_multiplier_2x20_win5_avx:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
vmovapd .Lones(%rip), $ones # broadcast ones
|
||||
vmovq $red_tbl_idx1, $tmp_xmm
|
||||
vpbroadcastq $tmp_xmm, $idx1
|
||||
vmovq $red_tbl_idx2, $tmp_xmm
|
||||
vpbroadcastq $tmp_xmm, $idx2
|
||||
leaq `(1<<5)*2*20*8`($red_tbl), %rax # holds end of the tbl
|
||||
|
||||
# zeroing t0..n, cur_idx
|
||||
vpxor $t0xmm, $t0xmm, $t0xmm
|
||||
vmovapd $t0, $cur_idx
|
||||
___
|
||||
foreach (1..9) {
|
||||
$code.="vmovapd $t0, $t[$_] \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
|
||||
.align 32
|
||||
.Lloop:
|
||||
vpcmpeqq $cur_idx, $idx1, $mask # mask of (idx1 == cur_idx)
|
||||
___
|
||||
foreach (0..4) {
|
||||
$code.=<<___;
|
||||
vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl
|
||||
vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
vpcmpeqq $cur_idx, $idx2, $mask # mask of (idx2 == cur_idx)
|
||||
___
|
||||
foreach (5..9) {
|
||||
$code.=<<___;
|
||||
vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl
|
||||
vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx
|
||||
addq \$`2*20*8`, $red_tbl
|
||||
cmpq $red_tbl, %rax
|
||||
jne .Lloop
|
||||
___
|
||||
# store t0..n
|
||||
foreach (0..9) {
|
||||
$code.="vmovdqu $t[$_], `${_}*32`($out) \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_extract_multiplier_2x20_win5_avx, .-ossl_extract_multiplier_2x20_win5_avx
|
||||
___
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lones:
|
||||
.quad 1,1,1,1
|
||||
.Lzeros:
|
||||
.quad 0,0,0,0
|
||||
___
|
||||
}
|
||||
|
||||
if ($win64) {
|
||||
$rec="%rcx";
|
||||
$frame="%rdx";
|
||||
$context="%r8";
|
||||
$disp="%r9";
|
||||
|
||||
$code.=<<___;
|
||||
.extern __imp_RtlVirtualUnwind
|
||||
.type rsaz_def_handler,\@abi-omnipotent
|
||||
.align 16
|
||||
rsaz_def_handler:
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %rbx
|
||||
push %rbp
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
pushfq
|
||||
sub \$64,%rsp
|
||||
|
||||
mov 120($context),%rax # pull context->Rax
|
||||
mov 248($context),%rbx # pull context->Rip
|
||||
|
||||
mov 8($disp),%rsi # disp->ImageBase
|
||||
mov 56($disp),%r11 # disp->HandlerData
|
||||
|
||||
mov 0(%r11),%r10d # HandlerData[0]
|
||||
lea (%rsi,%r10),%r10 # prologue label
|
||||
cmp %r10,%rbx # context->Rip<.Lprologue
|
||||
jb .Lcommon_seh_tail
|
||||
|
||||
mov 152($context),%rax # pull context->Rsp
|
||||
|
||||
mov 4(%r11),%r10d # HandlerData[1]
|
||||
lea (%rsi,%r10),%r10 # epilogue label
|
||||
cmp %r10,%rbx # context->Rip>=.Lepilogue
|
||||
jae .Lcommon_seh_tail
|
||||
|
||||
lea 48(%rax),%rax
|
||||
|
||||
mov -8(%rax),%rbx
|
||||
mov -16(%rax),%rbp
|
||||
mov -24(%rax),%r12
|
||||
mov -32(%rax),%r13
|
||||
mov -40(%rax),%r14
|
||||
mov -48(%rax),%r15
|
||||
mov %rbx,144($context) # restore context->Rbx
|
||||
mov %rbp,160($context) # restore context->Rbp
|
||||
mov %r12,216($context) # restore context->R12
|
||||
mov %r13,224($context) # restore context->R13
|
||||
mov %r14,232($context) # restore context->R14
|
||||
mov %r15,240($context) # restore context->R14
|
||||
|
||||
.Lcommon_seh_tail:
|
||||
mov 8(%rax),%rdi
|
||||
mov 16(%rax),%rsi
|
||||
mov %rax,152($context) # restore context->Rsp
|
||||
mov %rsi,168($context) # restore context->Rsi
|
||||
mov %rdi,176($context) # restore context->Rdi
|
||||
|
||||
mov 40($disp),%rdi # disp->ContextRecord
|
||||
mov $context,%rsi # context
|
||||
mov \$154,%ecx # sizeof(CONTEXT)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
mov $disp,%rsi
|
||||
xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
|
||||
mov 8(%rsi),%rdx # arg2, disp->ImageBase
|
||||
mov 0(%rsi),%r8 # arg3, disp->ControlPc
|
||||
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
|
||||
mov 40(%rsi),%r10 # disp->ContextRecord
|
||||
lea 56(%rsi),%r11 # &disp->HandlerData
|
||||
lea 24(%rsi),%r12 # &disp->EstablisherFrame
|
||||
mov %r10,32(%rsp) # arg5
|
||||
mov %r11,40(%rsp) # arg6
|
||||
mov %r12,48(%rsp) # arg7
|
||||
mov %rcx,56(%rsp) # arg8, (NULL)
|
||||
call *__imp_RtlVirtualUnwind(%rip)
|
||||
|
||||
mov \$1,%eax # ExceptionContinueSearch
|
||||
add \$64,%rsp
|
||||
popfq
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %rbp
|
||||
pop %rbx
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
ret
|
||||
.size rsaz_def_handler,.-rsaz_def_handler
|
||||
|
||||
.section .pdata
|
||||
.align 4
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x20_x1_avxifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x20_x1_avxifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x20_x1_avxifma256
|
||||
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x20_x2_avxifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x20_x2_avxifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x20_x2_avxifma256
|
||||
|
||||
.section .xdata
|
||||
.align 8
|
||||
.LSEH_info_ossl_rsaz_amm52x20_x1_avxifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_def_handler
|
||||
.rva .Lossl_rsaz_amm52x20_x1_avxifma256_body,.Lossl_rsaz_amm52x20_x1_avxifma256_epilogue
|
||||
.LSEH_info_ossl_rsaz_amm52x20_x2_avxifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_def_handler
|
||||
.rva .Lossl_rsaz_amm52x20_x2_avxifma256_body,.Lossl_rsaz_amm52x20_x2_avxifma256_epilogue
|
||||
___
|
||||
}
|
||||
}}} else {{{ # fallback for old assembler
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_avxifma_eligible
|
||||
.type ossl_rsaz_avxifma_eligible,\@abi-omnipotent
|
||||
ossl_rsaz_avxifma_eligible:
|
||||
xor %eax,%eax
|
||||
ret
|
||||
.size ossl_rsaz_avxifma_eligible, .-ossl_rsaz_avxifma_eligible
|
||||
|
||||
.globl ossl_rsaz_amm52x20_x1_avxifma256
|
||||
.globl ossl_rsaz_amm52x20_x2_avxifma256
|
||||
.globl ossl_extract_multiplier_2x20_win5_avx
|
||||
.type ossl_rsaz_amm52x20_x1_avxifma256,\@abi-omnipotent
|
||||
ossl_rsaz_amm52x20_x1_avxifma256:
|
||||
ossl_rsaz_amm52x20_x2_avxifma256:
|
||||
ossl_extract_multiplier_2x20_win5_avx:
|
||||
.byte 0x0f,0x0b # ud2
|
||||
ret
|
||||
.size ossl_rsaz_amm52x20_x1_avxifma256, .-ossl_rsaz_amm52x20_x1_avxifma256
|
||||
___
|
||||
}}}
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,883 @@
|
||||
# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
# Copyright (c) 2021, Intel Corporation. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
#
|
||||
# Originally written by Sergey Kirillov and Andrey Matyukov
|
||||
# Intel Corporation
|
||||
#
|
||||
# March 2021
|
||||
#
|
||||
# Initial release.
|
||||
#
|
||||
# Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues.
|
||||
#
|
||||
# IceLake-Client @ 1.3GHz
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
# | | OpenSSL 3.0.0-alpha15 | this | Unit |
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
# | rsa3072 | 6 397 637 | 2 866 593 | cycles/sign |
|
||||
# | | 203.2 | 453.5 / +123% | sign/s |
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
#
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
||||
$avx512ifma=0;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||||
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
||||
die "can't locate x86_64-xlate.pl";
|
||||
|
||||
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
||||
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
|
||||
$avx512ifma = ($1>=2.26);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
|
||||
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
|
||||
$avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && `$ENV{CC} -v 2>&1`
|
||||
=~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
|
||||
my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
|
||||
if ($1) {
|
||||
# Apple conditions, they use a different version series, see
|
||||
# https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2
|
||||
# clang 7.0.0 is Apple clang 10.0.1
|
||||
$avx512ifma = ($ver>=10.0001)
|
||||
} else {
|
||||
$avx512ifma = ($ver>=7.0);
|
||||
}
|
||||
}
|
||||
|
||||
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
|
||||
or die "can't call $xlate: $!";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
if ($avx512ifma>0) {{{
|
||||
@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
|
||||
|
||||
###############################################################################
|
||||
# Almost Montgomery Multiplication (AMM) for 30-digit number in radix 2^52.
|
||||
#
|
||||
# AMM is defined as presented in the paper [1].
|
||||
#
|
||||
# The input and output are presented in 2^52 radix domain, i.e.
|
||||
# |res|, |a|, |b|, |m| are arrays of 32 64-bit qwords with 12 high bits zeroed
|
||||
#
|
||||
# NOTE: the function uses zero-padded data - 2 high QWs is a padding.
|
||||
#
|
||||
# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
|
||||
#
|
||||
# NB: the AMM implementation does not perform "conditional" subtraction step
|
||||
# specified in the original algorithm as according to the Lemma 1 from the paper
|
||||
# [2], the result will be always < 2*m and can be used as a direct input to
|
||||
# the next AMM iteration. This post-condition is true, provided the correct
|
||||
# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k,
|
||||
# which matches our case: 1560 > 1536 + 2 * 1.
|
||||
#
|
||||
# [1] Gueron, S. Efficient software implementations of modular exponentiation.
|
||||
# DOI: 10.1007/s13389-012-0031-5
|
||||
# [2] Gueron, S. Enhanced Montgomery Multiplication.
|
||||
# DOI: 10.1007/3-540-36400-5_5
|
||||
#
|
||||
# void ossl_rsaz_amm52x30_x1_ifma256(BN_ULONG *res,
|
||||
# const BN_ULONG *a,
|
||||
# const BN_ULONG *b,
|
||||
# const BN_ULONG *m,
|
||||
# BN_ULONG k0);
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
|
||||
my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
|
||||
|
||||
my $mask52 = "%rax";
|
||||
my $acc0_0 = "%r9";
|
||||
my $acc0_0_low = "%r9d";
|
||||
my $acc0_1 = "%r15";
|
||||
my $acc0_1_low = "%r15d";
|
||||
my $b_ptr = "%r11";
|
||||
|
||||
my $iter = "%ebx";
|
||||
|
||||
my $zero = "%ymm0";
|
||||
my $Bi = "%ymm1";
|
||||
my $Yi = "%ymm2";
|
||||
my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h) = map("%ymm$_",(3..10));
|
||||
my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h) = map("%ymm$_",(11..18));
|
||||
|
||||
# Registers mapping for normalization
|
||||
my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (19..23)));
|
||||
|
||||
sub amm52x30_x1() {
|
||||
# _data_offset - offset in the |a| or |m| arrays pointing to the beginning
|
||||
# of data for corresponding AMM operation;
|
||||
# _b_offset - offset in the |b| array pointing to the next qword digit;
|
||||
my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_k0) = @_;
|
||||
my $_R0_xmm = $_R0;
|
||||
$_R0_xmm =~ s/%y/%x/;
|
||||
$code.=<<___;
|
||||
movq $_b_offset($b_ptr), %r13 # b[i]
|
||||
|
||||
vpbroadcastq %r13, $Bi # broadcast b[i]
|
||||
movq $_data_offset($a), %rdx
|
||||
mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2)
|
||||
addq %r13, $_acc # acc += t0
|
||||
movq %r12, %r10
|
||||
adcq \$0, %r10 # t2 += CF
|
||||
|
||||
movq $_k0, %r13
|
||||
imulq $_acc, %r13 # acc * k0
|
||||
andq $mask52, %r13 # yi = (acc * k0) & mask52
|
||||
|
||||
vpbroadcastq %r13, $Yi # broadcast y[i]
|
||||
movq $_data_offset($m), %rdx
|
||||
mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1)
|
||||
addq %r13, $_acc # acc += t0
|
||||
adcq %r12, %r10 # t2 += (t1 + CF)
|
||||
|
||||
shrq \$52, $_acc
|
||||
salq \$12, %r10
|
||||
or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12))
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h
|
||||
vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3
|
||||
vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h
|
||||
vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3
|
||||
vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h
|
||||
|
||||
# Shift accumulators right by 1 qword, zero extending the highest one
|
||||
valignq \$1, $_R0, $_R0h, $_R0
|
||||
valignq \$1, $_R0h, $_R1, $_R0h
|
||||
valignq \$1, $_R1, $_R1h, $_R1
|
||||
valignq \$1, $_R1h, $_R2, $_R1h
|
||||
valignq \$1, $_R2, $_R2h, $_R2
|
||||
valignq \$1, $_R2h, $_R3, $_R2h
|
||||
valignq \$1, $_R3, $_R3h, $_R3
|
||||
valignq \$1, $_R3h, $zero, $_R3h
|
||||
|
||||
vmovq $_R0_xmm, %r13
|
||||
addq %r13, $_acc # acc += R0[0]
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h
|
||||
vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3
|
||||
vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h
|
||||
vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3
|
||||
vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h
|
||||
___
|
||||
}
|
||||
|
||||
# Normalization routine: handles carry bits and gets bignum qwords to normalized
|
||||
# 2^52 representation.
|
||||
#
|
||||
# Uses %r8-14,%e[abcd]x
|
||||
sub amm52x30_x1_norm {
|
||||
my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h) = @_;
|
||||
$code.=<<___;
|
||||
# Put accumulator to low qword in R0
|
||||
vpbroadcastq $_acc, $T0
|
||||
vpblendd \$3, $T0, $_R0, $_R0
|
||||
|
||||
# Extract "carries" (12 high bits) from each QW of the bignum
|
||||
# Save them to LSB of QWs in T0..Tn
|
||||
vpsrlq \$52, $_R0, $T0
|
||||
vpsrlq \$52, $_R0h, $T0h
|
||||
vpsrlq \$52, $_R1, $T1
|
||||
vpsrlq \$52, $_R1h, $T1h
|
||||
vpsrlq \$52, $_R2, $T2
|
||||
vpsrlq \$52, $_R2h, $T2h
|
||||
vpsrlq \$52, $_R3, $T3
|
||||
vpsrlq \$52, $_R3h, $T3h
|
||||
|
||||
# "Shift left" T0..Tn by 1 QW
|
||||
valignq \$3, $T3, $T3h, $T3h
|
||||
valignq \$3, $T2h, $T3, $T3
|
||||
valignq \$3, $T2, $T2h, $T2h
|
||||
valignq \$3, $T1h, $T2, $T2
|
||||
valignq \$3, $T1, $T1h, $T1h
|
||||
valignq \$3, $T0h, $T1, $T1
|
||||
valignq \$3, $T0, $T0h, $T0h
|
||||
valignq \$3, .Lzeros(%rip), $T0, $T0
|
||||
|
||||
# Drop "carries" from R0..Rn QWs
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
vpandq .Lmask52x4(%rip), $_R2h, $_R2h
|
||||
vpandq .Lmask52x4(%rip), $_R3, $_R3
|
||||
vpandq .Lmask52x4(%rip), $_R3h, $_R3h
|
||||
|
||||
# Sum R0..Rn with corresponding adjusted carries
|
||||
vpaddq $T0, $_R0, $_R0
|
||||
vpaddq $T0h, $_R0h, $_R0h
|
||||
vpaddq $T1, $_R1, $_R1
|
||||
vpaddq $T1h, $_R1h, $_R1h
|
||||
vpaddq $T2, $_R2, $_R2
|
||||
vpaddq $T2h, $_R2h, $_R2h
|
||||
vpaddq $T3, $_R3, $_R3
|
||||
vpaddq $T3h, $_R3h, $_R3h
|
||||
|
||||
# Now handle carry bits from this addition
|
||||
# Get mask of QWs whose 52-bit parts overflow
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R0},%k1 # OP=nle (i.e. gt)
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R0h},%k2
|
||||
kmovb %k1,%r14d
|
||||
kmovb %k2,%r13d
|
||||
shl \$4,%r13b
|
||||
or %r13b,%r14b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R1},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R1h},%k2
|
||||
kmovb %k1,%r13d
|
||||
kmovb %k2,%r12d
|
||||
shl \$4,%r12b
|
||||
or %r12b,%r13b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R2},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R2h},%k2
|
||||
kmovb %k1,%r12d
|
||||
kmovb %k2,%r11d
|
||||
shl \$4,%r11b
|
||||
or %r11b,%r12b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R3},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R3h},%k2
|
||||
kmovb %k1,%r11d
|
||||
kmovb %k2,%r10d
|
||||
shl \$4,%r10b
|
||||
or %r10b,%r11b
|
||||
|
||||
addb %r14b,%r14b
|
||||
adcb %r13b,%r13b
|
||||
adcb %r12b,%r12b
|
||||
adcb %r11b,%r11b
|
||||
|
||||
# Get mask of QWs whose 52-bit parts saturated
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R0},%k1 # OP=eq
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R0h},%k2
|
||||
kmovb %k1,%r9d
|
||||
kmovb %k2,%r8d
|
||||
shl \$4,%r8b
|
||||
or %r8b,%r9b
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R1},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R1h},%k2
|
||||
kmovb %k1,%r8d
|
||||
kmovb %k2,%edx
|
||||
shl \$4,%dl
|
||||
or %dl,%r8b
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R2},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R2h},%k2
|
||||
kmovb %k1,%edx
|
||||
kmovb %k2,%ecx
|
||||
shl \$4,%cl
|
||||
or %cl,%dl
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R3},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R3h},%k2
|
||||
kmovb %k1,%ecx
|
||||
kmovb %k2,%ebx
|
||||
shl \$4,%bl
|
||||
or %bl,%cl
|
||||
|
||||
addb %r9b,%r14b
|
||||
adcb %r8b,%r13b
|
||||
adcb %dl,%r12b
|
||||
adcb %cl,%r11b
|
||||
|
||||
xor %r9b,%r14b
|
||||
xor %r8b,%r13b
|
||||
xor %dl,%r12b
|
||||
xor %cl,%r11b
|
||||
|
||||
kmovb %r14d,%k1
|
||||
shr \$4,%r14b
|
||||
kmovb %r14d,%k2
|
||||
kmovb %r13d,%k3
|
||||
shr \$4,%r13b
|
||||
kmovb %r13d,%k4
|
||||
kmovb %r12d,%k5
|
||||
shr \$4,%r12b
|
||||
kmovb %r12d,%k6
|
||||
kmovb %r11d,%k7
|
||||
|
||||
vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1}
|
||||
vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2}
|
||||
vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3}
|
||||
vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4}
|
||||
vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5}
|
||||
vpsubq .Lmask52x4(%rip), $_R2h, ${_R2h}{%k6}
|
||||
vpsubq .Lmask52x4(%rip), $_R3, ${_R3}{%k7}
|
||||
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
vpandq .Lmask52x4(%rip), $_R2h, $_R2h
|
||||
vpandq .Lmask52x4(%rip), $_R3, $_R3
|
||||
|
||||
shr \$4,%r11b
|
||||
kmovb %r11d,%k1
|
||||
|
||||
vpsubq .Lmask52x4(%rip), $_R3h, ${_R3h}{%k1}
|
||||
|
||||
vpandq .Lmask52x4(%rip), $_R3h, $_R3h
|
||||
___
|
||||
}
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x30_x1_ifma256
|
||||
.type ossl_rsaz_amm52x30_x1_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x30_x1_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment)
|
||||
vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers
|
||||
vmovdqa64 %xmm7, `1*16`(%rsp)
|
||||
vmovdqa64 %xmm8, `2*16`(%rsp)
|
||||
vmovdqa64 %xmm9, `3*16`(%rsp)
|
||||
vmovdqa64 %xmm10,`4*16`(%rsp)
|
||||
vmovdqa64 %xmm11,`5*16`(%rsp)
|
||||
vmovdqa64 %xmm12,`6*16`(%rsp)
|
||||
vmovdqa64 %xmm13,`7*16`(%rsp)
|
||||
vmovdqa64 %xmm14,`8*16`(%rsp)
|
||||
vmovdqa64 %xmm15,`9*16`(%rsp)
|
||||
.Lossl_rsaz_amm52x30_x1_ifma256_body:
|
||||
___
|
||||
$code.=<<___;
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
vmovdqa64 $zero, $R2_0h
|
||||
vmovdqa64 $zero, $R3_0
|
||||
vmovdqa64 $zero, $R3_0h
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
# Loop over 30 digits unrolled by 4
|
||||
mov \$7, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop7:
|
||||
___
|
||||
foreach my $idx (0..3) {
|
||||
&amm52x30_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0);
|
||||
}
|
||||
$code.=<<___;
|
||||
lea `4*8`($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop7
|
||||
___
|
||||
&amm52x30_x1(0,8*0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0);
|
||||
&amm52x30_x1(0,8*1,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0);
|
||||
|
||||
&amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
vmovdqu64 $R2_0h, `5*32`($res)
|
||||
vmovdqu64 $R3_0, `6*32`($res)
|
||||
vmovdqu64 $R3_0h, `7*32`($res)
|
||||
|
||||
vzeroupper
|
||||
lea (%rsp),%rax
|
||||
.cfi_def_cfa_register %rax
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
vmovdqa64 `0*16`(%rax),%xmm6
|
||||
vmovdqa64 `1*16`(%rax),%xmm7
|
||||
vmovdqa64 `2*16`(%rax),%xmm8
|
||||
vmovdqa64 `3*16`(%rax),%xmm9
|
||||
vmovdqa64 `4*16`(%rax),%xmm10
|
||||
vmovdqa64 `5*16`(%rax),%xmm11
|
||||
vmovdqa64 `6*16`(%rax),%xmm12
|
||||
vmovdqa64 `7*16`(%rax),%xmm13
|
||||
vmovdqa64 `8*16`(%rax),%xmm14
|
||||
vmovdqa64 `9*16`(%rax),%xmm15
|
||||
lea 168(%rsp),%rax
|
||||
___
|
||||
$code.=<<___;
|
||||
mov 0(%rax),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rax),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rax),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rax),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rax),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rax),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rax),%rsp # restore rsp
|
||||
.cfi_def_cfa %rsp,8
|
||||
.Lossl_rsaz_amm52x30_x1_ifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x30_x1_ifma256, .-ossl_rsaz_amm52x30_x1_ifma256
|
||||
___
|
||||
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lmask52x4:
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Dual Almost Montgomery Multiplication for 30-digit number in radix 2^52
|
||||
#
|
||||
# See description of ossl_rsaz_amm52x30_x1_ifma256() above for details about Almost
|
||||
# Montgomery Multiplication algorithm and function input parameters description.
|
||||
#
|
||||
# This function does two AMMs for two independent inputs, hence dual.
|
||||
#
|
||||
# NOTE: the function uses zero-padded data - 2 high QWs is a padding.
|
||||
#
|
||||
# void ossl_rsaz_amm52x30_x2_ifma256(BN_ULONG out[2][32],
|
||||
# const BN_ULONG a[2][32],
|
||||
# const BN_ULONG b[2][32],
|
||||
# const BN_ULONG m[2][32],
|
||||
# const BN_ULONG k0[2]);
|
||||
###############################################################################
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x30_x2_ifma256
|
||||
.type ossl_rsaz_amm52x30_x2_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x30_x2_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
lea -168(%rsp),%rsp
|
||||
vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers
|
||||
vmovdqa64 %xmm7, `1*16`(%rsp)
|
||||
vmovdqa64 %xmm8, `2*16`(%rsp)
|
||||
vmovdqa64 %xmm9, `3*16`(%rsp)
|
||||
vmovdqa64 %xmm10,`4*16`(%rsp)
|
||||
vmovdqa64 %xmm11,`5*16`(%rsp)
|
||||
vmovdqa64 %xmm12,`6*16`(%rsp)
|
||||
vmovdqa64 %xmm13,`7*16`(%rsp)
|
||||
vmovdqa64 %xmm14,`8*16`(%rsp)
|
||||
vmovdqa64 %xmm15,`9*16`(%rsp)
|
||||
.Lossl_rsaz_amm52x30_x2_ifma256_body:
|
||||
___
|
||||
$code.=<<___;
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
vmovdqa64 $zero, $R2_0h
|
||||
vmovdqa64 $zero, $R3_0
|
||||
vmovdqa64 $zero, $R3_0h
|
||||
|
||||
vmovdqa64 $zero, $R0_1
|
||||
vmovdqa64 $zero, $R0_1h
|
||||
vmovdqa64 $zero, $R1_1
|
||||
vmovdqa64 $zero, $R1_1h
|
||||
vmovdqa64 $zero, $R2_1
|
||||
vmovdqa64 $zero, $R2_1h
|
||||
vmovdqa64 $zero, $R3_1
|
||||
vmovdqa64 $zero, $R3_1h
|
||||
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
xorl $acc0_1_low, $acc0_1_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
mov \$30, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop30:
|
||||
___
|
||||
&amm52x30_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,"($k0)");
|
||||
# 32*8 = offset of the next dimension in two-dimension array
|
||||
&amm52x30_x1(32*8,32*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,"8($k0)");
|
||||
$code.=<<___;
|
||||
lea 8($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop30
|
||||
___
|
||||
&amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h);
|
||||
&amm52x30_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
vmovdqu64 $R2_0h, `5*32`($res)
|
||||
vmovdqu64 $R3_0, `6*32`($res)
|
||||
vmovdqu64 $R3_0h, `7*32`($res)
|
||||
|
||||
vmovdqu64 $R0_1, `8*32`($res)
|
||||
vmovdqu64 $R0_1h, `9*32`($res)
|
||||
vmovdqu64 $R1_1, `10*32`($res)
|
||||
vmovdqu64 $R1_1h, `11*32`($res)
|
||||
vmovdqu64 $R2_1, `12*32`($res)
|
||||
vmovdqu64 $R2_1h, `13*32`($res)
|
||||
vmovdqu64 $R3_1, `14*32`($res)
|
||||
vmovdqu64 $R3_1h, `15*32`($res)
|
||||
|
||||
vzeroupper
|
||||
lea (%rsp),%rax
|
||||
.cfi_def_cfa_register %rax
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
vmovdqa64 `0*16`(%rax),%xmm6
|
||||
vmovdqa64 `1*16`(%rax),%xmm7
|
||||
vmovdqa64 `2*16`(%rax),%xmm8
|
||||
vmovdqa64 `3*16`(%rax),%xmm9
|
||||
vmovdqa64 `4*16`(%rax),%xmm10
|
||||
vmovdqa64 `5*16`(%rax),%xmm11
|
||||
vmovdqa64 `6*16`(%rax),%xmm12
|
||||
vmovdqa64 `7*16`(%rax),%xmm13
|
||||
vmovdqa64 `8*16`(%rax),%xmm14
|
||||
vmovdqa64 `9*16`(%rax),%xmm15
|
||||
lea 168(%rsp),%rax
|
||||
___
|
||||
$code.=<<___;
|
||||
mov 0(%rax),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rax),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rax),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rax),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rax),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rax),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rax),%rsp
|
||||
.cfi_def_cfa %rsp,8
|
||||
.Lossl_rsaz_amm52x30_x2_ifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x30_x2_ifma256, .-ossl_rsaz_amm52x30_x2_ifma256
|
||||
___
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Constant time extraction from the precomputed table of powers base^i, where
|
||||
# i = 0..2^EXP_WIN_SIZE-1
|
||||
#
|
||||
# The input |red_table| contains precomputations for two independent base values.
|
||||
# |red_table_idx1| and |red_table_idx2| are corresponding power indexes.
|
||||
#
|
||||
# Extracted value (output) is 2 (30 + 2) digits numbers in 2^52 radix.
|
||||
# (2 high QW is zero padding)
|
||||
#
|
||||
# void ossl_extract_multiplier_2x30_win5(BN_ULONG *red_Y,
|
||||
# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][32],
|
||||
# int red_table_idx1, int red_table_idx2);
|
||||
#
|
||||
# EXP_WIN_SIZE = 5
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters
|
||||
my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
||||
("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
||||
|
||||
my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5));
|
||||
my ($t6,$t7,$t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15) = map("%ymm$_", (16..25));
|
||||
my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (26..30));
|
||||
|
||||
my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15);
|
||||
my $t0xmm = $t0;
|
||||
$t0xmm =~ s/%y/%x/;
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.align 32
|
||||
.globl ossl_extract_multiplier_2x30_win5
|
||||
.type ossl_extract_multiplier_2x30_win5,\@abi-omnipotent
|
||||
ossl_extract_multiplier_2x30_win5:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
vmovdqa64 .Lones(%rip), $ones # broadcast ones
|
||||
vpbroadcastq $red_tbl_idx1, $idx1
|
||||
vpbroadcastq $red_tbl_idx2, $idx2
|
||||
leaq `(1<<5)*2*32*8`($red_tbl), %rax # holds end of the tbl
|
||||
|
||||
# zeroing t0..n, cur_idx
|
||||
vpxor $t0xmm, $t0xmm, $t0xmm
|
||||
vmovdqa64 $t0, $cur_idx
|
||||
___
|
||||
foreach (1..15) {
|
||||
$code.="vmovdqa64 $t0, $t[$_] \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
|
||||
.align 32
|
||||
.Lloop:
|
||||
vpcmpq \$0, $cur_idx, $idx1, %k1 # mask of (idx1 == cur_idx)
|
||||
vpcmpq \$0, $cur_idx, $idx2, %k2 # mask of (idx2 == cur_idx)
|
||||
___
|
||||
foreach (0..15) {
|
||||
my $mask = $_<8?"%k1":"%k2";
|
||||
$code.=<<___;
|
||||
vmovdqu64 `${_}*32`($red_tbl), $tmp # load data from red_tbl
|
||||
vpblendmq $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx
|
||||
addq \$`2*32*8`, $red_tbl
|
||||
cmpq $red_tbl, %rax
|
||||
jne .Lloop
|
||||
___
|
||||
# store t0..n
|
||||
foreach (0..15) {
|
||||
$code.="vmovdqu64 $t[$_], `${_}*32`($out) \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_extract_multiplier_2x30_win5, .-ossl_extract_multiplier_2x30_win5
|
||||
___
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lones:
|
||||
.quad 1,1,1,1
|
||||
.Lzeros:
|
||||
.quad 0,0,0,0
|
||||
___
|
||||
}
|
||||
|
||||
if ($win64) {
|
||||
$rec="%rcx";
|
||||
$frame="%rdx";
|
||||
$context="%r8";
|
||||
$disp="%r9";
|
||||
|
||||
$code.=<<___;
|
||||
.extern __imp_RtlVirtualUnwind
|
||||
.type rsaz_avx_handler,\@abi-omnipotent
|
||||
.align 16
|
||||
rsaz_avx_handler:
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %rbx
|
||||
push %rbp
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
pushfq
|
||||
sub \$64,%rsp
|
||||
|
||||
mov 120($context),%rax # pull context->Rax
|
||||
mov 248($context),%rbx # pull context->Rip
|
||||
|
||||
mov 8($disp),%rsi # disp->ImageBase
|
||||
mov 56($disp),%r11 # disp->HandlerData
|
||||
|
||||
mov 0(%r11),%r10d # HandlerData[0]
|
||||
lea (%rsi,%r10),%r10 # prologue label
|
||||
cmp %r10,%rbx # context->Rip<.Lprologue
|
||||
jb .Lcommon_seh_tail
|
||||
|
||||
mov 4(%r11),%r10d # HandlerData[1]
|
||||
lea (%rsi,%r10),%r10 # epilogue label
|
||||
cmp %r10,%rbx # context->Rip>=.Lepilogue
|
||||
jae .Lcommon_seh_tail
|
||||
|
||||
mov 152($context),%rax # pull context->Rsp
|
||||
|
||||
lea (%rax),%rsi # %xmm save area
|
||||
lea 512($context),%rdi # & context.Xmm6
|
||||
mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
lea `48+168`(%rax),%rax
|
||||
|
||||
mov -8(%rax),%rbx
|
||||
mov -16(%rax),%rbp
|
||||
mov -24(%rax),%r12
|
||||
mov -32(%rax),%r13
|
||||
mov -40(%rax),%r14
|
||||
mov -48(%rax),%r15
|
||||
mov %rbx,144($context) # restore context->Rbx
|
||||
mov %rbp,160($context) # restore context->Rbp
|
||||
mov %r12,216($context) # restore context->R12
|
||||
mov %r13,224($context) # restore context->R13
|
||||
mov %r14,232($context) # restore context->R14
|
||||
mov %r15,240($context) # restore context->R14
|
||||
|
||||
.Lcommon_seh_tail:
|
||||
mov 8(%rax),%rdi
|
||||
mov 16(%rax),%rsi
|
||||
mov %rax,152($context) # restore context->Rsp
|
||||
mov %rsi,168($context) # restore context->Rsi
|
||||
mov %rdi,176($context) # restore context->Rdi
|
||||
|
||||
mov 40($disp),%rdi # disp->ContextRecord
|
||||
mov $context,%rsi # context
|
||||
mov \$154,%ecx # sizeof(CONTEXT)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
mov $disp,%rsi
|
||||
xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
|
||||
mov 8(%rsi),%rdx # arg2, disp->ImageBase
|
||||
mov 0(%rsi),%r8 # arg3, disp->ControlPc
|
||||
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
|
||||
mov 40(%rsi),%r10 # disp->ContextRecord
|
||||
lea 56(%rsi),%r11 # &disp->HandlerData
|
||||
lea 24(%rsi),%r12 # &disp->EstablisherFrame
|
||||
mov %r10,32(%rsp) # arg5
|
||||
mov %r11,40(%rsp) # arg6
|
||||
mov %r12,48(%rsp) # arg7
|
||||
mov %rcx,56(%rsp) # arg8, (NULL)
|
||||
call *__imp_RtlVirtualUnwind(%rip)
|
||||
|
||||
mov \$1,%eax # ExceptionContinueSearch
|
||||
add \$64,%rsp
|
||||
popfq
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %rbp
|
||||
pop %rbx
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
ret
|
||||
.size rsaz_avx_handler,.-rsaz_avx_handler
|
||||
|
||||
.section .pdata
|
||||
.align 4
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x30_x1_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x30_x1_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x30_x1_ifma256
|
||||
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x30_x2_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x30_x2_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x30_x2_ifma256
|
||||
|
||||
.section .xdata
|
||||
.align 8
|
||||
.LSEH_info_ossl_rsaz_amm52x30_x1_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_avx_handler
|
||||
.rva .Lossl_rsaz_amm52x30_x1_ifma256_body,.Lossl_rsaz_amm52x30_x1_ifma256_epilogue
|
||||
.LSEH_info_ossl_rsaz_amm52x30_x2_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_avx_handler
|
||||
.rva .Lossl_rsaz_amm52x30_x2_ifma256_body,.Lossl_rsaz_amm52x30_x2_ifma256_epilogue
|
||||
___
|
||||
}
|
||||
}}} else {{{ # fallback for old assembler
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x30_x1_ifma256
|
||||
.globl ossl_rsaz_amm52x30_x2_ifma256
|
||||
.globl ossl_extract_multiplier_2x30_win5
|
||||
.type ossl_rsaz_amm52x30_x1_ifma256,\@abi-omnipotent
|
||||
ossl_rsaz_amm52x30_x1_ifma256:
|
||||
ossl_rsaz_amm52x30_x2_ifma256:
|
||||
ossl_extract_multiplier_2x30_win5:
|
||||
.byte 0x0f,0x0b # ud2
|
||||
ret
|
||||
.size ossl_rsaz_amm52x30_x1_ifma256, .-ossl_rsaz_amm52x30_x1_ifma256
|
||||
___
|
||||
}}}
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,939 @@
|
||||
# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
# Copyright (c) 2021, Intel Corporation. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
#
|
||||
# Originally written by Sergey Kirillov and Andrey Matyukov
|
||||
# Intel Corporation
|
||||
#
|
||||
# March 2021
|
||||
#
|
||||
# Initial release.
|
||||
#
|
||||
# Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues.
|
||||
#
|
||||
# IceLake-Client @ 1.3GHz
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
# | | OpenSSL 3.0.0-alpha15 | this | Unit |
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
# | rsa4096 | 14 301 4300 | 5 813 953 | cycles/sign |
|
||||
# | | 90.9 | 223.6 / +146% | sign/s |
|
||||
# |---------+-----------------------+---------------+-------------|
|
||||
#
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
||||
$avx512ifma=0;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||||
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
||||
die "can't locate x86_64-xlate.pl";
|
||||
|
||||
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
||||
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
|
||||
$avx512ifma = ($1>=2.26);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
|
||||
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
|
||||
$avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12);
|
||||
}
|
||||
|
||||
if (!$avx512ifma && `$ENV{CC} -v 2>&1`
|
||||
=~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
|
||||
my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
|
||||
if ($1) {
|
||||
# Apple conditions, they use a different version series, see
|
||||
# https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2
|
||||
# clang 7.0.0 is Apple clang 10.0.1
|
||||
$avx512ifma = ($ver>=10.0001)
|
||||
} else {
|
||||
$avx512ifma = ($ver>=7.0);
|
||||
}
|
||||
}
|
||||
|
||||
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
|
||||
or die "can't call $xlate: $!";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
if ($avx512ifma>0) {{{
|
||||
@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9");
|
||||
|
||||
###############################################################################
|
||||
# Almost Montgomery Multiplication (AMM) for 40-digit number in radix 2^52.
|
||||
#
|
||||
# AMM is defined as presented in the paper [1].
|
||||
#
|
||||
# The input and output are presented in 2^52 radix domain, i.e.
|
||||
# |res|, |a|, |b|, |m| are arrays of 40 64-bit qwords with 12 high bits zeroed.
|
||||
# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64
|
||||
#
|
||||
# NB: the AMM implementation does not perform "conditional" subtraction step
|
||||
# specified in the original algorithm as according to the Lemma 1 from the paper
|
||||
# [2], the result will be always < 2*m and can be used as a direct input to
|
||||
# the next AMM iteration. This post-condition is true, provided the correct
|
||||
# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k,
|
||||
# which matches our case: 2080 > 2048 + 2 * 1.
|
||||
#
|
||||
# [1] Gueron, S. Efficient software implementations of modular exponentiation.
|
||||
# DOI: 10.1007/s13389-012-0031-5
|
||||
# [2] Gueron, S. Enhanced Montgomery Multiplication.
|
||||
# DOI: 10.1007/3-540-36400-5_5
|
||||
#
|
||||
# void ossl_rsaz_amm52x40_x1_ifma256(BN_ULONG *res,
|
||||
# const BN_ULONG *a,
|
||||
# const BN_ULONG *b,
|
||||
# const BN_ULONG *m,
|
||||
# BN_ULONG k0);
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8")
|
||||
my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI;
|
||||
|
||||
my $mask52 = "%rax";
|
||||
my $acc0_0 = "%r9";
|
||||
my $acc0_0_low = "%r9d";
|
||||
my $acc0_1 = "%r15";
|
||||
my $acc0_1_low = "%r15d";
|
||||
my $b_ptr = "%r11";
|
||||
|
||||
my $iter = "%ebx";
|
||||
|
||||
my $zero = "%ymm0";
|
||||
my $Bi = "%ymm1";
|
||||
my $Yi = "%ymm2";
|
||||
my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h) = map("%ymm$_",(3..12));
|
||||
my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h) = map("%ymm$_",(13..22));
|
||||
|
||||
# Registers mapping for normalization
|
||||
my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h,$T4,$T4h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (23..29)));
|
||||
|
||||
sub amm52x40_x1() {
|
||||
# _data_offset - offset in the |a| or |m| arrays pointing to the beginning
|
||||
# of data for corresponding AMM operation;
|
||||
# _b_offset - offset in the |b| array pointing to the next qword digit;
|
||||
my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h,$_k0) = @_;
|
||||
my $_R0_xmm = $_R0;
|
||||
$_R0_xmm =~ s/%y/%x/;
|
||||
$code.=<<___;
|
||||
movq $_b_offset($b_ptr), %r13 # b[i]
|
||||
|
||||
vpbroadcastq %r13, $Bi # broadcast b[i]
|
||||
movq $_data_offset($a), %rdx
|
||||
mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2)
|
||||
addq %r13, $_acc # acc += t0
|
||||
movq %r12, %r10
|
||||
adcq \$0, %r10 # t2 += CF
|
||||
|
||||
movq $_k0, %r13
|
||||
imulq $_acc, %r13 # acc * k0
|
||||
andq $mask52, %r13 # yi = (acc * k0) & mask52
|
||||
|
||||
vpbroadcastq %r13, $Yi # broadcast y[i]
|
||||
movq $_data_offset($m), %rdx
|
||||
mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1)
|
||||
addq %r13, $_acc # acc += t0
|
||||
adcq %r12, %r10 # t2 += (t1 + CF)
|
||||
|
||||
shrq \$52, $_acc
|
||||
salq \$12, %r10
|
||||
or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12))
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h
|
||||
vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3
|
||||
vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h
|
||||
vpmadd52luq `$_data_offset+64*4`($a), $Bi, $_R4
|
||||
vpmadd52luq `$_data_offset+64*4+32`($a), $Bi, $_R4h
|
||||
|
||||
vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h
|
||||
vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3
|
||||
vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h
|
||||
vpmadd52luq `$_data_offset+64*4`($m), $Yi, $_R4
|
||||
vpmadd52luq `$_data_offset+64*4+32`($m), $Yi, $_R4h
|
||||
|
||||
# Shift accumulators right by 1 qword, zero extending the highest one
|
||||
valignq \$1, $_R0, $_R0h, $_R0
|
||||
valignq \$1, $_R0h, $_R1, $_R0h
|
||||
valignq \$1, $_R1, $_R1h, $_R1
|
||||
valignq \$1, $_R1h, $_R2, $_R1h
|
||||
valignq \$1, $_R2, $_R2h, $_R2
|
||||
valignq \$1, $_R2h, $_R3, $_R2h
|
||||
valignq \$1, $_R3, $_R3h, $_R3
|
||||
valignq \$1, $_R3h, $_R4, $_R3h
|
||||
valignq \$1, $_R4, $_R4h, $_R4
|
||||
valignq \$1, $_R4h, $zero, $_R4h
|
||||
|
||||
vmovq $_R0_xmm, %r13
|
||||
addq %r13, $_acc # acc += R0[0]
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2
|
||||
vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h
|
||||
vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3
|
||||
vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h
|
||||
vpmadd52huq `$_data_offset+64*4`($a), $Bi, $_R4
|
||||
vpmadd52huq `$_data_offset+64*4+32`($a), $Bi, $_R4h
|
||||
|
||||
vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0
|
||||
vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h
|
||||
vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1
|
||||
vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h
|
||||
vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2
|
||||
vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h
|
||||
vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3
|
||||
vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h
|
||||
vpmadd52huq `$_data_offset+64*4`($m), $Yi, $_R4
|
||||
vpmadd52huq `$_data_offset+64*4+32`($m), $Yi, $_R4h
|
||||
___
|
||||
}
|
||||
|
||||
# Normalization routine: handles carry bits and gets bignum qwords to normalized
|
||||
# 2^52 representation.
|
||||
#
|
||||
# Uses %r8-14,%e[abcd]x
|
||||
sub amm52x40_x1_norm {
|
||||
my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h) = @_;
|
||||
$code.=<<___;
|
||||
# Put accumulator to low qword in R0
|
||||
vpbroadcastq $_acc, $T0
|
||||
vpblendd \$3, $T0, $_R0, $_R0
|
||||
|
||||
# Extract "carries" (12 high bits) from each QW of the bignum
|
||||
# Save them to LSB of QWs in T0..Tn
|
||||
vpsrlq \$52, $_R0, $T0
|
||||
vpsrlq \$52, $_R0h, $T0h
|
||||
vpsrlq \$52, $_R1, $T1
|
||||
vpsrlq \$52, $_R1h, $T1h
|
||||
vpsrlq \$52, $_R2, $T2
|
||||
vpsrlq \$52, $_R2h, $T2h
|
||||
vpsrlq \$52, $_R3, $T3
|
||||
vpsrlq \$52, $_R3h, $T3h
|
||||
vpsrlq \$52, $_R4, $T4
|
||||
vpsrlq \$52, $_R4h, $T4h
|
||||
|
||||
# "Shift left" T0..Tn by 1 QW
|
||||
valignq \$3, $T4, $T4h, $T4h
|
||||
valignq \$3, $T3h, $T4, $T4
|
||||
valignq \$3, $T3, $T3h, $T3h
|
||||
valignq \$3, $T2h, $T3, $T3
|
||||
valignq \$3, $T2, $T2h, $T2h
|
||||
valignq \$3, $T1h, $T2, $T2
|
||||
valignq \$3, $T1, $T1h, $T1h
|
||||
valignq \$3, $T0h, $T1, $T1
|
||||
valignq \$3, $T0, $T0h, $T0h
|
||||
valignq \$3, .Lzeros(%rip), $T0, $T0
|
||||
|
||||
# Drop "carries" from R0..Rn QWs
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
vpandq .Lmask52x4(%rip), $_R2h, $_R2h
|
||||
vpandq .Lmask52x4(%rip), $_R3, $_R3
|
||||
vpandq .Lmask52x4(%rip), $_R3h, $_R3h
|
||||
vpandq .Lmask52x4(%rip), $_R4, $_R4
|
||||
vpandq .Lmask52x4(%rip), $_R4h, $_R4h
|
||||
|
||||
# Sum R0..Rn with corresponding adjusted carries
|
||||
vpaddq $T0, $_R0, $_R0
|
||||
vpaddq $T0h, $_R0h, $_R0h
|
||||
vpaddq $T1, $_R1, $_R1
|
||||
vpaddq $T1h, $_R1h, $_R1h
|
||||
vpaddq $T2, $_R2, $_R2
|
||||
vpaddq $T2h, $_R2h, $_R2h
|
||||
vpaddq $T3, $_R3, $_R3
|
||||
vpaddq $T3h, $_R3h, $_R3h
|
||||
vpaddq $T4, $_R4, $_R4
|
||||
vpaddq $T4h, $_R4h, $_R4h
|
||||
|
||||
# Now handle carry bits from this addition
|
||||
# Get mask of QWs whose 52-bit parts overflow
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R0},%k1 # OP=nle (i.e. gt)
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R0h},%k2
|
||||
kmovb %k1,%r14d
|
||||
kmovb %k2,%r13d
|
||||
shl \$4,%r13b
|
||||
or %r13b,%r14b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R1},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R1h},%k2
|
||||
kmovb %k1,%r13d
|
||||
kmovb %k2,%r12d
|
||||
shl \$4,%r12b
|
||||
or %r12b,%r13b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R2},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R2h},%k2
|
||||
kmovb %k1,%r12d
|
||||
kmovb %k2,%r11d
|
||||
shl \$4,%r11b
|
||||
or %r11b,%r12b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R3},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R3h},%k2
|
||||
kmovb %k1,%r11d
|
||||
kmovb %k2,%r10d
|
||||
shl \$4,%r10b
|
||||
or %r10b,%r11b
|
||||
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R4},%k1
|
||||
vpcmpuq \$6,.Lmask52x4(%rip),${_R4h},%k2
|
||||
kmovb %k1,%r10d
|
||||
kmovb %k2,%r9d
|
||||
shl \$4,%r9b
|
||||
or %r9b,%r10b
|
||||
|
||||
addb %r14b,%r14b
|
||||
adcb %r13b,%r13b
|
||||
adcb %r12b,%r12b
|
||||
adcb %r11b,%r11b
|
||||
adcb %r10b,%r10b
|
||||
|
||||
# Get mask of QWs whose 52-bit parts saturated
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R0},%k1 # OP=eq
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R0h},%k2
|
||||
kmovb %k1,%r9d
|
||||
kmovb %k2,%r8d
|
||||
shl \$4,%r8b
|
||||
or %r8b,%r9b
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R1},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R1h},%k2
|
||||
kmovb %k1,%r8d
|
||||
kmovb %k2,%edx
|
||||
shl \$4,%dl
|
||||
or %dl,%r8b
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R2},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R2h},%k2
|
||||
kmovb %k1,%edx
|
||||
kmovb %k2,%ecx
|
||||
shl \$4,%cl
|
||||
or %cl,%dl
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R3},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R3h},%k2
|
||||
kmovb %k1,%ecx
|
||||
kmovb %k2,%ebx
|
||||
shl \$4,%bl
|
||||
or %bl,%cl
|
||||
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R4},%k1
|
||||
vpcmpuq \$0,.Lmask52x4(%rip),${_R4h},%k2
|
||||
kmovb %k1,%ebx
|
||||
kmovb %k2,%eax
|
||||
shl \$4,%al
|
||||
or %al,%bl
|
||||
|
||||
addb %r9b,%r14b
|
||||
adcb %r8b,%r13b
|
||||
adcb %dl,%r12b
|
||||
adcb %cl,%r11b
|
||||
adcb %bl,%r10b
|
||||
|
||||
xor %r9b,%r14b
|
||||
xor %r8b,%r13b
|
||||
xor %dl,%r12b
|
||||
xor %cl,%r11b
|
||||
xor %bl,%r10b
|
||||
|
||||
kmovb %r14d,%k1
|
||||
shr \$4,%r14b
|
||||
kmovb %r14d,%k2
|
||||
kmovb %r13d,%k3
|
||||
shr \$4,%r13b
|
||||
kmovb %r13d,%k4
|
||||
kmovb %r12d,%k5
|
||||
shr \$4,%r12b
|
||||
kmovb %r12d,%k6
|
||||
kmovb %r11d,%k7
|
||||
|
||||
vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1}
|
||||
vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2}
|
||||
vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3}
|
||||
vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4}
|
||||
vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5}
|
||||
vpsubq .Lmask52x4(%rip), $_R2h, ${_R2h}{%k6}
|
||||
vpsubq .Lmask52x4(%rip), $_R3, ${_R3}{%k7}
|
||||
|
||||
vpandq .Lmask52x4(%rip), $_R0, $_R0
|
||||
vpandq .Lmask52x4(%rip), $_R0h, $_R0h
|
||||
vpandq .Lmask52x4(%rip), $_R1, $_R1
|
||||
vpandq .Lmask52x4(%rip), $_R1h, $_R1h
|
||||
vpandq .Lmask52x4(%rip), $_R2, $_R2
|
||||
vpandq .Lmask52x4(%rip), $_R2h, $_R2h
|
||||
vpandq .Lmask52x4(%rip), $_R3, $_R3
|
||||
|
||||
shr \$4,%r11b
|
||||
kmovb %r11d,%k1
|
||||
kmovb %r10d,%k2
|
||||
shr \$4,%r10b
|
||||
kmovb %r10d,%k3
|
||||
|
||||
vpsubq .Lmask52x4(%rip), $_R3h, ${_R3h}{%k1}
|
||||
vpsubq .Lmask52x4(%rip), $_R4, ${_R4}{%k2}
|
||||
vpsubq .Lmask52x4(%rip), $_R4h, ${_R4h}{%k3}
|
||||
|
||||
vpandq .Lmask52x4(%rip), $_R3h, $_R3h
|
||||
vpandq .Lmask52x4(%rip), $_R4, $_R4
|
||||
vpandq .Lmask52x4(%rip), $_R4h, $_R4h
|
||||
___
|
||||
}
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x40_x1_ifma256
|
||||
.type ossl_rsaz_amm52x40_x1_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x40_x1_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment)
|
||||
vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers
|
||||
vmovdqa64 %xmm7, `1*16`(%rsp)
|
||||
vmovdqa64 %xmm8, `2*16`(%rsp)
|
||||
vmovdqa64 %xmm9, `3*16`(%rsp)
|
||||
vmovdqa64 %xmm10,`4*16`(%rsp)
|
||||
vmovdqa64 %xmm11,`5*16`(%rsp)
|
||||
vmovdqa64 %xmm12,`6*16`(%rsp)
|
||||
vmovdqa64 %xmm13,`7*16`(%rsp)
|
||||
vmovdqa64 %xmm14,`8*16`(%rsp)
|
||||
vmovdqa64 %xmm15,`9*16`(%rsp)
|
||||
.Lossl_rsaz_amm52x40_x1_ifma256_body:
|
||||
___
|
||||
$code.=<<___;
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
vmovdqa64 $zero, $R2_0h
|
||||
vmovdqa64 $zero, $R3_0
|
||||
vmovdqa64 $zero, $R3_0h
|
||||
vmovdqa64 $zero, $R4_0
|
||||
vmovdqa64 $zero, $R4_0h
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
# Loop over 40 digits unrolled by 4
|
||||
mov \$10, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop10:
|
||||
___
|
||||
foreach my $idx (0..3) {
|
||||
&amm52x40_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,$k0);
|
||||
}
|
||||
$code.=<<___;
|
||||
lea `4*8`($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop10
|
||||
___
|
||||
&amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
vmovdqu64 $R2_0h, `5*32`($res)
|
||||
vmovdqu64 $R3_0, `6*32`($res)
|
||||
vmovdqu64 $R3_0h, `7*32`($res)
|
||||
vmovdqu64 $R4_0, `8*32`($res)
|
||||
vmovdqu64 $R4_0h, `9*32`($res)
|
||||
|
||||
vzeroupper
|
||||
lea (%rsp),%rax
|
||||
.cfi_def_cfa_register %rax
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
vmovdqa64 `0*16`(%rax),%xmm6
|
||||
vmovdqa64 `1*16`(%rax),%xmm7
|
||||
vmovdqa64 `2*16`(%rax),%xmm8
|
||||
vmovdqa64 `3*16`(%rax),%xmm9
|
||||
vmovdqa64 `4*16`(%rax),%xmm10
|
||||
vmovdqa64 `5*16`(%rax),%xmm11
|
||||
vmovdqa64 `6*16`(%rax),%xmm12
|
||||
vmovdqa64 `7*16`(%rax),%xmm13
|
||||
vmovdqa64 `8*16`(%rax),%xmm14
|
||||
vmovdqa64 `9*16`(%rax),%xmm15
|
||||
lea 168(%rsp),%rax
|
||||
___
|
||||
$code.=<<___;
|
||||
mov 0(%rax),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rax),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rax),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rax),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rax),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rax),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rax),%rsp # restore rsp
|
||||
.cfi_def_cfa %rsp,8
|
||||
.Lossl_rsaz_amm52x40_x1_ifma256_epilogue:
|
||||
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256
|
||||
___
|
||||
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lmask52x4:
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
.quad 0xfffffffffffff
|
||||
___
|
||||
|
||||
###############################################################################
|
||||
# Dual Almost Montgomery Multiplication for 40-digit number in radix 2^52
|
||||
#
|
||||
# See description of ossl_rsaz_amm52x40_x1_ifma256() above for details about Almost
|
||||
# Montgomery Multiplication algorithm and function input parameters description.
|
||||
#
|
||||
# This function does two AMMs for two independent inputs, hence dual.
|
||||
#
|
||||
# void ossl_rsaz_amm52x40_x2_ifma256(BN_ULONG out[2][40],
|
||||
# const BN_ULONG a[2][40],
|
||||
# const BN_ULONG b[2][40],
|
||||
# const BN_ULONG m[2][40],
|
||||
# const BN_ULONG k0[2]);
|
||||
###############################################################################
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x40_x2_ifma256
|
||||
.type ossl_rsaz_amm52x40_x2_ifma256,\@function,5
|
||||
.align 32
|
||||
ossl_rsaz_amm52x40_x2_ifma256:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
push %rbx
|
||||
.cfi_push %rbx
|
||||
push %rbp
|
||||
.cfi_push %rbp
|
||||
push %r12
|
||||
.cfi_push %r12
|
||||
push %r13
|
||||
.cfi_push %r13
|
||||
push %r14
|
||||
.cfi_push %r14
|
||||
push %r15
|
||||
.cfi_push %r15
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
lea -168(%rsp),%rsp
|
||||
vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers
|
||||
vmovdqa64 %xmm7, `1*16`(%rsp)
|
||||
vmovdqa64 %xmm8, `2*16`(%rsp)
|
||||
vmovdqa64 %xmm9, `3*16`(%rsp)
|
||||
vmovdqa64 %xmm10,`4*16`(%rsp)
|
||||
vmovdqa64 %xmm11,`5*16`(%rsp)
|
||||
vmovdqa64 %xmm12,`6*16`(%rsp)
|
||||
vmovdqa64 %xmm13,`7*16`(%rsp)
|
||||
vmovdqa64 %xmm14,`8*16`(%rsp)
|
||||
vmovdqa64 %xmm15,`9*16`(%rsp)
|
||||
.Lossl_rsaz_amm52x40_x2_ifma256_body:
|
||||
___
|
||||
$code.=<<___;
|
||||
# Zeroing accumulators
|
||||
vpxord $zero, $zero, $zero
|
||||
vmovdqa64 $zero, $R0_0
|
||||
vmovdqa64 $zero, $R0_0h
|
||||
vmovdqa64 $zero, $R1_0
|
||||
vmovdqa64 $zero, $R1_0h
|
||||
vmovdqa64 $zero, $R2_0
|
||||
vmovdqa64 $zero, $R2_0h
|
||||
vmovdqa64 $zero, $R3_0
|
||||
vmovdqa64 $zero, $R3_0h
|
||||
vmovdqa64 $zero, $R4_0
|
||||
vmovdqa64 $zero, $R4_0h
|
||||
|
||||
vmovdqa64 $zero, $R0_1
|
||||
vmovdqa64 $zero, $R0_1h
|
||||
vmovdqa64 $zero, $R1_1
|
||||
vmovdqa64 $zero, $R1_1h
|
||||
vmovdqa64 $zero, $R2_1
|
||||
vmovdqa64 $zero, $R2_1h
|
||||
vmovdqa64 $zero, $R3_1
|
||||
vmovdqa64 $zero, $R3_1h
|
||||
vmovdqa64 $zero, $R4_1
|
||||
vmovdqa64 $zero, $R4_1h
|
||||
|
||||
|
||||
xorl $acc0_0_low, $acc0_0_low
|
||||
xorl $acc0_1_low, $acc0_1_low
|
||||
|
||||
movq $b, $b_ptr # backup address of b
|
||||
movq \$0xfffffffffffff, $mask52 # 52-bit mask
|
||||
|
||||
mov \$40, $iter
|
||||
|
||||
.align 32
|
||||
.Lloop40:
|
||||
___
|
||||
&amm52x40_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,"($k0)");
|
||||
# 40*8 = offset of the next dimension in two-dimension array
|
||||
&amm52x40_x1(40*8,40*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h,"8($k0)");
|
||||
$code.=<<___;
|
||||
lea 8($b_ptr), $b_ptr
|
||||
dec $iter
|
||||
jne .Lloop40
|
||||
___
|
||||
&amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h);
|
||||
&amm52x40_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h);
|
||||
$code.=<<___;
|
||||
|
||||
vmovdqu64 $R0_0, `0*32`($res)
|
||||
vmovdqu64 $R0_0h, `1*32`($res)
|
||||
vmovdqu64 $R1_0, `2*32`($res)
|
||||
vmovdqu64 $R1_0h, `3*32`($res)
|
||||
vmovdqu64 $R2_0, `4*32`($res)
|
||||
vmovdqu64 $R2_0h, `5*32`($res)
|
||||
vmovdqu64 $R3_0, `6*32`($res)
|
||||
vmovdqu64 $R3_0h, `7*32`($res)
|
||||
vmovdqu64 $R4_0, `8*32`($res)
|
||||
vmovdqu64 $R4_0h, `9*32`($res)
|
||||
|
||||
vmovdqu64 $R0_1, `10*32`($res)
|
||||
vmovdqu64 $R0_1h, `11*32`($res)
|
||||
vmovdqu64 $R1_1, `12*32`($res)
|
||||
vmovdqu64 $R1_1h, `13*32`($res)
|
||||
vmovdqu64 $R2_1, `14*32`($res)
|
||||
vmovdqu64 $R2_1h, `15*32`($res)
|
||||
vmovdqu64 $R3_1, `16*32`($res)
|
||||
vmovdqu64 $R3_1h, `17*32`($res)
|
||||
vmovdqu64 $R4_1, `18*32`($res)
|
||||
vmovdqu64 $R4_1h, `19*32`($res)
|
||||
|
||||
vzeroupper
|
||||
lea (%rsp),%rax
|
||||
.cfi_def_cfa_register %rax
|
||||
___
|
||||
$code.=<<___ if ($win64);
|
||||
vmovdqa64 `0*16`(%rax),%xmm6
|
||||
vmovdqa64 `1*16`(%rax),%xmm7
|
||||
vmovdqa64 `2*16`(%rax),%xmm8
|
||||
vmovdqa64 `3*16`(%rax),%xmm9
|
||||
vmovdqa64 `4*16`(%rax),%xmm10
|
||||
vmovdqa64 `5*16`(%rax),%xmm11
|
||||
vmovdqa64 `6*16`(%rax),%xmm12
|
||||
vmovdqa64 `7*16`(%rax),%xmm13
|
||||
vmovdqa64 `8*16`(%rax),%xmm14
|
||||
vmovdqa64 `9*16`(%rax),%xmm15
|
||||
lea 168(%rsp),%rax
|
||||
___
|
||||
$code.=<<___;
|
||||
mov 0(%rax),%r15
|
||||
.cfi_restore %r15
|
||||
mov 8(%rax),%r14
|
||||
.cfi_restore %r14
|
||||
mov 16(%rax),%r13
|
||||
.cfi_restore %r13
|
||||
mov 24(%rax),%r12
|
||||
.cfi_restore %r12
|
||||
mov 32(%rax),%rbp
|
||||
.cfi_restore %rbp
|
||||
mov 40(%rax),%rbx
|
||||
.cfi_restore %rbx
|
||||
lea 48(%rax),%rsp
|
||||
.cfi_def_cfa %rsp,8
|
||||
.Lossl_rsaz_amm52x40_x2_ifma256_epilogue:
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_rsaz_amm52x40_x2_ifma256, .-ossl_rsaz_amm52x40_x2_ifma256
|
||||
___
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Constant time extraction from the precomputed table of powers base^i, where
|
||||
# i = 0..2^EXP_WIN_SIZE-1
|
||||
#
|
||||
# The input |red_table| contains precomputations for two independent base values.
|
||||
# |red_table_idx1| and |red_table_idx2| are corresponding power indexes.
|
||||
#
|
||||
# Extracted value (output) is 2 40 digits numbers in 2^52 radix.
|
||||
#
|
||||
# void ossl_extract_multiplier_2x40_win5(BN_ULONG *red_Y,
|
||||
# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][40],
|
||||
# int red_table_idx1, int red_table_idx2);
|
||||
#
|
||||
# EXP_WIN_SIZE = 5
|
||||
###############################################################################
|
||||
{
|
||||
# input parameters
|
||||
my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
||||
("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
||||
|
||||
my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5));
|
||||
my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19));
|
||||
my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24));
|
||||
|
||||
my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9);
|
||||
my $t0xmm = $t0;
|
||||
$t0xmm =~ s/%y/%x/;
|
||||
|
||||
sub get_table_value_consttime() {
|
||||
my ($_idx,$_offset) = @_;
|
||||
$code.=<<___;
|
||||
vpxorq $cur_idx, $cur_idx, $cur_idx
|
||||
.align 32
|
||||
.Lloop_$_offset:
|
||||
vpcmpq \$0, $cur_idx, $_idx, %k1 # mask of (idx == cur_idx)
|
||||
___
|
||||
foreach (0..9) {
|
||||
$code.=<<___;
|
||||
vmovdqu64 `$_offset+${_}*32`($red_tbl), $tmp # load data from red_tbl
|
||||
vpblendmq $tmp, $t[$_], ${t[$_]}{%k1} # extract data when mask is not zero
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx
|
||||
addq \$`2*40*8`, $red_tbl
|
||||
cmpq $red_tbl, %rax
|
||||
jne .Lloop_$_offset
|
||||
___
|
||||
}
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.align 32
|
||||
.globl ossl_extract_multiplier_2x40_win5
|
||||
.type ossl_extract_multiplier_2x40_win5,\@abi-omnipotent
|
||||
ossl_extract_multiplier_2x40_win5:
|
||||
.cfi_startproc
|
||||
endbranch
|
||||
vmovdqa64 .Lones(%rip), $ones # broadcast ones
|
||||
vpbroadcastq $red_tbl_idx1, $idx1
|
||||
vpbroadcastq $red_tbl_idx2, $idx2
|
||||
leaq `(1<<5)*2*40*8`($red_tbl), %rax # holds end of the tbl
|
||||
|
||||
# backup red_tbl address
|
||||
movq $red_tbl, %r10
|
||||
|
||||
# zeroing t0..n, cur_idx
|
||||
vpxor $t0xmm, $t0xmm, $t0xmm
|
||||
___
|
||||
foreach (1..9) {
|
||||
$code.="vmovdqa64 $t0, $t[$_] \n";
|
||||
}
|
||||
|
||||
&get_table_value_consttime($idx1, 0);
|
||||
foreach (0..9) {
|
||||
$code.="vmovdqu64 $t[$_], `(0+$_)*32`($out) \n";
|
||||
}
|
||||
$code.="movq %r10, $red_tbl \n";
|
||||
&get_table_value_consttime($idx2, 40*8);
|
||||
foreach (0..9) {
|
||||
$code.="vmovdqu64 $t[$_], `(10+$_)*32`($out) \n";
|
||||
}
|
||||
$code.=<<___;
|
||||
|
||||
ret
|
||||
.cfi_endproc
|
||||
.size ossl_extract_multiplier_2x40_win5, .-ossl_extract_multiplier_2x40_win5
|
||||
___
|
||||
$code.=<<___;
|
||||
.section .rodata align=32
|
||||
.align 32
|
||||
.Lones:
|
||||
.quad 1,1,1,1
|
||||
.Lzeros:
|
||||
.quad 0,0,0,0
|
||||
___
|
||||
}
|
||||
|
||||
if ($win64) {
|
||||
$rec="%rcx";
|
||||
$frame="%rdx";
|
||||
$context="%r8";
|
||||
$disp="%r9";
|
||||
|
||||
$code.=<<___;
|
||||
.extern __imp_RtlVirtualUnwind
|
||||
.type rsaz_avx_handler,\@abi-omnipotent
|
||||
.align 16
|
||||
rsaz_avx_handler:
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %rbx
|
||||
push %rbp
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
pushfq
|
||||
sub \$64,%rsp
|
||||
|
||||
mov 120($context),%rax # pull context->Rax
|
||||
mov 248($context),%rbx # pull context->Rip
|
||||
|
||||
mov 8($disp),%rsi # disp->ImageBase
|
||||
mov 56($disp),%r11 # disp->HandlerData
|
||||
|
||||
mov 0(%r11),%r10d # HandlerData[0]
|
||||
lea (%rsi,%r10),%r10 # prologue label
|
||||
cmp %r10,%rbx # context->Rip<.Lprologue
|
||||
jb .Lcommon_seh_tail
|
||||
|
||||
mov 4(%r11),%r10d # HandlerData[1]
|
||||
lea (%rsi,%r10),%r10 # epilogue label
|
||||
cmp %r10,%rbx # context->Rip>=.Lepilogue
|
||||
jae .Lcommon_seh_tail
|
||||
|
||||
mov 152($context),%rax # pull context->Rsp
|
||||
|
||||
lea (%rax),%rsi # %xmm save area
|
||||
lea 512($context),%rdi # & context.Xmm6
|
||||
mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
lea `48+168`(%rax),%rax
|
||||
|
||||
mov -8(%rax),%rbx
|
||||
mov -16(%rax),%rbp
|
||||
mov -24(%rax),%r12
|
||||
mov -32(%rax),%r13
|
||||
mov -40(%rax),%r14
|
||||
mov -48(%rax),%r15
|
||||
mov %rbx,144($context) # restore context->Rbx
|
||||
mov %rbp,160($context) # restore context->Rbp
|
||||
mov %r12,216($context) # restore context->R12
|
||||
mov %r13,224($context) # restore context->R13
|
||||
mov %r14,232($context) # restore context->R14
|
||||
mov %r15,240($context) # restore context->R14
|
||||
|
||||
.Lcommon_seh_tail:
|
||||
mov 8(%rax),%rdi
|
||||
mov 16(%rax),%rsi
|
||||
mov %rax,152($context) # restore context->Rsp
|
||||
mov %rsi,168($context) # restore context->Rsi
|
||||
mov %rdi,176($context) # restore context->Rdi
|
||||
|
||||
mov 40($disp),%rdi # disp->ContextRecord
|
||||
mov $context,%rsi # context
|
||||
mov \$154,%ecx # sizeof(CONTEXT)
|
||||
.long 0xa548f3fc # cld; rep movsq
|
||||
|
||||
mov $disp,%rsi
|
||||
xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
|
||||
mov 8(%rsi),%rdx # arg2, disp->ImageBase
|
||||
mov 0(%rsi),%r8 # arg3, disp->ControlPc
|
||||
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
|
||||
mov 40(%rsi),%r10 # disp->ContextRecord
|
||||
lea 56(%rsi),%r11 # &disp->HandlerData
|
||||
lea 24(%rsi),%r12 # &disp->EstablisherFrame
|
||||
mov %r10,32(%rsp) # arg5
|
||||
mov %r11,40(%rsp) # arg6
|
||||
mov %r12,48(%rsp) # arg7
|
||||
mov %rcx,56(%rsp) # arg8, (NULL)
|
||||
call *__imp_RtlVirtualUnwind(%rip)
|
||||
|
||||
mov \$1,%eax # ExceptionContinueSearch
|
||||
add \$64,%rsp
|
||||
popfq
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %rbp
|
||||
pop %rbx
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
ret
|
||||
.size rsaz_avx_handler,.-rsaz_avx_handler
|
||||
|
||||
.section .pdata
|
||||
.align 4
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x40_x1_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x40_x1_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x40_x1_ifma256
|
||||
|
||||
.rva .LSEH_begin_ossl_rsaz_amm52x40_x2_ifma256
|
||||
.rva .LSEH_end_ossl_rsaz_amm52x40_x2_ifma256
|
||||
.rva .LSEH_info_ossl_rsaz_amm52x40_x2_ifma256
|
||||
|
||||
.section .xdata
|
||||
.align 8
|
||||
.LSEH_info_ossl_rsaz_amm52x40_x1_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_avx_handler
|
||||
.rva .Lossl_rsaz_amm52x40_x1_ifma256_body,.Lossl_rsaz_amm52x40_x1_ifma256_epilogue
|
||||
.LSEH_info_ossl_rsaz_amm52x40_x2_ifma256:
|
||||
.byte 9,0,0,0
|
||||
.rva rsaz_avx_handler
|
||||
.rva .Lossl_rsaz_amm52x40_x2_ifma256_body,.Lossl_rsaz_amm52x40_x2_ifma256_epilogue
|
||||
___
|
||||
}
|
||||
}}} else {{{ # fallback for old assembler
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl ossl_rsaz_amm52x40_x1_ifma256
|
||||
.globl ossl_rsaz_amm52x40_x2_ifma256
|
||||
.globl ossl_extract_multiplier_2x40_win5
|
||||
.type ossl_rsaz_amm52x40_x1_ifma256,\@abi-omnipotent
|
||||
ossl_rsaz_amm52x40_x1_ifma256:
|
||||
ossl_rsaz_amm52x40_x2_ifma256:
|
||||
ossl_extract_multiplier_2x40_win5:
|
||||
.byte 0x0f,0x0b # ud2
|
||||
ret
|
||||
.size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256
|
||||
___
|
||||
}}}
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "crypto/bn.h"
|
||||
#include "crypto/s390x_arch.h"
|
||||
|
||||
#ifdef S390X_MOD_EXP
|
||||
|
||||
# include <sys/types.h>
|
||||
# include <sys/stat.h>
|
||||
# include <fcntl.h>
|
||||
# include <asm/zcrypt.h>
|
||||
# include <sys/ioctl.h>
|
||||
# include <unistd.h>
|
||||
# include <errno.h>
|
||||
|
||||
static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
|
||||
const BIGNUM *m)
|
||||
{
|
||||
struct ica_rsa_modexpo me;
|
||||
unsigned char *buffer;
|
||||
size_t size;
|
||||
int res = 0;
|
||||
|
||||
if (OPENSSL_s390xcex == -1 || OPENSSL_s390xcex_nodev)
|
||||
return 0;
|
||||
size = BN_num_bytes(m);
|
||||
buffer = OPENSSL_zalloc(4 * size);
|
||||
if (buffer == NULL)
|
||||
return 0;
|
||||
me.inputdata = buffer;
|
||||
me.inputdatalength = size;
|
||||
me.outputdata = buffer + size;
|
||||
me.outputdatalength = size;
|
||||
me.b_key = buffer + 2 * size;
|
||||
me.n_modulus = buffer + 3 * size;
|
||||
if (BN_bn2binpad(a, me.inputdata, size) == -1
|
||||
|| BN_bn2binpad(p, me.b_key, size) == -1
|
||||
|| BN_bn2binpad(m, me.n_modulus, size) == -1)
|
||||
goto dealloc;
|
||||
if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) {
|
||||
if (BN_bin2bn(me.outputdata, size, r) != NULL)
|
||||
res = 1;
|
||||
} else if (errno == EBADF || errno == ENOTTY) {
|
||||
/*
|
||||
* In this cases, someone (e.g. a sandbox) closed the fd.
|
||||
* Make sure to not further use this hardware acceleration.
|
||||
* In case of ENOTTY the file descriptor was already reused for another
|
||||
* file. Do not attempt to use or close that file descriptor anymore.
|
||||
*/
|
||||
OPENSSL_s390xcex = -1;
|
||||
} else if (errno == ENODEV) {
|
||||
/*
|
||||
* No crypto card(s) available to handle RSA requests.
|
||||
* Make sure to not further use this hardware acceleration,
|
||||
* but do not close the file descriptor.
|
||||
*/
|
||||
OPENSSL_s390xcex_nodev = 1;
|
||||
}
|
||||
dealloc:
|
||||
OPENSSL_clear_free(buffer, 4 * size);
|
||||
return res;
|
||||
}
|
||||
|
||||
int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
|
||||
const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
|
||||
{
|
||||
if (s390x_mod_exp_hw(r, a, p, m) == 1)
|
||||
return 1;
|
||||
return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
|
||||
}
|
||||
|
||||
int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
|
||||
const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
|
||||
{
|
||||
struct ica_rsa_modexpo_crt crt;
|
||||
unsigned char *buffer, *part;
|
||||
size_t size, plen, qlen;
|
||||
int res = 0;
|
||||
|
||||
if (OPENSSL_s390xcex == -1 || OPENSSL_s390xcex_nodev)
|
||||
return 0;
|
||||
/*-
|
||||
* Hardware-accelerated CRT can only deal with p>q. Fall back to
|
||||
* software in the (hopefully rare) other cases.
|
||||
*/
|
||||
if (BN_ucmp(p, q) != 1)
|
||||
return 0;
|
||||
plen = BN_num_bytes(p);
|
||||
qlen = BN_num_bytes(q);
|
||||
size = (plen > qlen ? plen : qlen);
|
||||
buffer = OPENSSL_zalloc(9 * size + 24);
|
||||
if (buffer == NULL)
|
||||
return 0;
|
||||
part = buffer;
|
||||
crt.inputdata = part;
|
||||
crt.inputdatalength = 2 * size;
|
||||
part += 2 * size;
|
||||
crt.outputdata = part;
|
||||
crt.outputdatalength = 2 * size;
|
||||
part += 2 * size;
|
||||
crt.bp_key = part;
|
||||
part += size + 8;
|
||||
crt.bq_key = part;
|
||||
part += size;
|
||||
crt.np_prime = part;
|
||||
part += size + 8;
|
||||
crt.nq_prime = part;
|
||||
part += size;
|
||||
crt.u_mult_inv = part;
|
||||
if (BN_bn2binpad(i, crt.inputdata, crt.inputdatalength) == -1
|
||||
|| BN_bn2binpad(p, crt.np_prime, size + 8) == -1
|
||||
|| BN_bn2binpad(q, crt.nq_prime, size) == -1
|
||||
|| BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1
|
||||
|| BN_bn2binpad(dmq, crt.bq_key, size) == -1
|
||||
|| BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1)
|
||||
goto dealloc;
|
||||
if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) {
|
||||
if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL)
|
||||
res = 1;
|
||||
} else if (errno == EBADF || errno == ENOTTY) {
|
||||
/*
|
||||
* In this cases, someone (e.g. a sandbox) closed the fd.
|
||||
* Make sure to not further use this hardware acceleration.
|
||||
* In case of ENOTTY the file descriptor was already reused for another
|
||||
* file. Do not attempt to use or close that file descriptor anymore.
|
||||
*/
|
||||
OPENSSL_s390xcex = -1;
|
||||
} else if (errno == ENODEV) {
|
||||
/*
|
||||
* No crypto card(s) available to handle RSA requests.
|
||||
* Make sure to not further use this hardware acceleration,
|
||||
* but do not close the file descriptor.
|
||||
*/
|
||||
OPENSSL_s390xcex_nodev = 1;
|
||||
}
|
||||
dealloc:
|
||||
OPENSSL_clear_free(buffer, 9 * size + 24);
|
||||
return res;
|
||||
}
|
||||
|
||||
#else
|
||||
int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
|
||||
const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
|
||||
{
|
||||
return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
|
||||
}
|
||||
|
||||
int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
|
||||
const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
Executable
+1168
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Executable
+518
@@ -0,0 +1,518 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Jerry Shih <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Basic Bit-manipulation extension ('Zbb')
|
||||
# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)
|
||||
# Optional:
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
my $use_zvkb = $flavour && $flavour =~ /zvkb/i ? 1 : 0;
|
||||
my $isaext = "_v_zbb" . ( $use_zvkb ? "_zvkb" : "" );
|
||||
|
||||
$output and open STDOUT, ">$output";
|
||||
|
||||
my $code = <<___;
|
||||
.text
|
||||
___
|
||||
|
||||
# void ChaCha20_ctr32@{[$isaext]}(unsigned char *out, const unsigned char *inp,
|
||||
# size_t len, const unsigned int key[8],
|
||||
# const unsigned int counter[4]);
|
||||
################################################################################
|
||||
my ( $OUTPUT, $INPUT, $LEN, $KEY, $COUNTER ) = ( "a0", "a1", "a2", "a3", "a4" );
|
||||
my ( $CONST_DATA0, $CONST_DATA1, $CONST_DATA2, $CONST_DATA3 ) = ( "a5", "a6",
|
||||
"a7", "s0" );
|
||||
my ( $KEY0, $KEY1, $KEY2, $KEY3, $KEY4, $KEY5, $KEY6, $KEY7, $COUNTER0,
|
||||
$COUNTER1, $NONCE0, $NONCE1) = ( "s1", "s2", "s3", "s4", "s5", "s6", "s7",
|
||||
"s8", "s9", "s10", "s11", "t0" );
|
||||
my ( $STATE0, $STATE1, $STATE2, $STATE3,
|
||||
$STATE4, $STATE5, $STATE6, $STATE7,
|
||||
$STATE8, $STATE9, $STATE10, $STATE11,
|
||||
$STATE12, $STATE13, $STATE14, $STATE15) = (
|
||||
$CONST_DATA0, $CONST_DATA1, $CONST_DATA2, $CONST_DATA3,
|
||||
$KEY0, $KEY1, $KEY2, $KEY3,
|
||||
$KEY4, $KEY5, $KEY6, $KEY7,
|
||||
$COUNTER0, $COUNTER1, $NONCE0, $NONCE1 );
|
||||
my ( $VL ) = ( "t1" );
|
||||
my ( $CURRENT_COUNTER ) = ( "t2" );
|
||||
my ( $T0 ) = ( "t3" );
|
||||
my ( $T1 ) = ( "t4" );
|
||||
my ( $T2 ) = ( "t5" );
|
||||
my ( $T3 ) = ( "t6" );
|
||||
my (
|
||||
$V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, $V8, $V9, $V10,
|
||||
$V11, $V12, $V13, $V14, $V15, $V16, $V17, $V18, $V19, $V20, $V21,
|
||||
$V22, $V23, $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map( "v$_", ( 0 .. 31 ) );
|
||||
|
||||
sub chacha_sub_round {
|
||||
my (
|
||||
$A0, $B0, $C0,
|
||||
$A1, $B1, $C1,
|
||||
$A2, $B2, $C2,
|
||||
$A3, $B3, $C3,
|
||||
|
||||
$S_A0, $S_B0, $S_C0,
|
||||
$S_A1, $S_B1, $S_C1,
|
||||
$S_A2, $S_B2, $S_C2,
|
||||
$S_A3, $S_B3, $S_C3,
|
||||
|
||||
$ROL_SHIFT,
|
||||
|
||||
$V_T0, $V_T1, $V_T2, $V_T3,
|
||||
) = @_;
|
||||
|
||||
# a += b; c ^= a;
|
||||
my $code = <<___;
|
||||
@{[vadd_vv $A0, $A0, $B0]}
|
||||
add $S_A0, $S_A0, $S_B0
|
||||
@{[vadd_vv $A1, $A1, $B1]}
|
||||
add $S_A1, $S_A1, $S_B1
|
||||
@{[vadd_vv $A2, $A2, $B2]}
|
||||
add $S_A2, $S_A2, $S_B2
|
||||
@{[vadd_vv $A3, $A3, $B3]}
|
||||
add $S_A3, $S_A3, $S_B3
|
||||
@{[vxor_vv $C0, $C0, $A0]}
|
||||
xor $S_C0, $S_C0, $S_A0
|
||||
@{[vxor_vv $C1, $C1, $A1]}
|
||||
xor $S_C1, $S_C1, $S_A1
|
||||
@{[vxor_vv $C2, $C2, $A2]}
|
||||
xor $S_C2, $S_C2, $S_A2
|
||||
@{[vxor_vv $C3, $C3, $A3]}
|
||||
xor $S_C3, $S_C3, $S_A3
|
||||
___
|
||||
|
||||
# c <<<= $ROL_SHIFT;
|
||||
if ($use_zvkb) {
|
||||
my $ror_part = <<___;
|
||||
@{[vror_vi $C0, $C0, 32 - $ROL_SHIFT]}
|
||||
@{[roriw $S_C0, $S_C0, 32 - $ROL_SHIFT]}
|
||||
@{[vror_vi $C1, $C1, 32 - $ROL_SHIFT]}
|
||||
@{[roriw $S_C1, $S_C1, 32 - $ROL_SHIFT]}
|
||||
@{[vror_vi $C2, $C2, 32 - $ROL_SHIFT]}
|
||||
@{[roriw $S_C2, $S_C2, 32 - $ROL_SHIFT]}
|
||||
@{[vror_vi $C3, $C3, 32 - $ROL_SHIFT]}
|
||||
@{[roriw $S_C3, $S_C3, 32 - $ROL_SHIFT]}
|
||||
___
|
||||
|
||||
$code .= $ror_part;
|
||||
} else {
|
||||
my $ror_part = <<___;
|
||||
@{[vsll_vi $V_T0, $C0, $ROL_SHIFT]}
|
||||
@{[vsll_vi $V_T1, $C1, $ROL_SHIFT]}
|
||||
@{[vsll_vi $V_T2, $C2, $ROL_SHIFT]}
|
||||
@{[vsll_vi $V_T3, $C3, $ROL_SHIFT]}
|
||||
@{[vsrl_vi $C0, $C0, 32 - $ROL_SHIFT]}
|
||||
@{[vsrl_vi $C1, $C1, 32 - $ROL_SHIFT]}
|
||||
@{[vsrl_vi $C2, $C2, 32 - $ROL_SHIFT]}
|
||||
@{[vsrl_vi $C3, $C3, 32 - $ROL_SHIFT]}
|
||||
@{[vor_vv $C0, $C0, $V_T0]}
|
||||
@{[roriw $S_C0, $S_C0, 32 - $ROL_SHIFT]}
|
||||
@{[vor_vv $C1, $C1, $V_T1]}
|
||||
@{[roriw $S_C1, $S_C1, 32 - $ROL_SHIFT]}
|
||||
@{[vor_vv $C2, $C2, $V_T2]}
|
||||
@{[roriw $S_C2, $S_C2, 32 - $ROL_SHIFT]}
|
||||
@{[vor_vv $C3, $C3, $V_T3]}
|
||||
@{[roriw $S_C3, $S_C3, 32 - $ROL_SHIFT]}
|
||||
___
|
||||
|
||||
$code .= $ror_part;
|
||||
}
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
sub chacha_quad_round_group {
|
||||
my (
|
||||
$A0, $B0, $C0, $D0,
|
||||
$A1, $B1, $C1, $D1,
|
||||
$A2, $B2, $C2, $D2,
|
||||
$A3, $B3, $C3, $D3,
|
||||
|
||||
$S_A0, $S_B0, $S_C0, $S_D0,
|
||||
$S_A1, $S_B1, $S_C1, $S_D1,
|
||||
$S_A2, $S_B2, $S_C2, $S_D2,
|
||||
$S_A3, $S_B3, $S_C3, $S_D3,
|
||||
|
||||
$V_T0, $V_T1, $V_T2, $V_T3,
|
||||
) = @_;
|
||||
|
||||
my $code = <<___;
|
||||
# a += b; d ^= a; d <<<= 16;
|
||||
@{[chacha_sub_round
|
||||
$A0, $B0, $D0,
|
||||
$A1, $B1, $D1,
|
||||
$A2, $B2, $D2,
|
||||
$A3, $B3, $D3,
|
||||
$S_A0, $S_B0, $S_D0,
|
||||
$S_A1, $S_B1, $S_D1,
|
||||
$S_A2, $S_B2, $S_D2,
|
||||
$S_A3, $S_B3, $S_D3,
|
||||
16,
|
||||
$V_T0, $V_T1, $V_T2, $V_T3]}
|
||||
# c += d; b ^= c; b <<<= 12;
|
||||
@{[chacha_sub_round
|
||||
$C0, $D0, $B0,
|
||||
$C1, $D1, $B1,
|
||||
$C2, $D2, $B2,
|
||||
$C3, $D3, $B3,
|
||||
$S_C0, $S_D0, $S_B0,
|
||||
$S_C1, $S_D1, $S_B1,
|
||||
$S_C2, $S_D2, $S_B2,
|
||||
$S_C3, $S_D3, $S_B3,
|
||||
12,
|
||||
$V_T0, $V_T1, $V_T2, $V_T3]}
|
||||
# a += b; d ^= a; d <<<= 8;
|
||||
@{[chacha_sub_round
|
||||
$A0, $B0, $D0,
|
||||
$A1, $B1, $D1,
|
||||
$A2, $B2, $D2,
|
||||
$A3, $B3, $D3,
|
||||
$S_A0, $S_B0, $S_D0,
|
||||
$S_A1, $S_B1, $S_D1,
|
||||
$S_A2, $S_B2, $S_D2,
|
||||
$S_A3, $S_B3, $S_D3,
|
||||
8,
|
||||
$V_T0, $V_T1, $V_T2, $V_T3]}
|
||||
# c += d; b ^= c; b <<<= 7;
|
||||
@{[chacha_sub_round
|
||||
$C0, $D0, $B0,
|
||||
$C1, $D1, $B1,
|
||||
$C2, $D2, $B2,
|
||||
$C3, $D3, $B3,
|
||||
$S_C0, $S_D0, $S_B0,
|
||||
$S_C1, $S_D1, $S_B1,
|
||||
$S_C2, $S_D2, $S_B2,
|
||||
$S_C3, $S_D3, $S_B3,
|
||||
7,
|
||||
$V_T0, $V_T1, $V_T2, $V_T3]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl ChaCha20_ctr32@{[$isaext]}
|
||||
.type ChaCha20_ctr32@{[$isaext]},\@function
|
||||
ChaCha20_ctr32@{[$isaext]}:
|
||||
addi sp, sp, -96
|
||||
sd s0, 0(sp)
|
||||
sd s1, 8(sp)
|
||||
sd s2, 16(sp)
|
||||
sd s3, 24(sp)
|
||||
sd s4, 32(sp)
|
||||
sd s5, 40(sp)
|
||||
sd s6, 48(sp)
|
||||
sd s7, 56(sp)
|
||||
sd s8, 64(sp)
|
||||
sd s9, 72(sp)
|
||||
sd s10, 80(sp)
|
||||
sd s11, 88(sp)
|
||||
addi sp, sp, -64
|
||||
|
||||
lw $CURRENT_COUNTER, 0($COUNTER)
|
||||
|
||||
.Lblock_loop:
|
||||
# We will use the scalar ALU for 1 chacha block.
|
||||
srli $T0, $LEN, 6
|
||||
@{[vsetvli $VL, $T0, "e32", "m1", "ta", "ma"]}
|
||||
slli $T1, $VL, 6
|
||||
bltu $T1, $LEN, 1f
|
||||
# Since there is no more chacha block existed, we need to split 1 block
|
||||
# from vector ALU.
|
||||
addi $T1, $VL, -1
|
||||
@{[vsetvli $VL, $T1, "e32", "m1", "ta", "ma"]}
|
||||
1:
|
||||
|
||||
#### chacha block data
|
||||
# init chacha const states into $V0~$V3
|
||||
# "expa" little endian
|
||||
li $CONST_DATA0, 0x61707865
|
||||
@{[vmv_v_x $V0, $CONST_DATA0]}
|
||||
# "nd 3" little endian
|
||||
li $CONST_DATA1, 0x3320646e
|
||||
@{[vmv_v_x $V1, $CONST_DATA1]}
|
||||
# "2-by" little endian
|
||||
li $CONST_DATA2, 0x79622d32
|
||||
@{[vmv_v_x $V2, $CONST_DATA2]}
|
||||
# "te k" little endian
|
||||
li $CONST_DATA3, 0x6b206574
|
||||
lw $KEY0, 0($KEY)
|
||||
@{[vmv_v_x $V3, $CONST_DATA3]}
|
||||
|
||||
# init chacha key states into $V4~$V11
|
||||
lw $KEY1, 4($KEY)
|
||||
@{[vmv_v_x $V4, $KEY0]}
|
||||
lw $KEY2, 8($KEY)
|
||||
@{[vmv_v_x $V5, $KEY1]}
|
||||
lw $KEY3, 12($KEY)
|
||||
@{[vmv_v_x $V6, $KEY2]}
|
||||
lw $KEY4, 16($KEY)
|
||||
@{[vmv_v_x $V7, $KEY3]}
|
||||
lw $KEY5, 20($KEY)
|
||||
@{[vmv_v_x $V8, $KEY4]}
|
||||
lw $KEY6, 24($KEY)
|
||||
@{[vmv_v_x $V9, $KEY5]}
|
||||
lw $KEY7, 28($KEY)
|
||||
@{[vmv_v_x $V10, $KEY6]}
|
||||
@{[vmv_v_x $V11, $KEY7]}
|
||||
|
||||
# init chacha key states into $V12~$V13
|
||||
lw $COUNTER1, 4($COUNTER)
|
||||
@{[vid_v $V12]}
|
||||
lw $NONCE0, 8($COUNTER)
|
||||
@{[vadd_vx $V12, $V12, $CURRENT_COUNTER]}
|
||||
lw $NONCE1, 12($COUNTER)
|
||||
@{[vmv_v_x $V13, $COUNTER1]}
|
||||
add $COUNTER0, $CURRENT_COUNTER, $VL
|
||||
|
||||
# init chacha nonce states into $V14~$V15
|
||||
@{[vmv_v_x $V14, $NONCE0]}
|
||||
@{[vmv_v_x $V15, $NONCE1]}
|
||||
|
||||
li $T0, 64
|
||||
# load the top-half of input data into $V16~$V23
|
||||
@{[vlsseg_nf_e32_v 8, $V16, $INPUT, $T0]}
|
||||
|
||||
# till now in block_loop, we used:
|
||||
# - $V0~$V15 for chacha states.
|
||||
# - $V16~$V23 for top-half of input data.
|
||||
# - $V24~$V31 haven't been used yet.
|
||||
|
||||
# 20 round groups
|
||||
li $T0, 10
|
||||
.Lround_loop:
|
||||
# we can use $V24~$V31 as temporary registers in round_loop.
|
||||
addi $T0, $T0, -1
|
||||
@{[chacha_quad_round_group
|
||||
$V0, $V4, $V8, $V12,
|
||||
$V1, $V5, $V9, $V13,
|
||||
$V2, $V6, $V10, $V14,
|
||||
$V3, $V7, $V11, $V15,
|
||||
$STATE0, $STATE4, $STATE8, $STATE12,
|
||||
$STATE1, $STATE5, $STATE9, $STATE13,
|
||||
$STATE2, $STATE6, $STATE10, $STATE14,
|
||||
$STATE3, $STATE7, $STATE11, $STATE15,
|
||||
$V24, $V25, $V26, $V27]}
|
||||
@{[chacha_quad_round_group
|
||||
$V3, $V4, $V9, $V14,
|
||||
$V0, $V5, $V10, $V15,
|
||||
$V1, $V6, $V11, $V12,
|
||||
$V2, $V7, $V8, $V13,
|
||||
$STATE3, $STATE4, $STATE9, $STATE14,
|
||||
$STATE0, $STATE5, $STATE10, $STATE15,
|
||||
$STATE1, $STATE6, $STATE11, $STATE12,
|
||||
$STATE2, $STATE7, $STATE8, $STATE13,
|
||||
$V24, $V25, $V26, $V27]}
|
||||
bnez $T0, .Lround_loop
|
||||
|
||||
li $T0, 64
|
||||
# load the bottom-half of input data into $V24~$V31
|
||||
addi $T1, $INPUT, 32
|
||||
@{[vlsseg_nf_e32_v 8, $V24, $T1, $T0]}
|
||||
|
||||
# now, there are no free vector registers until the round_loop exits.
|
||||
|
||||
# add chacha top-half initial block states
|
||||
# "expa" little endian
|
||||
li $T0, 0x61707865
|
||||
@{[vadd_vx $V0, $V0, $T0]}
|
||||
add $STATE0, $STATE0, $T0
|
||||
# "nd 3" little endian
|
||||
li $T1, 0x3320646e
|
||||
@{[vadd_vx $V1, $V1, $T1]}
|
||||
add $STATE1, $STATE1, $T1
|
||||
lw $T0, 0($KEY)
|
||||
# "2-by" little endian
|
||||
li $T2, 0x79622d32
|
||||
@{[vadd_vx $V2, $V2, $T2]}
|
||||
add $STATE2, $STATE2, $T2
|
||||
lw $T1, 4($KEY)
|
||||
# "te k" little endian
|
||||
li $T3, 0x6b206574
|
||||
@{[vadd_vx $V3, $V3, $T3]}
|
||||
add $STATE3, $STATE3, $T3
|
||||
lw $T2, 8($KEY)
|
||||
@{[vadd_vx $V4, $V4, $T0]}
|
||||
add $STATE4, $STATE4, $T0
|
||||
lw $T3, 12($KEY)
|
||||
@{[vadd_vx $V5, $V5, $T1]}
|
||||
add $STATE5, $STATE5, $T1
|
||||
@{[vadd_vx $V6, $V6, $T2]}
|
||||
add $STATE6, $STATE6, $T2
|
||||
@{[vadd_vx $V7, $V7, $T3]}
|
||||
add $STATE7, $STATE7, $T3
|
||||
|
||||
# xor with the top-half input
|
||||
@{[vxor_vv $V16, $V16, $V0]}
|
||||
sw $STATE0, 0(sp)
|
||||
sw $STATE1, 4(sp)
|
||||
@{[vxor_vv $V17, $V17, $V1]}
|
||||
sw $STATE2, 8(sp)
|
||||
sw $STATE3, 12(sp)
|
||||
@{[vxor_vv $V18, $V18, $V2]}
|
||||
sw $STATE4, 16(sp)
|
||||
sw $STATE5, 20(sp)
|
||||
@{[vxor_vv $V19, $V19, $V3]}
|
||||
sw $STATE6, 24(sp)
|
||||
sw $STATE7, 28(sp)
|
||||
@{[vxor_vv $V20, $V20, $V4]}
|
||||
lw $T0, 16($KEY)
|
||||
@{[vxor_vv $V21, $V21, $V5]}
|
||||
lw $T1, 20($KEY)
|
||||
@{[vxor_vv $V22, $V22, $V6]}
|
||||
lw $T2, 24($KEY)
|
||||
@{[vxor_vv $V23, $V23, $V7]}
|
||||
|
||||
# save the top-half of output from $V16~$V23
|
||||
li $T3, 64
|
||||
@{[vssseg_nf_e32_v 8, $V16, $OUTPUT, $T3]}
|
||||
|
||||
# add chacha bottom-half initial block states
|
||||
@{[vadd_vx $V8, $V8, $T0]}
|
||||
add $STATE8, $STATE8, $T0
|
||||
lw $T3, 28($KEY)
|
||||
@{[vadd_vx $V9, $V9, $T1]}
|
||||
add $STATE9, $STATE9, $T1
|
||||
lw $T0, 4($COUNTER)
|
||||
@{[vadd_vx $V10, $V10, $T2]}
|
||||
add $STATE10, $STATE10, $T2
|
||||
lw $T1, 8($COUNTER)
|
||||
@{[vadd_vx $V11, $V11, $T3]}
|
||||
add $STATE11, $STATE11, $T3
|
||||
lw $T2, 12($COUNTER)
|
||||
@{[vid_v $V0]}
|
||||
add $STATE12, $STATE12, $CURRENT_COUNTER
|
||||
@{[vadd_vx $V12, $V12, $CURRENT_COUNTER]}
|
||||
add $STATE12, $STATE12, $VL
|
||||
@{[vadd_vx $V13, $V13, $T0]}
|
||||
add $STATE13, $STATE13, $T0
|
||||
@{[vadd_vx $V14, $V14, $T1]}
|
||||
add $STATE14, $STATE14, $T1
|
||||
@{[vadd_vx $V15, $V15, $T2]}
|
||||
add $STATE15, $STATE15, $T2
|
||||
@{[vadd_vv $V12, $V12, $V0]}
|
||||
# xor with the bottom-half input
|
||||
@{[vxor_vv $V24, $V24, $V8]}
|
||||
sw $STATE8, 32(sp)
|
||||
@{[vxor_vv $V25, $V25, $V9]}
|
||||
sw $STATE9, 36(sp)
|
||||
@{[vxor_vv $V26, $V26, $V10]}
|
||||
sw $STATE10, 40(sp)
|
||||
@{[vxor_vv $V27, $V27, $V11]}
|
||||
sw $STATE11, 44(sp)
|
||||
@{[vxor_vv $V29, $V29, $V13]}
|
||||
sw $STATE12, 48(sp)
|
||||
@{[vxor_vv $V28, $V28, $V12]}
|
||||
sw $STATE13, 52(sp)
|
||||
@{[vxor_vv $V30, $V30, $V14]}
|
||||
sw $STATE14, 56(sp)
|
||||
@{[vxor_vv $V31, $V31, $V15]}
|
||||
sw $STATE15, 60(sp)
|
||||
|
||||
# save the bottom-half of output from $V24~$V31
|
||||
li $T0, 64
|
||||
addi $T1, $OUTPUT, 32
|
||||
@{[vssseg_nf_e32_v 8, $V24, $T1, $T0]}
|
||||
|
||||
# the computed vector parts: `64 * VL`
|
||||
slli $T0, $VL, 6
|
||||
|
||||
add $INPUT, $INPUT, $T0
|
||||
add $OUTPUT, $OUTPUT, $T0
|
||||
sub $LEN, $LEN, $T0
|
||||
add $CURRENT_COUNTER, $CURRENT_COUNTER, $VL
|
||||
|
||||
# process the scalar data block
|
||||
addi $CURRENT_COUNTER, $CURRENT_COUNTER, 1
|
||||
li $T0, 64
|
||||
@{[minu $T1, $LEN, $T0]}
|
||||
sub $LEN, $LEN, $T1
|
||||
mv $T2, sp
|
||||
.Lscalar_data_loop:
|
||||
@{[vsetvli $VL, $T1, "e8", "m8", "ta", "ma"]}
|
||||
# from this on, vector registers are grouped with lmul = 8
|
||||
@{[vle8_v $V8, $INPUT]}
|
||||
@{[vle8_v $V16, $T2]}
|
||||
@{[vxor_vv $V8, $V8, $V16]}
|
||||
@{[vse8_v $V8, $OUTPUT]}
|
||||
add $INPUT, $INPUT, $VL
|
||||
add $OUTPUT, $OUTPUT, $VL
|
||||
add $T2, $T2, $VL
|
||||
sub $T1, $T1, $VL
|
||||
bnez $T1, .Lscalar_data_loop
|
||||
|
||||
bnez $LEN, .Lblock_loop
|
||||
|
||||
addi sp, sp, 64
|
||||
ld s0, 0(sp)
|
||||
ld s1, 8(sp)
|
||||
ld s2, 16(sp)
|
||||
ld s3, 24(sp)
|
||||
ld s4, 32(sp)
|
||||
ld s5, 40(sp)
|
||||
ld s6, 48(sp)
|
||||
ld s7, 56(sp)
|
||||
ld s8, 64(sp)
|
||||
ld s9, 72(sp)
|
||||
ld s10, 80(sp)
|
||||
ld s11, 88(sp)
|
||||
addi sp, sp, 96
|
||||
|
||||
ret
|
||||
.size ChaCha20_ctr32@{[$isaext]},.-ChaCha20_ctr32@{[$isaext]}
|
||||
___
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
Executable
+1290
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* This file is dual-licensed, meaning that you can use it under your
|
||||
* choice of either of the following two licenses:
|
||||
*
|
||||
* Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
* or
|
||||
*
|
||||
* Copyright (c) 2023, Jerry Shih <[email protected]>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <openssl/opensslconf.h>
|
||||
#include "crypto/chacha.h"
|
||||
#include "crypto/riscv_arch.h"
|
||||
|
||||
void ChaCha20_ctr32_v_zbb_zvkb(unsigned char *out, const unsigned char *inp,
|
||||
size_t len, const unsigned int key[8],
|
||||
const unsigned int counter[4]);
|
||||
|
||||
void ChaCha20_ctr32_v_zbb(unsigned char *out, const unsigned char *inp,
|
||||
size_t len, const unsigned int key[8],
|
||||
const unsigned int counter[4]);
|
||||
|
||||
void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len,
|
||||
const unsigned int key[8], const unsigned int counter[4])
|
||||
{
|
||||
if (len > CHACHA_BLK_SIZE && RISCV_HAS_ZBB() && riscv_vlen() >= 128) {
|
||||
if (RISCV_HAS_ZVKB()) {
|
||||
ChaCha20_ctr32_v_zbb_zvkb(out, inp, len, key, counter);
|
||||
} else {
|
||||
ChaCha20_ctr32_v_zbb(out, inp, len, key, counter);
|
||||
}
|
||||
} else {
|
||||
ChaCha20_ctr32_c(out, inp, len, key, counter);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
/*
|
||||
* Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
* Copyright Siemens AG 2022
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "cmp_local.h"
|
||||
#include <openssl/cmp_util.h>
|
||||
|
||||
static const X509_VERIFY_PARAM *get0_trustedStore_vpm(const OSSL_CMP_CTX *ctx)
|
||||
{
|
||||
const X509_STORE *ts = OSSL_CMP_CTX_get0_trustedStore(ctx);
|
||||
|
||||
return ts == NULL ? NULL : X509_STORE_get0_param(ts);
|
||||
}
|
||||
|
||||
static void cert_msg(const char *func, const char *file, int lineno,
|
||||
OSSL_CMP_severity level, OSSL_CMP_CTX *ctx,
|
||||
const char *source, X509 *cert, const char *msg)
|
||||
{
|
||||
char *subj = X509_NAME_oneline(X509_get_subject_name(cert), NULL, 0);
|
||||
|
||||
ossl_cmp_print_log(level, ctx, func, file, lineno,
|
||||
level == OSSL_CMP_LOG_WARNING ? "WARN" : "ERR",
|
||||
"certificate from '%s' with subject '%s' %s",
|
||||
source, subj, msg);
|
||||
OPENSSL_free(subj);
|
||||
}
|
||||
|
||||
/* use |type_CA| -1 (no CA type check) or 0 (must be EE) or 1 (must be CA) */
|
||||
static int ossl_X509_check(OSSL_CMP_CTX *ctx, const char *source, X509 *cert,
|
||||
int type_CA, const X509_VERIFY_PARAM *vpm)
|
||||
{
|
||||
uint32_t ex_flags = X509_get_extension_flags(cert);
|
||||
int res = X509_cmp_timeframe(vpm, X509_get0_notBefore(cert),
|
||||
X509_get0_notAfter(cert));
|
||||
int ret = res == 0;
|
||||
OSSL_CMP_severity level =
|
||||
vpm == NULL ? OSSL_CMP_LOG_WARNING : OSSL_CMP_LOG_ERR;
|
||||
|
||||
if (!ret)
|
||||
cert_msg(OPENSSL_FUNC, OPENSSL_FILE, OPENSSL_LINE, level, ctx,
|
||||
source, cert, res > 0 ? "has expired" : "not yet valid");
|
||||
if (type_CA >= 0 && (ex_flags & EXFLAG_V1) == 0) {
|
||||
int is_CA = (ex_flags & EXFLAG_CA) != 0;
|
||||
|
||||
if ((type_CA != 0) != is_CA) {
|
||||
cert_msg(OPENSSL_FUNC, OPENSSL_FILE, OPENSSL_LINE, level, ctx,
|
||||
source, cert,
|
||||
is_CA ? "is not an EE cert" : "is not a CA cert");
|
||||
ret = 0;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int ossl_X509_check_all(OSSL_CMP_CTX *ctx, const char *source,
|
||||
STACK_OF(X509) *certs,
|
||||
int type_CA, const X509_VERIFY_PARAM *vpm)
|
||||
{
|
||||
int i;
|
||||
int ret = 1;
|
||||
|
||||
for (i = 0; i < sk_X509_num(certs /* may be NULL */); i++)
|
||||
ret = ossl_X509_check(ctx, source,
|
||||
sk_X509_value(certs, i), type_CA, vpm)
|
||||
&& ret; /* Having 'ret' after the '&&', all certs are checked. */
|
||||
return ret;
|
||||
}
|
||||
|
||||
static OSSL_CMP_ITAV *get_genm_itav(OSSL_CMP_CTX *ctx,
|
||||
OSSL_CMP_ITAV *req, /* gets consumed */
|
||||
int expected, const char *desc)
|
||||
{
|
||||
STACK_OF(OSSL_CMP_ITAV) *itavs = NULL;
|
||||
int i, n;
|
||||
|
||||
if (ctx == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT);
|
||||
goto err;
|
||||
}
|
||||
if (OSSL_CMP_CTX_get_status(ctx) != OSSL_CMP_PKISTATUS_unspecified) {
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_UNCLEAN_CTX,
|
||||
"client context in unsuitable state; should call CMPclient_reinit() before");
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (!OSSL_CMP_CTX_push0_genm_ITAV(ctx, req))
|
||||
goto err;
|
||||
req = NULL;
|
||||
itavs = OSSL_CMP_exec_GENM_ses(ctx);
|
||||
if (itavs == NULL) {
|
||||
if (OSSL_CMP_CTX_get_status(ctx) != OSSL_CMP_PKISTATUS_request)
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_GETTING_GENP,
|
||||
"with infoType %s", desc);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if ((n = sk_OSSL_CMP_ITAV_num(itavs)) <= 0) {
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_GENP,
|
||||
"response on genm requesting infoType %s does not include suitable value", desc);
|
||||
sk_OSSL_CMP_ITAV_free(itavs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (n > 1)
|
||||
ossl_cmp_log2(WARN, ctx,
|
||||
"response on genm contains %d ITAVs; will use the first ITAV with infoType id-it-%s",
|
||||
n, desc);
|
||||
for (i = 0; i < n; i++) {
|
||||
OSSL_CMP_ITAV *itav = sk_OSSL_CMP_ITAV_shift(itavs);
|
||||
ASN1_OBJECT *obj = OSSL_CMP_ITAV_get0_type(itav);
|
||||
char name[128] = "genp contains InfoType '";
|
||||
size_t offset = strlen(name);
|
||||
|
||||
if (OBJ_obj2nid(obj) == expected) {
|
||||
for (i++; i < n; i++)
|
||||
OSSL_CMP_ITAV_free(sk_OSSL_CMP_ITAV_shift(itavs));
|
||||
sk_OSSL_CMP_ITAV_free(itavs);
|
||||
return itav;
|
||||
}
|
||||
|
||||
if (OBJ_obj2txt(name + offset, sizeof(name) - offset, obj, 0) < 0)
|
||||
strcat(name, "<unknown>");
|
||||
ossl_cmp_log2(WARN, ctx, "%s' while expecting 'id-it-%s'", name, desc);
|
||||
OSSL_CMP_ITAV_free(itav);
|
||||
}
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_GENP,
|
||||
"could not find any ITAV for %s", desc);
|
||||
|
||||
err:
|
||||
sk_OSSL_CMP_ITAV_free(itavs);
|
||||
OSSL_CMP_ITAV_free(req);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int OSSL_CMP_get1_caCerts(OSSL_CMP_CTX *ctx, STACK_OF(X509) **out)
|
||||
{
|
||||
OSSL_CMP_ITAV *req, *itav;
|
||||
STACK_OF(X509) *certs = NULL;
|
||||
int ret = 0;
|
||||
|
||||
if (out == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
*out = NULL;
|
||||
|
||||
if ((req = OSSL_CMP_ITAV_new_caCerts(NULL)) == NULL)
|
||||
return 0;
|
||||
if ((itav = get_genm_itav(ctx, req, NID_id_it_caCerts, "caCerts")) == NULL)
|
||||
return 0;
|
||||
if (!OSSL_CMP_ITAV_get0_caCerts(itav, &certs))
|
||||
goto end;
|
||||
ret = 1;
|
||||
if (certs == NULL) /* no CA certificate available */
|
||||
goto end;
|
||||
|
||||
if (!ossl_X509_check_all(ctx, "genp", certs, 1 /* CA */,
|
||||
get0_trustedStore_vpm(ctx))) {
|
||||
ret = 0;
|
||||
goto end;
|
||||
}
|
||||
*out = sk_X509_new_reserve(NULL, sk_X509_num(certs));
|
||||
if (!X509_add_certs(*out, certs,
|
||||
X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP)) {
|
||||
sk_X509_pop_free(*out, X509_free);
|
||||
*out = NULL;
|
||||
ret = 0;
|
||||
}
|
||||
|
||||
end:
|
||||
OSSL_CMP_ITAV_free(itav);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int selfsigned_verify_cb(int ok, X509_STORE_CTX *store_ctx)
|
||||
{
|
||||
if (ok == 0
|
||||
&& X509_STORE_CTX_get_error_depth(store_ctx) == 0
|
||||
&& X509_STORE_CTX_get_error(store_ctx)
|
||||
== X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT) {
|
||||
/* in this case, custom chain building */
|
||||
int i;
|
||||
STACK_OF(X509) *trust;
|
||||
STACK_OF(X509) *chain = X509_STORE_CTX_get0_chain(store_ctx);
|
||||
STACK_OF(X509) *untrusted = X509_STORE_CTX_get0_untrusted(store_ctx);
|
||||
X509_STORE_CTX_check_issued_fn check_issued =
|
||||
X509_STORE_CTX_get_check_issued(store_ctx);
|
||||
X509 *cert = sk_X509_value(chain, 0); /* target cert */
|
||||
X509 *issuer;
|
||||
|
||||
for (i = 0; i < sk_X509_num(untrusted); i++) {
|
||||
cert = sk_X509_value(untrusted, i);
|
||||
if (!X509_add_cert(chain, cert, X509_ADD_FLAG_UP_REF))
|
||||
return 0;
|
||||
}
|
||||
|
||||
trust = X509_STORE_get1_all_certs(X509_STORE_CTX_get0_store(store_ctx));
|
||||
for (i = 0; i < sk_X509_num(trust); i++) {
|
||||
issuer = sk_X509_value(trust, i);
|
||||
if ((*check_issued)(store_ctx, cert, issuer)) {
|
||||
if (X509_add_cert(chain, cert, X509_ADD_FLAG_UP_REF))
|
||||
ok = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
sk_X509_pop_free(trust, X509_free);
|
||||
return ok;
|
||||
} else {
|
||||
X509_STORE *ts = X509_STORE_CTX_get0_store(store_ctx);
|
||||
X509_STORE_CTX_verify_cb verify_cb;
|
||||
|
||||
if (ts == NULL || (verify_cb = X509_STORE_get_verify_cb(ts)) == NULL)
|
||||
return ok;
|
||||
return (*verify_cb)(ok, store_ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/* vanilla X509_verify_cert() does not support self-signed certs as target */
|
||||
static int verify_ss_cert(OSSL_LIB_CTX *libctx, const char *propq,
|
||||
X509_STORE *ts, STACK_OF(X509) *untrusted,
|
||||
X509 *target)
|
||||
{
|
||||
X509_STORE_CTX *csc = NULL;
|
||||
int ok = 0;
|
||||
|
||||
if (ts == NULL || target == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((csc = X509_STORE_CTX_new_ex(libctx, propq)) == NULL
|
||||
|| !X509_STORE_CTX_init(csc, ts, target, untrusted))
|
||||
goto err;
|
||||
X509_STORE_CTX_set_verify_cb(csc, selfsigned_verify_cb);
|
||||
ok = X509_verify_cert(csc) > 0;
|
||||
|
||||
err:
|
||||
X509_STORE_CTX_free(csc);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int
|
||||
verify_ss_cert_trans(OSSL_CMP_CTX *ctx, X509 *trusted /* may be NULL */,
|
||||
X509 *trans /* the only untrusted cert, may be NULL */,
|
||||
X509 *target, const char *desc)
|
||||
{
|
||||
X509_STORE *ts = OSSL_CMP_CTX_get0_trusted(ctx);
|
||||
STACK_OF(X509) *untrusted = NULL;
|
||||
int res = 0;
|
||||
|
||||
if (trusted != NULL) {
|
||||
X509_VERIFY_PARAM *vpm = X509_STORE_get0_param(ts);
|
||||
|
||||
if ((ts = X509_STORE_new()) == NULL)
|
||||
return 0;
|
||||
if (!X509_STORE_set1_param(ts, vpm)
|
||||
|| !X509_STORE_add_cert(ts, trusted))
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (trans != NULL
|
||||
&& !ossl_x509_add_cert_new(&untrusted, trans, X509_ADD_FLAG_UP_REF))
|
||||
goto err;
|
||||
|
||||
res = verify_ss_cert(OSSL_CMP_CTX_get0_libctx(ctx),
|
||||
OSSL_CMP_CTX_get0_propq(ctx),
|
||||
ts, untrusted, target);
|
||||
if (!res)
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE,
|
||||
"failed to validate %s certificate received in genp %s",
|
||||
desc, trusted == NULL ? "using trust store"
|
||||
: "with given certificate as trust anchor");
|
||||
|
||||
err:
|
||||
sk_X509_pop_free(untrusted, X509_free);
|
||||
if (trusted != NULL)
|
||||
X509_STORE_free(ts);
|
||||
return res;
|
||||
}
|
||||
|
||||
int OSSL_CMP_get1_rootCaKeyUpdate(OSSL_CMP_CTX *ctx,
|
||||
const X509 *oldWithOld, X509 **newWithNew,
|
||||
X509 **newWithOld, X509 **oldWithNew)
|
||||
{
|
||||
X509 *oldWithOld_copy = NULL, *my_newWithOld, *my_oldWithNew;
|
||||
OSSL_CMP_ITAV *req, *itav;
|
||||
int res = 0;
|
||||
|
||||
if (newWithNew == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return 0;
|
||||
}
|
||||
*newWithNew = NULL;
|
||||
|
||||
if ((req = OSSL_CMP_ITAV_new_rootCaCert(oldWithOld)) == NULL)
|
||||
return 0;
|
||||
itav = get_genm_itav(ctx, req, NID_id_it_rootCaKeyUpdate, "rootCaKeyUpdate");
|
||||
if (itav == NULL)
|
||||
return 0;
|
||||
|
||||
if (!OSSL_CMP_ITAV_get0_rootCaKeyUpdate(itav, newWithNew,
|
||||
&my_newWithOld, &my_oldWithNew))
|
||||
goto end;
|
||||
/* no root CA cert update available */
|
||||
if (*newWithNew == NULL) {
|
||||
res = 1;
|
||||
goto end;
|
||||
}
|
||||
if ((oldWithOld_copy = X509_dup(oldWithOld)) == NULL && oldWithOld != NULL)
|
||||
goto end;
|
||||
if (!verify_ss_cert_trans(ctx, oldWithOld_copy, my_newWithOld,
|
||||
*newWithNew, "newWithNew")) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE);
|
||||
goto end;
|
||||
}
|
||||
if (oldWithOld != NULL && my_oldWithNew != NULL
|
||||
&& !verify_ss_cert_trans(ctx, *newWithNew, my_oldWithNew,
|
||||
oldWithOld_copy, "oldWithOld")) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE);
|
||||
goto end;
|
||||
}
|
||||
|
||||
if (!X509_up_ref(*newWithNew))
|
||||
goto end;
|
||||
if (newWithOld != NULL &&
|
||||
(*newWithOld = my_newWithOld) != NULL && !X509_up_ref(*newWithOld))
|
||||
goto free;
|
||||
if (oldWithNew == NULL ||
|
||||
(*oldWithNew = my_oldWithNew) == NULL || X509_up_ref(*oldWithNew)) {
|
||||
res = 1;
|
||||
goto end;
|
||||
}
|
||||
if (newWithOld != NULL)
|
||||
X509_free(*newWithOld);
|
||||
free:
|
||||
X509_free(*newWithNew);
|
||||
|
||||
end:
|
||||
OSSL_CMP_ITAV_free(itav);
|
||||
X509_free(oldWithOld_copy);
|
||||
return res;
|
||||
}
|
||||
|
||||
int OSSL_CMP_get1_crlUpdate(OSSL_CMP_CTX *ctx, const X509 *crlcert,
|
||||
const X509_CRL *last_crl,
|
||||
X509_CRL **crl)
|
||||
{
|
||||
OSSL_CMP_CRLSTATUS *status = NULL;
|
||||
STACK_OF(OSSL_CMP_CRLSTATUS) *list = NULL;
|
||||
OSSL_CMP_ITAV *req = NULL, *itav = NULL;
|
||||
STACK_OF(X509_CRL) *crls = NULL;
|
||||
int res = 0;
|
||||
|
||||
if (crl == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
*crl = NULL;
|
||||
|
||||
if ((status = OSSL_CMP_CRLSTATUS_create(last_crl, crlcert, 1)) == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_GENERATE_CRLSTATUS);
|
||||
goto end;
|
||||
}
|
||||
if ((list = sk_OSSL_CMP_CRLSTATUS_new_reserve(NULL, 1)) == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_GENERATE_CRLSTATUS);
|
||||
goto end;
|
||||
}
|
||||
(void)sk_OSSL_CMP_CRLSTATUS_push(list, status); /* cannot fail */
|
||||
|
||||
if ((req = OSSL_CMP_ITAV_new0_crlStatusList(list)) == NULL)
|
||||
goto end;
|
||||
status = NULL;
|
||||
list = NULL;
|
||||
|
||||
if ((itav = get_genm_itav(ctx, req, NID_id_it_crls, "crl")) == NULL)
|
||||
goto end;
|
||||
|
||||
if (!OSSL_CMP_ITAV_get0_crls(itav, &crls))
|
||||
goto end;
|
||||
|
||||
if (crls == NULL) { /* no CRL update available */
|
||||
res = 1;
|
||||
goto end;
|
||||
}
|
||||
if (sk_X509_CRL_num(crls) != 1) {
|
||||
ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_GENP,
|
||||
"Unexpected number of CRLs in genp: %d",
|
||||
sk_X509_CRL_num(crls));
|
||||
goto end;
|
||||
}
|
||||
|
||||
if ((*crl = sk_X509_CRL_value(crls, 0)) == NULL || !X509_CRL_up_ref(*crl)) {
|
||||
*crl = NULL;
|
||||
goto end;
|
||||
}
|
||||
res = 1;
|
||||
end:
|
||||
OSSL_CMP_CRLSTATUS_free(status);
|
||||
sk_OSSL_CMP_CRLSTATUS_free(list);
|
||||
OSSL_CMP_ITAV_free(itav);
|
||||
return res;
|
||||
}
|
||||
|
||||
int OSSL_CMP_get1_certReqTemplate(OSSL_CMP_CTX *ctx,
|
||||
OSSL_CRMF_CERTTEMPLATE **certTemplate,
|
||||
OSSL_CMP_ATAVS **keySpec)
|
||||
{
|
||||
OSSL_CMP_ITAV *req, *itav = NULL;
|
||||
int res = 0;
|
||||
|
||||
if (keySpec != NULL)
|
||||
*keySpec = NULL;
|
||||
if (certTemplate == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
*certTemplate = NULL;
|
||||
|
||||
if ((req = OSSL_CMP_ITAV_new0_certReqTemplate(NULL, NULL)) == NULL) {
|
||||
ERR_raise(ERR_LIB_CMP, CMP_R_GENERATE_CERTREQTEMPLATE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((itav = get_genm_itav(ctx, req, NID_id_it_certReqTemplate,
|
||||
"certReqTemplate")) == NULL)
|
||||
return 0;
|
||||
|
||||
if (!OSSL_CMP_ITAV_get1_certReqTemplate(itav, certTemplate, keySpec))
|
||||
goto end;
|
||||
|
||||
res = 1;
|
||||
end:
|
||||
OSSL_CMP_ITAV_free(itav);
|
||||
return res;
|
||||
}
|
||||
@@ -0,0 +1,800 @@
|
||||
/*
|
||||
* Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
* Uses brotli compression library from https://github.com/google/brotli
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <openssl/objects.h>
|
||||
#include "internal/comp.h"
|
||||
#include <openssl/err.h>
|
||||
#include "crypto/cryptlib.h"
|
||||
#include "internal/bio.h"
|
||||
#include "internal/thread_once.h"
|
||||
#include "comp_local.h"
|
||||
|
||||
COMP_METHOD *COMP_brotli(void);
|
||||
|
||||
#ifdef OPENSSL_NO_BROTLI
|
||||
# undef BROTLI_SHARED
|
||||
#else
|
||||
|
||||
# include <brotli/decode.h>
|
||||
# include <brotli/encode.h>
|
||||
|
||||
/* memory allocations functions for brotli initialisation */
|
||||
static void *brotli_alloc(void *opaque, size_t size)
|
||||
{
|
||||
return OPENSSL_zalloc(size);
|
||||
}
|
||||
|
||||
static void brotli_free(void *opaque, void *address)
|
||||
{
|
||||
OPENSSL_free(address);
|
||||
}
|
||||
|
||||
/*
|
||||
* When OpenSSL is built on Windows, we do not want to require that
|
||||
* the BROTLI.DLL be available in order for the OpenSSL DLLs to
|
||||
* work. Therefore, all BROTLI routines are loaded at run time
|
||||
* and we do not link to a .LIB file when BROTLI_SHARED is set.
|
||||
*/
|
||||
# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32)
|
||||
# include <windows.h>
|
||||
# endif
|
||||
|
||||
# ifdef BROTLI_SHARED
|
||||
# include "internal/dso.h"
|
||||
|
||||
/* Function pointers */
|
||||
typedef BrotliEncoderState *(*encode_init_ft)(brotli_alloc_func, brotli_free_func, void *);
|
||||
typedef BROTLI_BOOL (*encode_stream_ft)(BrotliEncoderState *, BrotliEncoderOperation, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *);
|
||||
typedef BROTLI_BOOL (*encode_has_more_ft)(BrotliEncoderState *);
|
||||
typedef void (*encode_end_ft)(BrotliEncoderState *);
|
||||
typedef BROTLI_BOOL (*encode_oneshot_ft)(int, int, BrotliEncoderMode, size_t, const uint8_t in[], size_t *, uint8_t out[]);
|
||||
|
||||
typedef BrotliDecoderState *(*decode_init_ft)(brotli_alloc_func, brotli_free_func, void *);
|
||||
typedef BROTLI_BOOL (*decode_stream_ft)(BrotliDecoderState *, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *);
|
||||
typedef BROTLI_BOOL (*decode_has_more_ft)(BrotliDecoderState *);
|
||||
typedef void (*decode_end_ft)(BrotliDecoderState *);
|
||||
typedef BrotliDecoderErrorCode (*decode_error_ft)(BrotliDecoderState *);
|
||||
typedef const char *(*decode_error_string_ft)(BrotliDecoderErrorCode);
|
||||
typedef BROTLI_BOOL (*decode_is_finished_ft)(BrotliDecoderState *);
|
||||
typedef BrotliDecoderResult (*decode_oneshot_ft)(size_t, const uint8_t in[], size_t *, uint8_t out[]);
|
||||
|
||||
static encode_init_ft p_encode_init = NULL;
|
||||
static encode_stream_ft p_encode_stream = NULL;
|
||||
static encode_has_more_ft p_encode_has_more = NULL;
|
||||
static encode_end_ft p_encode_end = NULL;
|
||||
static encode_oneshot_ft p_encode_oneshot = NULL;
|
||||
|
||||
static decode_init_ft p_decode_init = NULL;
|
||||
static decode_stream_ft p_decode_stream = NULL;
|
||||
static decode_has_more_ft p_decode_has_more = NULL;
|
||||
static decode_end_ft p_decode_end = NULL;
|
||||
static decode_error_ft p_decode_error = NULL;
|
||||
static decode_error_string_ft p_decode_error_string = NULL;
|
||||
static decode_is_finished_ft p_decode_is_finished = NULL;
|
||||
static decode_oneshot_ft p_decode_oneshot = NULL;
|
||||
|
||||
static DSO *brotli_encode_dso = NULL;
|
||||
static DSO *brotli_decode_dso = NULL;
|
||||
|
||||
# define BrotliEncoderCreateInstance p_encode_init
|
||||
# define BrotliEncoderCompressStream p_encode_stream
|
||||
# define BrotliEncoderHasMoreOutput p_encode_has_more
|
||||
# define BrotliEncoderDestroyInstance p_encode_end
|
||||
# define BrotliEncoderCompress p_encode_oneshot
|
||||
|
||||
# define BrotliDecoderCreateInstance p_decode_init
|
||||
# define BrotliDecoderDecompressStream p_decode_stream
|
||||
# define BrotliDecoderHasMoreOutput p_decode_has_more
|
||||
# define BrotliDecoderDestroyInstance p_decode_end
|
||||
# define BrotliDecoderGetErrorCode p_decode_error
|
||||
# define BrotliDecoderErrorString p_decode_error_string
|
||||
# define BrotliDecoderIsFinished p_decode_is_finished
|
||||
# define BrotliDecoderDecompress p_decode_oneshot
|
||||
|
||||
# endif /* ifdef BROTLI_SHARED */
|
||||
|
||||
|
||||
struct brotli_state {
|
||||
BrotliEncoderState *encoder;
|
||||
BrotliDecoderState *decoder;
|
||||
};
|
||||
|
||||
static int brotli_stateful_init(COMP_CTX *ctx)
|
||||
{
|
||||
struct brotli_state *state = OPENSSL_zalloc(sizeof(*state));
|
||||
|
||||
if (state == NULL)
|
||||
return 0;
|
||||
|
||||
state->encoder = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL);
|
||||
if (state->encoder == NULL)
|
||||
goto err;
|
||||
|
||||
state->decoder = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL);
|
||||
if (state->decoder == NULL)
|
||||
goto err;
|
||||
|
||||
ctx->data = state;
|
||||
return 1;
|
||||
err:
|
||||
BrotliDecoderDestroyInstance(state->decoder);
|
||||
BrotliEncoderDestroyInstance(state->encoder);
|
||||
OPENSSL_free(state);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void brotli_stateful_finish(COMP_CTX *ctx)
|
||||
{
|
||||
struct brotli_state *state = ctx->data;
|
||||
|
||||
if (state != NULL) {
|
||||
BrotliDecoderDestroyInstance(state->decoder);
|
||||
BrotliEncoderDestroyInstance(state->encoder);
|
||||
OPENSSL_free(state);
|
||||
ctx->data = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static ossl_ssize_t brotli_stateful_compress_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
BROTLI_BOOL done;
|
||||
struct brotli_state *state = ctx->data;
|
||||
size_t in_avail = ilen;
|
||||
size_t out_avail = olen;
|
||||
|
||||
if (state == NULL || olen > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* The finish API does not provide a final output buffer,
|
||||
* so each compress operation has to be flushed, if all
|
||||
* the input data can't be accepted, or there is more output,
|
||||
* this has to be considered an error, since there is no more
|
||||
* output buffer space
|
||||
*/
|
||||
done = BrotliEncoderCompressStream(state->encoder, BROTLI_OPERATION_FLUSH,
|
||||
&in_avail, (const uint8_t**)&in,
|
||||
&out_avail, &out, NULL);
|
||||
if (done == BROTLI_FALSE
|
||||
|| in_avail != 0
|
||||
|| BrotliEncoderHasMoreOutput(state->encoder))
|
||||
return -1;
|
||||
|
||||
if (out_avail > olen)
|
||||
return -1;
|
||||
return (ossl_ssize_t)(olen - out_avail);
|
||||
}
|
||||
|
||||
static ossl_ssize_t brotli_stateful_expand_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
BrotliDecoderResult result;
|
||||
struct brotli_state *state = ctx->data;
|
||||
size_t in_avail = ilen;
|
||||
size_t out_avail = olen;
|
||||
|
||||
if (state == NULL || olen > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
result = BrotliDecoderDecompressStream(state->decoder, &in_avail,
|
||||
(const uint8_t**)&in, &out_avail,
|
||||
&out, NULL);
|
||||
if (result == BROTLI_DECODER_RESULT_ERROR
|
||||
|| in_avail != 0
|
||||
|| BrotliDecoderHasMoreOutput(state->decoder))
|
||||
return -1;
|
||||
|
||||
if (out_avail > olen)
|
||||
return -1;
|
||||
return (ossl_ssize_t)(olen - out_avail);
|
||||
}
|
||||
|
||||
static COMP_METHOD brotli_stateful_method = {
|
||||
NID_brotli,
|
||||
LN_brotli,
|
||||
brotli_stateful_init,
|
||||
brotli_stateful_finish,
|
||||
brotli_stateful_compress_block,
|
||||
brotli_stateful_expand_block
|
||||
};
|
||||
|
||||
static int brotli_oneshot_init(COMP_CTX *ctx)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void brotli_oneshot_finish(COMP_CTX *ctx)
|
||||
{
|
||||
}
|
||||
|
||||
static ossl_ssize_t brotli_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
size_t out_size = olen;
|
||||
ossl_ssize_t ret;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
if (BrotliEncoderCompress(BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW,
|
||||
BROTLI_DEFAULT_MODE, ilen, in,
|
||||
&out_size, out) == BROTLI_FALSE)
|
||||
return -1;
|
||||
|
||||
if (out_size > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
ret = (ossl_ssize_t)out_size;
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static ossl_ssize_t brotli_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
size_t out_size = olen;
|
||||
ossl_ssize_t ret;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
if (BrotliDecoderDecompress(ilen, in, &out_size, out) != BROTLI_DECODER_RESULT_SUCCESS)
|
||||
return -1;
|
||||
|
||||
if (out_size > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
ret = (ossl_ssize_t)out_size;
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static COMP_METHOD brotli_oneshot_method = {
|
||||
NID_brotli,
|
||||
LN_brotli,
|
||||
brotli_oneshot_init,
|
||||
brotli_oneshot_finish,
|
||||
brotli_oneshot_compress_block,
|
||||
brotli_oneshot_expand_block
|
||||
};
|
||||
|
||||
static CRYPTO_ONCE brotli_once = CRYPTO_ONCE_STATIC_INIT;
|
||||
DEFINE_RUN_ONCE_STATIC(ossl_comp_brotli_init)
|
||||
{
|
||||
# ifdef BROTLI_SHARED
|
||||
# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32)
|
||||
# define LIBBROTLIENC "BROTLIENC"
|
||||
# define LIBBROTLIDEC "BROTLIDEC"
|
||||
# else
|
||||
# define LIBBROTLIENC "brotlienc"
|
||||
# define LIBBROTLIDEC "brotlidec"
|
||||
# endif
|
||||
|
||||
brotli_encode_dso = DSO_load(NULL, LIBBROTLIENC, NULL, 0);
|
||||
if (brotli_encode_dso != NULL) {
|
||||
p_encode_init = (encode_init_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCreateInstance");
|
||||
p_encode_stream = (encode_stream_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompressStream");
|
||||
p_encode_has_more = (encode_has_more_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderHasMoreOutput");
|
||||
p_encode_end = (encode_end_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderDestroyInstance");
|
||||
p_encode_oneshot = (encode_oneshot_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompress");
|
||||
}
|
||||
|
||||
brotli_decode_dso = DSO_load(NULL, LIBBROTLIDEC, NULL, 0);
|
||||
if (brotli_decode_dso != NULL) {
|
||||
p_decode_init = (decode_init_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderCreateInstance");
|
||||
p_decode_stream = (decode_stream_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompressStream");
|
||||
p_decode_has_more = (decode_has_more_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderHasMoreOutput");
|
||||
p_decode_end = (decode_end_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDestroyInstance");
|
||||
p_decode_error = (decode_error_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderGetErrorCode");
|
||||
p_decode_error_string = (decode_error_string_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderErrorString");
|
||||
p_decode_is_finished = (decode_is_finished_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderIsFinished");
|
||||
p_decode_oneshot = (decode_oneshot_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompress");
|
||||
}
|
||||
|
||||
if (p_encode_init == NULL || p_encode_stream == NULL || p_encode_has_more == NULL
|
||||
|| p_encode_end == NULL || p_encode_oneshot == NULL || p_decode_init == NULL
|
||||
|| p_decode_stream == NULL || p_decode_has_more == NULL || p_decode_end == NULL
|
||||
|| p_decode_error == NULL || p_decode_error_string == NULL || p_decode_is_finished == NULL
|
||||
|| p_decode_oneshot == NULL) {
|
||||
ossl_comp_brotli_cleanup();
|
||||
return 0;
|
||||
}
|
||||
# endif
|
||||
return 1;
|
||||
}
|
||||
#endif /* ifndef BROTLI / else */
|
||||
|
||||
COMP_METHOD *COMP_brotli(void)
|
||||
{
|
||||
COMP_METHOD *meth = NULL;
|
||||
|
||||
#ifndef OPENSSL_NO_BROTLI
|
||||
if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init))
|
||||
meth = &brotli_stateful_method;
|
||||
#endif
|
||||
return meth;
|
||||
}
|
||||
|
||||
COMP_METHOD *COMP_brotli_oneshot(void)
|
||||
{
|
||||
COMP_METHOD *meth = NULL;
|
||||
|
||||
#ifndef OPENSSL_NO_BROTLI
|
||||
if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init))
|
||||
meth = &brotli_oneshot_method;
|
||||
#endif
|
||||
return meth;
|
||||
}
|
||||
|
||||
/* Also called from OPENSSL_cleanup() */
|
||||
void ossl_comp_brotli_cleanup(void)
|
||||
{
|
||||
#ifdef BROTLI_SHARED
|
||||
DSO_free(brotli_encode_dso);
|
||||
brotli_encode_dso = NULL;
|
||||
DSO_free(brotli_decode_dso);
|
||||
brotli_decode_dso = NULL;
|
||||
p_encode_init = NULL;
|
||||
p_encode_stream = NULL;
|
||||
p_encode_has_more = NULL;
|
||||
p_encode_end = NULL;
|
||||
p_encode_oneshot = NULL;
|
||||
p_decode_init = NULL;
|
||||
p_decode_stream = NULL;
|
||||
p_decode_has_more = NULL;
|
||||
p_decode_end = NULL;
|
||||
p_decode_error = NULL;
|
||||
p_decode_error_string = NULL;
|
||||
p_decode_is_finished = NULL;
|
||||
p_decode_oneshot = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef OPENSSL_NO_BROTLI
|
||||
|
||||
/* Brotli-based compression/decompression filter BIO */
|
||||
|
||||
typedef struct {
|
||||
struct { /* input structure */
|
||||
size_t avail_in;
|
||||
unsigned char *next_in;
|
||||
size_t avail_out;
|
||||
unsigned char *next_out;
|
||||
unsigned char *buf;
|
||||
size_t bufsize;
|
||||
BrotliDecoderState *state;
|
||||
} decode;
|
||||
struct { /* output structure */
|
||||
size_t avail_in;
|
||||
unsigned char *next_in;
|
||||
size_t avail_out;
|
||||
unsigned char *next_out;
|
||||
unsigned char *buf;
|
||||
size_t bufsize;
|
||||
BrotliEncoderState *state;
|
||||
int mode; /* Encoder mode to use */
|
||||
int done;
|
||||
unsigned char *ptr;
|
||||
size_t count;
|
||||
} encode;
|
||||
} BIO_BROTLI_CTX;
|
||||
|
||||
# define BROTLI_DEFAULT_BUFSIZE 1024
|
||||
|
||||
static int bio_brotli_new(BIO *bi);
|
||||
static int bio_brotli_free(BIO *bi);
|
||||
static int bio_brotli_read(BIO *b, char *out, int outl);
|
||||
static int bio_brotli_write(BIO *b, const char *in, int inl);
|
||||
static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr);
|
||||
static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp);
|
||||
|
||||
static const BIO_METHOD bio_meth_brotli = {
|
||||
BIO_TYPE_COMP,
|
||||
"brotli",
|
||||
/* TODO: Convert to new style write function */
|
||||
bwrite_conv,
|
||||
bio_brotli_write,
|
||||
/* TODO: Convert to new style read function */
|
||||
bread_conv,
|
||||
bio_brotli_read,
|
||||
NULL, /* bio_brotli_puts, */
|
||||
NULL, /* bio_brotli_gets, */
|
||||
bio_brotli_ctrl,
|
||||
bio_brotli_new,
|
||||
bio_brotli_free,
|
||||
bio_brotli_callback_ctrl
|
||||
};
|
||||
#endif
|
||||
|
||||
const BIO_METHOD *BIO_f_brotli(void)
|
||||
{
|
||||
#ifndef OPENSSL_NO_BROTLI
|
||||
if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init))
|
||||
return &bio_meth_brotli;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#ifndef OPENSSL_NO_BROTLI
|
||||
|
||||
static int bio_brotli_new(BIO *bi)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
|
||||
# ifdef BROTLI_SHARED
|
||||
if (!RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED);
|
||||
return 0;
|
||||
}
|
||||
# endif
|
||||
ctx = OPENSSL_zalloc(sizeof(*ctx));
|
||||
if (ctx == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
ctx->decode.bufsize = BROTLI_DEFAULT_BUFSIZE;
|
||||
ctx->decode.state = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL);
|
||||
if (ctx->decode.state == NULL)
|
||||
goto err;
|
||||
ctx->encode.bufsize = BROTLI_DEFAULT_BUFSIZE;
|
||||
ctx->encode.state = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL);
|
||||
if (ctx->encode.state == NULL)
|
||||
goto err;
|
||||
ctx->encode.mode = BROTLI_DEFAULT_MODE;
|
||||
BIO_set_init(bi, 1);
|
||||
BIO_set_data(bi, ctx);
|
||||
|
||||
return 1;
|
||||
|
||||
err:
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
BrotliDecoderDestroyInstance(ctx->decode.state);
|
||||
BrotliEncoderDestroyInstance(ctx->encode.state);
|
||||
OPENSSL_free(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int bio_brotli_free(BIO *bi)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
|
||||
if (bi == NULL)
|
||||
return 0;
|
||||
|
||||
ctx = BIO_get_data(bi);
|
||||
if (ctx != NULL) {
|
||||
BrotliDecoderDestroyInstance(ctx->decode.state);
|
||||
OPENSSL_free(ctx->decode.buf);
|
||||
BrotliEncoderDestroyInstance(ctx->encode.state);
|
||||
OPENSSL_free(ctx->encode.buf);
|
||||
OPENSSL_free(ctx);
|
||||
}
|
||||
BIO_set_data(bi, NULL);
|
||||
BIO_set_init(bi, 0);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int bio_brotli_read(BIO *b, char *out, int outl)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
BrotliDecoderResult bret;
|
||||
int ret;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (out == NULL || outl <= 0) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
#if INT_MAX > SIZE_MAX
|
||||
if ((unsigned int)outl > SIZE_MAX) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
BIO_clear_retry_flags(b);
|
||||
if (ctx->decode.buf == NULL) {
|
||||
ctx->decode.buf = OPENSSL_malloc(ctx->decode.bufsize);
|
||||
if (ctx->decode.buf == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
ctx->decode.next_in = ctx->decode.buf;
|
||||
ctx->decode.avail_in = 0;
|
||||
}
|
||||
|
||||
/* Copy output data directly to supplied buffer */
|
||||
ctx->decode.next_out = (unsigned char *)out;
|
||||
ctx->decode.avail_out = (size_t)outl;
|
||||
for (;;) {
|
||||
/* Decompress while data available */
|
||||
while (ctx->decode.avail_in > 0 || BrotliDecoderHasMoreOutput(ctx->decode.state)) {
|
||||
bret = BrotliDecoderDecompressStream(ctx->decode.state, &ctx->decode.avail_in, (const uint8_t**)&ctx->decode.next_in,
|
||||
&ctx->decode.avail_out, &ctx->decode.next_out, NULL);
|
||||
if (bret == BROTLI_DECODER_RESULT_ERROR) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR);
|
||||
ERR_add_error_data(1, BrotliDecoderErrorString(BrotliDecoderGetErrorCode(ctx->decode.state)));
|
||||
return 0;
|
||||
}
|
||||
/* If EOF or we've read everything then return */
|
||||
if (BrotliDecoderIsFinished(ctx->decode.state) || ctx->decode.avail_out == 0)
|
||||
return (int)(outl - ctx->decode.avail_out);
|
||||
}
|
||||
|
||||
/* If EOF */
|
||||
if (BrotliDecoderIsFinished(ctx->decode.state))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* No data in input buffer try to read some in, if an error then
|
||||
* return the total data read.
|
||||
*/
|
||||
ret = BIO_read(next, ctx->decode.buf, ctx->decode.bufsize);
|
||||
if (ret <= 0) {
|
||||
/* Total data read */
|
||||
int tot = outl - ctx->decode.avail_out;
|
||||
|
||||
BIO_copy_next_retry(b);
|
||||
if (ret < 0)
|
||||
return (tot > 0) ? tot : ret;
|
||||
return tot;
|
||||
}
|
||||
ctx->decode.avail_in = ret;
|
||||
ctx->decode.next_in = ctx->decode.buf;
|
||||
}
|
||||
}
|
||||
|
||||
static int bio_brotli_write(BIO *b, const char *in, int inl)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
BROTLI_BOOL brret;
|
||||
int ret;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (in == NULL || inl <= 0) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
#if INT_MAX > SIZE_MAX
|
||||
if ((unsigned int)inl > SIZE_MAX) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
if (ctx->encode.done)
|
||||
return 0;
|
||||
|
||||
BIO_clear_retry_flags(b);
|
||||
if (ctx->encode.buf == NULL) {
|
||||
ctx->encode.buf = OPENSSL_malloc(ctx->encode.bufsize);
|
||||
if (ctx->encode.buf == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
ctx->encode.ptr = ctx->encode.buf;
|
||||
ctx->encode.count = 0;
|
||||
ctx->encode.next_out = ctx->encode.buf;
|
||||
ctx->encode.avail_out = ctx->encode.bufsize;
|
||||
}
|
||||
/* Obtain input data directly from supplied buffer */
|
||||
ctx->encode.next_in = (unsigned char *)in;
|
||||
ctx->encode.avail_in = (size_t)inl;
|
||||
for (;;) {
|
||||
/* If data in output buffer write it first */
|
||||
while (ctx->encode.count > 0) {
|
||||
ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count);
|
||||
if (ret <= 0) {
|
||||
/* Total data written */
|
||||
int tot = inl - ctx->encode.avail_in;
|
||||
|
||||
BIO_copy_next_retry(b);
|
||||
if (ret < 0)
|
||||
return (tot > 0) ? tot : ret;
|
||||
return tot;
|
||||
}
|
||||
ctx->encode.ptr += ret;
|
||||
ctx->encode.count -= ret;
|
||||
}
|
||||
|
||||
/* Have we consumed all supplied data? */
|
||||
if (ctx->encode.avail_in == 0 && !BrotliEncoderHasMoreOutput(ctx->encode.state))
|
||||
return inl;
|
||||
|
||||
/* Compress some more */
|
||||
|
||||
/* Reset buffer */
|
||||
ctx->encode.ptr = ctx->encode.buf;
|
||||
ctx->encode.next_out = ctx->encode.buf;
|
||||
ctx->encode.avail_out = ctx->encode.bufsize;
|
||||
/* Compress some more */
|
||||
brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FLUSH, &ctx->encode.avail_in, (const uint8_t**)&ctx->encode.next_in,
|
||||
&ctx->encode.avail_out, &ctx->encode.next_out, NULL);
|
||||
if (brret != BROTLI_TRUE) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR);
|
||||
ERR_add_error_data(1, "brotli encoder error");
|
||||
return 0;
|
||||
}
|
||||
ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out;
|
||||
}
|
||||
}
|
||||
|
||||
static int bio_brotli_flush(BIO *b)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
BROTLI_BOOL brret;
|
||||
int ret;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
|
||||
/* If no data written or already flush show success */
|
||||
if (ctx->encode.buf == NULL || (ctx->encode.done && ctx->encode.count == 0))
|
||||
return 1;
|
||||
|
||||
BIO_clear_retry_flags(b);
|
||||
/* No more input data */
|
||||
ctx->encode.next_in = NULL;
|
||||
ctx->encode.avail_in = 0;
|
||||
for (;;) {
|
||||
/* If data in output buffer write it first */
|
||||
while (ctx->encode.count > 0) {
|
||||
ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count);
|
||||
if (ret <= 0) {
|
||||
BIO_copy_next_retry(b);
|
||||
return ret;
|
||||
}
|
||||
ctx->encode.ptr += ret;
|
||||
ctx->encode.count -= ret;
|
||||
}
|
||||
if (ctx->encode.done)
|
||||
return 1;
|
||||
|
||||
/* Compress some more */
|
||||
|
||||
/* Reset buffer */
|
||||
ctx->encode.ptr = ctx->encode.buf;
|
||||
ctx->encode.next_out = ctx->encode.buf;
|
||||
ctx->encode.avail_out = ctx->encode.bufsize;
|
||||
/* Compress some more */
|
||||
brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FINISH, &ctx->encode.avail_in,
|
||||
(const uint8_t**)&ctx->encode.next_in, &ctx->encode.avail_out, &ctx->encode.next_out, NULL);
|
||||
if (brret != BROTLI_TRUE) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR);
|
||||
ERR_add_error_data(1, "brotli encoder error");
|
||||
return 0;
|
||||
}
|
||||
if (!BrotliEncoderHasMoreOutput(ctx->encode.state) && ctx->encode.avail_in == 0)
|
||||
ctx->encode.done = 1;
|
||||
ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out;
|
||||
}
|
||||
}
|
||||
|
||||
static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr)
|
||||
{
|
||||
BIO_BROTLI_CTX *ctx;
|
||||
unsigned char *tmp;
|
||||
int ret = 0, *ip;
|
||||
size_t ibs, obs;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (next == NULL)
|
||||
return 0;
|
||||
ctx = BIO_get_data(b);
|
||||
switch (cmd) {
|
||||
|
||||
case BIO_CTRL_RESET:
|
||||
ctx->encode.count = 0;
|
||||
ctx->encode.done = 0;
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
case BIO_CTRL_FLUSH:
|
||||
ret = bio_brotli_flush(b);
|
||||
if (ret > 0) {
|
||||
ret = BIO_flush(next);
|
||||
BIO_copy_next_retry(b);
|
||||
}
|
||||
break;
|
||||
|
||||
case BIO_C_SET_BUFF_SIZE:
|
||||
ibs = ctx->decode.bufsize;
|
||||
obs = ctx->encode.bufsize;
|
||||
if (ptr != NULL) {
|
||||
ip = ptr;
|
||||
if (*ip == 0)
|
||||
ibs = (size_t)num;
|
||||
else
|
||||
obs = (size_t)num;
|
||||
} else {
|
||||
ibs = (size_t)num;
|
||||
obs = ibs;
|
||||
}
|
||||
|
||||
if (ibs > 0 && ibs != ctx->decode.bufsize) {
|
||||
/* Do not free/alloc, only reallocate */
|
||||
if (ctx->decode.buf != NULL) {
|
||||
tmp = OPENSSL_realloc(ctx->decode.buf, ibs);
|
||||
if (tmp == NULL)
|
||||
return 0;
|
||||
ctx->decode.buf = tmp;
|
||||
}
|
||||
ctx->decode.bufsize = ibs;
|
||||
}
|
||||
|
||||
if (obs > 0 && obs != ctx->encode.bufsize) {
|
||||
/* Do not free/alloc, only reallocate */
|
||||
if (ctx->encode.buf != NULL) {
|
||||
tmp = OPENSSL_realloc(ctx->encode.buf, obs);
|
||||
if (tmp == NULL)
|
||||
return 0;
|
||||
ctx->encode.buf = tmp;
|
||||
}
|
||||
ctx->encode.bufsize = obs;
|
||||
}
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
case BIO_C_DO_STATE_MACHINE:
|
||||
BIO_clear_retry_flags(b);
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
BIO_copy_next_retry(b);
|
||||
break;
|
||||
|
||||
case BIO_CTRL_WPENDING:
|
||||
if (BrotliEncoderHasMoreOutput(ctx->encode.state))
|
||||
ret = 1;
|
||||
else
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
case BIO_CTRL_PENDING:
|
||||
if (!BrotliDecoderIsFinished(ctx->decode.state))
|
||||
ret = 1;
|
||||
else
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp)
|
||||
{
|
||||
BIO *next = BIO_next(b);
|
||||
if (next == NULL)
|
||||
return 0;
|
||||
return BIO_callback_ctrl(next, cmd, fp);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,845 @@
|
||||
/*
|
||||
* Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
* Uses zstd compression library from https://github.com/facebook/zstd
|
||||
* Requires version 1.4.x (latest as of this writing is 1.4.5)
|
||||
* Using custom free functions require static linking, so that is disabled when
|
||||
* using the shared library.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <openssl/objects.h>
|
||||
#include "internal/comp.h"
|
||||
#include <openssl/err.h>
|
||||
#include "crypto/cryptlib.h"
|
||||
#include "internal/bio.h"
|
||||
#include "internal/thread_once.h"
|
||||
#include "comp_local.h"
|
||||
|
||||
COMP_METHOD *COMP_zstd(void);
|
||||
|
||||
#ifdef OPENSSL_NO_ZSTD
|
||||
# undef ZSTD_SHARED
|
||||
#else
|
||||
|
||||
# ifndef ZSTD_SHARED
|
||||
# define ZSTD_STATIC_LINKING_ONLY
|
||||
# endif
|
||||
# include <zstd.h>
|
||||
|
||||
/* Note: There is also a linux zstd.h file in the kernel source */
|
||||
# ifndef ZSTD_H_235446
|
||||
# error Wrong (i.e. linux) zstd.h included.
|
||||
# endif
|
||||
|
||||
# if ZSTD_VERSION_MAJOR != 1 && ZSTD_VERSION_MINOR < 4
|
||||
# error Expecting version 1.4 or greater of ZSTD
|
||||
# endif
|
||||
|
||||
# ifndef ZSTD_SHARED
|
||||
/* memory allocations functions for zstd initialisation */
|
||||
static void *zstd_alloc(void *opaque, size_t size)
|
||||
{
|
||||
return OPENSSL_zalloc(size);
|
||||
}
|
||||
|
||||
static void zstd_free(void *opaque, void *address)
|
||||
{
|
||||
OPENSSL_free(address);
|
||||
}
|
||||
|
||||
static ZSTD_customMem zstd_mem_funcs = {
|
||||
zstd_alloc,
|
||||
zstd_free,
|
||||
NULL
|
||||
};
|
||||
# endif
|
||||
|
||||
/*
|
||||
* When OpenSSL is built on Windows, we do not want to require that
|
||||
* the LIBZSTD.DLL be available in order for the OpenSSL DLLs to
|
||||
* work. Therefore, all ZSTD routines are loaded at run time
|
||||
* and we do not link to a .LIB file when ZSTD_SHARED is set.
|
||||
*/
|
||||
# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32)
|
||||
# include <windows.h>
|
||||
# endif
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
# include "internal/dso.h"
|
||||
|
||||
/* Function pointers */
|
||||
typedef ZSTD_CStream* (*createCStream_ft)(void);
|
||||
typedef size_t (*initCStream_ft)(ZSTD_CStream*, int);
|
||||
typedef size_t (*freeCStream_ft)(ZSTD_CStream*);
|
||||
typedef size_t (*compressStream2_ft)(ZSTD_CCtx*, ZSTD_outBuffer*, ZSTD_inBuffer*, ZSTD_EndDirective);
|
||||
typedef size_t (*flushStream_ft)(ZSTD_CStream*, ZSTD_outBuffer*);
|
||||
typedef size_t (*endStream_ft)(ZSTD_CStream*, ZSTD_outBuffer*);
|
||||
typedef size_t (*compress_ft)(void*, size_t, const void*, size_t, int);
|
||||
typedef ZSTD_DStream* (*createDStream_ft)(void);
|
||||
typedef size_t (*initDStream_ft)(ZSTD_DStream*);
|
||||
typedef size_t (*freeDStream_ft)(ZSTD_DStream*);
|
||||
typedef size_t (*decompressStream_ft)(ZSTD_DStream*, ZSTD_outBuffer*, ZSTD_inBuffer*);
|
||||
typedef size_t (*decompress_ft)(void*, size_t, const void*, size_t);
|
||||
typedef unsigned (*isError_ft)(size_t);
|
||||
typedef const char* (*getErrorName_ft)(size_t);
|
||||
typedef size_t (*DStreamInSize_ft)(void);
|
||||
typedef size_t (*CStreamInSize_ft)(void);
|
||||
|
||||
static createCStream_ft p_createCStream = NULL;
|
||||
static initCStream_ft p_initCStream = NULL;
|
||||
static freeCStream_ft p_freeCStream = NULL;
|
||||
static compressStream2_ft p_compressStream2 = NULL;
|
||||
static flushStream_ft p_flushStream = NULL;
|
||||
static endStream_ft p_endStream = NULL;
|
||||
static compress_ft p_compress = NULL;
|
||||
static createDStream_ft p_createDStream = NULL;
|
||||
static initDStream_ft p_initDStream = NULL;
|
||||
static freeDStream_ft p_freeDStream = NULL;
|
||||
static decompressStream_ft p_decompressStream = NULL;
|
||||
static decompress_ft p_decompress = NULL;
|
||||
static isError_ft p_isError = NULL;
|
||||
static getErrorName_ft p_getErrorName = NULL;
|
||||
static DStreamInSize_ft p_DStreamInSize = NULL;
|
||||
static CStreamInSize_ft p_CStreamInSize = NULL;
|
||||
|
||||
static DSO *zstd_dso = NULL;
|
||||
|
||||
# define ZSTD_createCStream p_createCStream
|
||||
# define ZSTD_initCStream p_initCStream
|
||||
# define ZSTD_freeCStream p_freeCStream
|
||||
# define ZSTD_compressStream2 p_compressStream2
|
||||
# define ZSTD_flushStream p_flushStream
|
||||
# define ZSTD_endStream p_endStream
|
||||
# define ZSTD_compress p_compress
|
||||
# define ZSTD_createDStream p_createDStream
|
||||
# define ZSTD_initDStream p_initDStream
|
||||
# define ZSTD_freeDStream p_freeDStream
|
||||
# define ZSTD_decompressStream p_decompressStream
|
||||
# define ZSTD_decompress p_decompress
|
||||
# define ZSTD_isError p_isError
|
||||
# define ZSTD_getErrorName p_getErrorName
|
||||
# define ZSTD_DStreamInSize p_DStreamInSize
|
||||
# define ZSTD_CStreamInSize p_CStreamInSize
|
||||
|
||||
# endif /* ifdef ZSTD_SHARED */
|
||||
|
||||
struct zstd_state {
|
||||
ZSTD_CStream *compressor;
|
||||
ZSTD_DStream *decompressor;
|
||||
};
|
||||
|
||||
static int zstd_stateful_init(COMP_CTX *ctx)
|
||||
{
|
||||
struct zstd_state *state = OPENSSL_zalloc(sizeof(*state));
|
||||
|
||||
if (state == NULL)
|
||||
return 0;
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
state->compressor = ZSTD_createCStream();
|
||||
# else
|
||||
state->compressor = ZSTD_createCStream_advanced(zstd_mem_funcs);
|
||||
# endif
|
||||
if (state->compressor == NULL)
|
||||
goto err;
|
||||
ZSTD_initCStream(state->compressor, ZSTD_CLEVEL_DEFAULT);
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
state->decompressor = ZSTD_createDStream();
|
||||
# else
|
||||
state->decompressor = ZSTD_createDStream_advanced(zstd_mem_funcs);
|
||||
# endif
|
||||
if (state->decompressor == NULL)
|
||||
goto err;
|
||||
ZSTD_initDStream(state->decompressor);
|
||||
|
||||
ctx->data = state;
|
||||
return 1;
|
||||
err:
|
||||
ZSTD_freeCStream(state->compressor);
|
||||
ZSTD_freeDStream(state->decompressor);
|
||||
OPENSSL_free(state);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void zstd_stateful_finish(COMP_CTX *ctx)
|
||||
{
|
||||
struct zstd_state *state = ctx->data;
|
||||
|
||||
if (state != NULL) {
|
||||
ZSTD_freeCStream(state->compressor);
|
||||
ZSTD_freeDStream(state->decompressor);
|
||||
OPENSSL_free(state);
|
||||
ctx->data = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static ossl_ssize_t zstd_stateful_compress_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
ZSTD_inBuffer inbuf;
|
||||
ZSTD_outBuffer outbuf;
|
||||
size_t ret;
|
||||
ossl_ssize_t fret;
|
||||
struct zstd_state *state = ctx->data;
|
||||
|
||||
inbuf.src = in;
|
||||
inbuf.size = ilen;
|
||||
inbuf.pos = 0;
|
||||
outbuf.dst = out;
|
||||
outbuf.size = olen;
|
||||
outbuf.pos = 0;
|
||||
|
||||
if (state == NULL)
|
||||
return -1;
|
||||
|
||||
/* If input length is zero, end the stream/frame ? */
|
||||
if (ilen == 0) {
|
||||
ret = ZSTD_endStream(state->compressor, &outbuf);
|
||||
if (ZSTD_isError(ret))
|
||||
return -1;
|
||||
goto end;
|
||||
}
|
||||
|
||||
/*
|
||||
* The finish API does not provide a final output buffer,
|
||||
* so each compress operation has to be ended, if all
|
||||
* the input data can't be accepted, or there is more output,
|
||||
* this has to be considered an error, since there is no more
|
||||
* output buffer space.
|
||||
*/
|
||||
do {
|
||||
ret = ZSTD_compressStream2(state->compressor, &outbuf, &inbuf, ZSTD_e_continue);
|
||||
if (ZSTD_isError(ret))
|
||||
return -1;
|
||||
/* do I need to check for ret == 0 ? */
|
||||
} while (inbuf.pos < inbuf.size);
|
||||
|
||||
/* Did not consume all the data */
|
||||
if (inbuf.pos < inbuf.size)
|
||||
return -1;
|
||||
|
||||
ret = ZSTD_flushStream(state->compressor, &outbuf);
|
||||
if (ZSTD_isError(ret))
|
||||
return -1;
|
||||
|
||||
end:
|
||||
if (outbuf.pos > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
fret = (ossl_ssize_t)outbuf.pos;
|
||||
if (fret < 0)
|
||||
return -1;
|
||||
return fret;
|
||||
}
|
||||
|
||||
static ossl_ssize_t zstd_stateful_expand_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
ZSTD_inBuffer inbuf;
|
||||
ZSTD_outBuffer outbuf;
|
||||
size_t ret;
|
||||
ossl_ssize_t fret;
|
||||
struct zstd_state *state = ctx->data;
|
||||
|
||||
inbuf.src = in;
|
||||
inbuf.size = ilen;
|
||||
inbuf.pos = 0;
|
||||
outbuf.dst = out;
|
||||
outbuf.size = olen;
|
||||
outbuf.pos = 0;
|
||||
|
||||
if (state == NULL)
|
||||
return -1;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
do {
|
||||
ret = ZSTD_decompressStream(state->decompressor, &outbuf, &inbuf);
|
||||
if (ZSTD_isError(ret))
|
||||
return -1;
|
||||
/* If we completed a frame, and there's more data, try again */
|
||||
} while (ret == 0 && inbuf.pos < inbuf.size);
|
||||
|
||||
/* Did not consume all the data */
|
||||
if (inbuf.pos < inbuf.size)
|
||||
return -1;
|
||||
|
||||
if (outbuf.pos > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
fret = (ossl_ssize_t)outbuf.pos;
|
||||
if (fret < 0)
|
||||
return -1;
|
||||
return fret;
|
||||
}
|
||||
|
||||
|
||||
static COMP_METHOD zstd_stateful_method = {
|
||||
NID_zstd,
|
||||
LN_zstd,
|
||||
zstd_stateful_init,
|
||||
zstd_stateful_finish,
|
||||
zstd_stateful_compress_block,
|
||||
zstd_stateful_expand_block
|
||||
};
|
||||
|
||||
static int zstd_oneshot_init(COMP_CTX *ctx)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void zstd_oneshot_finish(COMP_CTX *ctx)
|
||||
{
|
||||
}
|
||||
|
||||
static ossl_ssize_t zstd_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
size_t out_size;
|
||||
ossl_ssize_t ret;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
/* Note: uses STDLIB memory allocators */
|
||||
out_size = ZSTD_compress(out, olen, in, ilen, ZSTD_CLEVEL_DEFAULT);
|
||||
if (ZSTD_isError(out_size))
|
||||
return -1;
|
||||
|
||||
if (out_size > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
ret = (ossl_ssize_t)out_size;
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static ossl_ssize_t zstd_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out,
|
||||
size_t olen, unsigned char *in,
|
||||
size_t ilen)
|
||||
{
|
||||
size_t out_size;
|
||||
ossl_ssize_t ret;
|
||||
|
||||
if (ilen == 0)
|
||||
return 0;
|
||||
|
||||
/* Note: uses STDLIB memory allocators */
|
||||
out_size = ZSTD_decompress(out, olen, in, ilen);
|
||||
if (ZSTD_isError(out_size))
|
||||
return -1;
|
||||
|
||||
if (out_size > OSSL_SSIZE_MAX)
|
||||
return -1;
|
||||
ret = (ossl_ssize_t)out_size;
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static COMP_METHOD zstd_oneshot_method = {
|
||||
NID_zstd,
|
||||
LN_zstd,
|
||||
zstd_oneshot_init,
|
||||
zstd_oneshot_finish,
|
||||
zstd_oneshot_compress_block,
|
||||
zstd_oneshot_expand_block
|
||||
};
|
||||
|
||||
static CRYPTO_ONCE zstd_once = CRYPTO_ONCE_STATIC_INIT;
|
||||
DEFINE_RUN_ONCE_STATIC(ossl_comp_zstd_init)
|
||||
{
|
||||
# ifdef ZSTD_SHARED
|
||||
# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32)
|
||||
# define LIBZSTD "LIBZSTD"
|
||||
# else
|
||||
# define LIBZSTD "zstd"
|
||||
# endif
|
||||
|
||||
zstd_dso = DSO_load(NULL, LIBZSTD, NULL, 0);
|
||||
if (zstd_dso != NULL) {
|
||||
p_createCStream = (createCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_createCStream");
|
||||
p_initCStream = (initCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_initCStream");
|
||||
p_freeCStream = (freeCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_freeCStream");
|
||||
p_compressStream2 = (compressStream2_ft)DSO_bind_func(zstd_dso, "ZSTD_compressStream2");
|
||||
p_flushStream = (flushStream_ft)DSO_bind_func(zstd_dso, "ZSTD_flushStream");
|
||||
p_endStream = (endStream_ft)DSO_bind_func(zstd_dso, "ZSTD_endStream");
|
||||
p_compress = (compress_ft)DSO_bind_func(zstd_dso, "ZSTD_compress");
|
||||
p_createDStream = (createDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_createDStream");
|
||||
p_initDStream = (initDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_initDStream");
|
||||
p_freeDStream = (freeDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_freeDStream");
|
||||
p_decompressStream = (decompressStream_ft)DSO_bind_func(zstd_dso, "ZSTD_decompressStream");
|
||||
p_decompress = (decompress_ft)DSO_bind_func(zstd_dso, "ZSTD_decompress");
|
||||
p_isError = (isError_ft)DSO_bind_func(zstd_dso, "ZSTD_isError");
|
||||
p_getErrorName = (getErrorName_ft)DSO_bind_func(zstd_dso, "ZSTD_getErrorName");
|
||||
p_DStreamInSize = (DStreamInSize_ft)DSO_bind_func(zstd_dso, "ZSTD_DStreamInSize");
|
||||
p_CStreamInSize = (CStreamInSize_ft)DSO_bind_func(zstd_dso, "ZSTD_CStreamInSize");
|
||||
}
|
||||
|
||||
if (p_createCStream == NULL || p_initCStream == NULL || p_freeCStream == NULL
|
||||
|| p_compressStream2 == NULL || p_flushStream == NULL || p_endStream == NULL
|
||||
|| p_compress == NULL || p_createDStream == NULL || p_initDStream == NULL
|
||||
|| p_freeDStream == NULL || p_decompressStream == NULL || p_decompress == NULL
|
||||
|| p_isError == NULL || p_getErrorName == NULL || p_DStreamInSize == NULL
|
||||
|| p_CStreamInSize == NULL) {
|
||||
ossl_comp_zstd_cleanup();
|
||||
return 0;
|
||||
}
|
||||
# endif
|
||||
return 1;
|
||||
}
|
||||
#endif /* ifndef ZSTD / else */
|
||||
|
||||
COMP_METHOD *COMP_zstd(void)
|
||||
{
|
||||
COMP_METHOD *meth = NULL;
|
||||
|
||||
#ifndef OPENSSL_NO_ZSTD
|
||||
if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init))
|
||||
meth = &zstd_stateful_method;
|
||||
#endif
|
||||
return meth;
|
||||
}
|
||||
|
||||
COMP_METHOD *COMP_zstd_oneshot(void)
|
||||
{
|
||||
COMP_METHOD *meth = NULL;
|
||||
|
||||
#ifndef OPENSSL_NO_ZSTD
|
||||
if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init))
|
||||
meth = &zstd_oneshot_method;
|
||||
#endif
|
||||
return meth;
|
||||
}
|
||||
|
||||
/* Also called from OPENSSL_cleanup() */
|
||||
void ossl_comp_zstd_cleanup(void)
|
||||
{
|
||||
#ifdef ZSTD_SHARED
|
||||
DSO_free(zstd_dso);
|
||||
zstd_dso = NULL;
|
||||
p_createCStream = NULL;
|
||||
p_initCStream = NULL;
|
||||
p_freeCStream = NULL;
|
||||
p_compressStream2 = NULL;
|
||||
p_flushStream = NULL;
|
||||
p_endStream = NULL;
|
||||
p_compress = NULL;
|
||||
p_createDStream = NULL;
|
||||
p_initDStream = NULL;
|
||||
p_freeDStream = NULL;
|
||||
p_decompressStream = NULL;
|
||||
p_decompress = NULL;
|
||||
p_isError = NULL;
|
||||
p_getErrorName = NULL;
|
||||
p_DStreamInSize = NULL;
|
||||
p_CStreamInSize = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef OPENSSL_NO_ZSTD
|
||||
|
||||
/* Zstd-based compression/decompression filter BIO */
|
||||
|
||||
typedef struct {
|
||||
struct { /* input structure */
|
||||
ZSTD_DStream *state;
|
||||
ZSTD_inBuffer inbuf; /* has const src */
|
||||
size_t bufsize;
|
||||
void* buffer;
|
||||
} decompress;
|
||||
struct { /* output structure */
|
||||
ZSTD_CStream *state;
|
||||
ZSTD_outBuffer outbuf;
|
||||
size_t bufsize;
|
||||
size_t write_pos;
|
||||
} compress;
|
||||
} BIO_ZSTD_CTX;
|
||||
|
||||
# define ZSTD_DEFAULT_BUFSIZE 1024
|
||||
|
||||
static int bio_zstd_new(BIO *bi);
|
||||
static int bio_zstd_free(BIO *bi);
|
||||
static int bio_zstd_read(BIO *b, char *out, int outl);
|
||||
static int bio_zstd_write(BIO *b, const char *in, int inl);
|
||||
static long bio_zstd_ctrl(BIO *b, int cmd, long num, void *ptr);
|
||||
static long bio_zstd_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp);
|
||||
|
||||
static const BIO_METHOD bio_meth_zstd = {
|
||||
BIO_TYPE_COMP,
|
||||
"zstd",
|
||||
/* TODO: Convert to new style write function */
|
||||
bwrite_conv,
|
||||
bio_zstd_write,
|
||||
/* TODO: Convert to new style read function */
|
||||
bread_conv,
|
||||
bio_zstd_read,
|
||||
NULL, /* bio_zstd_puts, */
|
||||
NULL, /* bio_zstd_gets, */
|
||||
bio_zstd_ctrl,
|
||||
bio_zstd_new,
|
||||
bio_zstd_free,
|
||||
bio_zstd_callback_ctrl
|
||||
};
|
||||
#endif
|
||||
|
||||
const BIO_METHOD *BIO_f_zstd(void)
|
||||
{
|
||||
#ifndef OPENSSL_NO_ZSTD
|
||||
if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init))
|
||||
return &bio_meth_zstd;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#ifndef OPENSSL_NO_ZSTD
|
||||
static int bio_zstd_new(BIO *bi)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
(void)COMP_zstd();
|
||||
if (zstd_dso == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_NOT_SUPPORTED);
|
||||
return 0;
|
||||
}
|
||||
# endif
|
||||
ctx = OPENSSL_zalloc(sizeof(*ctx));
|
||||
if (ctx == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
ctx->decompress.state = ZSTD_createDStream();
|
||||
# else
|
||||
ctx->decompress.state = ZSTD_createDStream_advanced(zstd_mem_funcs);
|
||||
# endif
|
||||
if (ctx->decompress.state == NULL)
|
||||
goto err;
|
||||
ZSTD_initDStream(ctx->decompress.state);
|
||||
ctx->decompress.bufsize = ZSTD_DStreamInSize();
|
||||
|
||||
# ifdef ZSTD_SHARED
|
||||
ctx->compress.state = ZSTD_createCStream();
|
||||
# else
|
||||
ctx->compress.state = ZSTD_createCStream_advanced(zstd_mem_funcs);
|
||||
# endif
|
||||
if (ctx->compress.state == NULL)
|
||||
goto err;
|
||||
ZSTD_initCStream(ctx->compress.state, ZSTD_CLEVEL_DEFAULT);
|
||||
ctx->compress.bufsize = ZSTD_CStreamInSize();
|
||||
|
||||
BIO_set_init(bi, 1);
|
||||
BIO_set_data(bi, ctx);
|
||||
|
||||
return 1;
|
||||
err:
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
ZSTD_freeDStream(ctx->decompress.state);
|
||||
ZSTD_freeCStream(ctx->compress.state);
|
||||
OPENSSL_free(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int bio_zstd_free(BIO *bi)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
|
||||
if (bi == NULL)
|
||||
return 0;
|
||||
|
||||
ctx = BIO_get_data(bi);
|
||||
if (ctx != NULL) {
|
||||
ZSTD_freeDStream(ctx->decompress.state);
|
||||
OPENSSL_free(ctx->decompress.buffer);
|
||||
ZSTD_freeCStream(ctx->compress.state);
|
||||
OPENSSL_free(ctx->compress.outbuf.dst);
|
||||
OPENSSL_free(ctx);
|
||||
}
|
||||
BIO_set_data(bi, NULL);
|
||||
BIO_set_init(bi, 0);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int bio_zstd_read(BIO *b, char *out, int outl)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
size_t zret;
|
||||
int ret;
|
||||
ZSTD_outBuffer outBuf;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (out == NULL || outl <= 0)
|
||||
return 0;
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
BIO_clear_retry_flags(b);
|
||||
if (ctx->decompress.buffer == NULL) {
|
||||
ctx->decompress.buffer = OPENSSL_malloc(ctx->decompress.bufsize);
|
||||
if (ctx->decompress.buffer == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
ctx->decompress.inbuf.src = ctx->decompress.buffer;
|
||||
ctx->decompress.inbuf.size = 0;
|
||||
ctx->decompress.inbuf.pos = 0;
|
||||
}
|
||||
|
||||
/* Copy output data directly to supplied buffer */
|
||||
outBuf.dst = out;
|
||||
outBuf.size = (size_t)outl;
|
||||
outBuf.pos = 0;
|
||||
for (;;) {
|
||||
/* Decompress while data available */
|
||||
do {
|
||||
zret = ZSTD_decompressStream(ctx->decompress.state, &outBuf, &ctx->decompress.inbuf);
|
||||
if (ZSTD_isError(zret)) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_DECOMPRESS_ERROR);
|
||||
ERR_add_error_data(1, ZSTD_getErrorName(zret));
|
||||
return -1;
|
||||
}
|
||||
/* No more output space */
|
||||
if (outBuf.pos == outBuf.size)
|
||||
return outBuf.pos;
|
||||
} while (ctx->decompress.inbuf.pos < ctx->decompress.inbuf.size);
|
||||
|
||||
/*
|
||||
* No data in input buffer try to read some in, if an error then
|
||||
* return the total data read.
|
||||
*/
|
||||
ret = BIO_read(next, ctx->decompress.buffer, ctx->decompress.bufsize);
|
||||
if (ret <= 0) {
|
||||
BIO_copy_next_retry(b);
|
||||
if (ret < 0 && outBuf.pos == 0)
|
||||
return ret;
|
||||
return outBuf.pos;
|
||||
}
|
||||
ctx->decompress.inbuf.size = ret;
|
||||
ctx->decompress.inbuf.pos = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int bio_zstd_write(BIO *b, const char *in, int inl)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
size_t zret;
|
||||
ZSTD_inBuffer inBuf;
|
||||
int ret;
|
||||
int done = 0;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (in == NULL || inl <= 0)
|
||||
return 0;
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
|
||||
BIO_clear_retry_flags(b);
|
||||
if (ctx->compress.outbuf.dst == NULL) {
|
||||
ctx->compress.outbuf.dst = OPENSSL_malloc(ctx->compress.bufsize);
|
||||
if (ctx->compress.outbuf.dst == NULL) {
|
||||
ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE);
|
||||
return 0;
|
||||
}
|
||||
ctx->compress.outbuf.size = ctx->compress.bufsize;
|
||||
ctx->compress.outbuf.pos = 0;
|
||||
ctx->compress.write_pos = 0;
|
||||
}
|
||||
/* Obtain input data directly from supplied buffer */
|
||||
inBuf.src = in;
|
||||
inBuf.size = inl;
|
||||
inBuf.pos = 0;
|
||||
for (;;) {
|
||||
/* If data in output buffer write it first */
|
||||
while (ctx->compress.write_pos < ctx->compress.outbuf.pos) {
|
||||
ret = BIO_write(next, (unsigned char*)ctx->compress.outbuf.dst + ctx->compress.write_pos,
|
||||
ctx->compress.outbuf.pos - ctx->compress.write_pos);
|
||||
if (ret <= 0) {
|
||||
BIO_copy_next_retry(b);
|
||||
if (ret < 0 && inBuf.pos == 0)
|
||||
return ret;
|
||||
return inBuf.pos;
|
||||
}
|
||||
ctx->compress.write_pos += ret;
|
||||
}
|
||||
|
||||
/* Have we consumed all supplied data? */
|
||||
if (done)
|
||||
return inBuf.pos;
|
||||
|
||||
/* Reset buffer */
|
||||
ctx->compress.outbuf.pos = 0;
|
||||
ctx->compress.outbuf.size = ctx->compress.bufsize;
|
||||
ctx->compress.write_pos = 0;
|
||||
/* Compress some more */
|
||||
zret = ZSTD_compressStream2(ctx->compress.state, &ctx->compress.outbuf, &inBuf, ZSTD_e_end);
|
||||
if (ZSTD_isError(zret)) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_COMPRESS_ERROR);
|
||||
ERR_add_error_data(1, ZSTD_getErrorName(zret));
|
||||
return 0;
|
||||
} else if (zret == 0) {
|
||||
done = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int bio_zstd_flush(BIO *b)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
size_t zret;
|
||||
int ret;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
ctx = BIO_get_data(b);
|
||||
|
||||
/* If no data written or already flush show success */
|
||||
if (ctx->compress.outbuf.dst == NULL)
|
||||
return 1;
|
||||
|
||||
BIO_clear_retry_flags(b);
|
||||
/* No more input data */
|
||||
ctx->compress.outbuf.pos = 0;
|
||||
ctx->compress.outbuf.size = ctx->compress.bufsize;
|
||||
ctx->compress.write_pos = 0;
|
||||
for (;;) {
|
||||
/* If data in output buffer write it first */
|
||||
while (ctx->compress.write_pos < ctx->compress.outbuf.pos) {
|
||||
ret = BIO_write(next, (unsigned char*)ctx->compress.outbuf.dst + ctx->compress.write_pos,
|
||||
ctx->compress.outbuf.pos - ctx->compress.write_pos);
|
||||
if (ret <= 0) {
|
||||
BIO_copy_next_retry(b);
|
||||
return ret;
|
||||
}
|
||||
ctx->compress.write_pos += ret;
|
||||
}
|
||||
|
||||
/* Reset buffer */
|
||||
ctx->compress.outbuf.pos = 0;
|
||||
ctx->compress.outbuf.size = ctx->compress.bufsize;
|
||||
ctx->compress.write_pos = 0;
|
||||
/* Compress some more */
|
||||
zret = ZSTD_flushStream(ctx->compress.state, &ctx->compress.outbuf);
|
||||
if (ZSTD_isError(zret)) {
|
||||
ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_DECODE_ERROR);
|
||||
ERR_add_error_data(1, ZSTD_getErrorName(zret));
|
||||
return 0;
|
||||
}
|
||||
if (zret == 0)
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static long bio_zstd_ctrl(BIO *b, int cmd, long num, void *ptr)
|
||||
{
|
||||
BIO_ZSTD_CTX *ctx;
|
||||
int ret = 0, *ip;
|
||||
size_t ibs, obs;
|
||||
unsigned char *tmp;
|
||||
BIO *next = BIO_next(b);
|
||||
|
||||
if (next == NULL)
|
||||
return 0;
|
||||
ctx = BIO_get_data(b);
|
||||
switch (cmd) {
|
||||
|
||||
case BIO_CTRL_RESET:
|
||||
ctx->compress.write_pos = 0;
|
||||
ctx->compress.bufsize = 0;
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
case BIO_CTRL_FLUSH:
|
||||
ret = bio_zstd_flush(b);
|
||||
if (ret > 0) {
|
||||
ret = BIO_flush(next);
|
||||
BIO_copy_next_retry(b);
|
||||
}
|
||||
break;
|
||||
|
||||
case BIO_C_SET_BUFF_SIZE:
|
||||
ibs = ctx->decompress.bufsize;
|
||||
obs = ctx->compress.bufsize;
|
||||
if (ptr != NULL) {
|
||||
ip = ptr;
|
||||
if (*ip == 0)
|
||||
ibs = (size_t)num;
|
||||
else
|
||||
obs = (size_t)num;
|
||||
} else {
|
||||
obs = ibs = (size_t)num;
|
||||
}
|
||||
|
||||
if (ibs > 0 && ibs != ctx->decompress.bufsize) {
|
||||
if (ctx->decompress.buffer != NULL) {
|
||||
tmp = OPENSSL_realloc(ctx->decompress.buffer, ibs);
|
||||
if (tmp == NULL)
|
||||
return 0;
|
||||
if (ctx->decompress.inbuf.src == ctx->decompress.buffer)
|
||||
ctx->decompress.inbuf.src = tmp;
|
||||
ctx->decompress.buffer = tmp;
|
||||
}
|
||||
ctx->decompress.bufsize = ibs;
|
||||
}
|
||||
|
||||
if (obs > 0 && obs != ctx->compress.bufsize) {
|
||||
if (ctx->compress.outbuf.dst != NULL) {
|
||||
tmp = OPENSSL_realloc(ctx->compress.outbuf.dst, obs);
|
||||
if (tmp == NULL)
|
||||
return 0;
|
||||
ctx->compress.outbuf.dst = tmp;
|
||||
}
|
||||
ctx->compress.bufsize = obs;
|
||||
}
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
case BIO_C_DO_STATE_MACHINE:
|
||||
BIO_clear_retry_flags(b);
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
BIO_copy_next_retry(b);
|
||||
break;
|
||||
|
||||
case BIO_CTRL_WPENDING:
|
||||
if (ctx->compress.outbuf.pos < ctx->compress.outbuf.size)
|
||||
ret = 1;
|
||||
else
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
case BIO_CTRL_PENDING:
|
||||
if (ctx->decompress.inbuf.pos < ctx->decompress.inbuf.size)
|
||||
ret = 1;
|
||||
else
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = BIO_ctrl(next, cmd, num, ptr);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static long bio_zstd_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp)
|
||||
{
|
||||
BIO *next = BIO_next(b);
|
||||
if (next == NULL)
|
||||
return 0;
|
||||
return BIO_callback_ctrl(next, cmd, fp);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/crypto.h>
|
||||
#include <openssl/comp.h>
|
||||
#include <openssl/obj_mac.h>
|
||||
|
||||
#include "internal/cryptlib.h"
|
||||
#include "internal/comp.h"
|
||||
|
||||
#define SSL_COMP_NULL_IDX 0
|
||||
#define SSL_COMP_ZLIB_IDX 1
|
||||
#define SSL_COMP_NUM_IDX 2
|
||||
|
||||
#ifndef OPENSSL_NO_COMP
|
||||
static int sk_comp_cmp(const SSL_COMP *const *a, const SSL_COMP *const *b)
|
||||
{
|
||||
return ((*a)->id - (*b)->id);
|
||||
}
|
||||
#endif
|
||||
|
||||
STACK_OF(SSL_COMP) *ossl_load_builtin_compressions(void)
|
||||
{
|
||||
STACK_OF(SSL_COMP) *comp_methods = NULL;
|
||||
#ifndef OPENSSL_NO_COMP
|
||||
SSL_COMP *comp = NULL;
|
||||
COMP_METHOD *method = COMP_zlib();
|
||||
|
||||
comp_methods = sk_SSL_COMP_new(sk_comp_cmp);
|
||||
|
||||
if (COMP_get_type(method) != NID_undef && comp_methods != NULL) {
|
||||
comp = OPENSSL_malloc(sizeof(*comp));
|
||||
if (comp != NULL) {
|
||||
comp->method = method;
|
||||
comp->id = SSL_COMP_ZLIB_IDX;
|
||||
comp->name = COMP_get_name(method);
|
||||
if (!sk_SSL_COMP_push(comp_methods, comp))
|
||||
OPENSSL_free(comp);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return comp_methods;
|
||||
}
|
||||
|
||||
static void cmeth_free(SSL_COMP *cm)
|
||||
{
|
||||
OPENSSL_free(cm);
|
||||
}
|
||||
|
||||
void ossl_free_compression_methods_int(STACK_OF(SSL_COMP) *methods)
|
||||
{
|
||||
sk_SSL_COMP_pop_free(methods, cmeth_free);
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* Copyright 1995-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <openssl/opensslv.h>
|
||||
#include "internal/thread_once.h"
|
||||
#include "internal/cryptlib.h"
|
||||
#include "internal/e_os.h"
|
||||
|
||||
#if defined(_WIN32) && defined(OSSL_WINCTX)
|
||||
|
||||
# define TOSTR(x) #x
|
||||
# define MAKESTR(x) TOSTR(x)
|
||||
# define NOQUOTE(x) x
|
||||
# if defined(OSSL_WINCTX)
|
||||
# define REGISTRY_KEY "SOFTWARE\\WOW6432Node\\OpenSSL" "-" MAKESTR(OPENSSL_VERSION_MAJOR) "." MAKESTR(OPENSSL_VERSION_MINOR) "-" MAKESTR(OSSL_WINCTX)
|
||||
# endif
|
||||
|
||||
/**
|
||||
* @brief The directory where OpenSSL is installed.
|
||||
*/
|
||||
static char openssldir[MAX_PATH + 1];
|
||||
|
||||
/**
|
||||
* @brief The pointer to the openssldir buffer
|
||||
*/
|
||||
static char *openssldirptr = NULL;
|
||||
|
||||
/**
|
||||
* @brief The directory where OpenSSL engines are located.
|
||||
*/
|
||||
|
||||
static char enginesdir[MAX_PATH + 1];
|
||||
|
||||
/**
|
||||
* @brief The pointer to the enginesdir buffer
|
||||
*/
|
||||
static char *enginesdirptr = NULL;
|
||||
|
||||
/**
|
||||
* @brief The directory where OpenSSL modules are located.
|
||||
*/
|
||||
static char modulesdir[MAX_PATH + 1];
|
||||
|
||||
/**
|
||||
* @brief The pointer to the modulesdir buffer
|
||||
*/
|
||||
static char *modulesdirptr = NULL;
|
||||
|
||||
/**
|
||||
* @brief Get the list of Windows registry directories.
|
||||
*
|
||||
* This function retrieves a list of Windows registry directories.
|
||||
*
|
||||
* @return A pointer to a char array containing the registry directories.
|
||||
*/
|
||||
static char *get_windows_regdirs(char *dst, DWORD dstsizebytes, LPCWSTR valuename)
|
||||
{
|
||||
char *retval = NULL;
|
||||
# ifdef REGISTRY_KEY
|
||||
DWORD keysizebytes;
|
||||
DWORD ktype;
|
||||
HKEY hkey;
|
||||
LSTATUS ret;
|
||||
DWORD index = 0;
|
||||
LPCWSTR tempstr = NULL;
|
||||
|
||||
ret = RegOpenKeyEx(HKEY_LOCAL_MACHINE,
|
||||
TEXT(REGISTRY_KEY), KEY_WOW64_32KEY,
|
||||
KEY_QUERY_VALUE, &hkey);
|
||||
if (ret != ERROR_SUCCESS)
|
||||
goto out;
|
||||
|
||||
/* Always use wide call so we can avoid extra encoding conversions on the output */
|
||||
ret = RegQueryValueExW(hkey, valuename, NULL, &ktype, NULL,
|
||||
&keysizebytes);
|
||||
if (ret != ERROR_SUCCESS)
|
||||
goto out;
|
||||
if (ktype != REG_EXPAND_SZ && ktype != REG_SZ)
|
||||
goto out;
|
||||
if (keysizebytes > MAX_PATH * sizeof(WCHAR))
|
||||
goto out;
|
||||
|
||||
/*
|
||||
* RegQueryValueExW does not guarantee the buffer is null terminated,
|
||||
* so we make space for one in the allocation
|
||||
*/
|
||||
tempstr = OPENSSL_zalloc(keysizebytes + sizeof(WCHAR));
|
||||
|
||||
if (tempstr == NULL)
|
||||
goto out;
|
||||
|
||||
if (RegQueryValueExW(hkey, valuename,
|
||||
NULL, &ktype, (LPBYTE)tempstr, &keysizebytes) != ERROR_SUCCESS)
|
||||
goto out;
|
||||
|
||||
if (!WideCharToMultiByte(CP_UTF8, 0, tempstr, -1, dst, dstsizebytes,
|
||||
NULL, NULL))
|
||||
goto out;
|
||||
|
||||
retval = dst;
|
||||
out:
|
||||
OPENSSL_free(tempstr);
|
||||
RegCloseKey(hkey);
|
||||
# endif /* REGISTRY_KEY */
|
||||
return retval;
|
||||
}
|
||||
|
||||
static CRYPTO_ONCE defaults_setup_init = CRYPTO_ONCE_STATIC_INIT;
|
||||
|
||||
/**
|
||||
* @brief Function to setup default values to run once.
|
||||
* Only used in Windows environments. Does run time initialization
|
||||
* of openssldir/modulesdir/enginesdir from the registry
|
||||
*/
|
||||
DEFINE_RUN_ONCE_STATIC(do_defaults_setup)
|
||||
{
|
||||
get_windows_regdirs(openssldir, sizeof(openssldir), L"OPENSSLDIR");
|
||||
get_windows_regdirs(enginesdir, sizeof(enginesdir), L"ENGINESDIR");
|
||||
get_windows_regdirs(modulesdir, sizeof(modulesdir), L"MODULESDIR");
|
||||
|
||||
/*
|
||||
* Set our pointers only if the directories are fetched properly
|
||||
*/
|
||||
if (strlen(openssldir) > 0)
|
||||
openssldirptr = openssldir;
|
||||
|
||||
if (strlen(enginesdir) > 0)
|
||||
enginesdirptr = enginesdir;
|
||||
|
||||
if (strlen(modulesdir) > 0)
|
||||
modulesdirptr = modulesdir;
|
||||
|
||||
return 1;
|
||||
}
|
||||
#endif /* defined(_WIN32) && defined(OSSL_WINCTX) */
|
||||
|
||||
/**
|
||||
* @brief Get the directory where OpenSSL is installed.
|
||||
*
|
||||
* @return A pointer to a string containing the OpenSSL directory path.
|
||||
*/
|
||||
const char *ossl_get_openssldir(void)
|
||||
{
|
||||
#if defined(_WIN32) && defined (OSSL_WINCTX)
|
||||
if (!RUN_ONCE(&defaults_setup_init, do_defaults_setup))
|
||||
return NULL;
|
||||
return (const char *)openssldirptr;
|
||||
# else
|
||||
return OPENSSLDIR;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get the directory where OpenSSL engines are located.
|
||||
*
|
||||
* @return A pointer to a string containing the engines directory path.
|
||||
*/
|
||||
const char *ossl_get_enginesdir(void)
|
||||
{
|
||||
#if defined(_WIN32) && defined (OSSL_WINCTX)
|
||||
if (!RUN_ONCE(&defaults_setup_init, do_defaults_setup))
|
||||
return NULL;
|
||||
return (const char *)enginesdirptr;
|
||||
#else
|
||||
return ENGINESDIR;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get the directory where OpenSSL modules are located.
|
||||
*
|
||||
* @return A pointer to a string containing the modules directory path.
|
||||
*/
|
||||
const char *ossl_get_modulesdir(void)
|
||||
{
|
||||
#if defined(_WIN32) && defined(OSSL_WINCTX)
|
||||
if (!RUN_ONCE(&defaults_setup_init, do_defaults_setup))
|
||||
return NULL;
|
||||
return (const char *)modulesdirptr;
|
||||
#else
|
||||
return MODULESDIR;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get the build time defined windows installer context
|
||||
*
|
||||
* @return A char pointer to a string representing the windows install context
|
||||
*/
|
||||
const char *ossl_get_wininstallcontext(void)
|
||||
{
|
||||
#if defined(_WIN32) && defined (OSSL_WINCTX)
|
||||
return MAKESTR(OSSL_WINCTX);
|
||||
#else
|
||||
return "Undefined";
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
/*
|
||||
* Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <openssl/bn.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/core_names.h>
|
||||
#include <openssl/kdf.h>
|
||||
#include "internal/deterministic_nonce.h"
|
||||
#include "crypto/bn.h"
|
||||
|
||||
/*
|
||||
* Convert a Bit String to an Integer (See RFC 6979 Section 2.3.2)
|
||||
*
|
||||
* Params:
|
||||
* out The returned Integer as a BIGNUM
|
||||
* qlen_bits The maximum size of the returned integer in bits. The returned
|
||||
* Integer is shifted right if inlen is larger than qlen_bits..
|
||||
* in, inlen The input Bit String (in bytes).
|
||||
* Returns: 1 if successful, or 0 otherwise.
|
||||
*/
|
||||
static int bits2int(BIGNUM *out, int qlen_bits,
|
||||
const unsigned char *in, size_t inlen)
|
||||
{
|
||||
int blen_bits = inlen * 8;
|
||||
int shift;
|
||||
|
||||
if (BN_bin2bn(in, (int)inlen, out) == NULL)
|
||||
return 0;
|
||||
|
||||
shift = blen_bits - qlen_bits;
|
||||
if (shift > 0)
|
||||
return BN_rshift(out, out, shift);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert as above a Bit String in const time to an Integer w fixed top
|
||||
*
|
||||
* Params:
|
||||
* out The returned Integer as a BIGNUM
|
||||
* qlen_bits The maximum size of the returned integer in bits. The returned
|
||||
* Integer is shifted right if inlen is larger than qlen_bits..
|
||||
* in, inlen The input Bit String (in bytes). It has sizeof(BN_ULONG) bytes
|
||||
* prefix with all bits set that needs to be cleared out after
|
||||
* the conversion.
|
||||
* Returns: 1 if successful, or 0 otherwise.
|
||||
*/
|
||||
static int bits2int_consttime(BIGNUM *out, int qlen_bits,
|
||||
const unsigned char *in, size_t inlen)
|
||||
{
|
||||
int blen_bits = (inlen - sizeof(BN_ULONG)) * 8;
|
||||
int shift;
|
||||
|
||||
if (BN_bin2bn(in, (int)inlen, out) == NULL)
|
||||
return 0;
|
||||
|
||||
BN_set_flags(out, BN_FLG_CONSTTIME);
|
||||
ossl_bn_mask_bits_fixed_top(out, blen_bits);
|
||||
|
||||
shift = blen_bits - qlen_bits;
|
||||
if (shift > 0)
|
||||
return bn_rshift_fixed_top(out, out, shift);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert an Integer to an Octet String (See RFC 6979 2.3.3).
|
||||
* The value is zero padded if required.
|
||||
*
|
||||
* Params:
|
||||
* out The returned Octet String
|
||||
* num The input Integer
|
||||
* rlen The required size of the returned Octet String in bytes
|
||||
* Returns: 1 if successful, or 0 otherwise.
|
||||
*/
|
||||
static int int2octets(unsigned char *out, const BIGNUM *num, int rlen)
|
||||
{
|
||||
return BN_bn2binpad(num, out, rlen) >= 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert a Bit String to an Octet String (See RFC 6979 Section 2.3.4)
|
||||
*
|
||||
* Params:
|
||||
* out The returned octet string.
|
||||
* q The modulus
|
||||
* qlen_bits The length of q in bits
|
||||
* rlen The value of qlen_bits rounded up to the nearest 8 bits.
|
||||
* in, inlen The input bit string (in bytes)
|
||||
* Returns: 1 if successful, or 0 otherwise.
|
||||
*/
|
||||
static int bits2octets(unsigned char *out, const BIGNUM *q, int qlen_bits,
|
||||
int rlen, const unsigned char *in, size_t inlen)
|
||||
{
|
||||
int ret = 0;
|
||||
BIGNUM *z = BN_new();
|
||||
|
||||
if (z == NULL
|
||||
|| !bits2int(z, qlen_bits, in, inlen))
|
||||
goto err;
|
||||
|
||||
/* z2 = z1 mod q (Do a simple subtract, since z1 < 2^qlen_bits) */
|
||||
if (BN_cmp(z, q) >= 0
|
||||
&& !BN_usub(z, z, q))
|
||||
goto err;
|
||||
|
||||
ret = int2octets(out, z, rlen);
|
||||
err:
|
||||
BN_free(z);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Setup a KDF HMAC_DRBG object using fixed entropy and nonce data.
|
||||
*
|
||||
* Params:
|
||||
* digestname The digest name for the HMAC
|
||||
* entropy, entropylen A fixed input entropy buffer
|
||||
* nonce, noncelen A fixed input nonce buffer
|
||||
* libctx, propq Are used for fetching algorithms
|
||||
*
|
||||
* Returns: The created KDF HMAC_DRBG object if successful, or NULL otherwise.
|
||||
*/
|
||||
static EVP_KDF_CTX *kdf_setup(const char *digestname,
|
||||
const unsigned char *entropy, size_t entropylen,
|
||||
const unsigned char *nonce, size_t noncelen,
|
||||
OSSL_LIB_CTX *libctx, const char *propq)
|
||||
{
|
||||
EVP_KDF_CTX *ctx = NULL;
|
||||
EVP_KDF *kdf = NULL;
|
||||
OSSL_PARAM params[5], *p;
|
||||
|
||||
kdf = EVP_KDF_fetch(libctx, "HMAC-DRBG-KDF", propq);
|
||||
ctx = EVP_KDF_CTX_new(kdf);
|
||||
EVP_KDF_free(kdf);
|
||||
if (ctx == NULL)
|
||||
goto err;
|
||||
|
||||
p = params;
|
||||
*p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST,
|
||||
(char *)digestname, 0);
|
||||
if (propq != NULL)
|
||||
*p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_PROPERTIES,
|
||||
(char *)propq, 0);
|
||||
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_HMACDRBG_ENTROPY,
|
||||
(void *)entropy, entropylen);
|
||||
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_HMACDRBG_NONCE,
|
||||
(void *)nonce, noncelen);
|
||||
*p = OSSL_PARAM_construct_end();
|
||||
|
||||
if (EVP_KDF_CTX_set_params(ctx, params) <= 0)
|
||||
goto err;
|
||||
|
||||
return ctx;
|
||||
err:
|
||||
EVP_KDF_CTX_free(ctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Generate a Deterministic nonce 'k' for DSA/ECDSA as defined in
|
||||
* RFC 6979 Section 3.3. "Alternate Description of the Generation of k"
|
||||
*
|
||||
* Params:
|
||||
* out Returns the generated deterministic nonce 'k'
|
||||
* q A large prime number used for modulus operations for DSA and ECDSA.
|
||||
* priv The private key in the range [1, q-1]
|
||||
* hm, hmlen The digested message buffer in bytes
|
||||
* digestname The digest name used for signing. It is used as the HMAC digest.
|
||||
* libctx, propq Used for fetching algorithms
|
||||
*
|
||||
* Returns: 1 if successful, or 0 otherwise.
|
||||
*/
|
||||
int ossl_gen_deterministic_nonce_rfc6979(BIGNUM *out, const BIGNUM *q,
|
||||
const BIGNUM *priv,
|
||||
const unsigned char *hm, size_t hmlen,
|
||||
const char *digestname,
|
||||
OSSL_LIB_CTX *libctx,
|
||||
const char *propq)
|
||||
{
|
||||
EVP_KDF_CTX *kdfctx = NULL;
|
||||
int ret = 0, rlen = 0, qlen_bits = 0;
|
||||
unsigned char *entropyx = NULL, *nonceh = NULL, *rbits = NULL, *T = NULL;
|
||||
size_t allocsz = 0;
|
||||
const size_t prefsz = sizeof(BN_ULONG);
|
||||
|
||||
if (out == NULL)
|
||||
return 0;
|
||||
|
||||
qlen_bits = BN_num_bits(q);
|
||||
if (qlen_bits == 0)
|
||||
return 0;
|
||||
|
||||
/* Note rlen used here is in bytes since the input values are byte arrays */
|
||||
rlen = (qlen_bits + 7) / 8;
|
||||
allocsz = prefsz + 3 * rlen;
|
||||
|
||||
/* Use a single alloc for the buffers T, nonceh and entropyx */
|
||||
T = (unsigned char *)OPENSSL_zalloc(allocsz);
|
||||
if (T == NULL)
|
||||
return 0;
|
||||
rbits = T + prefsz;
|
||||
nonceh = rbits + rlen;
|
||||
entropyx = nonceh + rlen;
|
||||
|
||||
memset(T, 0xff, prefsz);
|
||||
|
||||
if (!int2octets(entropyx, priv, rlen)
|
||||
|| !bits2octets(nonceh, q, qlen_bits, rlen, hm, hmlen))
|
||||
goto end;
|
||||
|
||||
kdfctx = kdf_setup(digestname, entropyx, rlen, nonceh, rlen, libctx, propq);
|
||||
if (kdfctx == NULL)
|
||||
goto end;
|
||||
|
||||
do {
|
||||
if (!EVP_KDF_derive(kdfctx, rbits, rlen, NULL)
|
||||
|| !bits2int_consttime(out, qlen_bits, T, rlen + prefsz))
|
||||
goto end;
|
||||
} while (ossl_bn_is_word_fixed_top(out, 0)
|
||||
|| ossl_bn_is_word_fixed_top(out, 1)
|
||||
|| BN_ucmp(out, q) >= 0);
|
||||
#ifdef BN_DEBUG
|
||||
/* With BN_DEBUG on a fixed top number cannot be returned */
|
||||
bn_correct_top(out);
|
||||
#endif
|
||||
ret = 1;
|
||||
|
||||
end:
|
||||
EVP_KDF_CTX_free(kdfctx);
|
||||
OPENSSL_clear_free(T, allocsz);
|
||||
return ret;
|
||||
}
|
||||
Executable
+1532
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,831 @@
|
||||
#! /usr/bin/env perl
|
||||
# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
||||
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
|
||||
die "can't locate arm-xlate.pl";
|
||||
|
||||
open OUT,"| \"$^X\" $xlate $flavour \"$output\""
|
||||
or die "can't call $xlate: $!";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
my ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("x$_",(7..14));
|
||||
my ($a8,$a10,$a12,$a14,$a9,$a11,$a13,$a15)=map("x$_",(7..14));
|
||||
my ($t0,$t1,$t2,$t3)=map("x$_",(3..6));
|
||||
my ($t4,$t5,$t6,$t7,$t8)=map("x$_",(15..17,19,20));
|
||||
|
||||
sub bn_mod_add() {
|
||||
my $mod = shift;
|
||||
$code.=<<___;
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Addition
|
||||
adds $s0,$s0,$s4
|
||||
adcs $s1,$s1,$s5
|
||||
adcs $s2,$s2,$s6
|
||||
adcs $s3,$s3,$s7
|
||||
adc $t4,xzr,xzr
|
||||
|
||||
// Load polynomial
|
||||
adrp x2,$mod
|
||||
add x2,x2,:lo12:$mod
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Backup Addition
|
||||
mov $t0,$s0
|
||||
mov $t1,$s1
|
||||
mov $t2,$s2
|
||||
mov $t3,$s3
|
||||
|
||||
// Sub polynomial
|
||||
subs $t0,$t0,$s4
|
||||
sbcs $t1,$t1,$s5
|
||||
sbcs $t2,$t2,$s6
|
||||
sbcs $t3,$t3,$s7
|
||||
sbcs $t4,$t4,xzr
|
||||
|
||||
// Select based on carry
|
||||
csel $s0,$s0,$t0,cc
|
||||
csel $s1,$s1,$t1,cc
|
||||
csel $s2,$s2,$t2,cc
|
||||
csel $s3,$s3,$t3,cc
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
___
|
||||
}
|
||||
|
||||
sub bn_mod_sub() {
|
||||
my $mod = shift;
|
||||
$code.=<<___;
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Subtraction
|
||||
subs $s0,$s0,$s4
|
||||
sbcs $s1,$s1,$s5
|
||||
sbcs $s2,$s2,$s6
|
||||
sbcs $s3,$s3,$s7
|
||||
sbc $t4,xzr,xzr
|
||||
|
||||
// Load polynomial
|
||||
adrp x2,$mod
|
||||
add x2,x2,:lo12:$mod
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Backup subtraction
|
||||
mov $t0,$s0
|
||||
mov $t1,$s1
|
||||
mov $t2,$s2
|
||||
mov $t3,$s3
|
||||
|
||||
// Add polynomial
|
||||
adds $t0,$t0,$s4
|
||||
adcs $t1,$t1,$s5
|
||||
adcs $t2,$t2,$s6
|
||||
adcs $t3,$t3,$s7
|
||||
tst $t4,$t4
|
||||
|
||||
// Select based on carry
|
||||
csel $s0,$s0,$t0,eq
|
||||
csel $s1,$s1,$t1,eq
|
||||
csel $s2,$s2,$t2,eq
|
||||
csel $s3,$s3,$t3,eq
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
___
|
||||
}
|
||||
|
||||
sub bn_mod_div_by_2() {
|
||||
my $mod = shift;
|
||||
$code.=<<___;
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
|
||||
// Save the least significant bit
|
||||
mov $t0,$s0
|
||||
|
||||
// Right shift 1
|
||||
extr $s0,$s1,$s0,#1
|
||||
extr $s1,$s2,$s1,#1
|
||||
extr $s2,$s3,$s2,#1
|
||||
lsr $s3,$s3,#1
|
||||
|
||||
// Load mod
|
||||
adrp x2,$mod
|
||||
add x2,x2,:lo12:$mod
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Parity check
|
||||
tst $t0,#1
|
||||
csel $s4,xzr,$s4,eq
|
||||
csel $s5,xzr,$s5,eq
|
||||
csel $s6,xzr,$s6,eq
|
||||
csel $s7,xzr,$s7,eq
|
||||
|
||||
// Add
|
||||
adds $s0,$s0,$s4
|
||||
adcs $s1,$s1,$s5
|
||||
adcs $s2,$s2,$s6
|
||||
adc $s3,$s3,$s7
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
$code.=<<___;
|
||||
#include "arm_arch.h"
|
||||
.arch armv8-a
|
||||
.rodata
|
||||
|
||||
.align 5
|
||||
// The polynomial p
|
||||
.Lpoly:
|
||||
.quad 0xffffffffffffffff,0xffffffff00000000,0xffffffffffffffff,0xfffffffeffffffff
|
||||
// The order of polynomial n
|
||||
.Lord:
|
||||
.quad 0x53bbf40939d54123,0x7203df6b21c6052b,0xffffffffffffffff,0xfffffffeffffffff
|
||||
// (p + 1) / 2
|
||||
.Lpoly_div_2:
|
||||
.quad 0x8000000000000000,0xffffffff80000000,0xffffffffffffffff,0x7fffffff7fffffff
|
||||
// (n + 1) / 2
|
||||
.Lord_div_2:
|
||||
.quad 0xa9ddfa049ceaa092,0xb901efb590e30295,0xffffffffffffffff,0x7fffffff7fffffff
|
||||
|
||||
.text
|
||||
|
||||
// void bn_rshift1(BN_ULONG *a);
|
||||
.globl bn_rshift1
|
||||
.type bn_rshift1,%function
|
||||
.align 5
|
||||
bn_rshift1:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x0]
|
||||
ldp $s2,$s3,[x0,#16]
|
||||
|
||||
// Right shift
|
||||
extr $s0,$s1,$s0,#1
|
||||
extr $s1,$s2,$s1,#1
|
||||
extr $s2,$s3,$s2,#1
|
||||
lsr $s3,$s3,#1
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
|
||||
ret
|
||||
.size bn_rshift1,.-bn_rshift1
|
||||
|
||||
// void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
.globl bn_sub
|
||||
.type bn_sub,%function
|
||||
.align 5
|
||||
bn_sub:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// Subtraction
|
||||
subs $s0,$s0,$s4
|
||||
sbcs $s1,$s1,$s5
|
||||
sbcs $s2,$s2,$s6
|
||||
sbc $s3,$s3,$s7
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
|
||||
ret
|
||||
.size bn_sub,.-bn_sub
|
||||
|
||||
// void ecp_sm2p256_div_by_2(BN_ULONG *r,const BN_ULONG *a);
|
||||
.globl ecp_sm2p256_div_by_2
|
||||
.type ecp_sm2p256_div_by_2,%function
|
||||
.align 5
|
||||
ecp_sm2p256_div_by_2:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
___
|
||||
&bn_mod_div_by_2(".Lpoly_div_2");
|
||||
$code.=<<___;
|
||||
ret
|
||||
.size ecp_sm2p256_div_by_2,.-ecp_sm2p256_div_by_2
|
||||
|
||||
// void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r,const BN_ULONG *a);
|
||||
.globl ecp_sm2p256_div_by_2_mod_ord
|
||||
.type ecp_sm2p256_div_by_2_mod_ord,%function
|
||||
.align 5
|
||||
ecp_sm2p256_div_by_2_mod_ord:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
___
|
||||
&bn_mod_div_by_2(".Lord_div_2");
|
||||
$code.=<<___;
|
||||
ret
|
||||
.size ecp_sm2p256_div_by_2_mod_ord,.-ecp_sm2p256_div_by_2_mod_ord
|
||||
|
||||
// void ecp_sm2p256_mul_by_3(BN_ULONG *r,const BN_ULONG *a);
|
||||
.globl ecp_sm2p256_mul_by_3
|
||||
.type ecp_sm2p256_mul_by_3,%function
|
||||
.align 5
|
||||
ecp_sm2p256_mul_by_3:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
|
||||
// 2*a
|
||||
adds $s0,$s0,$s0
|
||||
adcs $s1,$s1,$s1
|
||||
adcs $s2,$s2,$s2
|
||||
adcs $s3,$s3,$s3
|
||||
adcs $t4,xzr,xzr
|
||||
|
||||
mov $t0,$s0
|
||||
mov $t1,$s1
|
||||
mov $t2,$s2
|
||||
mov $t3,$s3
|
||||
|
||||
// Sub polynomial
|
||||
adrp x2,.Lpoly
|
||||
add x2,x2,:lo12:.Lpoly
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
subs $s0,$s0,$s4
|
||||
sbcs $s1,$s1,$s5
|
||||
sbcs $s2,$s2,$s6
|
||||
sbcs $s3,$s3,$s7
|
||||
sbcs $t4,$t4,xzr
|
||||
|
||||
csel $s0,$s0,$t0,cs
|
||||
csel $s1,$s1,$t1,cs
|
||||
csel $s2,$s2,$t2,cs
|
||||
csel $s3,$s3,$t3,cs
|
||||
eor $t4,$t4,$t4
|
||||
|
||||
// 3*a
|
||||
ldp $s4,$s5,[x1]
|
||||
ldp $s6,$s7,[x1,#16]
|
||||
adds $s0,$s0,$s4
|
||||
adcs $s1,$s1,$s5
|
||||
adcs $s2,$s2,$s6
|
||||
adcs $s3,$s3,$s7
|
||||
adcs $t4,xzr,xzr
|
||||
|
||||
mov $t0,$s0
|
||||
mov $t1,$s1
|
||||
mov $t2,$s2
|
||||
mov $t3,$s3
|
||||
|
||||
// Sub polynomial
|
||||
adrp x2,.Lpoly
|
||||
add x2,x2,:lo12:.Lpoly
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
subs $s0,$s0,$s4
|
||||
sbcs $s1,$s1,$s5
|
||||
sbcs $s2,$s2,$s6
|
||||
sbcs $s3,$s3,$s7
|
||||
sbcs $t4,$t4,xzr
|
||||
|
||||
csel $s0,$s0,$t0,cs
|
||||
csel $s1,$s1,$t1,cs
|
||||
csel $s2,$s2,$t2,cs
|
||||
csel $s3,$s3,$t3,cs
|
||||
|
||||
// Store results
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
|
||||
ret
|
||||
.size ecp_sm2p256_mul_by_3,.-ecp_sm2p256_mul_by_3
|
||||
|
||||
// void ecp_sm2p256_add(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b);
|
||||
.globl ecp_sm2p256_add
|
||||
.type ecp_sm2p256_add,%function
|
||||
.align 5
|
||||
ecp_sm2p256_add:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
___
|
||||
&bn_mod_add(".Lpoly");
|
||||
$code.=<<___;
|
||||
ret
|
||||
.size ecp_sm2p256_add,.-ecp_sm2p256_add
|
||||
|
||||
// void ecp_sm2p256_sub(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b);
|
||||
.globl ecp_sm2p256_sub
|
||||
.type ecp_sm2p256_sub,%function
|
||||
.align 5
|
||||
ecp_sm2p256_sub:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
___
|
||||
&bn_mod_sub(".Lpoly");
|
||||
$code.=<<___;
|
||||
ret
|
||||
.size ecp_sm2p256_sub,.-ecp_sm2p256_sub
|
||||
|
||||
// void ecp_sm2p256_sub_mod_ord(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b);
|
||||
.globl ecp_sm2p256_sub_mod_ord
|
||||
.type ecp_sm2p256_sub_mod_ord,%function
|
||||
.align 5
|
||||
ecp_sm2p256_sub_mod_ord:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
___
|
||||
&bn_mod_sub(".Lord");
|
||||
$code.=<<___;
|
||||
ret
|
||||
.size ecp_sm2p256_sub_mod_ord,.-ecp_sm2p256_sub_mod_ord
|
||||
|
||||
.macro RDC
|
||||
// a = | s7 | ... | s0 |, where si are 64-bit quantities
|
||||
// = |a15|a14| ... |a1|a0|, where ai are 32-bit quantities
|
||||
// | s7 | s6 | s5 | s4 |
|
||||
// | a15 | a14 | a13 | a12 | a11 | a10 | a9 | a8 |
|
||||
// | s3 | s2 | s1 | s0 |
|
||||
// | a7 | a6 | a5 | a4 | a3 | a2 | a1 | a0 |
|
||||
// =================================================
|
||||
// | a8 | a11 | a10 | a9 | a8 | 0 | s4 | (+)
|
||||
// | a9 | a15 | s6 | a11 | 0 | a10 | a9 | (+)
|
||||
// | a10 | 0 | a14 | a13 | a12 | 0 | s5 | (+)
|
||||
// | a11 | 0 | s7 | a13 | 0 | a12 | a11 | (+)
|
||||
// | a12 | 0 | s7 | a13 | 0 | s6 | (+)
|
||||
// | a12 | 0 | 0 | a15 | a14 | 0 | a14 | a13 | (+)
|
||||
// | a13 | 0 | 0 | 0 | a15 | 0 | a14 | a13 | (+)
|
||||
// | a13 | 0 | 0 | 0 | 0 | 0 | s7 | (+)
|
||||
// | a14 | 0 | 0 | 0 | 0 | 0 | s7 | (+)
|
||||
// | a14 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+)
|
||||
// | a15 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+)
|
||||
// | a15 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (+)
|
||||
// | s7 | 0 | 0 | 0 | 0 | 0 | 0 | (+)
|
||||
// | 0 | 0 | 0 | 0 | 0 | a8 | 0 | 0 | (-)
|
||||
// | 0 | 0 | 0 | 0 | 0 | a9 | 0 | 0 | (-)
|
||||
// | 0 | 0 | 0 | 0 | 0 | a13 | 0 | 0 | (-)
|
||||
// | 0 | 0 | 0 | 0 | 0 | a14 | 0 | 0 | (-)
|
||||
// | U[7]| U[6]| U[5]| U[4]| U[3]| U[2]| U[1]| U[0]|
|
||||
// | V[3] | V[2] | V[1] | V[0] |
|
||||
|
||||
// 1. 64-bit addition
|
||||
// t2=s6+s7+s7
|
||||
adds $t2,$s6,$s7
|
||||
adcs $t1,xzr,xzr
|
||||
adds $t2,$t2,$s7
|
||||
adcs $t1,$t1,xzr
|
||||
// t3=s4+s5+t2
|
||||
adds $t3,$s4,$t2
|
||||
adcs $t4,$t1,xzr
|
||||
adds $t3,$t3,$s5
|
||||
adcs $t4,$t4,xzr
|
||||
// sum
|
||||
adds $s0,$s0,$t3
|
||||
adcs $s1,$s1,$t4
|
||||
adcs $s2,$s2,$t2
|
||||
adcs $s3,$s3,$s7
|
||||
adcs $t0,xzr,xzr
|
||||
adds $s3,$s3,$t1
|
||||
adcs $t0,$t0,xzr
|
||||
|
||||
stp $s0,$s1,[sp,#32]
|
||||
stp $s2,$s3,[sp,#48]
|
||||
|
||||
// 2. 64-bit to 32-bit spread
|
||||
mov $t1,#0xffffffff
|
||||
mov $s0,$s4
|
||||
mov $s1,$s5
|
||||
mov $s2,$s6
|
||||
mov $s3,$s7
|
||||
and $s0,$s0,$t1 // a8
|
||||
and $s1,$s1,$t1 // a10
|
||||
and $s2,$s2,$t1 // a12
|
||||
and $s3,$s3,$t1 // a14
|
||||
lsr $s4,$s4,#32 // a9
|
||||
lsr $s5,$s5,#32 // a11
|
||||
lsr $s6,$s6,#32 // a13
|
||||
lsr $s7,$s7,#32 // a15
|
||||
|
||||
// 3. 32-bit addition
|
||||
add $t1,$a14,$a12 // t1 <- a12 + a14
|
||||
add $t2,$a15,$a13 // t2 <- a13 + a15
|
||||
add $t3,$a8,$a9 // t3 <- a8 + a9
|
||||
add $t4,$a14,$a10 // t4 <- a10 + a14
|
||||
add $a15,$a15,$a11 // a15 <- a11 + a15
|
||||
add $a12,$t2,$t1 // a12 <- a12 + a13 + a14 + a15
|
||||
add $a10,$a10,$a12 // a10 <- a10 + a12 + a13 + a14 + a15
|
||||
add $a10,$a10,$a12 // a10 <- a10 + 2*(a12 + a13 + a14 + a15)
|
||||
add $a10,$a10,$t3 // a10 <- a8 + a9 + a10 + 2*(a12 + a13 + a14 + a15)
|
||||
add $a10,$a10,$a11 // a10 <- a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15)
|
||||
add $a12,$a12,$a13 // a12 <- a12 + 2*a13 + a14 + a15
|
||||
add $a12,$a12,$a11 // a12 <- a11 + a12 + 2*a13 + a14 + a15
|
||||
add $a12,$a12,$a8 // a12 <- a8 + a11 + a12 + 2*a13 + a14 + a15
|
||||
add $t3,$t3,$a14 // t3 <- a8 + a9 + a14
|
||||
add $t3,$t3,$a13 // t3 <- a8 + a9 + a13 + a14
|
||||
add $a9,$a9,$t2 // a9 <- a9 + a13 + a15
|
||||
add $a11,$a11,$a9 // a11 <- a9 + a11 + a13 + a15
|
||||
add $a11,$a11,$t2 // a11 <- a9 + a11 + 2*(a13 + a15)
|
||||
add $t1,$t1,$t4 // t1 <- a10 + a12 + 2*a14
|
||||
|
||||
// U[0] s5 a9 + a11 + 2*(a13 + a15)
|
||||
// U[1] t1 a10 + a12 + 2*a14
|
||||
// U[2] -t3 a8 + a9 + a13 + a14
|
||||
// U[3] s2 a8 + a11 + a12 + 2*a13 + a14 + a15
|
||||
// U[4] s4 a9 + a13 + a15
|
||||
// U[5] t4 a10 + a14
|
||||
// U[6] s7 a11 + a15
|
||||
// U[7] s1 a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15)
|
||||
|
||||
// 4. 32-bit to 64-bit
|
||||
lsl $s0,$t1,#32
|
||||
extr $t1,$s2,$t1,#32
|
||||
extr $s2,$t4,$s2,#32
|
||||
extr $t4,$s1,$t4,#32
|
||||
lsr $s1,$s1,#32
|
||||
|
||||
// 5. 64-bit addition
|
||||
adds $s5,$s5,$s0
|
||||
adcs $t1,$t1,xzr
|
||||
adcs $s4,$s4,$s2
|
||||
adcs $s7,$s7,$t4
|
||||
adcs $t0,$t0,$s1
|
||||
|
||||
// V[0] s5
|
||||
// V[1] t1
|
||||
// V[2] s4
|
||||
// V[3] s7
|
||||
// carry t0
|
||||
// sub t3
|
||||
|
||||
// 5. Process s0-s3
|
||||
ldp $s0,$s1,[sp,#32]
|
||||
ldp $s2,$s3,[sp,#48]
|
||||
// add with V0-V3
|
||||
adds $s0,$s0,$s5
|
||||
adcs $s1,$s1,$t1
|
||||
adcs $s2,$s2,$s4
|
||||
adcs $s3,$s3,$s7
|
||||
adcs $t0,$t0,xzr
|
||||
// sub with t3
|
||||
subs $s1,$s1,$t3
|
||||
sbcs $s2,$s2,xzr
|
||||
sbcs $s3,$s3,xzr
|
||||
sbcs $t0,$t0,xzr
|
||||
|
||||
// 6. MOD
|
||||
// First Mod
|
||||
lsl $t1,$t0,#32
|
||||
subs $t2,$t1,$t0
|
||||
|
||||
adds $s0,$s0,$t0
|
||||
adcs $s1,$s1,$t2
|
||||
adcs $s2,$s2,xzr
|
||||
adcs $s3,$s3,$t1
|
||||
|
||||
// Last Mod
|
||||
// return y - p if y > p else y
|
||||
mov $s4,$s0
|
||||
mov $s5,$s1
|
||||
mov $s6,$s2
|
||||
mov $s7,$s3
|
||||
|
||||
adrp $t0,.Lpoly
|
||||
add $t0,$t0,:lo12:.Lpoly
|
||||
ldp $t1,$t2,[$t0]
|
||||
ldp $t3,$t4,[$t0,#16]
|
||||
|
||||
adcs $t5,xzr,xzr
|
||||
|
||||
subs $s0,$s0,$t1
|
||||
sbcs $s1,$s1,$t2
|
||||
sbcs $s2,$s2,$t3
|
||||
sbcs $s3,$s3,$t4
|
||||
sbcs $t5,$t5,xzr
|
||||
|
||||
csel $s0,$s0,$s4,cs
|
||||
csel $s1,$s1,$s5,cs
|
||||
csel $s2,$s2,$s6,cs
|
||||
csel $s3,$s3,$s7,cs
|
||||
|
||||
.endm
|
||||
|
||||
// void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
.globl ecp_sm2p256_mul
|
||||
.type ecp_sm2p256_mul,%function
|
||||
.align 5
|
||||
ecp_sm2p256_mul:
|
||||
AARCH64_SIGN_LINK_REGISTER
|
||||
// Store scalar registers
|
||||
stp x29,x30,[sp,#-80]!
|
||||
add x29,sp,#0
|
||||
stp x16,x17,[sp,#16]
|
||||
stp x19,x20,[sp,#64]
|
||||
|
||||
// Load inputs
|
||||
ldp $s0,$s1,[x1]
|
||||
ldp $s2,$s3,[x1,#16]
|
||||
ldp $s4,$s5,[x2]
|
||||
ldp $s6,$s7,[x2,#16]
|
||||
|
||||
// ### multiplication ###
|
||||
// ========================
|
||||
// s3 s2 s1 s0
|
||||
// * s7 s6 s5 s4
|
||||
// ------------------------
|
||||
// + s0 s0 s0 s0
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s1 s1 s1 s1
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s2 s2 s2 s2
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s3 s3 s3 s3
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// ------------------------
|
||||
// s7 s6 s5 s4 s3 s2 s1 s0
|
||||
// ========================
|
||||
|
||||
// ### s0*s4 ###
|
||||
mul $t5,$s0,$s4
|
||||
umulh $t2,$s0,$s4
|
||||
|
||||
// ### s1*s4 + s0*s5 ###
|
||||
mul $t0,$s1,$s4
|
||||
umulh $t1,$s1,$s4
|
||||
adds $t2,$t2,$t0
|
||||
adcs $t3,$t1,xzr
|
||||
|
||||
mul $t0,$s0,$s5
|
||||
umulh $t1,$s0,$s5
|
||||
adds $t2,$t2,$t0
|
||||
adcs $t3,$t3,$t1
|
||||
adcs $t4,xzr,xzr
|
||||
|
||||
// ### s2*s4 + s1*s5 + s0*s6 ###
|
||||
mul $t0,$s2,$s4
|
||||
umulh $t1,$s2,$s4
|
||||
adds $t3,$t3,$t0
|
||||
adcs $t4,$t4,$t1
|
||||
|
||||
mul $t0,$s1,$s5
|
||||
umulh $t1,$s1,$s5
|
||||
adds $t3,$t3,$t0
|
||||
adcs $t4,$t4,$t1
|
||||
adcs $t6,xzr,xzr
|
||||
|
||||
mul $t0,$s0,$s6
|
||||
umulh $t1,$s0,$s6
|
||||
adds $t3,$t3,$t0
|
||||
adcs $t4,$t4,$t1
|
||||
adcs $t6,$t6,xzr
|
||||
|
||||
// ### s3*s4 + s2*s5 + s1*s6 + s0*s7 ###
|
||||
mul $t0,$s3,$s4
|
||||
umulh $t1,$s3,$s4
|
||||
adds $t4,$t4,$t0
|
||||
adcs $t6,$t6,$t1
|
||||
adcs $t7,xzr,xzr
|
||||
|
||||
mul $t0,$s2,$s5
|
||||
umulh $t1,$s2,$s5
|
||||
adds $t4,$t4,$t0
|
||||
adcs $t6,$t6,$t1
|
||||
adcs $t7,$t7,xzr
|
||||
|
||||
mul $t0,$s1,$s6
|
||||
umulh $t1,$s1,$s6
|
||||
adds $t4,$t4,$t0
|
||||
adcs $t6,$t6,$t1
|
||||
adcs $t7,$t7,xzr
|
||||
|
||||
mul $t0,$s0,$s7
|
||||
umulh $t1,$s0,$s7
|
||||
adds $t4,$t4,$t0
|
||||
adcs $t6,$t6,$t1
|
||||
adcs $t7,$t7,xzr
|
||||
|
||||
// ### s3*s5 + s2*s6 + s1*s7 ###
|
||||
mul $t0,$s3,$s5
|
||||
umulh $t1,$s3,$s5
|
||||
adds $t6,$t6,$t0
|
||||
adcs $t7,$t7,$t1
|
||||
adcs $t8,xzr,xzr
|
||||
|
||||
mul $t0,$s2,$s6
|
||||
umulh $t1,$s2,$s6
|
||||
adds $t6,$t6,$t0
|
||||
adcs $t7,$t7,$t1
|
||||
adcs $t8,$t8,xzr
|
||||
|
||||
mul $t0,$s1,$s7
|
||||
umulh $t1,$s1,$s7
|
||||
adds $s4,$t6,$t0
|
||||
adcs $t7,$t7,$t1
|
||||
adcs $t8,$t8,xzr
|
||||
|
||||
// ### s3*s6 + s2*s7 ###
|
||||
mul $t0,$s3,$s6
|
||||
umulh $t1,$s3,$s6
|
||||
adds $t7,$t7,$t0
|
||||
adcs $t8,$t8,$t1
|
||||
adcs $t6,xzr,xzr
|
||||
|
||||
mul $t0,$s2,$s7
|
||||
umulh $t1,$s2,$s7
|
||||
adds $s5,$t7,$t0
|
||||
adcs $t8,$t8,$t1
|
||||
adcs $t6,$t6,xzr
|
||||
|
||||
// ### s3*s7 ###
|
||||
mul $t0,$s3,$s7
|
||||
umulh $t1,$s3,$s7
|
||||
adds $s6,$t8,$t0
|
||||
adcs $s7,$t6,$t1
|
||||
|
||||
mov $s0,$t5
|
||||
mov $s1,$t2
|
||||
mov $s2,$t3
|
||||
mov $s3,$t4
|
||||
|
||||
// result of mul: s7 s6 s5 s4 s3 s2 s1 s0
|
||||
|
||||
// ### Reduction ###
|
||||
RDC
|
||||
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
|
||||
// Restore scalar registers
|
||||
ldp x16,x17,[sp,#16]
|
||||
ldp x19,x20,[sp,#64]
|
||||
ldp x29,x30,[sp],#80
|
||||
|
||||
AARCH64_VALIDATE_LINK_REGISTER
|
||||
ret
|
||||
.size ecp_sm2p256_mul,.-ecp_sm2p256_mul
|
||||
|
||||
// void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a);
|
||||
.globl ecp_sm2p256_sqr
|
||||
.type ecp_sm2p256_sqr,%function
|
||||
.align 5
|
||||
|
||||
ecp_sm2p256_sqr:
|
||||
AARCH64_SIGN_LINK_REGISTER
|
||||
// Store scalar registers
|
||||
stp x29,x30,[sp,#-80]!
|
||||
add x29,sp,#0
|
||||
stp x16,x17,[sp,#16]
|
||||
stp x19,x20,[sp,#64]
|
||||
|
||||
// Load inputs
|
||||
ldp $s4,$s5,[x1]
|
||||
ldp $s6,$s7,[x1,#16]
|
||||
|
||||
// ### square ###
|
||||
// ========================
|
||||
// s7 s6 s5 s4
|
||||
// * s7 s6 s5 s4
|
||||
// ------------------------
|
||||
// + s4 s4 s4 s4
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s5 s5 s5 s5
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s6 s6 s6 s6
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// s7 s7 s7 s7
|
||||
// * * * *
|
||||
// s7 s6 s5 s4
|
||||
// ------------------------
|
||||
// s7 s6 s5 s4 s3 s2 s1 s0
|
||||
// ========================
|
||||
|
||||
// ### s4*s5 ###
|
||||
mul $s1,$s4,$s5
|
||||
umulh $s2,$s4,$s5
|
||||
|
||||
// ### s4*s6 ###
|
||||
mul $t0,$s6,$s4
|
||||
umulh $s3,$s6,$s4
|
||||
adds $s2,$s2,$t0
|
||||
adcs $s3,$s3,xzr
|
||||
|
||||
// ### s4*s7 + s5*s6 ###
|
||||
mul $t0,$s7,$s4
|
||||
umulh $t1,$s7,$s4
|
||||
adds $s3,$s3,$t0
|
||||
adcs $s0,$t1,xzr
|
||||
|
||||
mul $t0,$s6,$s5
|
||||
umulh $t1,$s6,$s5
|
||||
adds $s3,$s3,$t0
|
||||
adcs $s0,$s0,$t1
|
||||
adcs $t2,xzr,xzr
|
||||
|
||||
// ### s5*s7 ###
|
||||
mul $t0,$s7,$s5
|
||||
umulh $t1,$s7,$s5
|
||||
adds $s0,$s0,$t0
|
||||
adcs $t2,$t2,$t1
|
||||
|
||||
// ### s6*s7 ###
|
||||
mul $t0,$s7,$s6
|
||||
umulh $t1,$s7,$s6
|
||||
adds $t2,$t2,$t0
|
||||
adcs $t3,$t1,xzr
|
||||
|
||||
// ### 2*(t3,t2,s0,s3,s2,s1) ###
|
||||
adds $s1,$s1,$s1
|
||||
adcs $s2,$s2,$s2
|
||||
adcs $s3,$s3,$s3
|
||||
adcs $s0,$s0,$s0
|
||||
adcs $t2,$t2,$t2
|
||||
adcs $t3,$t3,$t3
|
||||
adcs $t4,xzr,xzr
|
||||
|
||||
// ### s4*s4 ###
|
||||
mul $t5,$s4,$s4
|
||||
umulh $t6,$s4,$s4
|
||||
|
||||
// ### s5*s5 ###
|
||||
mul $s4,$s5,$s5
|
||||
umulh $s5,$s5,$s5
|
||||
|
||||
// ### s6*s6 ###
|
||||
mul $t0,$s6,$s6
|
||||
umulh $t1,$s6,$s6
|
||||
|
||||
// ### s7*s7 ###
|
||||
mul $t7,$s7,$s7
|
||||
umulh $t8,$s7,$s7
|
||||
|
||||
adds $s1,$s1,$t6
|
||||
adcs $s2,$s2,$s4
|
||||
adcs $s3,$s3,$s5
|
||||
adcs $s0,$s0,$t0
|
||||
adcs $t2,$t2,$t1
|
||||
adcs $t3,$t3,$t7
|
||||
adcs $t4,$t4,$t8
|
||||
|
||||
mov $s4,$s0
|
||||
mov $s0,$t5
|
||||
mov $s5,$t2
|
||||
mov $s6,$t3
|
||||
mov $s7,$t4
|
||||
|
||||
// result of mul: s7 s6 s5 s4 s3 s2 s1 s0
|
||||
|
||||
// ### Reduction ###
|
||||
RDC
|
||||
|
||||
stp $s0,$s1,[x0]
|
||||
stp $s2,$s3,[x0,#16]
|
||||
|
||||
// Restore scalar registers
|
||||
ldp x16,x17,[sp,#16]
|
||||
ldp x19,x20,[sp,#64]
|
||||
ldp x29,x30,[sp],#80
|
||||
|
||||
AARCH64_VALIDATE_LINK_REGISTER
|
||||
ret
|
||||
.size ecp_sm2p256_sqr,.-ecp_sm2p256_sqr
|
||||
___
|
||||
}
|
||||
|
||||
foreach (split("\n",$code)) {
|
||||
s/\`([^\`]*)\`/eval $1/ge;
|
||||
|
||||
print $_,"\n";
|
||||
}
|
||||
close STDOUT or die "error closing STDOUT: $!"; # enforce flush
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,790 @@
|
||||
/*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* SM2 low level APIs are deprecated for public use, but still ok for
|
||||
* internal use.
|
||||
*/
|
||||
#include "internal/deprecated.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <openssl/err.h>
|
||||
#include "crypto/bn.h"
|
||||
#include "ec_local.h"
|
||||
#include "internal/common.h"
|
||||
#include "internal/constant_time.h"
|
||||
|
||||
#define P256_LIMBS (256 / BN_BITS2)
|
||||
|
||||
#if !defined(OPENSSL_NO_SM2_PRECOMP)
|
||||
extern const BN_ULONG ecp_sm2p256_precomputed[8 * 32 * 256];
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
BN_ULONG X[P256_LIMBS];
|
||||
BN_ULONG Y[P256_LIMBS];
|
||||
BN_ULONG Z[P256_LIMBS];
|
||||
} P256_POINT;
|
||||
|
||||
typedef struct {
|
||||
BN_ULONG X[P256_LIMBS];
|
||||
BN_ULONG Y[P256_LIMBS];
|
||||
} P256_POINT_AFFINE;
|
||||
|
||||
#if !defined(OPENSSL_NO_SM2_PRECOMP)
|
||||
/* Coordinates of G, for which we have precomputed tables */
|
||||
ALIGN32 static const BN_ULONG def_xG[P256_LIMBS] = {
|
||||
0x715a4589334c74c7, 0x8fe30bbff2660be1,
|
||||
0x5f9904466a39c994, 0x32c4ae2c1f198119
|
||||
};
|
||||
|
||||
ALIGN32 static const BN_ULONG def_yG[P256_LIMBS] = {
|
||||
0x02df32e52139f0a0, 0xd0a9877cc62a4740,
|
||||
0x59bdcee36b692153, 0xbc3736a2f4f6779c,
|
||||
};
|
||||
#endif
|
||||
|
||||
/* p and order for SM2 according to GB/T 32918.5-2017 */
|
||||
ALIGN32 static const BN_ULONG def_p[P256_LIMBS] = {
|
||||
0xffffffffffffffff, 0xffffffff00000000,
|
||||
0xffffffffffffffff, 0xfffffffeffffffff
|
||||
};
|
||||
ALIGN32 static const BN_ULONG def_ord[P256_LIMBS] = {
|
||||
0x53bbf40939d54123, 0x7203df6b21c6052b,
|
||||
0xffffffffffffffff, 0xfffffffeffffffff
|
||||
};
|
||||
|
||||
ALIGN32 static const BN_ULONG ONE[P256_LIMBS] = {1, 0, 0, 0};
|
||||
|
||||
/* Functions implemented in assembly */
|
||||
/*
|
||||
* Most of below mentioned functions *preserve* the property of inputs
|
||||
* being fully reduced, i.e. being in [0, modulus) range. Simply put if
|
||||
* inputs are fully reduced, then output is too.
|
||||
*/
|
||||
/* Right shift: a >> 1 */
|
||||
void bn_rshift1(BN_ULONG *a);
|
||||
/* Sub: r = a - b */
|
||||
void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
/* Modular div by 2: r = a / 2 mod p */
|
||||
void ecp_sm2p256_div_by_2(BN_ULONG *r, const BN_ULONG *a);
|
||||
/* Modular div by 2: r = a / 2 mod n, where n = ord(p) */
|
||||
void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r, const BN_ULONG *a);
|
||||
/* Modular add: r = a + b mod p */
|
||||
void ecp_sm2p256_add(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
/* Modular sub: r = a - b mod p */
|
||||
void ecp_sm2p256_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
/* Modular sub: r = a - b mod n, where n = ord(p) */
|
||||
void ecp_sm2p256_sub_mod_ord(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
/* Modular mul by 3: out = 3 * a mod p */
|
||||
void ecp_sm2p256_mul_by_3(BN_ULONG *r, const BN_ULONG *a);
|
||||
/* Modular mul: r = a * b mod p */
|
||||
void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b);
|
||||
/* Modular sqr: r = a ^ 2 mod p */
|
||||
void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a);
|
||||
|
||||
static ossl_inline BN_ULONG is_zeros(const BN_ULONG *a)
|
||||
{
|
||||
BN_ULONG res;
|
||||
|
||||
res = a[0] | a[1] | a[2] | a[3];
|
||||
|
||||
return constant_time_is_zero_64(res);
|
||||
}
|
||||
|
||||
static ossl_inline int is_equal(const BN_ULONG *a, const BN_ULONG *b)
|
||||
{
|
||||
BN_ULONG res;
|
||||
|
||||
res = a[0] ^ b[0];
|
||||
res |= a[1] ^ b[1];
|
||||
res |= a[2] ^ b[2];
|
||||
res |= a[3] ^ b[3];
|
||||
|
||||
return constant_time_is_zero_64(res);
|
||||
}
|
||||
|
||||
static ossl_inline int is_greater(const BN_ULONG *a, const BN_ULONG *b)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = P256_LIMBS - 1; i >= 0; --i) {
|
||||
if (a[i] > b[i])
|
||||
return 1;
|
||||
if (a[i] < b[i])
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define is_one(a) is_equal(a, ONE)
|
||||
#define is_even(a) !(a[0] & 1)
|
||||
#define is_point_equal(a, b) \
|
||||
is_equal(a->X, b->X) && \
|
||||
is_equal(a->Y, b->Y) && \
|
||||
is_equal(a->Z, b->Z)
|
||||
|
||||
/* Bignum and field elements conversion */
|
||||
#define ecp_sm2p256_bignum_field_elem(out, in) \
|
||||
bn_copy_words(out, in, P256_LIMBS)
|
||||
|
||||
/* Binary algorithm for inversion in Fp */
|
||||
#define BN_MOD_INV(out, in, mod_div, mod_sub, mod) \
|
||||
do { \
|
||||
ALIGN32 BN_ULONG u[4]; \
|
||||
ALIGN32 BN_ULONG v[4]; \
|
||||
ALIGN32 BN_ULONG x1[4] = {1, 0, 0, 0}; \
|
||||
ALIGN32 BN_ULONG x2[4] = {0}; \
|
||||
\
|
||||
if (is_zeros(in)) \
|
||||
return; \
|
||||
memcpy(u, in, 32); \
|
||||
memcpy(v, mod, 32); \
|
||||
while (!is_one(u) && !is_one(v)) { \
|
||||
while (is_even(u)) { \
|
||||
bn_rshift1(u); \
|
||||
mod_div(x1, x1); \
|
||||
} \
|
||||
while (is_even(v)) { \
|
||||
bn_rshift1(v); \
|
||||
mod_div(x2, x2); \
|
||||
} \
|
||||
if (is_greater(u, v) == 1) { \
|
||||
bn_sub(u, u, v); \
|
||||
mod_sub(x1, x1, x2); \
|
||||
} else { \
|
||||
bn_sub(v, v, u); \
|
||||
mod_sub(x2, x2, x1); \
|
||||
} \
|
||||
} \
|
||||
if (is_one(u)) \
|
||||
memcpy(out, x1, 32); \
|
||||
else \
|
||||
memcpy(out, x2, 32); \
|
||||
} while (0)
|
||||
|
||||
/* Modular inverse |out| = |in|^(-1) mod |p|. */
|
||||
static ossl_inline void ecp_sm2p256_mod_inverse(BN_ULONG* out,
|
||||
const BN_ULONG* in) {
|
||||
BN_MOD_INV(out, in, ecp_sm2p256_div_by_2, ecp_sm2p256_sub, def_p);
|
||||
}
|
||||
|
||||
/* Modular inverse mod order |out| = |in|^(-1) % |ord|. */
|
||||
static ossl_inline void ecp_sm2p256_mod_ord_inverse(BN_ULONG* out,
|
||||
const BN_ULONG* in) {
|
||||
BN_MOD_INV(out, in, ecp_sm2p256_div_by_2_mod_ord, ecp_sm2p256_sub_mod_ord,
|
||||
def_ord);
|
||||
}
|
||||
|
||||
/* Point double: R <- P + P */
|
||||
static void ecp_sm2p256_point_double(P256_POINT *R, const P256_POINT *P)
|
||||
{
|
||||
unsigned int i;
|
||||
ALIGN32 BN_ULONG tmp0[P256_LIMBS];
|
||||
ALIGN32 BN_ULONG tmp1[P256_LIMBS];
|
||||
ALIGN32 BN_ULONG tmp2[P256_LIMBS];
|
||||
|
||||
/* zero-check P->Z */
|
||||
if (is_zeros(P->Z)) {
|
||||
for (i = 0; i < P256_LIMBS; ++i)
|
||||
R->Z[i] = 0;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
ecp_sm2p256_sqr(tmp0, P->Z);
|
||||
ecp_sm2p256_sub(tmp1, P->X, tmp0);
|
||||
ecp_sm2p256_add(tmp0, P->X, tmp0);
|
||||
ecp_sm2p256_mul(tmp1, tmp1, tmp0);
|
||||
ecp_sm2p256_mul_by_3(tmp1, tmp1);
|
||||
ecp_sm2p256_add(R->Y, P->Y, P->Y);
|
||||
ecp_sm2p256_mul(R->Z, R->Y, P->Z);
|
||||
ecp_sm2p256_sqr(R->Y, R->Y);
|
||||
ecp_sm2p256_mul(tmp2, R->Y, P->X);
|
||||
ecp_sm2p256_sqr(R->Y, R->Y);
|
||||
ecp_sm2p256_div_by_2(R->Y, R->Y);
|
||||
ecp_sm2p256_sqr(R->X, tmp1);
|
||||
ecp_sm2p256_add(tmp0, tmp2, tmp2);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp0);
|
||||
ecp_sm2p256_sub(tmp0, tmp2, R->X);
|
||||
ecp_sm2p256_mul(tmp0, tmp0, tmp1);
|
||||
ecp_sm2p256_sub(tmp1, tmp0, R->Y);
|
||||
memcpy(R->Y, tmp1, 32);
|
||||
}
|
||||
|
||||
/* Point add affine: R <- P + Q */
|
||||
static void ecp_sm2p256_point_add_affine(P256_POINT *R, const P256_POINT *P,
|
||||
const P256_POINT_AFFINE *Q)
|
||||
{
|
||||
unsigned int i;
|
||||
ALIGN32 BN_ULONG tmp0[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG tmp1[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG tmp2[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG tmp3[P256_LIMBS] = {0};
|
||||
|
||||
/* zero-check P->Z */
|
||||
if (is_zeros(P->Z)) {
|
||||
for (i = 0; i < P256_LIMBS; ++i) {
|
||||
R->X[i] = Q->X[i];
|
||||
R->Y[i] = Q->Y[i];
|
||||
R->Z[i] = 0;
|
||||
}
|
||||
R->Z[0] = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
ecp_sm2p256_sqr(tmp0, P->Z);
|
||||
ecp_sm2p256_mul(tmp1, tmp0, P->Z);
|
||||
ecp_sm2p256_mul(tmp0, tmp0, Q->X);
|
||||
ecp_sm2p256_mul(tmp1, tmp1, Q->Y);
|
||||
ecp_sm2p256_sub(tmp0, tmp0, P->X);
|
||||
ecp_sm2p256_sub(tmp1, tmp1, P->Y);
|
||||
|
||||
/* zero-check tmp0, tmp1 */
|
||||
if (is_zeros(tmp0)) {
|
||||
if (is_zeros(tmp1)) {
|
||||
P256_POINT K;
|
||||
|
||||
for (i = 0; i < P256_LIMBS; ++i) {
|
||||
K.X[i] = Q->X[i];
|
||||
K.Y[i] = Q->Y[i];
|
||||
K.Z[i] = 0;
|
||||
}
|
||||
K.Z[0] = 1;
|
||||
ecp_sm2p256_point_double(R, &K);
|
||||
} else {
|
||||
for (i = 0; i < P256_LIMBS; ++i)
|
||||
R->Z[i] = 0;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
ecp_sm2p256_mul(R->Z, P->Z, tmp0);
|
||||
ecp_sm2p256_sqr(tmp2, tmp0);
|
||||
ecp_sm2p256_mul(tmp3, tmp2, tmp0);
|
||||
ecp_sm2p256_mul(tmp2, tmp2, P->X);
|
||||
ecp_sm2p256_add(tmp0, tmp2, tmp2);
|
||||
ecp_sm2p256_sqr(R->X, tmp1);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp0);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp3);
|
||||
ecp_sm2p256_sub(tmp2, tmp2, R->X);
|
||||
ecp_sm2p256_mul(tmp2, tmp2, tmp1);
|
||||
ecp_sm2p256_mul(tmp3, tmp3, P->Y);
|
||||
ecp_sm2p256_sub(R->Y, tmp2, tmp3);
|
||||
}
|
||||
|
||||
/* Point add: R <- P + Q */
|
||||
static void ecp_sm2p256_point_add(P256_POINT *R, const P256_POINT *P,
|
||||
const P256_POINT *Q)
|
||||
{
|
||||
unsigned int i;
|
||||
ALIGN32 BN_ULONG tmp0[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG tmp1[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG tmp2[P256_LIMBS] = {0};
|
||||
|
||||
/* zero-check P | Q ->Z */
|
||||
if (is_zeros(P->Z)) {
|
||||
for (i = 0; i < P256_LIMBS; ++i) {
|
||||
R->X[i] = Q->X[i];
|
||||
R->Y[i] = Q->Y[i];
|
||||
R->Z[i] = Q->Z[i];
|
||||
}
|
||||
|
||||
return;
|
||||
} else if (is_zeros(Q->Z)) {
|
||||
for (i = 0; i < P256_LIMBS; ++i) {
|
||||
R->X[i] = P->X[i];
|
||||
R->Y[i] = P->Y[i];
|
||||
R->Z[i] = P->Z[i];
|
||||
}
|
||||
|
||||
return;
|
||||
} else if (is_point_equal(P, Q)) {
|
||||
ecp_sm2p256_point_double(R, Q);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
ecp_sm2p256_sqr(tmp0, P->Z);
|
||||
ecp_sm2p256_mul(tmp1, tmp0, P->Z);
|
||||
ecp_sm2p256_mul(tmp0, tmp0, Q->X);
|
||||
ecp_sm2p256_mul(tmp1, tmp1, Q->Y);
|
||||
ecp_sm2p256_mul(R->Y, P->Y, Q->Z);
|
||||
ecp_sm2p256_mul(R->Z, Q->Z, P->Z);
|
||||
ecp_sm2p256_sqr(tmp2, Q->Z);
|
||||
ecp_sm2p256_mul(R->Y, tmp2, R->Y);
|
||||
ecp_sm2p256_mul(R->X, tmp2, P->X);
|
||||
ecp_sm2p256_sub(tmp0, tmp0, R->X);
|
||||
ecp_sm2p256_mul(R->Z, tmp0, R->Z);
|
||||
ecp_sm2p256_sub(tmp1, tmp1, R->Y);
|
||||
ecp_sm2p256_sqr(tmp2, tmp0);
|
||||
ecp_sm2p256_mul(tmp0, tmp0, tmp2);
|
||||
ecp_sm2p256_mul(tmp2, tmp2, R->X);
|
||||
ecp_sm2p256_sqr(R->X, tmp1);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp2);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp2);
|
||||
ecp_sm2p256_sub(R->X, R->X, tmp0);
|
||||
ecp_sm2p256_sub(tmp2, tmp2, R->X);
|
||||
ecp_sm2p256_mul(tmp2, tmp1, tmp2);
|
||||
ecp_sm2p256_mul(tmp0, tmp0, R->Y);
|
||||
ecp_sm2p256_sub(R->Y, tmp2, tmp0);
|
||||
}
|
||||
|
||||
#if !defined(OPENSSL_NO_SM2_PRECOMP)
|
||||
/* Base point mul by scalar: k - scalar, G - base point */
|
||||
static void ecp_sm2p256_point_G_mul_by_scalar(P256_POINT *R, const BN_ULONG *k)
|
||||
{
|
||||
unsigned int i, index, mask = 0xff;
|
||||
P256_POINT_AFFINE Q;
|
||||
|
||||
memset(R, 0, sizeof(P256_POINT));
|
||||
|
||||
if (is_zeros(k))
|
||||
return;
|
||||
|
||||
index = k[0] & mask;
|
||||
if (index) {
|
||||
index = index * 8;
|
||||
memcpy(R->X, ecp_sm2p256_precomputed + index, 32);
|
||||
memcpy(R->Y, ecp_sm2p256_precomputed + index + P256_LIMBS, 32);
|
||||
R->Z[0] = 1;
|
||||
}
|
||||
|
||||
for (i = 1; i < 32; ++i) {
|
||||
index = (k[i / 8] >> (8 * (i % 8))) & mask;
|
||||
|
||||
if (index) {
|
||||
index = index + i * 256;
|
||||
index = index * 8;
|
||||
memcpy(Q.X, ecp_sm2p256_precomputed + index, 32);
|
||||
memcpy(Q.Y, ecp_sm2p256_precomputed + index + P256_LIMBS, 32);
|
||||
ecp_sm2p256_point_add_affine(R, R, &Q);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Affine point mul by scalar: k - scalar, P - affine point
|
||||
*/
|
||||
static void ecp_sm2p256_point_P_mul_by_scalar(P256_POINT *R, const BN_ULONG *k,
|
||||
P256_POINT_AFFINE P)
|
||||
{
|
||||
int i, init = 0;
|
||||
unsigned int index, mask = 0x0f;
|
||||
ALIGN64 P256_POINT precomputed[16];
|
||||
|
||||
memset(R, 0, sizeof(P256_POINT));
|
||||
|
||||
if (is_zeros(k))
|
||||
return;
|
||||
|
||||
/* The first value of the precomputed table is P. */
|
||||
memcpy(precomputed[1].X, P.X, 32);
|
||||
memcpy(precomputed[1].Y, P.Y, 32);
|
||||
precomputed[1].Z[0] = 1;
|
||||
precomputed[1].Z[1] = 0;
|
||||
precomputed[1].Z[2] = 0;
|
||||
precomputed[1].Z[3] = 0;
|
||||
|
||||
/* The second value of the precomputed table is 2P. */
|
||||
ecp_sm2p256_point_double(&precomputed[2], &precomputed[1]);
|
||||
|
||||
/* The subsequent elements are 3P, 4P, and so on. */
|
||||
for (i = 3; i < 16; ++i)
|
||||
ecp_sm2p256_point_add_affine(&precomputed[i], &precomputed[i - 1], &P);
|
||||
|
||||
for (i = 64 - 1; i >= 0; --i) {
|
||||
index = (k[i / 16] >> (4 * (i % 16))) & mask;
|
||||
|
||||
if (init == 0) {
|
||||
if (index) {
|
||||
memcpy(R, &precomputed[index], sizeof(P256_POINT));
|
||||
init = 1;
|
||||
}
|
||||
} else {
|
||||
ecp_sm2p256_point_double(R, R);
|
||||
ecp_sm2p256_point_double(R, R);
|
||||
ecp_sm2p256_point_double(R, R);
|
||||
ecp_sm2p256_point_double(R, R);
|
||||
if (index)
|
||||
ecp_sm2p256_point_add(R, R, &precomputed[index]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Get affine point */
|
||||
static void ecp_sm2p256_point_get_affine(P256_POINT_AFFINE *R,
|
||||
const P256_POINT *P)
|
||||
{
|
||||
ALIGN32 BN_ULONG z_inv3[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG z_inv2[P256_LIMBS] = {0};
|
||||
|
||||
if (is_one(P->Z)) {
|
||||
memcpy(R->X, P->X, 32);
|
||||
memcpy(R->Y, P->Y, 32);
|
||||
return;
|
||||
}
|
||||
|
||||
ecp_sm2p256_mod_inverse(z_inv3, P->Z);
|
||||
ecp_sm2p256_sqr(z_inv2, z_inv3);
|
||||
ecp_sm2p256_mul(R->X, P->X, z_inv2);
|
||||
ecp_sm2p256_mul(z_inv3, z_inv3, z_inv2);
|
||||
ecp_sm2p256_mul(R->Y, P->Y, z_inv3);
|
||||
}
|
||||
|
||||
#if !defined(OPENSSL_NO_SM2_PRECOMP)
|
||||
static int ecp_sm2p256_is_affine_G(const EC_POINT *generator)
|
||||
{
|
||||
return (bn_get_top(generator->X) == P256_LIMBS)
|
||||
&& (bn_get_top(generator->Y) == P256_LIMBS)
|
||||
&& is_equal(bn_get_words(generator->X), def_xG)
|
||||
&& is_equal(bn_get_words(generator->Y), def_yG)
|
||||
&& (generator->Z_is_one == 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Convert Jacobian coordinate point into affine coordinate (x,y)
|
||||
*/
|
||||
static int ecp_sm2p256_get_affine(const EC_GROUP *group,
|
||||
const EC_POINT *point,
|
||||
BIGNUM *x, BIGNUM *y, BN_CTX *ctx)
|
||||
{
|
||||
ALIGN32 BN_ULONG z_inv2[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG z_inv3[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG x_aff[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG y_aff[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG point_x[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG point_y[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG point_z[P256_LIMBS] = {0};
|
||||
|
||||
if (EC_POINT_is_at_infinity(group, point)) {
|
||||
ECerr(ERR_LIB_EC, EC_R_POINT_AT_INFINITY);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (ecp_sm2p256_bignum_field_elem(point_x, point->X) <= 0
|
||||
|| ecp_sm2p256_bignum_field_elem(point_y, point->Y) <= 0
|
||||
|| ecp_sm2p256_bignum_field_elem(point_z, point->Z) <= 0) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ecp_sm2p256_mod_inverse(z_inv3, point_z);
|
||||
ecp_sm2p256_sqr(z_inv2, z_inv3);
|
||||
|
||||
if (x != NULL) {
|
||||
ecp_sm2p256_mul(x_aff, point_x, z_inv2);
|
||||
if (!bn_set_words(x, x_aff, P256_LIMBS))
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (y != NULL) {
|
||||
ecp_sm2p256_mul(z_inv3, z_inv3, z_inv2);
|
||||
ecp_sm2p256_mul(y_aff, point_y, z_inv3);
|
||||
if (!bn_set_words(y, y_aff, P256_LIMBS))
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* r = sum(scalar[i]*point[i]) */
|
||||
static int ecp_sm2p256_windowed_mul(const EC_GROUP *group,
|
||||
P256_POINT *r,
|
||||
const BIGNUM **scalar,
|
||||
const EC_POINT **point,
|
||||
size_t num, BN_CTX *ctx)
|
||||
{
|
||||
unsigned int i;
|
||||
int ret = 0;
|
||||
const BIGNUM **scalars = NULL;
|
||||
ALIGN32 BN_ULONG k[P256_LIMBS] = {0};
|
||||
P256_POINT kP;
|
||||
ALIGN32 union {
|
||||
P256_POINT p;
|
||||
P256_POINT_AFFINE a;
|
||||
} t, p;
|
||||
|
||||
if (num > OPENSSL_MALLOC_MAX_NELEMS(P256_POINT)
|
||||
|| (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) {
|
||||
ECerr(ERR_LIB_EC, ERR_R_MALLOC_FAILURE);
|
||||
goto err;
|
||||
}
|
||||
|
||||
memset(r, 0, sizeof(P256_POINT));
|
||||
|
||||
for (i = 0; i < num; i++) {
|
||||
if (EC_POINT_is_at_infinity(group, point[i]))
|
||||
continue;
|
||||
|
||||
if ((BN_num_bits(scalar[i]) > 256) || BN_is_negative(scalar[i])) {
|
||||
BIGNUM *tmp;
|
||||
|
||||
if ((tmp = BN_CTX_get(ctx)) == NULL)
|
||||
goto err;
|
||||
if (!BN_nnmod(tmp, scalar[i], group->order, ctx)) {
|
||||
ECerr(ERR_LIB_EC, ERR_R_BN_LIB);
|
||||
goto err;
|
||||
}
|
||||
scalars[i] = tmp;
|
||||
} else {
|
||||
scalars[i] = scalar[i];
|
||||
}
|
||||
|
||||
if (ecp_sm2p256_bignum_field_elem(k, scalars[i]) <= 0
|
||||
|| ecp_sm2p256_bignum_field_elem(p.p.X, point[i]->X) <= 0
|
||||
|| ecp_sm2p256_bignum_field_elem(p.p.Y, point[i]->Y) <= 0
|
||||
|| ecp_sm2p256_bignum_field_elem(p.p.Z, point[i]->Z) <= 0) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
goto err;
|
||||
}
|
||||
|
||||
ecp_sm2p256_point_get_affine(&t.a, &p.p);
|
||||
ecp_sm2p256_point_P_mul_by_scalar(&kP, k, t.a);
|
||||
ecp_sm2p256_point_add(r, r, &kP);
|
||||
}
|
||||
|
||||
ret = 1;
|
||||
err:
|
||||
OPENSSL_free(scalars);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* r = scalar*G + sum(scalars[i]*points[i]) */
|
||||
static int ecp_sm2p256_points_mul(const EC_GROUP *group,
|
||||
EC_POINT *r,
|
||||
const BIGNUM *scalar,
|
||||
size_t num,
|
||||
const EC_POINT *points[],
|
||||
const BIGNUM *scalars[], BN_CTX *ctx)
|
||||
{
|
||||
int ret = 0, p_is_infinity = 0;
|
||||
const EC_POINT *generator = NULL;
|
||||
ALIGN32 BN_ULONG k[P256_LIMBS] = {0};
|
||||
ALIGN32 union {
|
||||
P256_POINT p;
|
||||
P256_POINT_AFFINE a;
|
||||
} t, p;
|
||||
|
||||
if ((num + 1) == 0 || (num + 1) > OPENSSL_MALLOC_MAX_NELEMS(void *)) {
|
||||
ECerr(ERR_LIB_EC, ERR_R_MALLOC_FAILURE);
|
||||
goto err;
|
||||
}
|
||||
|
||||
BN_CTX_start(ctx);
|
||||
|
||||
if (scalar) {
|
||||
generator = EC_GROUP_get0_generator(group);
|
||||
if (generator == NULL) {
|
||||
ECerr(ERR_LIB_EC, EC_R_UNDEFINED_GENERATOR);
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (!ecp_sm2p256_bignum_field_elem(k, scalar)) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
goto err;
|
||||
}
|
||||
#if !defined(OPENSSL_NO_SM2_PRECOMP)
|
||||
if (ecp_sm2p256_is_affine_G(generator)) {
|
||||
ecp_sm2p256_point_G_mul_by_scalar(&p.p, k);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
/* if no precomputed table */
|
||||
const EC_POINT *new_generator[1];
|
||||
const BIGNUM *g_scalars[1];
|
||||
|
||||
new_generator[0] = generator;
|
||||
g_scalars[0] = scalar;
|
||||
|
||||
if (!ecp_sm2p256_windowed_mul(group, &p.p, g_scalars, new_generator,
|
||||
(new_generator[0] != NULL
|
||||
&& g_scalars[0] != NULL), ctx))
|
||||
goto err;
|
||||
}
|
||||
} else {
|
||||
p_is_infinity = 1;
|
||||
}
|
||||
if (num) {
|
||||
P256_POINT *out = &t.p;
|
||||
|
||||
if (p_is_infinity)
|
||||
out = &p.p;
|
||||
|
||||
if (!ecp_sm2p256_windowed_mul(group, out, scalars, points, num, ctx))
|
||||
goto err;
|
||||
|
||||
if (!p_is_infinity)
|
||||
ecp_sm2p256_point_add(&p.p, &p.p, out);
|
||||
}
|
||||
|
||||
/* Not constant-time, but we're only operating on the public output. */
|
||||
if (!bn_set_words(r->X, p.p.X, P256_LIMBS)
|
||||
|| !bn_set_words(r->Y, p.p.Y, P256_LIMBS)
|
||||
|| !bn_set_words(r->Z, p.p.Z, P256_LIMBS))
|
||||
goto err;
|
||||
r->Z_is_one = is_equal(bn_get_words(r->Z), ONE) & 1;
|
||||
|
||||
ret = 1;
|
||||
err:
|
||||
BN_CTX_end(ctx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int ecp_sm2p256_field_mul(const EC_GROUP *group, BIGNUM *r,
|
||||
const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
|
||||
{
|
||||
ALIGN32 BN_ULONG a_fe[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG b_fe[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG r_fe[P256_LIMBS] = {0};
|
||||
|
||||
if (a == NULL || b == NULL || r == NULL)
|
||||
return 0;
|
||||
|
||||
if (!ecp_sm2p256_bignum_field_elem(a_fe, a)
|
||||
|| !ecp_sm2p256_bignum_field_elem(b_fe, b)) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ecp_sm2p256_mul(r_fe, a_fe, b_fe);
|
||||
|
||||
if (!bn_set_words(r, r_fe, P256_LIMBS))
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int ecp_sm2p256_field_sqr(const EC_GROUP *group, BIGNUM *r,
|
||||
const BIGNUM *a, BN_CTX *ctx)
|
||||
{
|
||||
ALIGN32 BN_ULONG a_fe[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG r_fe[P256_LIMBS] = {0};
|
||||
|
||||
if (a == NULL || r == NULL)
|
||||
return 0;
|
||||
|
||||
if (!ecp_sm2p256_bignum_field_elem(a_fe, a)) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ecp_sm2p256_sqr(r_fe, a_fe);
|
||||
|
||||
if (!bn_set_words(r, r_fe, P256_LIMBS))
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int ecp_sm2p256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r,
|
||||
const BIGNUM *x, BN_CTX *ctx)
|
||||
{
|
||||
int ret = 0;
|
||||
ALIGN32 BN_ULONG t[P256_LIMBS] = {0};
|
||||
ALIGN32 BN_ULONG out[P256_LIMBS] = {0};
|
||||
|
||||
if (bn_wexpand(r, P256_LIMBS) == NULL) {
|
||||
ECerr(ERR_LIB_EC, ERR_R_BN_LIB);
|
||||
goto err;
|
||||
}
|
||||
|
||||
if ((BN_num_bits(x) > 256) || BN_is_negative(x)) {
|
||||
BIGNUM *tmp;
|
||||
|
||||
if ((tmp = BN_CTX_get(ctx)) == NULL
|
||||
|| !BN_nnmod(tmp, x, group->order, ctx)) {
|
||||
ECerr(ERR_LIB_EC, ERR_R_BN_LIB);
|
||||
goto err;
|
||||
}
|
||||
x = tmp;
|
||||
}
|
||||
|
||||
if (!ecp_sm2p256_bignum_field_elem(t, x)) {
|
||||
ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE);
|
||||
goto err;
|
||||
}
|
||||
|
||||
ecp_sm2p256_mod_ord_inverse(out, t);
|
||||
|
||||
if (!bn_set_words(r, out, P256_LIMBS))
|
||||
goto err;
|
||||
|
||||
ret = 1;
|
||||
err:
|
||||
return ret;
|
||||
}
|
||||
|
||||
const EC_METHOD *EC_GFp_sm2p256_method(void)
|
||||
{
|
||||
static const EC_METHOD ret = {
|
||||
EC_FLAGS_DEFAULT_OCT,
|
||||
NID_X9_62_prime_field,
|
||||
ossl_ec_GFp_simple_group_init,
|
||||
ossl_ec_GFp_simple_group_finish,
|
||||
ossl_ec_GFp_simple_group_clear_finish,
|
||||
ossl_ec_GFp_simple_group_copy,
|
||||
ossl_ec_GFp_simple_group_set_curve,
|
||||
ossl_ec_GFp_simple_group_get_curve,
|
||||
ossl_ec_GFp_simple_group_get_degree,
|
||||
ossl_ec_group_simple_order_bits,
|
||||
ossl_ec_GFp_simple_group_check_discriminant,
|
||||
ossl_ec_GFp_simple_point_init,
|
||||
ossl_ec_GFp_simple_point_finish,
|
||||
ossl_ec_GFp_simple_point_clear_finish,
|
||||
ossl_ec_GFp_simple_point_copy,
|
||||
ossl_ec_GFp_simple_point_set_to_infinity,
|
||||
ossl_ec_GFp_simple_point_set_affine_coordinates,
|
||||
ecp_sm2p256_get_affine,
|
||||
0, 0, 0,
|
||||
ossl_ec_GFp_simple_add,
|
||||
ossl_ec_GFp_simple_dbl,
|
||||
ossl_ec_GFp_simple_invert,
|
||||
ossl_ec_GFp_simple_is_at_infinity,
|
||||
ossl_ec_GFp_simple_is_on_curve,
|
||||
ossl_ec_GFp_simple_cmp,
|
||||
ossl_ec_GFp_simple_make_affine,
|
||||
ossl_ec_GFp_simple_points_make_affine,
|
||||
ecp_sm2p256_points_mul, /* mul */
|
||||
0 /* precompute_mult */,
|
||||
0 /* have_precompute_mult */,
|
||||
ecp_sm2p256_field_mul,
|
||||
ecp_sm2p256_field_sqr,
|
||||
0 /* field_div */,
|
||||
0 /* field_inv */,
|
||||
0 /* field_encode */,
|
||||
0 /* field_decode */,
|
||||
0 /* field_set_to_one */,
|
||||
ossl_ec_key_simple_priv2oct,
|
||||
ossl_ec_key_simple_oct2priv,
|
||||
0, /* set private */
|
||||
ossl_ec_key_simple_generate_key,
|
||||
ossl_ec_key_simple_check_key,
|
||||
ossl_ec_key_simple_generate_public_key,
|
||||
0, /* keycopy */
|
||||
0, /* keyfinish */
|
||||
ossl_ecdh_simple_compute_key,
|
||||
ossl_ecdsa_simple_sign_setup,
|
||||
ossl_ecdsa_simple_sign_sig,
|
||||
ossl_ecdsa_simple_verify_sig,
|
||||
ecp_sm2p256_inv_mod_ord,
|
||||
0, /* blind_coordinates */
|
||||
0, /* ladder_pre */
|
||||
0, /* ladder_step */
|
||||
0 /* ladder_post */
|
||||
};
|
||||
|
||||
return &ret;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2003-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#define OSSL_FORCE_ERR_STATE
|
||||
|
||||
#include <openssl/err.h>
|
||||
#include "err_local.h"
|
||||
|
||||
int ERR_set_mark(void)
|
||||
{
|
||||
ERR_STATE *es;
|
||||
|
||||
es = ossl_err_get_state_int();
|
||||
if (es == NULL)
|
||||
return 0;
|
||||
|
||||
if (es->bottom == es->top)
|
||||
return 0;
|
||||
es->err_marks[es->top]++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ERR_pop(void)
|
||||
{
|
||||
ERR_STATE *es;
|
||||
|
||||
es = ossl_err_get_state_int();
|
||||
if (es == NULL || es->bottom == es->top)
|
||||
return 0;
|
||||
|
||||
err_clear(es, es->top, 0);
|
||||
es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ERR_pop_to_mark(void)
|
||||
{
|
||||
ERR_STATE *es;
|
||||
|
||||
es = ossl_err_get_state_int();
|
||||
if (es == NULL)
|
||||
return 0;
|
||||
|
||||
while (es->bottom != es->top
|
||||
&& es->err_marks[es->top] == 0) {
|
||||
err_clear(es, es->top, 0);
|
||||
es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1;
|
||||
}
|
||||
|
||||
if (es->bottom == es->top)
|
||||
return 0;
|
||||
es->err_marks[es->top]--;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ERR_count_to_mark(void)
|
||||
{
|
||||
ERR_STATE *es;
|
||||
int count = 0, top;
|
||||
|
||||
es = ossl_err_get_state_int();
|
||||
if (es == NULL)
|
||||
return 0;
|
||||
|
||||
top = es->top;
|
||||
while (es->bottom != top
|
||||
&& es->err_marks[top] == 0) {
|
||||
++count;
|
||||
top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
int ERR_clear_last_mark(void)
|
||||
{
|
||||
ERR_STATE *es;
|
||||
int top;
|
||||
|
||||
es = ossl_err_get_state_int();
|
||||
if (es == NULL)
|
||||
return 0;
|
||||
|
||||
top = es->top;
|
||||
while (es->bottom != top
|
||||
&& es->err_marks[top] == 0) {
|
||||
top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1;
|
||||
}
|
||||
|
||||
if (es->bottom == top)
|
||||
return 0;
|
||||
es->err_marks[top]--;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#define OSSL_FORCE_ERR_STATE
|
||||
|
||||
#include <openssl/err.h>
|
||||
#include "err_local.h"
|
||||
|
||||
/*
|
||||
* Save and restore error state.
|
||||
* We are using CRYPTO_zalloc(.., NULL, 0) instead of OPENSSL_malloc() in
|
||||
* these functions to prevent mem alloc error loop.
|
||||
*/
|
||||
|
||||
ERR_STATE *OSSL_ERR_STATE_new(void)
|
||||
{
|
||||
return CRYPTO_zalloc(sizeof(ERR_STATE), NULL, 0);
|
||||
}
|
||||
|
||||
void OSSL_ERR_STATE_save(ERR_STATE *es)
|
||||
{
|
||||
size_t i;
|
||||
ERR_STATE *thread_es;
|
||||
|
||||
if (es == NULL)
|
||||
return;
|
||||
|
||||
for (i = 0; i < ERR_NUM_ERRORS; i++)
|
||||
err_clear(es, i, 1);
|
||||
|
||||
thread_es = ossl_err_get_state_int();
|
||||
if (thread_es == NULL)
|
||||
return;
|
||||
|
||||
memcpy(es, thread_es, sizeof(*es));
|
||||
/* Taking over the pointers, just clear the thread state. */
|
||||
memset(thread_es, 0, sizeof(*thread_es));
|
||||
}
|
||||
|
||||
void OSSL_ERR_STATE_save_to_mark(ERR_STATE *es)
|
||||
{
|
||||
size_t i, j, count;
|
||||
int top;
|
||||
ERR_STATE *thread_es;
|
||||
|
||||
if (es == NULL)
|
||||
return;
|
||||
|
||||
thread_es = ossl_err_get_state_int();
|
||||
if (thread_es == NULL) {
|
||||
for (i = 0; i < ERR_NUM_ERRORS; ++i)
|
||||
err_clear(es, i, 1);
|
||||
|
||||
es->top = es->bottom = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Determine number of errors we are going to move. */
|
||||
for (count = 0, top = thread_es->top;
|
||||
thread_es->bottom != top
|
||||
&& thread_es->err_marks[top] == 0;
|
||||
++count)
|
||||
top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1;
|
||||
|
||||
/* Move the errors, preserving order. */
|
||||
for (i = 0, j = top; i < count; ++i) {
|
||||
j = (j + 1) % ERR_NUM_ERRORS;
|
||||
|
||||
err_clear(es, i, 1);
|
||||
|
||||
/* Move the error entry to the given ERR_STATE. */
|
||||
es->err_flags[i] = thread_es->err_flags[j];
|
||||
es->err_marks[i] = 0;
|
||||
es->err_buffer[i] = thread_es->err_buffer[j];
|
||||
es->err_data[i] = thread_es->err_data[j];
|
||||
es->err_data_size[i] = thread_es->err_data_size[j];
|
||||
es->err_data_flags[i] = thread_es->err_data_flags[j];
|
||||
es->err_file[i] = thread_es->err_file[j];
|
||||
es->err_line[i] = thread_es->err_line[j];
|
||||
es->err_func[i] = thread_es->err_func[j];
|
||||
|
||||
thread_es->err_flags[j] = 0;
|
||||
thread_es->err_buffer[j] = 0;
|
||||
thread_es->err_data[j] = NULL;
|
||||
thread_es->err_data_size[j] = 0;
|
||||
thread_es->err_data_flags[j] = 0;
|
||||
thread_es->err_file[j] = NULL;
|
||||
thread_es->err_line[j] = 0;
|
||||
thread_es->err_func[j] = NULL;
|
||||
}
|
||||
|
||||
if (i > 0) {
|
||||
thread_es->top = top;
|
||||
/* If we moved anything, es's stack always starts at [0]. */
|
||||
es->top = i - 1;
|
||||
es->bottom = ERR_NUM_ERRORS - 1;
|
||||
} else {
|
||||
/* Didn't move anything - empty stack */
|
||||
es->top = es->bottom = 0;
|
||||
}
|
||||
|
||||
/* Erase extra space as a precaution. */
|
||||
for (; i < ERR_NUM_ERRORS; ++i)
|
||||
err_clear(es, i, 1);
|
||||
}
|
||||
|
||||
void OSSL_ERR_STATE_restore(const ERR_STATE *es)
|
||||
{
|
||||
size_t i;
|
||||
ERR_STATE *thread_es;
|
||||
|
||||
if (es == NULL || es->bottom == es->top)
|
||||
return;
|
||||
|
||||
thread_es = ossl_err_get_state_int();
|
||||
if (thread_es == NULL)
|
||||
return;
|
||||
|
||||
for (i = (size_t)es->bottom; i != (size_t)es->top;) {
|
||||
size_t top;
|
||||
|
||||
i = (i + 1) % ERR_NUM_ERRORS;
|
||||
if ((es->err_flags[i] & ERR_FLAG_CLEAR) != 0)
|
||||
continue;
|
||||
|
||||
err_get_slot(thread_es);
|
||||
top = thread_es->top;
|
||||
err_clear(thread_es, top, 0);
|
||||
|
||||
thread_es->err_flags[top] = es->err_flags[i];
|
||||
thread_es->err_buffer[top] = es->err_buffer[i];
|
||||
|
||||
err_set_debug(thread_es, top, es->err_file[i], es->err_line[i],
|
||||
es->err_func[i]);
|
||||
|
||||
if (es->err_data[i] != NULL && es->err_data_size[i] != 0) {
|
||||
void *data;
|
||||
size_t data_sz = es->err_data_size[i];
|
||||
|
||||
data = CRYPTO_malloc(data_sz, NULL, 0);
|
||||
if (data != NULL) {
|
||||
memcpy(data, es->err_data[i], data_sz);
|
||||
err_set_data(thread_es, top, data, data_sz,
|
||||
es->err_data_flags[i] | ERR_TXT_MALLOCED);
|
||||
}
|
||||
} else {
|
||||
err_clear_data(thread_es, top, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
/*
|
||||
* Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <openssl/params.h>
|
||||
#include <openssl/param_build.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/core_names.h>
|
||||
|
||||
#include "internal/common.h"
|
||||
#include "internal/provider.h"
|
||||
#include "crypto/evp.h"
|
||||
#include "evp_local.h"
|
||||
|
||||
int EVP_SKEY_export(const EVP_SKEY *skey, int selection,
|
||||
OSSL_CALLBACK *export_cb, void *export_cbarg)
|
||||
{
|
||||
if (skey == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return evp_skeymgmt_export(skey->skeymgmt, skey->keydata, selection, export_cb, export_cbarg);
|
||||
}
|
||||
|
||||
static EVP_SKEY *evp_skey_alloc(EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
EVP_SKEY *skey;
|
||||
|
||||
if (!ossl_assert(skeymgmt != NULL))
|
||||
return NULL;
|
||||
|
||||
if ((skey = OPENSSL_zalloc(sizeof(*skey))) == NULL)
|
||||
return NULL;
|
||||
|
||||
if (!CRYPTO_NEW_REF(&skey->references, 1))
|
||||
goto err;
|
||||
|
||||
skey->lock = CRYPTO_THREAD_lock_new();
|
||||
if (skey->lock == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB);
|
||||
goto err;
|
||||
}
|
||||
skey->skeymgmt = skeymgmt;
|
||||
return skey;
|
||||
|
||||
err:
|
||||
CRYPTO_FREE_REF(&skey->references);
|
||||
CRYPTO_THREAD_lock_free(skey->lock);
|
||||
OPENSSL_free(skey);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static EVP_SKEY *evp_skey_alloc_fetch(OSSL_LIB_CTX *libctx,
|
||||
const char *skeymgmtname,
|
||||
const char *propquery)
|
||||
{
|
||||
EVP_SKEYMGMT *skeymgmt;
|
||||
EVP_SKEY *skey;
|
||||
|
||||
skeymgmt = EVP_SKEYMGMT_fetch(libctx, skeymgmtname, propquery);
|
||||
if (skeymgmt == NULL) {
|
||||
/*
|
||||
* if the specific key_type is unknown, attempt to use the generic
|
||||
* key management
|
||||
*/
|
||||
skeymgmt = EVP_SKEYMGMT_fetch(libctx, OSSL_SKEY_TYPE_GENERIC, propquery);
|
||||
if (skeymgmt == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_FETCH_FAILED);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
skey = evp_skey_alloc(skeymgmt);
|
||||
if (skey == NULL)
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
|
||||
return skey;
|
||||
}
|
||||
|
||||
EVP_SKEY *EVP_SKEY_import(OSSL_LIB_CTX *libctx, const char *skeymgmtname, const char *propquery,
|
||||
int selection, const OSSL_PARAM *params)
|
||||
{
|
||||
EVP_SKEY *skey = evp_skey_alloc_fetch(libctx, skeymgmtname, propquery);
|
||||
|
||||
if (skey == NULL)
|
||||
return NULL;
|
||||
|
||||
skey->keydata = evp_skeymgmt_import(skey->skeymgmt, selection, params);
|
||||
if (skey->keydata == NULL)
|
||||
goto err;
|
||||
|
||||
return skey;
|
||||
|
||||
err:
|
||||
EVP_SKEY_free(skey);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
EVP_SKEY *EVP_SKEY_generate(OSSL_LIB_CTX *libctx, const char *skeymgmtname,
|
||||
const char *propquery, const OSSL_PARAM *params)
|
||||
{
|
||||
EVP_SKEY *skey = evp_skey_alloc_fetch(libctx, skeymgmtname, propquery);
|
||||
|
||||
if (skey == NULL)
|
||||
return NULL;
|
||||
|
||||
skey->keydata = evp_skeymgmt_generate(skey->skeymgmt, params);
|
||||
if (skey->keydata == NULL)
|
||||
goto err;
|
||||
|
||||
return skey;
|
||||
|
||||
err:
|
||||
EVP_SKEY_free(skey);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct raw_key_details_st {
|
||||
const void **key;
|
||||
size_t *len;
|
||||
};
|
||||
|
||||
static int get_secret_key(const OSSL_PARAM params[], void *arg)
|
||||
{
|
||||
const OSSL_PARAM *p = NULL;
|
||||
struct raw_key_details_st *raw_key = arg;
|
||||
|
||||
if ((p = OSSL_PARAM_locate_const(params, OSSL_SKEY_PARAM_RAW_BYTES)) != NULL)
|
||||
return OSSL_PARAM_get_octet_string_ptr(p, raw_key->key, raw_key->len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int EVP_SKEY_get0_raw_key(const EVP_SKEY *skey, const unsigned char **key,
|
||||
size_t *len)
|
||||
{
|
||||
struct raw_key_details_st raw_key;
|
||||
|
||||
if (skey == NULL || key == NULL || len == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return 0;
|
||||
}
|
||||
|
||||
raw_key.key = (const void **)key;
|
||||
raw_key.len = len;
|
||||
|
||||
return evp_skeymgmt_export(skey->skeymgmt, skey->keydata,
|
||||
OSSL_SKEYMGMT_SELECT_SECRET_KEY,
|
||||
get_secret_key, &raw_key);
|
||||
}
|
||||
|
||||
EVP_SKEY *EVP_SKEY_import_raw_key(OSSL_LIB_CTX *libctx, const char *skeymgmtname,
|
||||
unsigned char *key, size_t keylen,
|
||||
const char *propquery)
|
||||
{
|
||||
OSSL_PARAM params[2];
|
||||
|
||||
params[0] = OSSL_PARAM_construct_octet_string(OSSL_SKEY_PARAM_RAW_BYTES,
|
||||
(void *)key, keylen);
|
||||
params[1] = OSSL_PARAM_construct_end();
|
||||
|
||||
return EVP_SKEY_import(libctx, skeymgmtname, propquery,
|
||||
OSSL_SKEYMGMT_SELECT_SECRET_KEY, params);
|
||||
}
|
||||
|
||||
int EVP_SKEY_up_ref(EVP_SKEY *skey)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (CRYPTO_UP_REF(&skey->references, &i) <= 0)
|
||||
return 0;
|
||||
|
||||
REF_PRINT_COUNT("EVP_SKEY", i, skey);
|
||||
REF_ASSERT_ISNT(i < 2);
|
||||
return i > 1 ? 1 : 0;
|
||||
}
|
||||
|
||||
void EVP_SKEY_free(EVP_SKEY *skey)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (skey == NULL)
|
||||
return;
|
||||
|
||||
CRYPTO_DOWN_REF(&skey->references, &i);
|
||||
REF_PRINT_COUNT("EVP_SKEY", i, skey);
|
||||
if (i > 0)
|
||||
return;
|
||||
REF_ASSERT_ISNT(i < 0);
|
||||
evp_skeymgmt_freedata(skey->skeymgmt, skey->keydata);
|
||||
|
||||
EVP_SKEYMGMT_free(skey->skeymgmt);
|
||||
|
||||
CRYPTO_THREAD_lock_free(skey->lock);
|
||||
CRYPTO_FREE_REF(&skey->references);
|
||||
OPENSSL_free(skey);
|
||||
}
|
||||
|
||||
const char *EVP_SKEY_get0_key_id(const EVP_SKEY *skey)
|
||||
{
|
||||
if (skey == NULL)
|
||||
return NULL;
|
||||
|
||||
if (skey->skeymgmt->get_key_id)
|
||||
return skey->skeymgmt->get_key_id(skey->keydata);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char *EVP_SKEY_get0_skeymgmt_name(const EVP_SKEY *skey)
|
||||
{
|
||||
if (skey == NULL)
|
||||
return NULL;
|
||||
|
||||
return skey->skeymgmt->type_name;
|
||||
|
||||
}
|
||||
|
||||
const char *EVP_SKEY_get0_provider_name(const EVP_SKEY *skey)
|
||||
{
|
||||
if (skey == NULL)
|
||||
return NULL;
|
||||
|
||||
return ossl_provider_name(skey->skeymgmt->prov);
|
||||
}
|
||||
|
||||
int EVP_SKEY_is_a(const EVP_SKEY *skey, const char *name)
|
||||
{
|
||||
if (skey == NULL)
|
||||
return 0;
|
||||
|
||||
return EVP_SKEYMGMT_is_a(skey->skeymgmt, name);
|
||||
}
|
||||
|
||||
struct transfer_cb_ctx {
|
||||
int selection;
|
||||
EVP_SKEYMGMT *skeymgmt;
|
||||
void *keydata;
|
||||
};
|
||||
|
||||
static int transfer_cb(const OSSL_PARAM params[], void *arg)
|
||||
{
|
||||
struct transfer_cb_ctx *ctx = arg;
|
||||
|
||||
ctx->keydata = evp_skeymgmt_import(ctx->skeymgmt, ctx->selection, params);
|
||||
return 1;
|
||||
}
|
||||
|
||||
EVP_SKEY *EVP_SKEY_to_provider(EVP_SKEY *skey, OSSL_LIB_CTX *libctx,
|
||||
OSSL_PROVIDER *prov, const char *propquery)
|
||||
{
|
||||
struct transfer_cb_ctx ctx = { 0 };
|
||||
EVP_SKEYMGMT *skeymgmt = NULL;
|
||||
EVP_SKEY *ret = NULL;
|
||||
|
||||
if (skey == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (prov != NULL) {
|
||||
if (skey->skeymgmt->prov == prov)
|
||||
skeymgmt = skey->skeymgmt;
|
||||
else
|
||||
skeymgmt = evp_skeymgmt_fetch_from_prov(prov, skey->skeymgmt->type_name,
|
||||
propquery);
|
||||
} else {
|
||||
/* If no provider, get the default skeymgmt */
|
||||
skeymgmt = EVP_SKEYMGMT_fetch(libctx, skey->skeymgmt->type_name,
|
||||
propquery);
|
||||
}
|
||||
if (skeymgmt == NULL) {
|
||||
ERR_raise(ERR_LIB_EVP, ERR_R_FETCH_FAILED);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Short-circuit if destination provider is the same as origin */
|
||||
if (skey->skeymgmt->name_id == skeymgmt->name_id
|
||||
&& skey->skeymgmt->prov == skeymgmt->prov) {
|
||||
if (!EVP_SKEY_up_ref(skey))
|
||||
goto err;
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
return skey;
|
||||
}
|
||||
|
||||
ctx.selection = OSSL_SKEYMGMT_SELECT_ALL;
|
||||
ctx.skeymgmt = skeymgmt;
|
||||
|
||||
if (!EVP_SKEY_export(skey, ctx.selection, transfer_cb, &ctx))
|
||||
goto err;
|
||||
|
||||
if (ctx.keydata == NULL)
|
||||
goto err;
|
||||
|
||||
ret = evp_skey_alloc(skeymgmt);
|
||||
if (ret == NULL)
|
||||
goto err;
|
||||
|
||||
ret->keydata = ctx.keydata;
|
||||
|
||||
return ret;
|
||||
|
||||
err:
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
EVP_SKEY_free(ret);
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
/*
|
||||
* Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/crypto.h>
|
||||
#include <openssl/core_dispatch.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/err.h>
|
||||
#include "internal/core.h"
|
||||
#include "internal/provider.h"
|
||||
#include "internal/refcount.h"
|
||||
#include "crypto/evp.h"
|
||||
#include "evp_local.h"
|
||||
|
||||
void *evp_skeymgmt_generate(const EVP_SKEYMGMT *skeymgmt, const OSSL_PARAM params[])
|
||||
{
|
||||
void *provctx = ossl_provider_ctx(EVP_SKEYMGMT_get0_provider(skeymgmt));
|
||||
|
||||
return (skeymgmt->generate != NULL) ? skeymgmt->generate(provctx, params) : NULL;
|
||||
}
|
||||
|
||||
void *evp_skeymgmt_import(const EVP_SKEYMGMT *skeymgmt, int selection, const OSSL_PARAM params[])
|
||||
{
|
||||
void *provctx = ossl_provider_ctx(EVP_SKEYMGMT_get0_provider(skeymgmt));
|
||||
|
||||
/* This is mandatory, no need to check for its presence */
|
||||
return skeymgmt->import(provctx, selection, params);
|
||||
}
|
||||
|
||||
int evp_skeymgmt_export(const EVP_SKEYMGMT *skeymgmt, void *keydata,
|
||||
int selection, OSSL_CALLBACK *param_cb, void *cbarg)
|
||||
{
|
||||
/* This is mandatory, no need to check for its presence */
|
||||
return skeymgmt->export(keydata, selection, param_cb, cbarg);
|
||||
}
|
||||
|
||||
void evp_skeymgmt_freedata(const EVP_SKEYMGMT *skeymgmt, void *keydata)
|
||||
{
|
||||
/* This is mandatory, no need to check for its presence */
|
||||
skeymgmt->free(keydata);
|
||||
}
|
||||
|
||||
static void *skeymgmt_new(void)
|
||||
{
|
||||
EVP_SKEYMGMT *skeymgmt = NULL;
|
||||
|
||||
if ((skeymgmt = OPENSSL_zalloc(sizeof(*skeymgmt))) == NULL)
|
||||
return NULL;
|
||||
if (!CRYPTO_NEW_REF(&skeymgmt->refcnt, 1)) {
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
return NULL;
|
||||
}
|
||||
return skeymgmt;
|
||||
}
|
||||
|
||||
static void *skeymgmt_from_algorithm(int name_id,
|
||||
const OSSL_ALGORITHM *algodef,
|
||||
OSSL_PROVIDER *prov)
|
||||
{
|
||||
const OSSL_DISPATCH *fns = algodef->implementation;
|
||||
EVP_SKEYMGMT *skeymgmt = NULL;
|
||||
|
||||
if ((skeymgmt = skeymgmt_new()) == NULL)
|
||||
return NULL;
|
||||
|
||||
skeymgmt->name_id = name_id;
|
||||
if ((skeymgmt->type_name = ossl_algorithm_get1_first_name(algodef)) == NULL) {
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
return NULL;
|
||||
}
|
||||
skeymgmt->description = algodef->algorithm_description;
|
||||
|
||||
for (; fns->function_id != 0; fns++) {
|
||||
switch (fns->function_id) {
|
||||
case OSSL_FUNC_SKEYMGMT_FREE:
|
||||
if (skeymgmt->free == NULL)
|
||||
skeymgmt->free = OSSL_FUNC_skeymgmt_free(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_IMPORT:
|
||||
if (skeymgmt->import == NULL)
|
||||
skeymgmt->import = OSSL_FUNC_skeymgmt_import(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_EXPORT:
|
||||
if (skeymgmt->export == NULL)
|
||||
skeymgmt->export = OSSL_FUNC_skeymgmt_export(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_GENERATE:
|
||||
if (skeymgmt->generate == NULL)
|
||||
skeymgmt->generate = OSSL_FUNC_skeymgmt_generate(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_GET_KEY_ID:
|
||||
if (skeymgmt->get_key_id == NULL)
|
||||
skeymgmt->get_key_id = OSSL_FUNC_skeymgmt_get_key_id(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_IMP_SETTABLE_PARAMS:
|
||||
if (skeymgmt->imp_params == NULL)
|
||||
skeymgmt->imp_params = OSSL_FUNC_skeymgmt_imp_settable_params(fns);
|
||||
break;
|
||||
case OSSL_FUNC_SKEYMGMT_GEN_SETTABLE_PARAMS:
|
||||
if (skeymgmt->gen_params == NULL)
|
||||
skeymgmt->gen_params = OSSL_FUNC_skeymgmt_gen_settable_params(fns);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Check that the provider is sensible */
|
||||
if (skeymgmt->free == NULL
|
||||
|| skeymgmt->import == NULL
|
||||
|| skeymgmt->export == NULL) {
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_PROVIDER_FUNCTIONS);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!ossl_provider_up_ref(prov)) {
|
||||
EVP_SKEYMGMT_free(skeymgmt);
|
||||
ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
skeymgmt->prov = prov;
|
||||
|
||||
return skeymgmt;
|
||||
}
|
||||
|
||||
EVP_SKEYMGMT *evp_skeymgmt_fetch_from_prov(OSSL_PROVIDER *prov,
|
||||
const char *name,
|
||||
const char *properties)
|
||||
{
|
||||
return evp_generic_fetch_from_prov(prov,
|
||||
OSSL_OP_SKEYMGMT,
|
||||
name, properties,
|
||||
skeymgmt_from_algorithm,
|
||||
(int (*)(void *))EVP_SKEYMGMT_up_ref,
|
||||
(void (*)(void *))EVP_SKEYMGMT_free);
|
||||
}
|
||||
|
||||
EVP_SKEYMGMT *EVP_SKEYMGMT_fetch(OSSL_LIB_CTX *ctx, const char *algorithm,
|
||||
const char *properties)
|
||||
{
|
||||
return evp_generic_fetch(ctx, OSSL_OP_SKEYMGMT, algorithm, properties,
|
||||
skeymgmt_from_algorithm,
|
||||
(int (*)(void *))EVP_SKEYMGMT_up_ref,
|
||||
(void (*)(void *))EVP_SKEYMGMT_free);
|
||||
}
|
||||
|
||||
int EVP_SKEYMGMT_up_ref(EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
int ref = 0;
|
||||
|
||||
CRYPTO_UP_REF(&skeymgmt->refcnt, &ref);
|
||||
return 1;
|
||||
}
|
||||
|
||||
void EVP_SKEYMGMT_free(EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
int ref = 0;
|
||||
|
||||
if (skeymgmt == NULL)
|
||||
return;
|
||||
|
||||
CRYPTO_DOWN_REF(&skeymgmt->refcnt, &ref);
|
||||
if (ref > 0)
|
||||
return;
|
||||
OPENSSL_free(skeymgmt->type_name);
|
||||
ossl_provider_free(skeymgmt->prov);
|
||||
CRYPTO_FREE_REF(&skeymgmt->refcnt);
|
||||
OPENSSL_free(skeymgmt);
|
||||
}
|
||||
|
||||
const OSSL_PROVIDER *EVP_SKEYMGMT_get0_provider(const EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
return (skeymgmt != NULL) ? skeymgmt->prov : NULL;
|
||||
}
|
||||
|
||||
const char *EVP_SKEYMGMT_get0_description(const EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
return (skeymgmt != NULL) ? skeymgmt->description : NULL;
|
||||
}
|
||||
|
||||
const char *EVP_SKEYMGMT_get0_name(const EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
return (skeymgmt != NULL) ? skeymgmt->type_name : NULL;
|
||||
}
|
||||
|
||||
int EVP_SKEYMGMT_is_a(const EVP_SKEYMGMT *skeymgmt, const char *name)
|
||||
{
|
||||
return skeymgmt != NULL
|
||||
&& evp_is_a(skeymgmt->prov, skeymgmt->name_id, NULL, name);
|
||||
}
|
||||
|
||||
void EVP_SKEYMGMT_do_all_provided(OSSL_LIB_CTX *libctx,
|
||||
void (*fn)(EVP_SKEYMGMT *skeymgmt, void *arg),
|
||||
void *arg)
|
||||
{
|
||||
evp_generic_do_all(libctx, OSSL_OP_KEYMGMT,
|
||||
(void (*)(void *, void *))fn, arg,
|
||||
skeymgmt_from_algorithm,
|
||||
(int (*)(void *))EVP_SKEYMGMT_up_ref,
|
||||
(void (*)(void *))EVP_SKEYMGMT_free);
|
||||
}
|
||||
|
||||
int EVP_SKEYMGMT_names_do_all(const EVP_SKEYMGMT *skeymgmt,
|
||||
void (*fn)(const char *name, void *data),
|
||||
void *data)
|
||||
{
|
||||
if (skeymgmt == NULL)
|
||||
return 0;
|
||||
|
||||
if (skeymgmt->prov != NULL)
|
||||
return evp_names_do_all(skeymgmt->prov, skeymgmt->name_id, fn, data);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
const OSSL_PARAM *EVP_SKEYMGMT_get0_gen_settable_params(const EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
void *provctx = NULL;
|
||||
|
||||
if (skeymgmt == NULL)
|
||||
return 0;
|
||||
|
||||
provctx = ossl_provider_ctx(EVP_SKEYMGMT_get0_provider(skeymgmt));
|
||||
|
||||
return (skeymgmt->gen_params != NULL) ? skeymgmt->gen_params(provctx) : NULL;
|
||||
}
|
||||
|
||||
const OSSL_PARAM *EVP_SKEYMGMT_get0_imp_settable_params(const EVP_SKEYMGMT *skeymgmt)
|
||||
{
|
||||
void *provctx = NULL;
|
||||
|
||||
if (skeymgmt == NULL)
|
||||
return 0;
|
||||
|
||||
provctx = ossl_provider_ctx(EVP_SKEYMGMT_get0_provider(skeymgmt));
|
||||
|
||||
return (skeymgmt->imp_params != NULL) ? skeymgmt->imp_params(provctx) : NULL;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
LIBS=../../libcrypto
|
||||
$COMMON=hashtable.c hashfunc.c
|
||||
|
||||
SOURCE[../../libcrypto]=$COMMON
|
||||
SOURCE[../../providers/libfips.a]=$COMMON
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
*/
|
||||
|
||||
#include "internal/hashfunc.h"
|
||||
|
||||
ossl_unused uint64_t ossl_fnv1a_hash(uint8_t *key, size_t len)
|
||||
{
|
||||
uint64_t hash = 0xcbf29ce484222325ULL;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
hash ^= key[i];
|
||||
hash *= 0x00000100000001B3ULL;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
@@ -0,0 +1,749 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*
|
||||
*
|
||||
*
|
||||
* Notes On hash table design and layout
|
||||
* This hashtable uses a hopscotch algorithm to do indexing. The data structure
|
||||
* looks as follows:
|
||||
*
|
||||
* hash +--------------+
|
||||
* value+------->+ HT_VALUE |
|
||||
* + +--------------+
|
||||
* +-------+
|
||||
* | |
|
||||
* +---------------------------------------------------------+
|
||||
* | | | | | |
|
||||
* | entry | entry | entry | entry | |
|
||||
* | | | | | |
|
||||
* +---------------------------------------------------------+
|
||||
* | | |
|
||||
* | | |
|
||||
* +---------------------------------------------------------+
|
||||
* | + + +
|
||||
* | neighborhood[0] neighborhood[1] |
|
||||
* | |
|
||||
* | |
|
||||
* +---------------------------------------------------------+
|
||||
* |
|
||||
* +
|
||||
* neighborhoods
|
||||
*
|
||||
* On lookup/insert/delete, the items key is hashed to a 64 bit value
|
||||
* and the result is masked to provide an index into the neighborhoods
|
||||
* table. Once a neighborhood is determined, an in-order search is done
|
||||
* of the elements in the neighborhood indexes entries for a matching hash
|
||||
* value, if found, the corresponding HT_VALUE is used for the respective
|
||||
* operation. The number of entries in a neighborhood is determined at build
|
||||
* time based on the cacheline size of the target CPU. The intent is for a
|
||||
* neighborhood to have all entries in the neighborhood fit into a single cache
|
||||
* line to speed up lookups. If all entries in a neighborhood are in use at the
|
||||
* time of an insert, the table is expanded and rehashed.
|
||||
*
|
||||
* Lockless reads hash table is based on the same design but does not
|
||||
* allow growing and deletion. Thus subsequent neighborhoods are always
|
||||
* searched for a match until an empty entry is found.
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <internal/rcu.h>
|
||||
#include <internal/hashtable.h>
|
||||
#include <internal/hashfunc.h>
|
||||
#include <openssl/rand.h>
|
||||
|
||||
/*
|
||||
* gcc defines __SANITIZE_THREAD__
|
||||
* but clang uses the feature attributes api
|
||||
* map the latter to the former
|
||||
*/
|
||||
#if defined(__clang__) && defined(__has_feature)
|
||||
# if __has_feature(thread_sanitizer)
|
||||
# define __SANITIZE_THREADS__
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef __SANITIZE_THREADS__
|
||||
# include <sanitizer/tsan_interface.h>
|
||||
#endif
|
||||
|
||||
#include "internal/numbers.h"
|
||||
/*
|
||||
* When we do a lookup/insert/delete, there is a high likelihood
|
||||
* that we will iterate over at least part of the neighborhood list
|
||||
* As such, because we design a neighborhood entry to fit into a single
|
||||
* cache line it is advantageous, when supported to fetch the entire
|
||||
* structure for faster lookups
|
||||
*/
|
||||
#if defined(__GNUC__) || defined(__CLANG__)
|
||||
# define PREFETCH_NEIGHBORHOOD(x) __builtin_prefetch(x.entries)
|
||||
# define PREFETCH(x) __builtin_prefetch(x)
|
||||
#else
|
||||
# define PREFETCH_NEIGHBORHOOD(x)
|
||||
# define PREFETCH(x)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Define our neighborhood list length
|
||||
* Note: It should always be a power of 2
|
||||
*/
|
||||
#define DEFAULT_NEIGH_LEN_LOG 4
|
||||
#define DEFAULT_NEIGH_LEN (1 << DEFAULT_NEIGH_LEN_LOG)
|
||||
|
||||
/*
|
||||
* For now assume cache line size is 64 bytes
|
||||
*/
|
||||
#define CACHE_LINE_BYTES 64
|
||||
#define CACHE_LINE_ALIGNMENT CACHE_LINE_BYTES
|
||||
|
||||
#define NEIGHBORHOOD_LEN (CACHE_LINE_BYTES / sizeof(struct ht_neighborhood_entry_st))
|
||||
/*
|
||||
* Defines our chains of values
|
||||
*/
|
||||
struct ht_internal_value_st {
|
||||
HT_VALUE value;
|
||||
HT *ht;
|
||||
};
|
||||
|
||||
struct ht_neighborhood_entry_st {
|
||||
uint64_t hash;
|
||||
struct ht_internal_value_st *value;
|
||||
};
|
||||
|
||||
struct ht_neighborhood_st {
|
||||
struct ht_neighborhood_entry_st entries[NEIGHBORHOOD_LEN];
|
||||
};
|
||||
|
||||
/*
|
||||
* Updates to data in this struct
|
||||
* require an rcu sync after modification
|
||||
* prior to free
|
||||
*/
|
||||
struct ht_mutable_data_st {
|
||||
struct ht_neighborhood_st *neighborhoods;
|
||||
void *neighborhood_ptr_to_free;
|
||||
uint64_t neighborhood_mask;
|
||||
};
|
||||
|
||||
/*
|
||||
* Private data may be updated on the write
|
||||
* side only, and so do not require rcu sync
|
||||
*/
|
||||
struct ht_write_private_data_st {
|
||||
size_t neighborhood_len;
|
||||
size_t value_count;
|
||||
int need_sync;
|
||||
};
|
||||
|
||||
struct ht_internal_st {
|
||||
HT_CONFIG config;
|
||||
CRYPTO_RCU_LOCK *lock;
|
||||
CRYPTO_RWLOCK *atomic_lock;
|
||||
struct ht_mutable_data_st *md;
|
||||
struct ht_write_private_data_st wpd;
|
||||
};
|
||||
|
||||
static void free_value(struct ht_internal_value_st *v);
|
||||
|
||||
static struct ht_neighborhood_st *alloc_new_neighborhood_list(size_t len,
|
||||
void **freeptr)
|
||||
{
|
||||
struct ht_neighborhood_st *ret;
|
||||
|
||||
ret = OPENSSL_aligned_alloc(sizeof(struct ht_neighborhood_st) * len,
|
||||
CACHE_LINE_BYTES, freeptr);
|
||||
|
||||
/* fall back to regular malloc */
|
||||
if (ret == NULL) {
|
||||
ret = *freeptr = OPENSSL_malloc(sizeof(struct ht_neighborhood_st) * len);
|
||||
if (ret == NULL)
|
||||
return NULL;
|
||||
}
|
||||
memset(ret, 0, sizeof(struct ht_neighborhood_st) * len);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void internal_free_nop(HT_VALUE *v)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
HT *ossl_ht_new(const HT_CONFIG *conf)
|
||||
{
|
||||
HT *new = OPENSSL_zalloc(sizeof(*new));
|
||||
|
||||
if (new == NULL)
|
||||
return NULL;
|
||||
|
||||
new->atomic_lock = CRYPTO_THREAD_lock_new();
|
||||
if (new->atomic_lock == NULL)
|
||||
goto err;
|
||||
|
||||
memcpy(&new->config, conf, sizeof(*conf));
|
||||
|
||||
if (new->config.init_neighborhoods != 0) {
|
||||
new->wpd.neighborhood_len = new->config.init_neighborhoods;
|
||||
/* round up to the next power of 2 */
|
||||
new->wpd.neighborhood_len--;
|
||||
new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 1;
|
||||
new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 2;
|
||||
new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 4;
|
||||
new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 8;
|
||||
new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 16;
|
||||
new->wpd.neighborhood_len++;
|
||||
} else {
|
||||
new->wpd.neighborhood_len = DEFAULT_NEIGH_LEN;
|
||||
}
|
||||
|
||||
if (new->config.ht_free_fn == NULL)
|
||||
new->config.ht_free_fn = internal_free_nop;
|
||||
|
||||
new->md = OPENSSL_zalloc(sizeof(*new->md));
|
||||
if (new->md == NULL)
|
||||
goto err;
|
||||
|
||||
new->md->neighborhoods =
|
||||
alloc_new_neighborhood_list(new->wpd.neighborhood_len,
|
||||
&new->md->neighborhood_ptr_to_free);
|
||||
if (new->md->neighborhoods == NULL)
|
||||
goto err;
|
||||
new->md->neighborhood_mask = new->wpd.neighborhood_len - 1;
|
||||
|
||||
new->lock = ossl_rcu_lock_new(1, conf->ctx);
|
||||
if (new->lock == NULL)
|
||||
goto err;
|
||||
|
||||
if (new->config.ht_hash_fn == NULL)
|
||||
new->config.ht_hash_fn = ossl_fnv1a_hash;
|
||||
|
||||
return new;
|
||||
|
||||
err:
|
||||
CRYPTO_THREAD_lock_free(new->atomic_lock);
|
||||
ossl_rcu_lock_free(new->lock);
|
||||
if (new->md != NULL)
|
||||
OPENSSL_free(new->md->neighborhood_ptr_to_free);
|
||||
OPENSSL_free(new->md);
|
||||
OPENSSL_free(new);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void ossl_ht_read_lock(HT *htable)
|
||||
{
|
||||
ossl_rcu_read_lock(htable->lock);
|
||||
}
|
||||
|
||||
void ossl_ht_read_unlock(HT *htable)
|
||||
{
|
||||
ossl_rcu_read_unlock(htable->lock);
|
||||
}
|
||||
|
||||
void ossl_ht_write_lock(HT *htable)
|
||||
{
|
||||
ossl_rcu_write_lock(htable->lock);
|
||||
htable->wpd.need_sync = 0;
|
||||
}
|
||||
|
||||
void ossl_ht_write_unlock(HT *htable)
|
||||
{
|
||||
int need_sync = htable->wpd.need_sync;
|
||||
|
||||
htable->wpd.need_sync = 0;
|
||||
ossl_rcu_write_unlock(htable->lock);
|
||||
if (need_sync)
|
||||
ossl_synchronize_rcu(htable->lock);
|
||||
}
|
||||
|
||||
static void free_oldmd(void *arg)
|
||||
{
|
||||
struct ht_mutable_data_st *oldmd = arg;
|
||||
size_t i, j;
|
||||
size_t neighborhood_len = (size_t)oldmd->neighborhood_mask + 1;
|
||||
struct ht_internal_value_st *v;
|
||||
|
||||
for (i = 0; i < neighborhood_len; i++) {
|
||||
PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[i + 1]);
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
if (oldmd->neighborhoods[i].entries[j].value != NULL) {
|
||||
v = oldmd->neighborhoods[i].entries[j].value;
|
||||
v->ht->config.ht_free_fn((HT_VALUE *)v);
|
||||
free_value(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
OPENSSL_free(oldmd->neighborhood_ptr_to_free);
|
||||
OPENSSL_free(oldmd);
|
||||
}
|
||||
|
||||
static int ossl_ht_flush_internal(HT *h)
|
||||
{
|
||||
struct ht_mutable_data_st *newmd = NULL;
|
||||
struct ht_mutable_data_st *oldmd = NULL;
|
||||
|
||||
newmd = OPENSSL_zalloc(sizeof(*newmd));
|
||||
if (newmd == NULL)
|
||||
return 0;
|
||||
|
||||
newmd->neighborhoods = alloc_new_neighborhood_list(DEFAULT_NEIGH_LEN,
|
||||
&newmd->neighborhood_ptr_to_free);
|
||||
if (newmd->neighborhoods == NULL) {
|
||||
OPENSSL_free(newmd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
newmd->neighborhood_mask = DEFAULT_NEIGH_LEN - 1;
|
||||
|
||||
/* Swap the old and new mutable data sets */
|
||||
oldmd = ossl_rcu_deref(&h->md);
|
||||
ossl_rcu_assign_ptr(&h->md, &newmd);
|
||||
|
||||
/* Set the number of entries to 0 */
|
||||
h->wpd.value_count = 0;
|
||||
h->wpd.neighborhood_len = DEFAULT_NEIGH_LEN;
|
||||
|
||||
ossl_rcu_call(h->lock, free_oldmd, oldmd);
|
||||
h->wpd.need_sync = 1;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ossl_ht_flush(HT *h)
|
||||
{
|
||||
return ossl_ht_flush_internal(h);
|
||||
}
|
||||
|
||||
void ossl_ht_free(HT *h)
|
||||
{
|
||||
if (h == NULL)
|
||||
return;
|
||||
|
||||
ossl_ht_write_lock(h);
|
||||
ossl_ht_flush_internal(h);
|
||||
ossl_ht_write_unlock(h);
|
||||
/* Freeing the lock does a final sync for us */
|
||||
CRYPTO_THREAD_lock_free(h->atomic_lock);
|
||||
ossl_rcu_lock_free(h->lock);
|
||||
OPENSSL_free(h->md->neighborhood_ptr_to_free);
|
||||
OPENSSL_free(h->md);
|
||||
OPENSSL_free(h);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t ossl_ht_count(HT *h)
|
||||
{
|
||||
size_t count;
|
||||
|
||||
count = h->wpd.value_count;
|
||||
return count;
|
||||
}
|
||||
|
||||
void ossl_ht_foreach_until(HT *h, int (*cb)(HT_VALUE *obj, void *arg),
|
||||
void *arg)
|
||||
{
|
||||
size_t i, j;
|
||||
struct ht_mutable_data_st *md;
|
||||
|
||||
md = ossl_rcu_deref(&h->md);
|
||||
for (i = 0; i < md->neighborhood_mask + 1; i++) {
|
||||
PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]);
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
if (md->neighborhoods[i].entries[j].value != NULL) {
|
||||
if (!cb((HT_VALUE *)md->neighborhoods[i].entries[j].value, arg))
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
}
|
||||
out:
|
||||
return;
|
||||
}
|
||||
|
||||
HT_VALUE_LIST *ossl_ht_filter(HT *h, size_t max_len,
|
||||
int (*filter)(HT_VALUE *obj, void *arg),
|
||||
void *arg)
|
||||
{
|
||||
struct ht_mutable_data_st *md;
|
||||
HT_VALUE_LIST *list = OPENSSL_zalloc(sizeof(HT_VALUE_LIST)
|
||||
+ (sizeof(HT_VALUE *) * max_len));
|
||||
size_t i, j;
|
||||
struct ht_internal_value_st *v;
|
||||
|
||||
if (list == NULL)
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* The list array lives just beyond the end of
|
||||
* the struct
|
||||
*/
|
||||
list->list = (HT_VALUE **)(list + 1);
|
||||
|
||||
md = ossl_rcu_deref(&h->md);
|
||||
for (i = 0; i < md->neighborhood_mask + 1; i++) {
|
||||
PREFETCH_NEIGHBORHOOD(md->neighborhoods[i+1]);
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
v = md->neighborhoods[i].entries[j].value;
|
||||
if (v != NULL && filter((HT_VALUE *)v, arg)) {
|
||||
list->list[list->list_len++] = (HT_VALUE *)v;
|
||||
if (list->list_len == max_len)
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
}
|
||||
out:
|
||||
return list;
|
||||
}
|
||||
|
||||
void ossl_ht_value_list_free(HT_VALUE_LIST *list)
|
||||
{
|
||||
OPENSSL_free(list);
|
||||
}
|
||||
|
||||
static int compare_hash(uint64_t hash1, uint64_t hash2)
|
||||
{
|
||||
return (hash1 == hash2);
|
||||
}
|
||||
|
||||
static void free_old_neigh_table(void *arg)
|
||||
{
|
||||
struct ht_mutable_data_st *oldmd = arg;
|
||||
|
||||
OPENSSL_free(oldmd->neighborhood_ptr_to_free);
|
||||
OPENSSL_free(oldmd);
|
||||
}
|
||||
|
||||
/*
|
||||
* Increase hash table bucket list
|
||||
* must be called with write_lock held
|
||||
*/
|
||||
static int grow_hashtable(HT *h, size_t oldsize)
|
||||
{
|
||||
struct ht_mutable_data_st *newmd;
|
||||
struct ht_mutable_data_st *oldmd = ossl_rcu_deref(&h->md);
|
||||
int rc = 0;
|
||||
uint64_t oldi, oldj, newi, newj;
|
||||
uint64_t oldhash;
|
||||
struct ht_internal_value_st *oldv;
|
||||
int rehashed;
|
||||
size_t newsize = oldsize * 2;
|
||||
|
||||
if (h->config.lockless_reads)
|
||||
goto out;
|
||||
|
||||
if ((newmd = OPENSSL_zalloc(sizeof(*newmd))) == NULL)
|
||||
goto out;
|
||||
|
||||
/* bucket list is always a power of 2 */
|
||||
newmd->neighborhoods = alloc_new_neighborhood_list(oldsize * 2,
|
||||
&newmd->neighborhood_ptr_to_free);
|
||||
if (newmd->neighborhoods == NULL)
|
||||
goto out_free;
|
||||
|
||||
/* being a power of 2 makes for easy mask computation */
|
||||
newmd->neighborhood_mask = (newsize - 1);
|
||||
|
||||
/*
|
||||
* Now we need to start rehashing entries
|
||||
* Note we don't need to use atomics here as the new
|
||||
* mutable data hasn't been published
|
||||
*/
|
||||
for (oldi = 0; oldi < h->wpd.neighborhood_len; oldi++) {
|
||||
PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[oldi + 1]);
|
||||
for (oldj = 0; oldj < NEIGHBORHOOD_LEN; oldj++) {
|
||||
oldv = oldmd->neighborhoods[oldi].entries[oldj].value;
|
||||
if (oldv == NULL)
|
||||
continue;
|
||||
oldhash = oldmd->neighborhoods[oldi].entries[oldj].hash;
|
||||
newi = oldhash & newmd->neighborhood_mask;
|
||||
rehashed = 0;
|
||||
for (newj = 0; newj < NEIGHBORHOOD_LEN; newj++) {
|
||||
if (newmd->neighborhoods[newi].entries[newj].value == NULL) {
|
||||
newmd->neighborhoods[newi].entries[newj].value = oldv;
|
||||
newmd->neighborhoods[newi].entries[newj].hash = oldhash;
|
||||
rehashed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (rehashed == 0) {
|
||||
/* we ran out of space in a neighborhood, grow again */
|
||||
OPENSSL_free(newmd->neighborhoods);
|
||||
OPENSSL_free(newmd);
|
||||
return grow_hashtable(h, newsize);
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Now that our entries are all hashed into the new bucket list
|
||||
* update our bucket_len and target_max_load
|
||||
*/
|
||||
h->wpd.neighborhood_len = newsize;
|
||||
|
||||
/*
|
||||
* Now we replace the old mutable data with the new
|
||||
*/
|
||||
ossl_rcu_assign_ptr(&h->md, &newmd);
|
||||
ossl_rcu_call(h->lock, free_old_neigh_table, oldmd);
|
||||
h->wpd.need_sync = 1;
|
||||
/*
|
||||
* And we're done
|
||||
*/
|
||||
rc = 1;
|
||||
|
||||
out:
|
||||
return rc;
|
||||
out_free:
|
||||
OPENSSL_free(newmd->neighborhoods);
|
||||
OPENSSL_free(newmd);
|
||||
goto out;
|
||||
}
|
||||
|
||||
static void free_old_ht_value(void *arg)
|
||||
{
|
||||
HT_VALUE *h = (HT_VALUE *)arg;
|
||||
|
||||
/*
|
||||
* Note, this is only called on replacement,
|
||||
* the caller is responsible for freeing the
|
||||
* held data, we just need to free the wrapping
|
||||
* struct here
|
||||
*/
|
||||
OPENSSL_free(h);
|
||||
}
|
||||
|
||||
static ossl_inline int match_key(HT_KEY *a, HT_KEY *b)
|
||||
{
|
||||
/*
|
||||
* keys match if they are both present, the same size
|
||||
* and compare equal in memory
|
||||
*/
|
||||
PREFETCH(a->keybuf);
|
||||
PREFETCH(b->keybuf);
|
||||
if (a->keybuf != NULL && b->keybuf != NULL && a->keysize == b->keysize)
|
||||
return !memcmp(a->keybuf, b->keybuf, a->keysize);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int ossl_ht_insert_locked(HT *h, uint64_t hash,
|
||||
struct ht_internal_value_st *newval,
|
||||
HT_VALUE **olddata)
|
||||
{
|
||||
struct ht_mutable_data_st *md = h->md;
|
||||
uint64_t neigh_idx_start = hash & md->neighborhood_mask;
|
||||
uint64_t neigh_idx = neigh_idx_start;
|
||||
size_t j;
|
||||
uint64_t ihash;
|
||||
HT_VALUE *ival;
|
||||
size_t empty_idx = SIZE_MAX;
|
||||
int lockless_reads = h->config.lockless_reads;
|
||||
|
||||
do {
|
||||
PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]);
|
||||
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value);
|
||||
if (ival == NULL) {
|
||||
empty_idx = j;
|
||||
/* lockless_reads implies no deletion, we can break out */
|
||||
if (lockless_reads)
|
||||
goto not_found;
|
||||
continue;
|
||||
}
|
||||
if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash,
|
||||
&ihash, h->atomic_lock))
|
||||
return 0;
|
||||
if (compare_hash(hash, ihash) && match_key(&newval->value.key,
|
||||
&ival->key)) {
|
||||
if (olddata == NULL) {
|
||||
/* This would insert a duplicate -> fail */
|
||||
return 0;
|
||||
}
|
||||
/* Do a replacement */
|
||||
if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[j].hash,
|
||||
hash, h->atomic_lock))
|
||||
return 0;
|
||||
*olddata = (HT_VALUE *)md->neighborhoods[neigh_idx].entries[j].value;
|
||||
ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[j].value,
|
||||
&newval);
|
||||
ossl_rcu_call(h->lock, free_old_ht_value, *olddata);
|
||||
h->wpd.need_sync = 1;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (!lockless_reads)
|
||||
break;
|
||||
/* Continue search in subsequent neighborhoods */
|
||||
neigh_idx = (neigh_idx + 1) & md->neighborhood_mask;
|
||||
} while (neigh_idx != neigh_idx_start);
|
||||
|
||||
not_found:
|
||||
/* If we get to here, its just an insert */
|
||||
if (empty_idx == SIZE_MAX)
|
||||
return -1; /* out of space */
|
||||
if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[empty_idx].hash,
|
||||
hash, h->atomic_lock))
|
||||
return 0;
|
||||
h->wpd.value_count++;
|
||||
ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[empty_idx].value,
|
||||
&newval);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static struct ht_internal_value_st *alloc_new_value(HT *h, HT_KEY *key,
|
||||
void *data,
|
||||
uintptr_t *type)
|
||||
{
|
||||
struct ht_internal_value_st *tmp;
|
||||
size_t nvsize = sizeof(*tmp);
|
||||
|
||||
if (h->config.collision_check == 1)
|
||||
nvsize += key->keysize;
|
||||
|
||||
tmp = OPENSSL_malloc(nvsize);
|
||||
|
||||
if (tmp == NULL)
|
||||
return NULL;
|
||||
|
||||
tmp->ht = h;
|
||||
tmp->value.value = data;
|
||||
tmp->value.type_id = type;
|
||||
tmp->value.key.keybuf = NULL;
|
||||
if (h->config.collision_check) {
|
||||
tmp->value.key.keybuf = (uint8_t *)(tmp + 1);
|
||||
tmp->value.key.keysize = key->keysize;
|
||||
memcpy(tmp->value.key.keybuf, key->keybuf, key->keysize);
|
||||
}
|
||||
|
||||
|
||||
return tmp;
|
||||
}
|
||||
|
||||
static void free_value(struct ht_internal_value_st *v)
|
||||
{
|
||||
OPENSSL_free(v);
|
||||
}
|
||||
|
||||
int ossl_ht_insert(HT *h, HT_KEY *key, HT_VALUE *data, HT_VALUE **olddata)
|
||||
{
|
||||
struct ht_internal_value_st *newval = NULL;
|
||||
uint64_t hash;
|
||||
int rc = 0;
|
||||
int i;
|
||||
|
||||
if (data->value == NULL)
|
||||
goto out;
|
||||
|
||||
newval = alloc_new_value(h, key, data->value, data->type_id);
|
||||
if (newval == NULL)
|
||||
goto out;
|
||||
|
||||
/*
|
||||
* we have to take our lock here to prevent other changes
|
||||
* to the bucket list
|
||||
*/
|
||||
hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
|
||||
|
||||
for (i = 0;
|
||||
(rc = ossl_ht_insert_locked(h, hash, newval, olddata)) == -1
|
||||
&& i < 4;
|
||||
++i)
|
||||
if (!grow_hashtable(h, h->wpd.neighborhood_len)) {
|
||||
rc = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (rc <= 0)
|
||||
free_value(newval);
|
||||
|
||||
out:
|
||||
return rc;
|
||||
}
|
||||
|
||||
HT_VALUE *ossl_ht_get(HT *h, HT_KEY *key)
|
||||
{
|
||||
struct ht_mutable_data_st *md;
|
||||
uint64_t hash;
|
||||
uint64_t neigh_idx_start;
|
||||
uint64_t neigh_idx;
|
||||
struct ht_internal_value_st *ival = NULL;
|
||||
size_t j;
|
||||
uint64_t ehash;
|
||||
int lockless_reads = h->config.lockless_reads;
|
||||
|
||||
hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
|
||||
|
||||
md = ossl_rcu_deref(&h->md);
|
||||
neigh_idx = neigh_idx_start = hash & md->neighborhood_mask;
|
||||
do {
|
||||
PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]);
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value);
|
||||
if (ival == NULL) {
|
||||
if (lockless_reads)
|
||||
/* lockless_reads implies no deletion, we can break out */
|
||||
return NULL;
|
||||
continue;
|
||||
}
|
||||
if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash,
|
||||
&ehash, h->atomic_lock))
|
||||
return NULL;
|
||||
if (compare_hash(hash, ehash) && match_key(&ival->value.key, key))
|
||||
return (HT_VALUE *)ival;
|
||||
}
|
||||
if (!lockless_reads)
|
||||
break;
|
||||
/* Continue search in subsequent neighborhoods */
|
||||
neigh_idx = (neigh_idx + 1) & md->neighborhood_mask;
|
||||
} while (neigh_idx != neigh_idx_start);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void free_old_entry(void *arg)
|
||||
{
|
||||
struct ht_internal_value_st *v = arg;
|
||||
|
||||
v->ht->config.ht_free_fn((HT_VALUE *)v);
|
||||
free_value(v);
|
||||
}
|
||||
|
||||
int ossl_ht_delete(HT *h, HT_KEY *key)
|
||||
{
|
||||
uint64_t hash;
|
||||
uint64_t neigh_idx;
|
||||
size_t j;
|
||||
struct ht_internal_value_st *v = NULL;
|
||||
HT_VALUE *nv = NULL;
|
||||
int rc = 0;
|
||||
|
||||
if (h->config.lockless_reads)
|
||||
return 0;
|
||||
|
||||
hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
|
||||
|
||||
neigh_idx = hash & h->md->neighborhood_mask;
|
||||
PREFETCH_NEIGHBORHOOD(h->md->neighborhoods[neigh_idx]);
|
||||
for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
|
||||
v = (struct ht_internal_value_st *)h->md->neighborhoods[neigh_idx].entries[j].value;
|
||||
if (v == NULL)
|
||||
continue;
|
||||
if (compare_hash(hash, h->md->neighborhoods[neigh_idx].entries[j].hash)
|
||||
&& match_key(key, &v->value.key)) {
|
||||
if (!CRYPTO_atomic_store(&h->md->neighborhoods[neigh_idx].entries[j].hash,
|
||||
0, h->atomic_lock))
|
||||
break;
|
||||
h->wpd.value_count--;
|
||||
ossl_rcu_assign_ptr(&h->md->neighborhoods[neigh_idx].entries[j].value,
|
||||
&nv);
|
||||
rc = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (rc == 1) {
|
||||
ossl_rcu_call(h->lock, free_old_entry, v);
|
||||
h->wpd.need_sync = 1;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
/*
|
||||
* Copyright 2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
/* We need to use some engine deprecated APIs */
|
||||
#define OPENSSL_SUPPRESS_DEPRECATED
|
||||
|
||||
#include "crypto/s390x_arch.h"
|
||||
#include "hmac_local.h"
|
||||
#include "openssl/obj_mac.h"
|
||||
#include "openssl/evp.h"
|
||||
#if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE)
|
||||
# include <openssl/engine.h>
|
||||
#endif
|
||||
|
||||
#ifdef OPENSSL_HMAC_S390X
|
||||
|
||||
static int s390x_fc_from_md(const EVP_MD *md)
|
||||
{
|
||||
int fc;
|
||||
|
||||
if (EVP_MD_is_a(md, "SHA2-224"))
|
||||
fc = S390X_HMAC_SHA_224;
|
||||
else if (EVP_MD_is_a(md, "SHA2-256"))
|
||||
fc = S390X_HMAC_SHA_256;
|
||||
else if (EVP_MD_is_a(md, "SHA2-384"))
|
||||
fc = S390X_HMAC_SHA_384;
|
||||
else if (EVP_MD_is_a(md, "SHA2-512"))
|
||||
fc = S390X_HMAC_SHA_512;
|
||||
else
|
||||
return 0;
|
||||
|
||||
if ((OPENSSL_s390xcap_P.kmac[1] & S390X_CAPBIT(fc)) == 0)
|
||||
return 0;
|
||||
|
||||
return fc;
|
||||
}
|
||||
|
||||
static void s390x_call_kmac(HMAC_CTX *ctx, const unsigned char *in, size_t len)
|
||||
{
|
||||
unsigned int fc = ctx->plat.s390x.fc;
|
||||
|
||||
if (ctx->plat.s390x.ikp)
|
||||
fc |= S390X_KMAC_IKP;
|
||||
|
||||
if (ctx->plat.s390x.iimp)
|
||||
fc |= S390X_KMAC_IIMP;
|
||||
|
||||
switch (ctx->plat.s390x.fc) {
|
||||
case S390X_HMAC_SHA_224:
|
||||
case S390X_HMAC_SHA_256:
|
||||
ctx->plat.s390x.param.hmac_224_256.imbl += ((uint64_t)len * 8);
|
||||
break;
|
||||
case S390X_HMAC_SHA_384:
|
||||
case S390X_HMAC_SHA_512:
|
||||
ctx->plat.s390x.param.hmac_384_512.imbl += ((uint128_t)len * 8);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
s390x_kmac(in, len, fc, &ctx->plat.s390x.param);
|
||||
|
||||
ctx->plat.s390x.ikp = 1;
|
||||
}
|
||||
|
||||
static int s390x_check_engine_used(const EVP_MD *md, ENGINE *impl)
|
||||
{
|
||||
# if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE)
|
||||
const EVP_MD *d;
|
||||
|
||||
if (impl != NULL) {
|
||||
if (!ENGINE_init(impl))
|
||||
return 0;
|
||||
} else {
|
||||
impl = ENGINE_get_digest_engine(EVP_MD_get_type(md));
|
||||
}
|
||||
|
||||
if (impl == NULL)
|
||||
return 0;
|
||||
|
||||
d = ENGINE_get_digest(impl, EVP_MD_get_type(md));
|
||||
ENGINE_finish(impl);
|
||||
|
||||
if (d != NULL)
|
||||
return 1;
|
||||
# endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int s390x_HMAC_init(HMAC_CTX *ctx, const void *key, int key_len, ENGINE *impl)
|
||||
{
|
||||
unsigned char *key_param;
|
||||
unsigned int key_param_len;
|
||||
|
||||
ctx->plat.s390x.fc = s390x_fc_from_md(ctx->md);
|
||||
if (ctx->plat.s390x.fc == 0)
|
||||
return -1; /* Not supported by kmac instruction */
|
||||
|
||||
if (s390x_check_engine_used(ctx->md, impl)) {
|
||||
ctx->plat.s390x.fc = 0;
|
||||
return -1; /* An engine handles the digest, disable acceleration */
|
||||
}
|
||||
|
||||
ctx->plat.s390x.blk_size = EVP_MD_get_block_size(ctx->md);
|
||||
if (ctx->plat.s390x.blk_size < 0)
|
||||
return 0;
|
||||
|
||||
if (ctx->plat.s390x.size !=
|
||||
(size_t)(ctx->plat.s390x.blk_size * HMAC_S390X_BUF_NUM_BLOCKS)) {
|
||||
OPENSSL_clear_free(ctx->plat.s390x.buf, ctx->plat.s390x.size);
|
||||
ctx->plat.s390x.size = 0;
|
||||
ctx->plat.s390x.buf = OPENSSL_zalloc(ctx->plat.s390x.blk_size *
|
||||
HMAC_S390X_BUF_NUM_BLOCKS);
|
||||
if (ctx->plat.s390x.buf == NULL)
|
||||
return 0;
|
||||
ctx->plat.s390x.size = ctx->plat.s390x.blk_size *
|
||||
HMAC_S390X_BUF_NUM_BLOCKS;
|
||||
}
|
||||
ctx->plat.s390x.num = 0;
|
||||
|
||||
ctx->plat.s390x.ikp = 0;
|
||||
ctx->plat.s390x.iimp = 1;
|
||||
|
||||
switch (ctx->plat.s390x.fc) {
|
||||
case S390X_HMAC_SHA_224:
|
||||
case S390X_HMAC_SHA_256:
|
||||
ctx->plat.s390x.param.hmac_224_256.imbl = 0;
|
||||
OPENSSL_cleanse(ctx->plat.s390x.param.hmac_224_256.h,
|
||||
sizeof(ctx->plat.s390x.param.hmac_224_256.h));
|
||||
break;
|
||||
case S390X_HMAC_SHA_384:
|
||||
case S390X_HMAC_SHA_512:
|
||||
ctx->plat.s390x.param.hmac_384_512.imbl = 0;
|
||||
OPENSSL_cleanse(ctx->plat.s390x.param.hmac_384_512.h,
|
||||
sizeof(ctx->plat.s390x.param.hmac_384_512.h));
|
||||
break;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (key != NULL) {
|
||||
switch (ctx->plat.s390x.fc) {
|
||||
case S390X_HMAC_SHA_224:
|
||||
case S390X_HMAC_SHA_256:
|
||||
OPENSSL_cleanse(&ctx->plat.s390x.param.hmac_224_256.key,
|
||||
sizeof(ctx->plat.s390x.param.hmac_224_256.key));
|
||||
key_param = ctx->plat.s390x.param.hmac_224_256.key;
|
||||
key_param_len = sizeof(ctx->plat.s390x.param.hmac_224_256.key);
|
||||
break;
|
||||
case S390X_HMAC_SHA_384:
|
||||
case S390X_HMAC_SHA_512:
|
||||
OPENSSL_cleanse(&ctx->plat.s390x.param.hmac_384_512.key,
|
||||
sizeof(ctx->plat.s390x.param.hmac_384_512.key));
|
||||
key_param = ctx->plat.s390x.param.hmac_384_512.key;
|
||||
key_param_len = sizeof(ctx->plat.s390x.param.hmac_384_512.key);
|
||||
break;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!ossl_assert(ctx->plat.s390x.blk_size <= (int)key_param_len))
|
||||
return 0;
|
||||
|
||||
if (key_len > ctx->plat.s390x.blk_size) {
|
||||
if (!EVP_DigestInit_ex(ctx->md_ctx, ctx->md, impl)
|
||||
|| !EVP_DigestUpdate(ctx->md_ctx, key, key_len)
|
||||
|| !EVP_DigestFinal_ex(ctx->md_ctx, key_param,
|
||||
&key_param_len))
|
||||
return 0;
|
||||
} else {
|
||||
if (key_len < 0 || key_len > (int)key_param_len)
|
||||
return 0;
|
||||
memcpy(key_param, key, key_len);
|
||||
/* remaining key bytes already zeroed out above */
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int s390x_HMAC_update(HMAC_CTX *ctx, const unsigned char *data, size_t len)
|
||||
{
|
||||
size_t remain, num;
|
||||
|
||||
if (len == 0)
|
||||
return 1;
|
||||
|
||||
/* buffer is full, process it now */
|
||||
if (ctx->plat.s390x.num == ctx->plat.s390x.size) {
|
||||
s390x_call_kmac(ctx, ctx->plat.s390x.buf, ctx->plat.s390x.num);
|
||||
|
||||
ctx->plat.s390x.num = 0;
|
||||
}
|
||||
|
||||
remain = ctx->plat.s390x.size - ctx->plat.s390x.num;
|
||||
if (len > remain) {
|
||||
/* data does not fit into buffer */
|
||||
if (ctx->plat.s390x.num > 0) {
|
||||
/* first fill buffer and process it */
|
||||
memcpy(&ctx->plat.s390x.buf[ctx->plat.s390x.num], data, remain);
|
||||
ctx->plat.s390x.num += remain;
|
||||
|
||||
s390x_call_kmac(ctx, ctx->plat.s390x.buf, ctx->plat.s390x.num);
|
||||
|
||||
ctx->plat.s390x.num = 0;
|
||||
|
||||
data += remain;
|
||||
len -= remain;
|
||||
}
|
||||
|
||||
if (!ossl_assert(ctx->plat.s390x.num == 0))
|
||||
return 0;
|
||||
|
||||
if (len > ctx->plat.s390x.size) {
|
||||
/*
|
||||
* remaining data is still larger than buffer, process remaining
|
||||
* full blocks of input directly
|
||||
*/
|
||||
remain = len % ctx->plat.s390x.blk_size;
|
||||
num = len - remain;
|
||||
|
||||
s390x_call_kmac(ctx, data, num);
|
||||
|
||||
data += num;
|
||||
len -= num;
|
||||
}
|
||||
}
|
||||
|
||||
/* add remaining input data (which is < buffer size) to buffer */
|
||||
if (!ossl_assert(len <= ctx->plat.s390x.size))
|
||||
return 0;
|
||||
|
||||
if (len > 0) {
|
||||
memcpy(&ctx->plat.s390x.buf[ctx->plat.s390x.num], data, len);
|
||||
ctx->plat.s390x.num += len;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int s390x_HMAC_final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len)
|
||||
{
|
||||
void *result;
|
||||
unsigned int res_len;
|
||||
|
||||
ctx->plat.s390x.iimp = 0; /* last block */
|
||||
s390x_call_kmac(ctx, ctx->plat.s390x.buf, ctx->plat.s390x.num);
|
||||
|
||||
ctx->plat.s390x.num = 0;
|
||||
|
||||
switch (ctx->plat.s390x.fc) {
|
||||
case S390X_HMAC_SHA_224:
|
||||
result = &ctx->plat.s390x.param.hmac_224_256.h[0];
|
||||
res_len = SHA224_DIGEST_LENGTH;
|
||||
break;
|
||||
case S390X_HMAC_SHA_256:
|
||||
result = &ctx->plat.s390x.param.hmac_224_256.h[0];
|
||||
res_len = SHA256_DIGEST_LENGTH;
|
||||
break;
|
||||
case S390X_HMAC_SHA_384:
|
||||
result = &ctx->plat.s390x.param.hmac_384_512.h[0];
|
||||
res_len = SHA384_DIGEST_LENGTH;
|
||||
break;
|
||||
case S390X_HMAC_SHA_512:
|
||||
result = &ctx->plat.s390x.param.hmac_384_512.h[0];
|
||||
res_len = SHA512_DIGEST_LENGTH;
|
||||
break;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy(md, result, res_len);
|
||||
if (len != NULL)
|
||||
*len = res_len;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int s390x_HMAC_CTX_copy(HMAC_CTX *dctx, HMAC_CTX *sctx)
|
||||
{
|
||||
dctx->plat.s390x.fc = sctx->plat.s390x.fc;
|
||||
dctx->plat.s390x.blk_size = sctx->plat.s390x.blk_size;
|
||||
dctx->plat.s390x.ikp = sctx->plat.s390x.ikp;
|
||||
dctx->plat.s390x.iimp = sctx->plat.s390x.iimp;
|
||||
|
||||
memcpy(&dctx->plat.s390x.param, &sctx->plat.s390x.param,
|
||||
sizeof(dctx->plat.s390x.param));
|
||||
|
||||
OPENSSL_clear_free(dctx->plat.s390x.buf, dctx->plat.s390x.size);
|
||||
dctx->plat.s390x.buf = NULL;
|
||||
if (sctx->plat.s390x.buf != NULL) {
|
||||
dctx->plat.s390x.buf = OPENSSL_memdup(sctx->plat.s390x.buf,
|
||||
sctx->plat.s390x.size);
|
||||
if (dctx->plat.s390x.buf == NULL)
|
||||
return 0;
|
||||
}
|
||||
|
||||
dctx->plat.s390x.size = sctx->plat.s390x.size;
|
||||
dctx->plat.s390x.num = sctx->plat.s390x.num;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int s390x_HMAC_CTX_cleanup(HMAC_CTX *ctx)
|
||||
{
|
||||
OPENSSL_clear_free(ctx->plat.s390x.buf, ctx->plat.s390x.size);
|
||||
ctx->plat.s390x.buf = NULL;
|
||||
ctx->plat.s390x.size = 0;
|
||||
ctx->plat.s390x.num = 0;
|
||||
|
||||
OPENSSL_cleanse(&ctx->plat.s390x.param, sizeof(ctx->plat.s390x.param));
|
||||
|
||||
ctx->plat.s390x.blk_size = 0;
|
||||
ctx->plat.s390x.ikp = 0;
|
||||
ctx->plat.s390x.iimp = 1;
|
||||
|
||||
ctx->plat.s390x.fc = 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,5 @@
|
||||
LIBS=../../libcrypto
|
||||
|
||||
$COMMON=hpke_util.c hpke.c
|
||||
|
||||
SOURCE[../../libcrypto]=$COMMON
|
||||
+1463
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,528 @@
|
||||
/*
|
||||
* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <openssl/core_names.h>
|
||||
#include <openssl/kdf.h>
|
||||
#include <openssl/params.h>
|
||||
#include <openssl/err.h>
|
||||
#include <openssl/proverr.h>
|
||||
#include <openssl/hpke.h>
|
||||
#include <openssl/sha.h>
|
||||
#include <openssl/rand.h>
|
||||
#include "crypto/ecx.h"
|
||||
#include "crypto/rand.h"
|
||||
#include "internal/hpke_util.h"
|
||||
#include "internal/packet.h"
|
||||
#include "internal/nelem.h"
|
||||
#include "internal/common.h"
|
||||
|
||||
/*
|
||||
* Delimiter used in OSSL_HPKE_str2suite
|
||||
*/
|
||||
#define OSSL_HPKE_STR_DELIMCHAR ','
|
||||
|
||||
/*
|
||||
* table with identifier and synonym strings
|
||||
* right now, there are 4 synonyms for each - a name, a hex string
|
||||
* a hex string with a leading zero and a decimal string - more
|
||||
* could be added but that seems like enough
|
||||
*/
|
||||
typedef struct {
|
||||
uint16_t id;
|
||||
char *synonyms[4];
|
||||
} synonymttab_t;
|
||||
|
||||
/* max length of string we'll try map to a suite */
|
||||
#define OSSL_HPKE_MAX_SUITESTR 38
|
||||
|
||||
/* Define HPKE labels from RFC9180 in hex for EBCDIC compatibility */
|
||||
/* ASCII: "HPKE-v1", in hex for EBCDIC compatibility */
|
||||
static const char LABEL_HPKEV1[] = "\x48\x50\x4B\x45\x2D\x76\x31";
|
||||
|
||||
/*
|
||||
* Note that if additions are made to the set of IANA codepoints
|
||||
* and the tables below, corresponding additions should also be
|
||||
* made to the synonymtab tables a little further down so that
|
||||
* OSSL_HPKE_str2suite() continues to function correctly.
|
||||
*
|
||||
* The canonical place to check for IANA registered codepoints
|
||||
* is: https://www.iana.org/assignments/hpke/hpke.xhtml
|
||||
*/
|
||||
|
||||
/*
|
||||
* @brief table of KEMs
|
||||
* See RFC9180 Section 7.1 "Table 2 KEM IDs"
|
||||
*/
|
||||
static const OSSL_HPKE_KEM_INFO hpke_kem_tab[] = {
|
||||
#ifndef OPENSSL_NO_EC
|
||||
{ OSSL_HPKE_KEM_ID_P256, "EC", OSSL_HPKE_KEMSTR_P256,
|
||||
LN_sha256, SHA256_DIGEST_LENGTH, 65, 65, 32, 0xFF },
|
||||
{ OSSL_HPKE_KEM_ID_P384, "EC", OSSL_HPKE_KEMSTR_P384,
|
||||
LN_sha384, SHA384_DIGEST_LENGTH, 97, 97, 48, 0xFF },
|
||||
{ OSSL_HPKE_KEM_ID_P521, "EC", OSSL_HPKE_KEMSTR_P521,
|
||||
LN_sha512, SHA512_DIGEST_LENGTH, 133, 133, 66, 0x01 },
|
||||
# ifndef OPENSSL_NO_ECX
|
||||
{ OSSL_HPKE_KEM_ID_X25519, OSSL_HPKE_KEMSTR_X25519, NULL,
|
||||
LN_sha256, SHA256_DIGEST_LENGTH,
|
||||
X25519_KEYLEN, X25519_KEYLEN, X25519_KEYLEN, 0x00 },
|
||||
{ OSSL_HPKE_KEM_ID_X448, OSSL_HPKE_KEMSTR_X448, NULL,
|
||||
LN_sha512, SHA512_DIGEST_LENGTH,
|
||||
X448_KEYLEN, X448_KEYLEN, X448_KEYLEN, 0x00 }
|
||||
# endif
|
||||
#else
|
||||
{ OSSL_HPKE_KEM_ID_RESERVED, NULL, NULL, NULL, 0, 0, 0, 0, 0x00 }
|
||||
#endif
|
||||
};
|
||||
|
||||
/*
|
||||
* @brief table of AEADs
|
||||
* See RFC9180 Section 7.2 "Table 3 KDF IDs"
|
||||
*/
|
||||
static const OSSL_HPKE_AEAD_INFO hpke_aead_tab[] = {
|
||||
{ OSSL_HPKE_AEAD_ID_AES_GCM_128, LN_aes_128_gcm, 16, 16,
|
||||
OSSL_HPKE_MAX_NONCELEN },
|
||||
{ OSSL_HPKE_AEAD_ID_AES_GCM_256, LN_aes_256_gcm, 16, 32,
|
||||
OSSL_HPKE_MAX_NONCELEN },
|
||||
#if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
|
||||
{ OSSL_HPKE_AEAD_ID_CHACHA_POLY1305, LN_chacha20_poly1305, 16, 32,
|
||||
OSSL_HPKE_MAX_NONCELEN },
|
||||
#endif
|
||||
{ OSSL_HPKE_AEAD_ID_EXPORTONLY, NULL, 0, 0, 0 }
|
||||
};
|
||||
|
||||
/*
|
||||
* @brief table of KDFs
|
||||
* See RFC9180 Section 7.3 "Table 5 AEAD IDs"
|
||||
*/
|
||||
static const OSSL_HPKE_KDF_INFO hpke_kdf_tab[] = {
|
||||
{ OSSL_HPKE_KDF_ID_HKDF_SHA256, LN_sha256, SHA256_DIGEST_LENGTH },
|
||||
{ OSSL_HPKE_KDF_ID_HKDF_SHA384, LN_sha384, SHA384_DIGEST_LENGTH },
|
||||
{ OSSL_HPKE_KDF_ID_HKDF_SHA512, LN_sha512, SHA512_DIGEST_LENGTH }
|
||||
};
|
||||
|
||||
/**
|
||||
* Synonym tables for KEMs, KDFs and AEADs: idea is to allow
|
||||
* mapping strings to suites with a little flexibility in terms
|
||||
* of allowing a name or a couple of forms of number (for
|
||||
* the IANA codepoint). If new IANA codepoints are allocated
|
||||
* then these tables should be updated at the same time as the
|
||||
* others above.
|
||||
*
|
||||
* The function to use these is ossl_hpke_str2suite() further down
|
||||
* this file and shouldn't need modification so long as the table
|
||||
* sizes (i.e. allow exactly 4 synonyms) don't change.
|
||||
*/
|
||||
static const synonymttab_t kemstrtab[] = {
|
||||
{OSSL_HPKE_KEM_ID_P256,
|
||||
{OSSL_HPKE_KEMSTR_P256, "0x10", "0x10", "16" }},
|
||||
{OSSL_HPKE_KEM_ID_P384,
|
||||
{OSSL_HPKE_KEMSTR_P384, "0x11", "0x11", "17" }},
|
||||
{OSSL_HPKE_KEM_ID_P521,
|
||||
{OSSL_HPKE_KEMSTR_P521, "0x12", "0x12", "18" }},
|
||||
# ifndef OPENSSL_NO_ECX
|
||||
{OSSL_HPKE_KEM_ID_X25519,
|
||||
{OSSL_HPKE_KEMSTR_X25519, "0x20", "0x20", "32" }},
|
||||
{OSSL_HPKE_KEM_ID_X448,
|
||||
{OSSL_HPKE_KEMSTR_X448, "0x21", "0x21", "33" }}
|
||||
# endif
|
||||
};
|
||||
static const synonymttab_t kdfstrtab[] = {
|
||||
{OSSL_HPKE_KDF_ID_HKDF_SHA256,
|
||||
{OSSL_HPKE_KDFSTR_256, "0x1", "0x01", "1"}},
|
||||
{OSSL_HPKE_KDF_ID_HKDF_SHA384,
|
||||
{OSSL_HPKE_KDFSTR_384, "0x2", "0x02", "2"}},
|
||||
{OSSL_HPKE_KDF_ID_HKDF_SHA512,
|
||||
{OSSL_HPKE_KDFSTR_512, "0x3", "0x03", "3"}}
|
||||
};
|
||||
static const synonymttab_t aeadstrtab[] = {
|
||||
{OSSL_HPKE_AEAD_ID_AES_GCM_128,
|
||||
{OSSL_HPKE_AEADSTR_AES128GCM, "0x1", "0x01", "1"}},
|
||||
{OSSL_HPKE_AEAD_ID_AES_GCM_256,
|
||||
{OSSL_HPKE_AEADSTR_AES256GCM, "0x2", "0x02", "2"}},
|
||||
{OSSL_HPKE_AEAD_ID_CHACHA_POLY1305,
|
||||
{OSSL_HPKE_AEADSTR_CP, "0x3", "0x03", "3"}},
|
||||
{OSSL_HPKE_AEAD_ID_EXPORTONLY,
|
||||
{OSSL_HPKE_AEADSTR_EXP, "ff", "0xff", "255"}}
|
||||
};
|
||||
|
||||
/* Return an object containing KEM constants associated with a EC curve name */
|
||||
const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_curve(const char *curve)
|
||||
{
|
||||
int i, sz = OSSL_NELEM(hpke_kem_tab);
|
||||
|
||||
for (i = 0; i < sz; ++i) {
|
||||
const char *group = hpke_kem_tab[i].groupname;
|
||||
|
||||
if (group == NULL)
|
||||
group = hpke_kem_tab[i].keytype;
|
||||
if (OPENSSL_strcasecmp(curve, group) == 0)
|
||||
return &hpke_kem_tab[i];
|
||||
}
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_id(uint16_t kemid)
|
||||
{
|
||||
int i, sz = OSSL_NELEM(hpke_kem_tab);
|
||||
|
||||
/*
|
||||
* this check can happen if we're in a no-ec build and there are no
|
||||
* KEMS available
|
||||
*/
|
||||
if (kemid == OSSL_HPKE_KEM_ID_RESERVED) {
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE);
|
||||
return NULL;
|
||||
}
|
||||
for (i = 0; i != sz; ++i) {
|
||||
if (hpke_kem_tab[i].kem_id == kemid)
|
||||
return &hpke_kem_tab[i];
|
||||
}
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_random(OSSL_LIB_CTX *ctx)
|
||||
{
|
||||
uint32_t rval = 0;
|
||||
int err = 0;
|
||||
size_t sz = OSSL_NELEM(hpke_kem_tab);
|
||||
|
||||
rval = ossl_rand_uniform_uint32(ctx, sz, &err);
|
||||
return (err == 1 ? NULL : &hpke_kem_tab[rval]);
|
||||
}
|
||||
|
||||
const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_id(uint16_t kdfid)
|
||||
{
|
||||
int i, sz = OSSL_NELEM(hpke_kdf_tab);
|
||||
|
||||
for (i = 0; i != sz; ++i) {
|
||||
if (hpke_kdf_tab[i].kdf_id == kdfid)
|
||||
return &hpke_kdf_tab[i];
|
||||
}
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KDF);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_random(OSSL_LIB_CTX *ctx)
|
||||
{
|
||||
uint32_t rval = 0;
|
||||
int err = 0;
|
||||
size_t sz = OSSL_NELEM(hpke_kdf_tab);
|
||||
|
||||
rval = ossl_rand_uniform_uint32(ctx, sz, &err);
|
||||
return (err == 1 ? NULL : &hpke_kdf_tab[rval]);
|
||||
}
|
||||
|
||||
const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_id(uint16_t aeadid)
|
||||
{
|
||||
int i, sz = OSSL_NELEM(hpke_aead_tab);
|
||||
|
||||
for (i = 0; i != sz; ++i) {
|
||||
if (hpke_aead_tab[i].aead_id == aeadid)
|
||||
return &hpke_aead_tab[i];
|
||||
}
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_AEAD);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_random(OSSL_LIB_CTX *ctx)
|
||||
{
|
||||
uint32_t rval = 0;
|
||||
int err = 0;
|
||||
/* the minus 1 below is so we don't pick the EXPORTONLY codepoint */
|
||||
size_t sz = OSSL_NELEM(hpke_aead_tab) - 1;
|
||||
|
||||
rval = ossl_rand_uniform_uint32(ctx, sz, &err);
|
||||
return (err == 1 ? NULL : &hpke_aead_tab[rval]);
|
||||
}
|
||||
|
||||
static int kdf_derive(EVP_KDF_CTX *kctx,
|
||||
unsigned char *out, size_t outlen, int mode,
|
||||
const unsigned char *salt, size_t saltlen,
|
||||
const unsigned char *ikm, size_t ikmlen,
|
||||
const unsigned char *info, size_t infolen)
|
||||
{
|
||||
int ret;
|
||||
OSSL_PARAM params[5], *p = params;
|
||||
|
||||
*p++ = OSSL_PARAM_construct_int(OSSL_KDF_PARAM_MODE, &mode);
|
||||
if (salt != NULL)
|
||||
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT,
|
||||
(char *)salt, saltlen);
|
||||
if (ikm != NULL)
|
||||
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_KEY,
|
||||
(char *)ikm, ikmlen);
|
||||
if (info != NULL)
|
||||
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_INFO,
|
||||
(char *)info, infolen);
|
||||
*p = OSSL_PARAM_construct_end();
|
||||
ret = EVP_KDF_derive(kctx, out, outlen, params) > 0;
|
||||
if (!ret)
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ossl_hpke_kdf_extract(EVP_KDF_CTX *kctx,
|
||||
unsigned char *prk, size_t prklen,
|
||||
const unsigned char *salt, size_t saltlen,
|
||||
const unsigned char *ikm, size_t ikmlen)
|
||||
{
|
||||
return kdf_derive(kctx, prk, prklen, EVP_KDF_HKDF_MODE_EXTRACT_ONLY,
|
||||
salt, saltlen, ikm, ikmlen, NULL, 0);
|
||||
}
|
||||
|
||||
/* Common code to perform a HKDF expand */
|
||||
int ossl_hpke_kdf_expand(EVP_KDF_CTX *kctx,
|
||||
unsigned char *okm, size_t okmlen,
|
||||
const unsigned char *prk, size_t prklen,
|
||||
const unsigned char *info, size_t infolen)
|
||||
{
|
||||
return kdf_derive(kctx, okm, okmlen, EVP_KDF_HKDF_MODE_EXPAND_ONLY,
|
||||
NULL, 0, prk, prklen, info, infolen);
|
||||
}
|
||||
|
||||
/*
|
||||
* See RFC 9180 Section 4 LabelExtract()
|
||||
*/
|
||||
int ossl_hpke_labeled_extract(EVP_KDF_CTX *kctx,
|
||||
unsigned char *prk, size_t prklen,
|
||||
const unsigned char *salt, size_t saltlen,
|
||||
const char *protocol_label,
|
||||
const unsigned char *suiteid, size_t suiteidlen,
|
||||
const char *label,
|
||||
const unsigned char *ikm, size_t ikmlen)
|
||||
{
|
||||
int ret = 0;
|
||||
size_t label_hpkev1len = 0;
|
||||
size_t protocol_labellen = 0;
|
||||
size_t labellen = 0;
|
||||
size_t labeled_ikmlen = 0;
|
||||
unsigned char *labeled_ikm = NULL;
|
||||
WPACKET pkt;
|
||||
|
||||
label_hpkev1len = strlen(LABEL_HPKEV1);
|
||||
protocol_labellen = strlen(protocol_label);
|
||||
labellen = strlen(label);
|
||||
labeled_ikmlen = label_hpkev1len + protocol_labellen
|
||||
+ suiteidlen + labellen + ikmlen;
|
||||
labeled_ikm = OPENSSL_malloc(labeled_ikmlen);
|
||||
if (labeled_ikm == NULL)
|
||||
return 0;
|
||||
|
||||
/* labeled_ikm = concat("HPKE-v1", suiteid, label, ikm) */
|
||||
if (!WPACKET_init_static_len(&pkt, labeled_ikm, labeled_ikmlen, 0)
|
||||
|| !WPACKET_memcpy(&pkt, LABEL_HPKEV1, label_hpkev1len)
|
||||
|| !WPACKET_memcpy(&pkt, protocol_label, protocol_labellen)
|
||||
|| !WPACKET_memcpy(&pkt, suiteid, suiteidlen)
|
||||
|| !WPACKET_memcpy(&pkt, label, labellen)
|
||||
|| !WPACKET_memcpy(&pkt, ikm, ikmlen)
|
||||
|| !WPACKET_get_total_written(&pkt, &labeled_ikmlen)
|
||||
|| !WPACKET_finish(&pkt)) {
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
|
||||
goto end;
|
||||
}
|
||||
|
||||
ret = ossl_hpke_kdf_extract(kctx, prk, prklen, salt, saltlen,
|
||||
labeled_ikm, labeled_ikmlen);
|
||||
end:
|
||||
WPACKET_cleanup(&pkt);
|
||||
OPENSSL_cleanse(labeled_ikm, labeled_ikmlen);
|
||||
OPENSSL_free(labeled_ikm);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* See RFC 9180 Section 4 LabelExpand()
|
||||
*/
|
||||
int ossl_hpke_labeled_expand(EVP_KDF_CTX *kctx,
|
||||
unsigned char *okm, size_t okmlen,
|
||||
const unsigned char *prk, size_t prklen,
|
||||
const char *protocol_label,
|
||||
const unsigned char *suiteid, size_t suiteidlen,
|
||||
const char *label,
|
||||
const unsigned char *info, size_t infolen)
|
||||
{
|
||||
int ret = 0;
|
||||
size_t label_hpkev1len = 0;
|
||||
size_t protocol_labellen = 0;
|
||||
size_t labellen = 0;
|
||||
size_t labeled_infolen = 0;
|
||||
unsigned char *labeled_info = NULL;
|
||||
WPACKET pkt;
|
||||
|
||||
label_hpkev1len = strlen(LABEL_HPKEV1);
|
||||
protocol_labellen = strlen(protocol_label);
|
||||
labellen = strlen(label);
|
||||
labeled_infolen = 2 + okmlen + prklen + label_hpkev1len
|
||||
+ protocol_labellen + suiteidlen + labellen + infolen;
|
||||
labeled_info = OPENSSL_malloc(labeled_infolen);
|
||||
if (labeled_info == NULL)
|
||||
return 0;
|
||||
|
||||
/* labeled_info = concat(okmlen, "HPKE-v1", suiteid, label, info) */
|
||||
if (!WPACKET_init_static_len(&pkt, labeled_info, labeled_infolen, 0)
|
||||
|| !WPACKET_put_bytes_u16(&pkt, okmlen)
|
||||
|| !WPACKET_memcpy(&pkt, LABEL_HPKEV1, label_hpkev1len)
|
||||
|| !WPACKET_memcpy(&pkt, protocol_label, protocol_labellen)
|
||||
|| !WPACKET_memcpy(&pkt, suiteid, suiteidlen)
|
||||
|| !WPACKET_memcpy(&pkt, label, labellen)
|
||||
|| !WPACKET_memcpy(&pkt, info, infolen)
|
||||
|| !WPACKET_get_total_written(&pkt, &labeled_infolen)
|
||||
|| !WPACKET_finish(&pkt)) {
|
||||
ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
|
||||
goto end;
|
||||
}
|
||||
|
||||
ret = ossl_hpke_kdf_expand(kctx, okm, okmlen,
|
||||
prk, prklen, labeled_info, labeled_infolen);
|
||||
end:
|
||||
WPACKET_cleanup(&pkt);
|
||||
OPENSSL_free(labeled_info);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Common code to create a HKDF ctx */
|
||||
EVP_KDF_CTX *ossl_kdf_ctx_create(const char *kdfname, const char *mdname,
|
||||
OSSL_LIB_CTX *libctx, const char *propq)
|
||||
{
|
||||
EVP_KDF *kdf;
|
||||
EVP_KDF_CTX *kctx = NULL;
|
||||
|
||||
kdf = EVP_KDF_fetch(libctx, kdfname, propq);
|
||||
if (kdf == NULL) {
|
||||
ERR_raise(ERR_LIB_CRYPTO, ERR_R_FETCH_FAILED);
|
||||
return NULL;
|
||||
}
|
||||
kctx = EVP_KDF_CTX_new(kdf);
|
||||
EVP_KDF_free(kdf);
|
||||
if (kctx != NULL && mdname != NULL) {
|
||||
OSSL_PARAM params[3], *p = params;
|
||||
|
||||
if (mdname != NULL)
|
||||
*p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST,
|
||||
(char *)mdname, 0);
|
||||
if (propq != NULL)
|
||||
*p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_PROPERTIES,
|
||||
(char *)propq, 0);
|
||||
*p = OSSL_PARAM_construct_end();
|
||||
if (EVP_KDF_CTX_set_params(kctx, params) <= 0) {
|
||||
EVP_KDF_CTX_free(kctx);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
return kctx;
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief look for a label into the synonym tables, and return its id
|
||||
* @param st is the string value
|
||||
* @param synp is the synonyms labels array
|
||||
* @param arrsize is the previous array size
|
||||
* @return 0 when not found, else the matching item id.
|
||||
*/
|
||||
static uint16_t synonyms_name2id(const char *st, const synonymttab_t *synp,
|
||||
size_t arrsize)
|
||||
{
|
||||
size_t i, j;
|
||||
|
||||
for (i = 0; i < arrsize; ++i) {
|
||||
for (j = 0; j < OSSL_NELEM(synp[i].synonyms); ++j) {
|
||||
if (OPENSSL_strcasecmp(st, synp[i].synonyms[j]) == 0)
|
||||
return synp[i].id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief map a string to a HPKE suite based on synonym tables
|
||||
* @param str is the string value
|
||||
* @param suite is the resulting suite
|
||||
* @return 1 for success, otherwise failure
|
||||
*/
|
||||
int ossl_hpke_str2suite(const char *suitestr, OSSL_HPKE_SUITE *suite)
|
||||
{
|
||||
uint16_t kem = 0, kdf = 0, aead = 0;
|
||||
char *st = NULL, *instrcp = NULL;
|
||||
size_t inplen;
|
||||
int labels = 0, result = 0;
|
||||
int delim_count = 0;
|
||||
|
||||
if (suitestr == NULL || suitestr[0] == 0x00 || suite == NULL) {
|
||||
ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER);
|
||||
return 0;
|
||||
}
|
||||
inplen = OPENSSL_strnlen(suitestr, OSSL_HPKE_MAX_SUITESTR);
|
||||
if (inplen >= OSSL_HPKE_MAX_SUITESTR) {
|
||||
ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* we don't want a delimiter at the end of the string;
|
||||
* strtok_r/s() doesn't care about that, so we should
|
||||
*/
|
||||
if (suitestr[inplen - 1] == OSSL_HPKE_STR_DELIMCHAR)
|
||||
return 0;
|
||||
/* We want exactly two delimiters in the input string */
|
||||
for (st = (char *)suitestr; *st != '\0'; st++) {
|
||||
if (*st == OSSL_HPKE_STR_DELIMCHAR)
|
||||
delim_count++;
|
||||
}
|
||||
if (delim_count != 2)
|
||||
return 0;
|
||||
|
||||
/* Duplicate `suitestr` to allow its parsing */
|
||||
instrcp = OPENSSL_memdup(suitestr, inplen + 1);
|
||||
if (instrcp == NULL)
|
||||
goto fail;
|
||||
|
||||
/* See if it contains a mix of our strings and numbers */
|
||||
st = instrcp;
|
||||
|
||||
while (st != NULL && labels < 3) {
|
||||
char *cp = strchr(st, OSSL_HPKE_STR_DELIMCHAR);
|
||||
|
||||
/* add a NUL like strtok would if we're not at the end */
|
||||
if (cp != NULL)
|
||||
*cp = '\0';
|
||||
|
||||
/* check if string is known or number and if so handle appropriately */
|
||||
if (labels == 0
|
||||
&& (kem = synonyms_name2id(st, kemstrtab,
|
||||
OSSL_NELEM(kemstrtab))) == 0)
|
||||
goto fail;
|
||||
else if (labels == 1
|
||||
&& (kdf = synonyms_name2id(st, kdfstrtab,
|
||||
OSSL_NELEM(kdfstrtab))) == 0)
|
||||
goto fail;
|
||||
else if (labels == 2
|
||||
&& (aead = synonyms_name2id(st, aeadstrtab,
|
||||
OSSL_NELEM(aeadstrtab))) == 0)
|
||||
goto fail;
|
||||
|
||||
if (cp == NULL)
|
||||
st = NULL;
|
||||
else
|
||||
st = cp + 1;
|
||||
++labels;
|
||||
}
|
||||
if (st != NULL || labels != 3)
|
||||
goto fail;
|
||||
suite->kem_id = kem;
|
||||
suite->kdf_id = kdf;
|
||||
suite->aead_id = aead;
|
||||
result = 1;
|
||||
|
||||
fail:
|
||||
OPENSSL_free(instrcp);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/indicator.h>
|
||||
#include <openssl/core_names.h>
|
||||
#include <openssl/params.h>
|
||||
#include "internal/cryptlib.h"
|
||||
#include "crypto/context.h"
|
||||
|
||||
typedef struct indicator_cb_st
|
||||
{
|
||||
OSSL_INDICATOR_CALLBACK *cb;
|
||||
} INDICATOR_CB;
|
||||
|
||||
void *ossl_indicator_set_callback_new(OSSL_LIB_CTX *ctx)
|
||||
{
|
||||
INDICATOR_CB *cb;
|
||||
|
||||
cb = OPENSSL_zalloc(sizeof(*cb));
|
||||
return cb;
|
||||
}
|
||||
|
||||
void ossl_indicator_set_callback_free(void *cb)
|
||||
{
|
||||
OPENSSL_free(cb);
|
||||
}
|
||||
|
||||
static INDICATOR_CB *get_indicator_callback(OSSL_LIB_CTX *libctx)
|
||||
{
|
||||
return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_INDICATOR_CB_INDEX);
|
||||
}
|
||||
|
||||
void OSSL_INDICATOR_set_callback(OSSL_LIB_CTX *libctx,
|
||||
OSSL_INDICATOR_CALLBACK *cb)
|
||||
{
|
||||
INDICATOR_CB *icb = get_indicator_callback(libctx);
|
||||
|
||||
if (icb != NULL)
|
||||
icb->cb = cb;
|
||||
}
|
||||
|
||||
void OSSL_INDICATOR_get_callback(OSSL_LIB_CTX *libctx,
|
||||
OSSL_INDICATOR_CALLBACK **cb)
|
||||
{
|
||||
INDICATOR_CB *icb = get_indicator_callback(libctx);
|
||||
|
||||
if (cb != NULL)
|
||||
*cb = (icb != NULL ? icb->cb : NULL);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
#! /usr/bin/env perl
|
||||
# Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
($zero,$ra,$tp,$sp)=map("\$r$_",(0..3));
|
||||
($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11));
|
||||
($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9)=map("\$r$_",(12..21));
|
||||
($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$r$_",(23..30));
|
||||
($vr0,$vr1,$vr2,$vr3,$vr4,$vr5,$vr6,$vr7,$vr8,$vr9,$vr10,$vr11,$vr12,$vr13,$vr14,$vr15,$vr16,$vr17,$vr18,$vr19)=map("\$vr$_",(0..19));
|
||||
($fp)=map("\$r$_",(22));
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
my $output;
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
open STDOUT,">$output";
|
||||
|
||||
{
|
||||
my ($in_a,$in_b,$len,$m,$temp1,$temp2) = ($a0,$a1,$a2,$t0,$t1,$t2);
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl CRYPTO_memcmp
|
||||
.type CRYPTO_memcmp,\@function
|
||||
CRYPTO_memcmp:
|
||||
li.d $m,0
|
||||
beqz $len,2f # len == 0
|
||||
1:
|
||||
ld.bu $temp1,$in_a,0
|
||||
ld.bu $temp2,$in_b,0
|
||||
addi.d $in_a,$in_a,1
|
||||
addi.d $in_b,$in_b,1
|
||||
addi.d $len,$len,-1
|
||||
xor $temp1,$temp1,$temp2
|
||||
or $m,$m,$temp1
|
||||
blt $zero,$len,1b
|
||||
2:
|
||||
move $a0,$m
|
||||
jr $ra
|
||||
___
|
||||
}
|
||||
{
|
||||
my ($ptr,$len,$temp1,$temp2) = ($a0,$a1,$t0,$t1);
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# void OPENSSL_cleanse(void *ptr, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl OPENSSL_cleanse
|
||||
.type OPENSSL_cleanse,\@function
|
||||
OPENSSL_cleanse:
|
||||
beqz $len,2f # len == 0, return
|
||||
srli.d $temp1,$len,4
|
||||
bnez $temp1,3f # len > 15
|
||||
|
||||
1: # Store <= 15 individual bytes
|
||||
st.b $zero,$ptr,0
|
||||
addi.d $ptr,$ptr,1
|
||||
addi.d $len,$len,-1
|
||||
bnez $len,1b
|
||||
2:
|
||||
jr $ra
|
||||
|
||||
3: # Store individual bytes until we are aligned
|
||||
andi $temp1,$ptr,0x7
|
||||
beqz $temp1,4f
|
||||
st.b $zero,$ptr,0
|
||||
addi.d $ptr,$ptr,1
|
||||
addi.d $len,$len,-1
|
||||
b 3b
|
||||
|
||||
4: # Store aligned dwords
|
||||
li.d $temp2,8
|
||||
4:
|
||||
st.d $zero,$ptr,0
|
||||
addi.d $ptr,$ptr,8
|
||||
addi.d $len,$len,-8
|
||||
bge $len,$temp2,4b # if len>=8 loop
|
||||
bnez $len,1b # if len<8 and len != 0, store remaining bytes
|
||||
jr $ra
|
||||
___
|
||||
}
|
||||
{
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# uint32_t OPENSSL_rdtsc(void)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl OPENSSL_rdtsc
|
||||
.type OPENSSL_rdtsc,\@function
|
||||
OPENSSL_rdtsc:
|
||||
rdtimel.w $a0,$zero
|
||||
jr $ra
|
||||
___
|
||||
}
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval($1)/gem;
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
#ifndef OSSL_CRYPTO_LOONGARCH_ARCH_H
|
||||
# define OSSL_CRYPTO_LOONGARCH_ARCH_H
|
||||
|
||||
# ifndef __ASSEMBLER__
|
||||
extern unsigned int OPENSSL_loongarch_hwcap_P;
|
||||
# endif
|
||||
|
||||
# define LOONGARCH_HWCAP_LSX (1 << 4)
|
||||
# define LOONGARCH_HWCAP_LASX (1 << 5)
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
#include <sys/auxv.h>
|
||||
#include "loongarch_arch.h"
|
||||
|
||||
unsigned int OPENSSL_loongarch_hwcap_P = 0;
|
||||
|
||||
void OPENSSL_cpuid_setup(void)
|
||||
{
|
||||
OPENSSL_loongarch_hwcap_P = getauxval(AT_HWCAP);
|
||||
}
|
||||
Executable
+715
@@ -0,0 +1,715 @@
|
||||
#! /usr/bin/env perl
|
||||
# Copyright 2021-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
# MD5 optimized for aarch64.
|
||||
|
||||
use strict;
|
||||
|
||||
my $code;
|
||||
|
||||
#no warnings qw(uninitialized);
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; my $xlate;
|
||||
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
|
||||
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
|
||||
die "can't locate arm-xlate.pl";
|
||||
|
||||
open OUT,"| \"$^X\" $xlate $flavour \"$output\""
|
||||
or die "can't call $xlate: $1";
|
||||
*STDOUT=*OUT;
|
||||
|
||||
$code .= <<EOF;
|
||||
#include "arm_arch.h"
|
||||
|
||||
.text
|
||||
.globl ossl_md5_block_asm_data_order
|
||||
.type ossl_md5_block_asm_data_order,\@function
|
||||
ossl_md5_block_asm_data_order:
|
||||
AARCH64_VALID_CALL_TARGET
|
||||
// Save all callee-saved registers
|
||||
stp x19,x20,[sp,#-80]!
|
||||
stp x21,x22,[sp,#16]
|
||||
stp x23,x24,[sp,#32]
|
||||
stp x25,x26,[sp,#48]
|
||||
stp x27,x28,[sp,#64]
|
||||
|
||||
ldp w10, w11, [x0, #0] // Load MD5 state->A and state->B
|
||||
ldp w12, w13, [x0, #8] // Load MD5 state->C and state->D
|
||||
.align 5
|
||||
ossl_md5_blocks_loop:
|
||||
eor x17, x12, x13 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
and x16, x17, x11 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
ldp w15, w20, [x1] // Load 2 words of input data0 M[0],M[1]
|
||||
ldp w3, w21, [x1, #8] // Load 2 words of input data0 M[2],M[3]
|
||||
#ifdef __AARCH64EB__
|
||||
rev w15, w15
|
||||
rev w20, w20
|
||||
rev w3, w3
|
||||
rev w21, w21
|
||||
#endif
|
||||
eor x14, x16, x13 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x9, #0xa478 // Load lower half of constant 0xd76aa478
|
||||
movk x9, #0xd76a, lsl #16 // Load upper half of constant 0xd76aa478
|
||||
add w8, w10, w15 // Add dest value
|
||||
add w7, w8, w9 // Add constant 0xd76aa478
|
||||
add w6, w7, w14 // Add aux function result
|
||||
ror w6, w6, #25 // Rotate left s=7 bits
|
||||
eor x5, x11, x12 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w4, w11, w6 // Add X parameter round 1 A=FF(A, B, C, D, 0xd76aa478, s=7, M[0])
|
||||
and x8, x5, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x17, x8, x12 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x16, #0xb756 // Load lower half of constant 0xe8c7b756
|
||||
movk x16, #0xe8c7, lsl #16 // Load upper half of constant 0xe8c7b756
|
||||
add w9, w13, w20 // Add dest value
|
||||
add w7, w9, w16 // Add constant 0xe8c7b756
|
||||
add w14, w7, w17 // Add aux function result
|
||||
ror w14, w14, #20 // Rotate left s=12 bits
|
||||
eor x6, x4, x11 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w5, w4, w14 // Add X parameter round 1 D=FF(D, A, B, C, 0xe8c7b756, s=12, M[1])
|
||||
and x8, x6, x5 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x9, x8, x11 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x16, #0x70db // Load lower half of constant 0x242070db
|
||||
movk x16, #0x2420, lsl #16 // Load upper half of constant 0x242070db
|
||||
add w7, w12, w3 // Add dest value
|
||||
add w17, w7, w16 // Add constant 0x242070db
|
||||
add w14, w17, w9 // Add aux function result
|
||||
ror w14, w14, #15 // Rotate left s=17 bits
|
||||
eor x6, x5, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w8, w5, w14 // Add X parameter round 1 C=FF(C, D, A, B, 0x242070db, s=17, M[2])
|
||||
and x7, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x16, x7, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x9, #0xceee // Load lower half of constant 0xc1bdceee
|
||||
movk x9, #0xc1bd, lsl #16 // Load upper half of constant 0xc1bdceee
|
||||
add w14, w11, w21 // Add dest value
|
||||
add w6, w14, w9 // Add constant 0xc1bdceee
|
||||
add w7, w6, w16 // Add aux function result
|
||||
ror w7, w7, #10 // Rotate left s=22 bits
|
||||
eor x17, x8, x5 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w9, w8, w7 // Add X parameter round 1 B=FF(B, C, D, A, 0xc1bdceee, s=22, M[3])
|
||||
ldp w14, w22, [x1, #16] // Load 2 words of input data0 M[4],M[5]
|
||||
ldp w7, w23, [x1, #24] // Load 2 words of input data0 M[6],M[7]
|
||||
#ifdef __AARCH64EB__
|
||||
rev w14, w14
|
||||
rev w22, w22
|
||||
rev w7, w7
|
||||
rev w23, w23
|
||||
#endif
|
||||
and x16, x17, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x16, x5 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x16, #0xfaf // Load lower half of constant 0xf57c0faf
|
||||
movk x16, #0xf57c, lsl #16 // Load upper half of constant 0xf57c0faf
|
||||
add w17, w4, w14 // Add dest value
|
||||
add w16, w17, w16 // Add constant 0xf57c0faf
|
||||
add w4, w16, w6 // Add aux function result
|
||||
ror w4, w4, #25 // Rotate left s=7 bits
|
||||
eor x16, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w17, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0xf57c0faf, s=7, M[4])
|
||||
and x16, x16, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x16, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x4, #0xc62a // Load lower half of constant 0x4787c62a
|
||||
movk x4, #0x4787, lsl #16 // Load upper half of constant 0x4787c62a
|
||||
add w16, w5, w22 // Add dest value
|
||||
add w16, w16, w4 // Add constant 0x4787c62a
|
||||
add w5, w16, w6 // Add aux function result
|
||||
ror w5, w5, #20 // Rotate left s=12 bits
|
||||
eor x4, x17, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w19, w17, w5 // Add X parameter round 1 D=FF(D, A, B, C, 0x4787c62a, s=12, M[5])
|
||||
and x6, x4, x19 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x5, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x4, #0x4613 // Load lower half of constant 0xa8304613
|
||||
movk x4, #0xa830, lsl #16 // Load upper half of constant 0xa8304613
|
||||
add w6, w8, w7 // Add dest value
|
||||
add w8, w6, w4 // Add constant 0xa8304613
|
||||
add w4, w8, w5 // Add aux function result
|
||||
ror w4, w4, #15 // Rotate left s=17 bits
|
||||
eor x6, x19, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w8, w19, w4 // Add X parameter round 1 C=FF(C, D, A, B, 0xa8304613, s=17, M[6])
|
||||
and x5, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x4, x5, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x6, #0x9501 // Load lower half of constant 0xfd469501
|
||||
movk x6, #0xfd46, lsl #16 // Load upper half of constant 0xfd469501
|
||||
add w9, w9, w23 // Add dest value
|
||||
add w5, w9, w6 // Add constant 0xfd469501
|
||||
add w9, w5, w4 // Add aux function result
|
||||
ror w9, w9, #10 // Rotate left s=22 bits
|
||||
eor x6, x8, x19 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w4, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0xfd469501, s=22, M[7])
|
||||
ldp w5, w24, [x1, #32] // Load 2 words of input data0 M[8],M[9]
|
||||
ldp w16, w25, [x1, #40] // Load 2 words of input data0 M[10],M[11]
|
||||
#ifdef __AARCH64EB__
|
||||
rev w5, w5
|
||||
rev w24, w24
|
||||
rev w16, w16
|
||||
rev w25, w25
|
||||
#endif
|
||||
and x9, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x9, x19 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x9, #0x98d8 // Load lower half of constant 0x698098d8
|
||||
movk x9, #0x6980, lsl #16 // Load upper half of constant 0x698098d8
|
||||
add w17, w17, w5 // Add dest value
|
||||
add w9, w17, w9 // Add constant 0x698098d8
|
||||
add w17, w9, w6 // Add aux function result
|
||||
ror w17, w17, #25 // Rotate left s=7 bits
|
||||
eor x9, x4, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w6, w4, w17 // Add X parameter round 1 A=FF(A, B, C, D, 0x698098d8, s=7, M[8])
|
||||
and x17, x9, x6 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x9, x17, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x17, #0xf7af // Load lower half of constant 0x8b44f7af
|
||||
movk x17, #0x8b44, lsl #16 // Load upper half of constant 0x8b44f7af
|
||||
add w19, w19, w24 // Add dest value
|
||||
add w17, w19, w17 // Add constant 0x8b44f7af
|
||||
add w19, w17, w9 // Add aux function result
|
||||
ror w19, w19, #20 // Rotate left s=12 bits
|
||||
eor x9, x6, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w17, w6, w19 // Add X parameter round 1 D=FF(D, A, B, C, 0x8b44f7af, s=12, M[9])
|
||||
and x9, x9, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x9, x9, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x11, #0x5bb1 // Load lower half of constant 0xffff5bb1
|
||||
movk x11, #0xffff, lsl #16 // Load upper half of constant 0xffff5bb1
|
||||
add w8, w8, w16 // Add dest value
|
||||
add w8, w8, w11 // Add constant 0xffff5bb1
|
||||
add w8, w8, w9 // Add aux function result
|
||||
ror w8, w8, #15 // Rotate left s=17 bits
|
||||
eor x9, x17, x6 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xffff5bb1, s=17, M[10])
|
||||
and x9, x9, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x9, x9, x6 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x11, #0xd7be // Load lower half of constant 0x895cd7be
|
||||
movk x11, #0x895c, lsl #16 // Load upper half of constant 0x895cd7be
|
||||
add w4, w4, w25 // Add dest value
|
||||
add w4, w4, w11 // Add constant 0x895cd7be
|
||||
add w9, w4, w9 // Add aux function result
|
||||
ror w9, w9, #10 // Rotate left s=22 bits
|
||||
eor x4, x8, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x895cd7be, s=22, M[11])
|
||||
ldp w11, w26, [x1, #48] // Load 2 words of input data0 M[12],M[13]
|
||||
ldp w12, w27, [x1, #56] // Load 2 words of input data0 M[14],M[15]
|
||||
#ifdef __AARCH64EB__
|
||||
rev w11, w11
|
||||
rev w26, w26
|
||||
rev w12, w12
|
||||
rev w27, w27
|
||||
#endif
|
||||
and x4, x4, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x4, x4, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x19, #0x1122 // Load lower half of constant 0x6b901122
|
||||
movk x19, #0x6b90, lsl #16 // Load upper half of constant 0x6b901122
|
||||
add w6, w6, w11 // Add dest value
|
||||
add w6, w6, w19 // Add constant 0x6b901122
|
||||
add w4, w6, w4 // Add aux function result
|
||||
ror w4, w4, #25 // Rotate left s=7 bits
|
||||
eor x6, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w4, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0x6b901122, s=7, M[12])
|
||||
and x6, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x6, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x19, #0x7193 // Load lower half of constant 0xfd987193
|
||||
movk x19, #0xfd98, lsl #16 // Load upper half of constant 0xfd987193
|
||||
add w17, w17, w26 // Add dest value
|
||||
add w17, w17, w19 // Add constant 0xfd987193
|
||||
add w17, w17, w6 // Add aux function result
|
||||
ror w17, w17, #20 // Rotate left s=12 bits
|
||||
eor x6, x4, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w17, w4, w17 // Add X parameter round 1 D=FF(D, A, B, C, 0xfd987193, s=12, M[13])
|
||||
and x6, x6, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x13, #0x438e // Load lower half of constant 0xa679438e
|
||||
movk x13, #0xa679, lsl #16 // Load upper half of constant 0xa679438e
|
||||
add w8, w8, w12 // Add dest value
|
||||
add w8, w8, w13 // Add constant 0xa679438e
|
||||
add w8, w8, w6 // Add aux function result
|
||||
ror w8, w8, #15 // Rotate left s=17 bits
|
||||
eor x6, x17, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xa679438e, s=17, M[14])
|
||||
and x6, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
eor x6, x6, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z)
|
||||
movz x13, #0x821 // Load lower half of constant 0x49b40821
|
||||
movk x13, #0x49b4, lsl #16 // Load upper half of constant 0x49b40821
|
||||
add w9, w9, w27 // Add dest value
|
||||
add w9, w9, w13 // Add constant 0x49b40821
|
||||
add w9, w9, w6 // Add aux function result
|
||||
ror w9, w9, #10 // Rotate left s=22 bits
|
||||
bic x6, x8, x17 // Aux function round 2 (~z & y)
|
||||
add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x49b40821, s=22, M[15])
|
||||
movz x13, #0x2562 // Load lower half of constant 0xf61e2562
|
||||
movk x13, #0xf61e, lsl #16 // Load upper half of constant 0xf61e2562
|
||||
add w4, w4, w20 // Add dest value
|
||||
add w4, w4, w13 // Add constant 0xf61e2562
|
||||
and x13, x9, x17 // Aux function round 2 (x & z)
|
||||
add w4, w4, w6 // Add (~z & y)
|
||||
add w4, w4, w13 // Add (x & z)
|
||||
ror w4, w4, #27 // Rotate left s=5 bits
|
||||
bic x6, x9, x8 // Aux function round 2 (~z & y)
|
||||
add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xf61e2562, s=5, M[1])
|
||||
movz x13, #0xb340 // Load lower half of constant 0xc040b340
|
||||
movk x13, #0xc040, lsl #16 // Load upper half of constant 0xc040b340
|
||||
add w17, w17, w7 // Add dest value
|
||||
add w17, w17, w13 // Add constant 0xc040b340
|
||||
and x13, x4, x8 // Aux function round 2 (x & z)
|
||||
add w17, w17, w6 // Add (~z & y)
|
||||
add w17, w17, w13 // Add (x & z)
|
||||
ror w17, w17, #23 // Rotate left s=9 bits
|
||||
bic x6, x4, x9 // Aux function round 2 (~z & y)
|
||||
add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc040b340, s=9, M[6])
|
||||
movz x13, #0x5a51 // Load lower half of constant 0x265e5a51
|
||||
movk x13, #0x265e, lsl #16 // Load upper half of constant 0x265e5a51
|
||||
add w8, w8, w25 // Add dest value
|
||||
add w8, w8, w13 // Add constant 0x265e5a51
|
||||
and x13, x17, x9 // Aux function round 2 (x & z)
|
||||
add w8, w8, w6 // Add (~z & y)
|
||||
add w8, w8, w13 // Add (x & z)
|
||||
ror w8, w8, #18 // Rotate left s=14 bits
|
||||
bic x6, x17, x4 // Aux function round 2 (~z & y)
|
||||
add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x265e5a51, s=14, M[11])
|
||||
movz x13, #0xc7aa // Load lower half of constant 0xe9b6c7aa
|
||||
movk x13, #0xe9b6, lsl #16 // Load upper half of constant 0xe9b6c7aa
|
||||
add w9, w9, w15 // Add dest value
|
||||
add w9, w9, w13 // Add constant 0xe9b6c7aa
|
||||
and x13, x8, x4 // Aux function round 2 (x & z)
|
||||
add w9, w9, w6 // Add (~z & y)
|
||||
add w9, w9, w13 // Add (x & z)
|
||||
ror w9, w9, #12 // Rotate left s=20 bits
|
||||
bic x6, x8, x17 // Aux function round 2 (~z & y)
|
||||
add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe9b6c7aa, s=20, M[0])
|
||||
movz x13, #0x105d // Load lower half of constant 0xd62f105d
|
||||
movk x13, #0xd62f, lsl #16 // Load upper half of constant 0xd62f105d
|
||||
add w4, w4, w22 // Add dest value
|
||||
add w4, w4, w13 // Add constant 0xd62f105d
|
||||
and x13, x9, x17 // Aux function round 2 (x & z)
|
||||
add w4, w4, w6 // Add (~z & y)
|
||||
add w4, w4, w13 // Add (x & z)
|
||||
ror w4, w4, #27 // Rotate left s=5 bits
|
||||
bic x6, x9, x8 // Aux function round 2 (~z & y)
|
||||
add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xd62f105d, s=5, M[5])
|
||||
movz x13, #0x1453 // Load lower half of constant 0x2441453
|
||||
movk x13, #0x244, lsl #16 // Load upper half of constant 0x2441453
|
||||
add w17, w17, w16 // Add dest value
|
||||
add w17, w17, w13 // Add constant 0x2441453
|
||||
and x13, x4, x8 // Aux function round 2 (x & z)
|
||||
add w17, w17, w6 // Add (~z & y)
|
||||
add w17, w17, w13 // Add (x & z)
|
||||
ror w17, w17, #23 // Rotate left s=9 bits
|
||||
bic x6, x4, x9 // Aux function round 2 (~z & y)
|
||||
add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0x2441453, s=9, M[10])
|
||||
movz x13, #0xe681 // Load lower half of constant 0xd8a1e681
|
||||
movk x13, #0xd8a1, lsl #16 // Load upper half of constant 0xd8a1e681
|
||||
add w8, w8, w27 // Add dest value
|
||||
add w8, w8, w13 // Add constant 0xd8a1e681
|
||||
and x13, x17, x9 // Aux function round 2 (x & z)
|
||||
add w8, w8, w6 // Add (~z & y)
|
||||
add w8, w8, w13 // Add (x & z)
|
||||
ror w8, w8, #18 // Rotate left s=14 bits
|
||||
bic x6, x17, x4 // Aux function round 2 (~z & y)
|
||||
add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xd8a1e681, s=14, M[15])
|
||||
movz x13, #0xfbc8 // Load lower half of constant 0xe7d3fbc8
|
||||
movk x13, #0xe7d3, lsl #16 // Load upper half of constant 0xe7d3fbc8
|
||||
add w9, w9, w14 // Add dest value
|
||||
add w9, w9, w13 // Add constant 0xe7d3fbc8
|
||||
and x13, x8, x4 // Aux function round 2 (x & z)
|
||||
add w9, w9, w6 // Add (~z & y)
|
||||
add w9, w9, w13 // Add (x & z)
|
||||
ror w9, w9, #12 // Rotate left s=20 bits
|
||||
bic x6, x8, x17 // Aux function round 2 (~z & y)
|
||||
add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe7d3fbc8, s=20, M[4])
|
||||
movz x13, #0xcde6 // Load lower half of constant 0x21e1cde6
|
||||
movk x13, #0x21e1, lsl #16 // Load upper half of constant 0x21e1cde6
|
||||
add w4, w4, w24 // Add dest value
|
||||
add w4, w4, w13 // Add constant 0x21e1cde6
|
||||
and x13, x9, x17 // Aux function round 2 (x & z)
|
||||
add w4, w4, w6 // Add (~z & y)
|
||||
add w4, w4, w13 // Add (x & z)
|
||||
ror w4, w4, #27 // Rotate left s=5 bits
|
||||
bic x6, x9, x8 // Aux function round 2 (~z & y)
|
||||
add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0x21e1cde6, s=5, M[9])
|
||||
movz x13, #0x7d6 // Load lower half of constant 0xc33707d6
|
||||
movk x13, #0xc337, lsl #16 // Load upper half of constant 0xc33707d6
|
||||
add w17, w17, w12 // Add dest value
|
||||
add w17, w17, w13 // Add constant 0xc33707d6
|
||||
and x13, x4, x8 // Aux function round 2 (x & z)
|
||||
add w17, w17, w6 // Add (~z & y)
|
||||
add w17, w17, w13 // Add (x & z)
|
||||
ror w17, w17, #23 // Rotate left s=9 bits
|
||||
bic x6, x4, x9 // Aux function round 2 (~z & y)
|
||||
add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc33707d6, s=9, M[14])
|
||||
movz x13, #0xd87 // Load lower half of constant 0xf4d50d87
|
||||
movk x13, #0xf4d5, lsl #16 // Load upper half of constant 0xf4d50d87
|
||||
add w8, w8, w21 // Add dest value
|
||||
add w8, w8, w13 // Add constant 0xf4d50d87
|
||||
and x13, x17, x9 // Aux function round 2 (x & z)
|
||||
add w8, w8, w6 // Add (~z & y)
|
||||
add w8, w8, w13 // Add (x & z)
|
||||
ror w8, w8, #18 // Rotate left s=14 bits
|
||||
bic x6, x17, x4 // Aux function round 2 (~z & y)
|
||||
add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xf4d50d87, s=14, M[3])
|
||||
movz x13, #0x14ed // Load lower half of constant 0x455a14ed
|
||||
movk x13, #0x455a, lsl #16 // Load upper half of constant 0x455a14ed
|
||||
add w9, w9, w5 // Add dest value
|
||||
add w9, w9, w13 // Add constant 0x455a14ed
|
||||
and x13, x8, x4 // Aux function round 2 (x & z)
|
||||
add w9, w9, w6 // Add (~z & y)
|
||||
add w9, w9, w13 // Add (x & z)
|
||||
ror w9, w9, #12 // Rotate left s=20 bits
|
||||
bic x6, x8, x17 // Aux function round 2 (~z & y)
|
||||
add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x455a14ed, s=20, M[8])
|
||||
movz x13, #0xe905 // Load lower half of constant 0xa9e3e905
|
||||
movk x13, #0xa9e3, lsl #16 // Load upper half of constant 0xa9e3e905
|
||||
add w4, w4, w26 // Add dest value
|
||||
add w4, w4, w13 // Add constant 0xa9e3e905
|
||||
and x13, x9, x17 // Aux function round 2 (x & z)
|
||||
add w4, w4, w6 // Add (~z & y)
|
||||
add w4, w4, w13 // Add (x & z)
|
||||
ror w4, w4, #27 // Rotate left s=5 bits
|
||||
bic x6, x9, x8 // Aux function round 2 (~z & y)
|
||||
add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xa9e3e905, s=5, M[13])
|
||||
movz x13, #0xa3f8 // Load lower half of constant 0xfcefa3f8
|
||||
movk x13, #0xfcef, lsl #16 // Load upper half of constant 0xfcefa3f8
|
||||
add w17, w17, w3 // Add dest value
|
||||
add w17, w17, w13 // Add constant 0xfcefa3f8
|
||||
and x13, x4, x8 // Aux function round 2 (x & z)
|
||||
add w17, w17, w6 // Add (~z & y)
|
||||
add w17, w17, w13 // Add (x & z)
|
||||
ror w17, w17, #23 // Rotate left s=9 bits
|
||||
bic x6, x4, x9 // Aux function round 2 (~z & y)
|
||||
add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xfcefa3f8, s=9, M[2])
|
||||
movz x13, #0x2d9 // Load lower half of constant 0x676f02d9
|
||||
movk x13, #0x676f, lsl #16 // Load upper half of constant 0x676f02d9
|
||||
add w8, w8, w23 // Add dest value
|
||||
add w8, w8, w13 // Add constant 0x676f02d9
|
||||
and x13, x17, x9 // Aux function round 2 (x & z)
|
||||
add w8, w8, w6 // Add (~z & y)
|
||||
add w8, w8, w13 // Add (x & z)
|
||||
ror w8, w8, #18 // Rotate left s=14 bits
|
||||
bic x6, x17, x4 // Aux function round 2 (~z & y)
|
||||
add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x676f02d9, s=14, M[7])
|
||||
movz x13, #0x4c8a // Load lower half of constant 0x8d2a4c8a
|
||||
movk x13, #0x8d2a, lsl #16 // Load upper half of constant 0x8d2a4c8a
|
||||
add w9, w9, w11 // Add dest value
|
||||
add w9, w9, w13 // Add constant 0x8d2a4c8a
|
||||
and x13, x8, x4 // Aux function round 2 (x & z)
|
||||
add w9, w9, w6 // Add (~z & y)
|
||||
add w9, w9, w13 // Add (x & z)
|
||||
eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w9, w9, #12 // Rotate left s=20 bits
|
||||
movz x10, #0x3942 // Load lower half of constant 0xfffa3942
|
||||
add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x8d2a4c8a, s=20, M[12])
|
||||
movk x10, #0xfffa, lsl #16 // Load upper half of constant 0xfffa3942
|
||||
add w4, w4, w22 // Add dest value
|
||||
eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w4, w4, w10 // Add constant 0xfffa3942
|
||||
add w4, w4, w6 // Add aux function result
|
||||
ror w4, w4, #28 // Rotate left s=4 bits
|
||||
eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x10, #0xf681 // Load lower half of constant 0x8771f681
|
||||
add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xfffa3942, s=4, M[5])
|
||||
movk x10, #0x8771, lsl #16 // Load upper half of constant 0x8771f681
|
||||
add w17, w17, w5 // Add dest value
|
||||
eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w17, w17, w10 // Add constant 0x8771f681
|
||||
add w17, w17, w6 // Add aux function result
|
||||
eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w17, w17, #21 // Rotate left s=11 bits
|
||||
movz x13, #0x6122 // Load lower half of constant 0x6d9d6122
|
||||
add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x8771f681, s=11, M[8])
|
||||
movk x13, #0x6d9d, lsl #16 // Load upper half of constant 0x6d9d6122
|
||||
add w8, w8, w25 // Add dest value
|
||||
eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w8, w8, w13 // Add constant 0x6d9d6122
|
||||
add w8, w8, w6 // Add aux function result
|
||||
ror w8, w8, #16 // Rotate left s=16 bits
|
||||
eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x13, #0x380c // Load lower half of constant 0xfde5380c
|
||||
add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x6d9d6122, s=16, M[11])
|
||||
movk x13, #0xfde5, lsl #16 // Load upper half of constant 0xfde5380c
|
||||
add w9, w9, w12 // Add dest value
|
||||
eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w9, w9, w13 // Add constant 0xfde5380c
|
||||
add w9, w9, w6 // Add aux function result
|
||||
eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w9, w9, #9 // Rotate left s=23 bits
|
||||
movz x10, #0xea44 // Load lower half of constant 0xa4beea44
|
||||
add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xfde5380c, s=23, M[14])
|
||||
movk x10, #0xa4be, lsl #16 // Load upper half of constant 0xa4beea44
|
||||
add w4, w4, w20 // Add dest value
|
||||
eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w4, w4, w10 // Add constant 0xa4beea44
|
||||
add w4, w4, w6 // Add aux function result
|
||||
ror w4, w4, #28 // Rotate left s=4 bits
|
||||
eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x10, #0xcfa9 // Load lower half of constant 0x4bdecfa9
|
||||
add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xa4beea44, s=4, M[1])
|
||||
movk x10, #0x4bde, lsl #16 // Load upper half of constant 0x4bdecfa9
|
||||
add w17, w17, w14 // Add dest value
|
||||
eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w17, w17, w10 // Add constant 0x4bdecfa9
|
||||
add w17, w17, w6 // Add aux function result
|
||||
eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w17, w17, #21 // Rotate left s=11 bits
|
||||
movz x13, #0x4b60 // Load lower half of constant 0xf6bb4b60
|
||||
add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x4bdecfa9, s=11, M[4])
|
||||
movk x13, #0xf6bb, lsl #16 // Load upper half of constant 0xf6bb4b60
|
||||
add w8, w8, w23 // Add dest value
|
||||
eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w8, w8, w13 // Add constant 0xf6bb4b60
|
||||
add w8, w8, w6 // Add aux function result
|
||||
ror w8, w8, #16 // Rotate left s=16 bits
|
||||
eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x13, #0xbc70 // Load lower half of constant 0xbebfbc70
|
||||
add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xf6bb4b60, s=16, M[7])
|
||||
movk x13, #0xbebf, lsl #16 // Load upper half of constant 0xbebfbc70
|
||||
add w9, w9, w16 // Add dest value
|
||||
eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w9, w9, w13 // Add constant 0xbebfbc70
|
||||
add w9, w9, w6 // Add aux function result
|
||||
eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w9, w9, #9 // Rotate left s=23 bits
|
||||
movz x10, #0x7ec6 // Load lower half of constant 0x289b7ec6
|
||||
add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xbebfbc70, s=23, M[10])
|
||||
movk x10, #0x289b, lsl #16 // Load upper half of constant 0x289b7ec6
|
||||
add w4, w4, w26 // Add dest value
|
||||
eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w4, w4, w10 // Add constant 0x289b7ec6
|
||||
add w4, w4, w6 // Add aux function result
|
||||
ror w4, w4, #28 // Rotate left s=4 bits
|
||||
eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x10, #0x27fa // Load lower half of constant 0xeaa127fa
|
||||
add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0x289b7ec6, s=4, M[13])
|
||||
movk x10, #0xeaa1, lsl #16 // Load upper half of constant 0xeaa127fa
|
||||
add w17, w17, w15 // Add dest value
|
||||
eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w17, w17, w10 // Add constant 0xeaa127fa
|
||||
add w17, w17, w6 // Add aux function result
|
||||
eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w17, w17, #21 // Rotate left s=11 bits
|
||||
movz x13, #0x3085 // Load lower half of constant 0xd4ef3085
|
||||
add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xeaa127fa, s=11, M[0])
|
||||
movk x13, #0xd4ef, lsl #16 // Load upper half of constant 0xd4ef3085
|
||||
add w8, w8, w21 // Add dest value
|
||||
eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w8, w8, w13 // Add constant 0xd4ef3085
|
||||
add w8, w8, w6 // Add aux function result
|
||||
ror w8, w8, #16 // Rotate left s=16 bits
|
||||
eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x13, #0x1d05 // Load lower half of constant 0x4881d05
|
||||
add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xd4ef3085, s=16, M[3])
|
||||
movk x13, #0x488, lsl #16 // Load upper half of constant 0x4881d05
|
||||
add w9, w9, w7 // Add dest value
|
||||
eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w9, w9, w13 // Add constant 0x4881d05
|
||||
add w9, w9, w6 // Add aux function result
|
||||
eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w9, w9, #9 // Rotate left s=23 bits
|
||||
movz x10, #0xd039 // Load lower half of constant 0xd9d4d039
|
||||
add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0x4881d05, s=23, M[6])
|
||||
movk x10, #0xd9d4, lsl #16 // Load upper half of constant 0xd9d4d039
|
||||
add w4, w4, w24 // Add dest value
|
||||
eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w4, w4, w10 // Add constant 0xd9d4d039
|
||||
add w4, w4, w6 // Add aux function result
|
||||
ror w4, w4, #28 // Rotate left s=4 bits
|
||||
eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x10, #0x99e5 // Load lower half of constant 0xe6db99e5
|
||||
add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xd9d4d039, s=4, M[9])
|
||||
movk x10, #0xe6db, lsl #16 // Load upper half of constant 0xe6db99e5
|
||||
add w17, w17, w11 // Add dest value
|
||||
eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w17, w17, w10 // Add constant 0xe6db99e5
|
||||
add w17, w17, w6 // Add aux function result
|
||||
eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
ror w17, w17, #21 // Rotate left s=11 bits
|
||||
movz x13, #0x7cf8 // Load lower half of constant 0x1fa27cf8
|
||||
add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xe6db99e5, s=11, M[12])
|
||||
movk x13, #0x1fa2, lsl #16 // Load upper half of constant 0x1fa27cf8
|
||||
add w8, w8, w27 // Add dest value
|
||||
eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w8, w8, w13 // Add constant 0x1fa27cf8
|
||||
add w8, w8, w6 // Add aux function result
|
||||
ror w8, w8, #16 // Rotate left s=16 bits
|
||||
eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z)
|
||||
movz x13, #0x5665 // Load lower half of constant 0xc4ac5665
|
||||
add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x1fa27cf8, s=16, M[15])
|
||||
movk x13, #0xc4ac, lsl #16 // Load upper half of constant 0xc4ac5665
|
||||
add w9, w9, w3 // Add dest value
|
||||
eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z)
|
||||
add w9, w9, w13 // Add constant 0xc4ac5665
|
||||
add w9, w9, w6 // Add aux function result
|
||||
ror w9, w9, #9 // Rotate left s=23 bits
|
||||
movz x6, #0x2244 // Load lower half of constant 0xf4292244
|
||||
movk x6, #0xf429, lsl #16 // Load upper half of constant 0xf4292244
|
||||
add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xc4ac5665, s=23, M[2])
|
||||
add w4, w4, w15 // Add dest value
|
||||
orn x13, x9, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w4, w4, w6 // Add constant 0xf4292244
|
||||
eor x6, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w4, w4, w6 // Add aux function result
|
||||
ror w4, w4, #26 // Rotate left s=6 bits
|
||||
movz x6, #0xff97 // Load lower half of constant 0x432aff97
|
||||
movk x6, #0x432a, lsl #16 // Load upper half of constant 0x432aff97
|
||||
add w4, w9, w4 // Add X parameter round 4 A=II(A, B, C, D, 0xf4292244, s=6, M[0])
|
||||
orn x10, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w17, w17, w23 // Add dest value
|
||||
eor x10, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w17, w17, w6 // Add constant 0x432aff97
|
||||
add w6, w17, w10 // Add aux function result
|
||||
ror w6, w6, #22 // Rotate left s=10 bits
|
||||
movz x17, #0x23a7 // Load lower half of constant 0xab9423a7
|
||||
movk x17, #0xab94, lsl #16 // Load upper half of constant 0xab9423a7
|
||||
add w6, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x432aff97, s=10, M[7])
|
||||
add w8, w8, w12 // Add dest value
|
||||
orn x10, x6, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w8, w17 // Add constant 0xab9423a7
|
||||
eor x17, x4, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w8, w17 // Add aux function result
|
||||
ror w8, w8, #17 // Rotate left s=15 bits
|
||||
movz x17, #0xa039 // Load lower half of constant 0xfc93a039
|
||||
movk x17, #0xfc93, lsl #16 // Load upper half of constant 0xfc93a039
|
||||
add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xab9423a7, s=15, M[14])
|
||||
orn x13, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w9, w22 // Add dest value
|
||||
eor x13, x6, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w9, w17 // Add constant 0xfc93a039
|
||||
add w17, w9, w13 // Add aux function result
|
||||
ror w17, w17, #11 // Rotate left s=21 bits
|
||||
movz x9, #0x59c3 // Load lower half of constant 0x655b59c3
|
||||
movk x9, #0x655b, lsl #16 // Load upper half of constant 0x655b59c3
|
||||
add w17, w8, w17 // Add X parameter round 4 B=II(B, C, D, A, 0xfc93a039, s=21, M[5])
|
||||
add w4, w4, w11 // Add dest value
|
||||
orn x13, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w4, w9 // Add constant 0x655b59c3
|
||||
eor x4, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w9, w4 // Add aux function result
|
||||
ror w9, w9, #26 // Rotate left s=6 bits
|
||||
movz x4, #0xcc92 // Load lower half of constant 0x8f0ccc92
|
||||
movk x4, #0x8f0c, lsl #16 // Load upper half of constant 0x8f0ccc92
|
||||
add w9, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x655b59c3, s=6, M[12])
|
||||
orn x10, x9, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w6, w6, w21 // Add dest value
|
||||
eor x10, x17, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w4, w6, w4 // Add constant 0x8f0ccc92
|
||||
add w6, w4, w10 // Add aux function result
|
||||
ror w6, w6, #22 // Rotate left s=10 bits
|
||||
movz x4, #0xf47d // Load lower half of constant 0xffeff47d
|
||||
movk x4, #0xffef, lsl #16 // Load upper half of constant 0xffeff47d
|
||||
add w6, w9, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x8f0ccc92, s=10, M[3])
|
||||
add w8, w8, w16 // Add dest value
|
||||
orn x10, x6, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w8, w4 // Add constant 0xffeff47d
|
||||
eor x4, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w8, w4 // Add aux function result
|
||||
ror w8, w8, #17 // Rotate left s=15 bits
|
||||
movz x4, #0x5dd1 // Load lower half of constant 0x85845dd1
|
||||
movk x4, #0x8584, lsl #16 // Load upper half of constant 0x85845dd1
|
||||
add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xffeff47d, s=15, M[10])
|
||||
orn x10, x8, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w17, w20 // Add dest value
|
||||
eor x17, x6, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w15, w4 // Add constant 0x85845dd1
|
||||
add w4, w15, w17 // Add aux function result
|
||||
ror w4, w4, #11 // Rotate left s=21 bits
|
||||
movz x15, #0x7e4f // Load lower half of constant 0x6fa87e4f
|
||||
movk x15, #0x6fa8, lsl #16 // Load upper half of constant 0x6fa87e4f
|
||||
add w17, w8, w4 // Add X parameter round 4 B=II(B, C, D, A, 0x85845dd1, s=21, M[1])
|
||||
add w4, w9, w5 // Add dest value
|
||||
orn x9, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w4, w15 // Add constant 0x6fa87e4f
|
||||
eor x4, x8, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w15, w4 // Add aux function result
|
||||
ror w9, w9, #26 // Rotate left s=6 bits
|
||||
movz x15, #0xe6e0 // Load lower half of constant 0xfe2ce6e0
|
||||
movk x15, #0xfe2c, lsl #16 // Load upper half of constant 0xfe2ce6e0
|
||||
add w4, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x6fa87e4f, s=6, M[8])
|
||||
orn x9, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w6, w6, w27 // Add dest value
|
||||
eor x9, x17, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w6, w15 // Add constant 0xfe2ce6e0
|
||||
add w6, w15, w9 // Add aux function result
|
||||
ror w6, w6, #22 // Rotate left s=10 bits
|
||||
movz x9, #0x4314 // Load lower half of constant 0xa3014314
|
||||
movk x9, #0xa301, lsl #16 // Load upper half of constant 0xa3014314
|
||||
add w15, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0xfe2ce6e0, s=10, M[15])
|
||||
add w6, w8, w7 // Add dest value
|
||||
orn x7, x15, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w6, w9 // Add constant 0xa3014314
|
||||
eor x9, x4, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w6, w8, w9 // Add aux function result
|
||||
ror w6, w6, #17 // Rotate left s=15 bits
|
||||
movz x7, #0x11a1 // Load lower half of constant 0x4e0811a1
|
||||
movk x7, #0x4e08, lsl #16 // Load upper half of constant 0x4e0811a1
|
||||
add w8, w15, w6 // Add X parameter round 4 C=II(C, D, A, B, 0xa3014314, s=15, M[6])
|
||||
orn x9, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w6, w17, w26 // Add dest value
|
||||
eor x17, x15, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w9, w6, w7 // Add constant 0x4e0811a1
|
||||
add w7, w9, w17 // Add aux function result
|
||||
ror w7, w7, #11 // Rotate left s=21 bits
|
||||
movz x6, #0x7e82 // Load lower half of constant 0xf7537e82
|
||||
movk x6, #0xf753, lsl #16 // Load upper half of constant 0xf7537e82
|
||||
add w9, w8, w7 // Add X parameter round 4 B=II(B, C, D, A, 0x4e0811a1, s=21, M[13])
|
||||
add w17, w4, w14 // Add dest value
|
||||
orn x7, x9, x15 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w14, w17, w6 // Add constant 0xf7537e82
|
||||
eor x4, x8, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w17, w14, w4 // Add aux function result
|
||||
ror w17, w17, #26 // Rotate left s=6 bits
|
||||
movz x6, #0xf235 // Load lower half of constant 0xbd3af235
|
||||
movk x6, #0xbd3a, lsl #16 // Load upper half of constant 0xbd3af235
|
||||
add w7, w9, w17 // Add X parameter round 4 A=II(A, B, C, D, 0xf7537e82, s=6, M[4])
|
||||
orn x14, x7, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w4, w15, w25 // Add dest value
|
||||
eor x17, x9, x14 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w4, w6 // Add constant 0xbd3af235
|
||||
add w16, w15, w17 // Add aux function result
|
||||
ror w16, w16, #22 // Rotate left s=10 bits
|
||||
movz x14, #0xd2bb // Load lower half of constant 0x2ad7d2bb
|
||||
movk x14, #0x2ad7, lsl #16 // Load upper half of constant 0x2ad7d2bb
|
||||
add w4, w7, w16 // Add X parameter round 4 D=II(D, A, B, C, 0xbd3af235, s=10, M[11])
|
||||
add w6, w8, w3 // Add dest value
|
||||
orn x15, x4, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w17, w6, w14 // Add constant 0x2ad7d2bb
|
||||
eor x16, x7, x15 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w8, w17, w16 // Add aux function result
|
||||
ror w8, w8, #17 // Rotate left s=15 bits
|
||||
movz x3, #0xd391 // Load lower half of constant 0xeb86d391
|
||||
movk x3, #0xeb86, lsl #16 // Load upper half of constant 0xeb86d391
|
||||
add w14, w4, w8 // Add X parameter round 4 C=II(C, D, A, B, 0x2ad7d2bb, s=15, M[2])
|
||||
orn x6, x14, x7 // Begin aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w15, w9, w24 // Add dest value
|
||||
eor x17, x4, x6 // End aux function round 4 I(x,y,z)=((~z|x)^y)
|
||||
add w16, w15, w3 // Add constant 0xeb86d391
|
||||
add w8, w16, w17 // Add aux function result
|
||||
ror w8, w8, #11 // Rotate left s=21 bits
|
||||
ldp w6, w15, [x0] // Reload MD5 state->A and state->B
|
||||
ldp w5, w9, [x0, #8] // Reload MD5 state->C and state->D
|
||||
add w3, w14, w8 // Add X parameter round 4 B=II(B, C, D, A, 0xeb86d391, s=21, M[9])
|
||||
add w13, w4, w9 // Add result of MD5 rounds to state->D
|
||||
add w12, w14, w5 // Add result of MD5 rounds to state->C
|
||||
add w10, w7, w6 // Add result of MD5 rounds to state->A
|
||||
add w11, w3, w15 // Add result of MD5 rounds to state->B
|
||||
stp w12, w13, [x0, #8] // Store MD5 states C,D
|
||||
stp w10, w11, [x0] // Store MD5 states A,B
|
||||
add x1, x1, #64 // Increment data pointer
|
||||
subs w2, w2, #1 // Decrement block counter
|
||||
b.ne ossl_md5_blocks_loop
|
||||
|
||||
ldp x21,x22,[sp,#16]
|
||||
ldp x23,x24,[sp,#32]
|
||||
ldp x25,x26,[sp,#48]
|
||||
ldp x27,x28,[sp,#64]
|
||||
ldp x19,x20,[sp],#80
|
||||
ret
|
||||
|
||||
EOF
|
||||
|
||||
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
|
||||
# CONTEXT *context,DISPATCHER_CONTEXT *disp)
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
Executable
+298
@@ -0,0 +1,298 @@
|
||||
#! /usr/bin/env perl
|
||||
# Author: Min Zhou <[email protected]>
|
||||
# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the OpenSSL license (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
# Reference to crypto/md5/asm/md5-x86_64.pl
|
||||
# MD5 optimized for LoongArch.
|
||||
|
||||
use strict;
|
||||
|
||||
my $code;
|
||||
|
||||
my ($zero,$ra,$tp,$sp,$fp)=map("\$r$_",(0..3,22));
|
||||
my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11));
|
||||
my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$x)=map("\$r$_",(12..21));
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
my $output;
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
open STDOUT,">$output";
|
||||
|
||||
# round1_step() does:
|
||||
# dst = x + ((dst + F(x,y,z) + X[k] + T_i) <<< s)
|
||||
# $t1 = y ^ z
|
||||
# $t2 = dst + X[k_next]
|
||||
sub round1_step
|
||||
{
|
||||
my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
|
||||
my $T_i_h = ($T_i & 0xfffff000) >> 12;
|
||||
my $T_i_l = $T_i & 0xfff;
|
||||
|
||||
# In LoongArch we have to use two instructions of lu12i.w and ori to load a
|
||||
# 32-bit immediate into a general register. Meanwhile, the instruction lu12i.w
|
||||
# treats the 20-bit immediate as a signed number. So if the T_i_h is greater
|
||||
# than or equal to (1<<19), we need provide lu12i.w a corresponding negative
|
||||
# number whose complement equals to the sign extension of T_i_h.
|
||||
|
||||
# The details of the instruction lu12i.w can be found as following:
|
||||
# https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_lu12i_w_lu32i_d_lu52i_d
|
||||
|
||||
$T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19));
|
||||
|
||||
$code .= " ld.w $t0,$a1,0 /* (NEXT STEP) X[0] */\n" if ($pos == -1);
|
||||
$code .= " xor $t1,$y,$z /* y ^ z */\n" if ($pos == -1);
|
||||
$code .= " add.w $t2,$dst,$t0 /* dst + X[k] */\n" if ($pos == -1);
|
||||
$code .= <<EOF;
|
||||
lu12i.w $t8,$T_i_h /* load bits [31:12] of constant */
|
||||
and $t1,$x,$t1 /* x & ... */
|
||||
ori $t8,$t8,$T_i_l /* load bits [11:0] of constant */
|
||||
xor $t1,$z,$t1 /* z ^ ... */
|
||||
add.w $t7,$t2,$t8 /* dst + X[k] + Const */
|
||||
ld.w $t0,$a1,$k_next*4 /* (NEXT STEP) X[$k_next] */
|
||||
add.w $dst,$t7,$t1 /* dst += ... */
|
||||
add.w $t2,$z,$t0 /* (NEXT STEP) dst + X[$k_next] */
|
||||
EOF
|
||||
|
||||
$code .= " rotri.w $dst,$dst,32-$s /* dst <<< s */\n";
|
||||
if ($pos != 1) {
|
||||
$code .= " xor $t1,$x,$y /* (NEXT STEP) y ^ z */\n";
|
||||
} else {
|
||||
$code .= " move $t0,$a7 /* (NEXT ROUND) $t0 = z' (copy of z) */\n";
|
||||
$code .= " nor $t1,$zero,$a7 /* (NEXT ROUND) $t1 = not z' (copy of not z) */\n";
|
||||
}
|
||||
$code .= " add.w $dst,$dst,$x /* dst += x */\n";
|
||||
}
|
||||
|
||||
# round2_step() does:
|
||||
# dst = x + ((dst + G(x,y,z) + X[k] + T_i) <<< s)
|
||||
# $t0 = z' (copy of z for the next step)
|
||||
# $t1 = not z' (copy of not z for the next step)
|
||||
# $t2 = dst + X[k_next]
|
||||
sub round2_step
|
||||
{
|
||||
my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
|
||||
my $T_i_h = ($T_i & 0xfffff000) >> 12;
|
||||
my $T_i_l = $T_i & 0xfff;
|
||||
$T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19));
|
||||
|
||||
$code .= <<EOF;
|
||||
lu12i.w $t8,$T_i_h /* load bits [31:12] of Constant */
|
||||
and $t0,$x,$t0 /* x & z */
|
||||
ori $t8,$t8,$T_i_l /* load bits [11:0] of Constant */
|
||||
and $t1,$y,$t1 /* y & (not z) */
|
||||
add.w $t7,$t2,$t8 /* dst + X[k] + Const */
|
||||
or $t1,$t0,$t1 /* (y & (not z)) | (x & z) */
|
||||
ld.w $t0,$a1,$k_next*4 /* (NEXT STEP) X[$k_next] */
|
||||
add.w $dst,$t7,$t1 /* dst += ... */
|
||||
add.w $t2,$z,$t0 /* (NEXT STEP) dst + X[$k_next] */
|
||||
EOF
|
||||
|
||||
$code .= " rotri.w $dst,$dst,32-$s /* dst <<< s */\n";
|
||||
if ($pos != 1) {
|
||||
$code .= " move $t0,$y /* (NEXT STEP) z' = $y */\n";
|
||||
$code .= " nor $t1,$zero,$y /* (NEXT STEP) not z' = not $y */\n";
|
||||
} else {
|
||||
$code .= " xor $t1,$a6,$a7 /* (NEXT ROUND) $t1 = y ^ z */\n";
|
||||
}
|
||||
$code .= " add.w $dst,$dst,$x /* dst += x */\n";
|
||||
}
|
||||
|
||||
# round3_step() does:
|
||||
# dst = x + ((dst + H(x,y,z) + X[k] + T_i) <<< s)
|
||||
# $t1 = y ^ z
|
||||
# $t2 = dst + X[k_next]
|
||||
sub round3_step
|
||||
{
|
||||
my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
|
||||
my $T_i_h = ($T_i & 0xfffff000) >> 12;
|
||||
my $T_i_l = $T_i & 0xfff;
|
||||
$T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19));
|
||||
|
||||
$code .= <<EOF;
|
||||
lu12i.w $t8,$T_i_h /* load bits [31:12] of Constant */
|
||||
xor $t1,$x,$t1 /* x ^ ... */
|
||||
ori $t8,$t8,$T_i_l /* load bits [11:0] of Constant */
|
||||
add.w $t7,$t2,$t8 /* dst + X[k] + Const */
|
||||
ld.w $t0,$a1,$k_next*4 /* (NEXT STEP) X[$k_next] */
|
||||
add.w $dst,$t7,$t1 /* dst += ... */
|
||||
add.w $t2,$z,$t0 /* (NEXT STEP) dst + X[$k_next] */
|
||||
EOF
|
||||
|
||||
$code .= " rotri.w $dst,$dst,32-$s /* dst <<< s */\n";
|
||||
if ($pos != 1) {
|
||||
$code .= " xor $t1,$x,$y /* (NEXT STEP) y ^ z */\n";
|
||||
} else {
|
||||
$code .= " nor $t1,$zero,$a7 /* (NEXT ROUND) $t1 = not z */\n";
|
||||
}
|
||||
$code .= " add.w $dst,$dst,$x /* dst += x */\n";
|
||||
}
|
||||
|
||||
# round4_step() does:
|
||||
# dst = x + ((dst + I(x,y,z) + X[k] + T_i) <<< s)
|
||||
# $t1 = not z' (copy of not z for the next step)
|
||||
# $t2 = dst + X[k_next]
|
||||
sub round4_step
|
||||
{
|
||||
my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
|
||||
my $T_i_h = ($T_i & 0xfffff000) >> 12;
|
||||
my $T_i_l = $T_i & 0xfff;
|
||||
$T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19));
|
||||
|
||||
$code .= <<EOF;
|
||||
lu12i.w $t8,$T_i_h /* load bits [31:12] of Constant */
|
||||
or $t1,$x,$t1 /* x | ... */
|
||||
ori $t8,$t8,$T_i_l /* load bits [11:0] of Constant */
|
||||
xor $t1,$y,$t1 /* y ^ ... */
|
||||
add.w $t7,$t2,$t8 /* dst + X[k] + Const */
|
||||
EOF
|
||||
|
||||
if ($pos != 1) {
|
||||
$code .= " ld.w $t0,$a1,$k_next*4 /* (NEXT STEP) X[$k_next] */\n";
|
||||
$code .= " add.w $dst,$t7,$t1 /* dst += ... */\n";
|
||||
$code .= " add.w $t2,$z,$t0 /* (NEXT STEP) dst + X[$k_next] */\n";
|
||||
$code .= " rotri.w $dst,$dst,32-$s /* dst <<< s */\n";
|
||||
$code .= " nor $t1,$zero,$y /* (NEXT STEP) not z' = not $y */\n";
|
||||
$code .= " add.w $dst,$dst,$x /* dst += x */\n";
|
||||
} else {
|
||||
$code .= " add.w $a4,$t3,$a4 /* (NEXT LOOP) add old value of A */\n";
|
||||
$code .= " add.w $dst,$t7,$t1 /* dst += ... */\n";
|
||||
$code .= " add.w $a7,$t6,$a7 /* (NEXT LOOP) add old value of D */\n";
|
||||
$code .= " rotri.w $dst,$dst,32-$s /* dst <<< s */\n";
|
||||
$code .= " addi.d $a1,$a1,64 /* (NEXT LOOP) ptr += 64 */\n";
|
||||
$code .= " add.w $dst,$dst,$x /* dst += x */\n";
|
||||
}
|
||||
}
|
||||
|
||||
$code .= <<EOF;
|
||||
.text
|
||||
|
||||
.globl ossl_md5_block_asm_data_order
|
||||
.type ossl_md5_block_asm_data_order function
|
||||
ossl_md5_block_asm_data_order:
|
||||
# $a0 = arg #1 (ctx, MD5_CTX pointer)
|
||||
# $a1 = arg #2 (ptr, data pointer)
|
||||
# $a2 = arg #3 (nbr, number of 16-word blocks to process)
|
||||
beqz $a2,.Lend # cmp nbr with 0, jmp if nbr == 0
|
||||
|
||||
# ptr is '$a1'
|
||||
# end is '$a3'
|
||||
slli.d $t0,$a2,6
|
||||
add.d $a3,$a1,$t0
|
||||
|
||||
# A is '$a4'
|
||||
# B is '$a5'
|
||||
# C is '$a6'
|
||||
# D is '$a7'
|
||||
ld.w $a4,$a0,0 # a4 = ctx->A
|
||||
ld.w $a5,$a0,4 # a5 = ctx->B
|
||||
ld.w $a6,$a0,8 # a6 = ctx->C
|
||||
ld.w $a7,$a0,12 # a7 = ctx->D
|
||||
|
||||
# BEGIN of loop over 16-word blocks
|
||||
.align 6
|
||||
.Lloop:
|
||||
# save old values of A, B, C, D
|
||||
move $t3,$a4
|
||||
move $t4,$a5
|
||||
move $t5,$a6
|
||||
move $t6,$a7
|
||||
|
||||
preld 0,$a1,0
|
||||
preld 0,$a1,64
|
||||
EOF
|
||||
|
||||
round1_step(-1, $a4, $a5, $a6, $a7, '1', 0xd76aa478, '7');
|
||||
round1_step(0, $a7, $a4, $a5, $a6, '2', 0xe8c7b756, '12');
|
||||
round1_step(0, $a6, $a7, $a4, $a5, '3', 0x242070db, '17');
|
||||
round1_step(0, $a5, $a6, $a7, $a4, '4', 0xc1bdceee, '22');
|
||||
round1_step(0, $a4, $a5, $a6, $a7, '5', 0xf57c0faf, '7');
|
||||
round1_step(0, $a7, $a4, $a5, $a6, '6', 0x4787c62a, '12');
|
||||
round1_step(0, $a6, $a7, $a4, $a5, '7', 0xa8304613, '17');
|
||||
round1_step(0, $a5, $a6, $a7, $a4, '8', 0xfd469501, '22');
|
||||
round1_step(0, $a4, $a5, $a6, $a7, '9', 0x698098d8, '7');
|
||||
round1_step(0, $a7, $a4, $a5, $a6, '10', 0x8b44f7af, '12');
|
||||
round1_step(0, $a6, $a7, $a4, $a5, '11', 0xffff5bb1, '17');
|
||||
round1_step(0, $a5, $a6, $a7, $a4, '12', 0x895cd7be, '22');
|
||||
round1_step(0, $a4, $a5, $a6, $a7, '13', 0x6b901122, '7');
|
||||
round1_step(0, $a7, $a4, $a5, $a6, '14', 0xfd987193, '12');
|
||||
round1_step(0, $a6, $a7, $a4, $a5, '15', 0xa679438e, '17');
|
||||
round1_step(1, $a5, $a6, $a7, $a4, '1', 0x49b40821, '22');
|
||||
|
||||
round2_step(-1, $a4, $a5, $a6, $a7, '6', 0xf61e2562, '5');
|
||||
round2_step(0, $a7, $a4, $a5, $a6, '11', 0xc040b340, '9');
|
||||
round2_step(0, $a6, $a7, $a4, $a5, '0', 0x265e5a51, '14');
|
||||
round2_step(0, $a5, $a6, $a7, $a4, '5', 0xe9b6c7aa, '20');
|
||||
round2_step(0, $a4, $a5, $a6, $a7, '10', 0xd62f105d, '5');
|
||||
round2_step(0, $a7, $a4, $a5, $a6, '15', 0x2441453, '9');
|
||||
round2_step(0, $a6, $a7, $a4, $a5, '4', 0xd8a1e681, '14');
|
||||
round2_step(0, $a5, $a6, $a7, $a4, '9', 0xe7d3fbc8, '20');
|
||||
round2_step(0, $a4, $a5, $a6, $a7, '14', 0x21e1cde6, '5');
|
||||
round2_step(0, $a7, $a4, $a5, $a6, '3', 0xc33707d6, '9');
|
||||
round2_step(0, $a6, $a7, $a4, $a5, '8', 0xf4d50d87, '14');
|
||||
round2_step(0, $a5, $a6, $a7, $a4, '13', 0x455a14ed, '20');
|
||||
round2_step(0, $a4, $a5, $a6, $a7, '2', 0xa9e3e905, '5');
|
||||
round2_step(0, $a7, $a4, $a5, $a6, '7', 0xfcefa3f8, '9');
|
||||
round2_step(0, $a6, $a7, $a4, $a5, '12', 0x676f02d9, '14');
|
||||
round2_step(1, $a5, $a6, $a7, $a4, '5', 0x8d2a4c8a, '20');
|
||||
|
||||
round3_step(-1, $a4, $a5, $a6, $a7, '8', 0xfffa3942, '4');
|
||||
round3_step(0, $a7, $a4, $a5, $a6, '11', 0x8771f681, '11');
|
||||
round3_step(0, $a6, $a7, $a4, $a5, '14', 0x6d9d6122, '16');
|
||||
round3_step(0, $a5, $a6, $a7, $a4, '1', 0xfde5380c, '23');
|
||||
round3_step(0, $a4, $a5, $a6, $a7, '4', 0xa4beea44, '4');
|
||||
round3_step(0, $a7, $a4, $a5, $a6, '7', 0x4bdecfa9, '11');
|
||||
round3_step(0, $a6, $a7, $a4, $a5, '10', 0xf6bb4b60, '16');
|
||||
round3_step(0, $a5, $a6, $a7, $a4, '13', 0xbebfbc70, '23');
|
||||
round3_step(0, $a4, $a5, $a6, $a7, '0', 0x289b7ec6, '4');
|
||||
round3_step(0, $a7, $a4, $a5, $a6, '3', 0xeaa127fa, '11');
|
||||
round3_step(0, $a6, $a7, $a4, $a5, '6', 0xd4ef3085, '16');
|
||||
round3_step(0, $a5, $a6, $a7, $a4, '9', 0x4881d05, '23');
|
||||
round3_step(0, $a4, $a5, $a6, $a7, '12', 0xd9d4d039, '4');
|
||||
round3_step(0, $a7, $a4, $a5, $a6, '15', 0xe6db99e5, '11');
|
||||
round3_step(0, $a6, $a7, $a4, $a5, '2', 0x1fa27cf8, '16');
|
||||
round3_step(1, $a5, $a6, $a7, $a4, '0', 0xc4ac5665, '23');
|
||||
|
||||
round4_step(-1, $a4, $a5, $a6, $a7, '7', 0xf4292244, '6');
|
||||
round4_step(0, $a7, $a4, $a5, $a6, '14', 0x432aff97, '10');
|
||||
round4_step(0, $a6, $a7, $a4, $a5, '5', 0xab9423a7, '15');
|
||||
round4_step(0, $a5, $a6, $a7, $a4, '12', 0xfc93a039, '21');
|
||||
round4_step(0, $a4, $a5, $a6, $a7, '3', 0x655b59c3, '6');
|
||||
round4_step(0, $a7, $a4, $a5, $a6, '10', 0x8f0ccc92, '10');
|
||||
round4_step(0, $a6, $a7, $a4, $a5, '1', 0xffeff47d, '15');
|
||||
round4_step(0, $a5, $a6, $a7, $a4, '8', 0x85845dd1, '21');
|
||||
round4_step(0, $a4, $a5, $a6, $a7, '15', 0x6fa87e4f, '6');
|
||||
round4_step(0, $a7, $a4, $a5, $a6, '6', 0xfe2ce6e0, '10');
|
||||
round4_step(0, $a6, $a7, $a4, $a5, '13', 0xa3014314, '15');
|
||||
round4_step(0, $a5, $a6, $a7, $a4, '4', 0x4e0811a1, '21');
|
||||
round4_step(0, $a4, $a5, $a6, $a7, '11', 0xf7537e82, '6');
|
||||
round4_step(0, $a7, $a4, $a5, $a6, '2', 0xbd3af235, '10');
|
||||
round4_step(0, $a6, $a7, $a4, $a5, '9', 0x2ad7d2bb, '15');
|
||||
round4_step(1, $a5, $a6, $a7, $a4, '0', 0xeb86d391, '21');
|
||||
|
||||
$code .= <<EOF;
|
||||
# add old values of B, C
|
||||
add.w $a5,$t4,$a5
|
||||
add.w $a6,$t5,$a6
|
||||
|
||||
bltu $a1,$a3,.Lloop # jmp if ptr < end
|
||||
|
||||
st.w $a4,$a0,0 # ctx->A = A
|
||||
st.w $a5,$a0,4 # ctx->B = B
|
||||
st.w $a6,$a0,8 # ctx->C = C
|
||||
st.w $a7,$a0,12 # ctx->D = D
|
||||
|
||||
.Lend:
|
||||
jr $ra
|
||||
.size ossl_md5_block_asm_data_order,.-ossl_md5_block_asm_data_order
|
||||
EOF
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval($1)/gem;
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT;
|
||||
@@ -0,0 +1,10 @@
|
||||
LIBS=../../libcrypto
|
||||
|
||||
$COMMON=ml_dsa_encoders.c ml_dsa_key_compress.c ml_dsa_key.c \
|
||||
ml_dsa_matrix.c ml_dsa_ntt.c ml_dsa_params.c ml_dsa_sample.c \
|
||||
ml_dsa_sign.c
|
||||
|
||||
IF[{- !$disabled{'ml-dsa'} -}]
|
||||
SOURCE[../../libcrypto]=$COMMON
|
||||
SOURCE[../../providers/libfips.a]=$COMMON
|
||||
ENDIF
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/evp.h>
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
shake_xof(EVP_MD_CTX *ctx, const EVP_MD *md, const uint8_t *in, size_t in_len,
|
||||
uint8_t *out, size_t out_len)
|
||||
{
|
||||
return (EVP_DigestInit_ex2(ctx, md, NULL) == 1
|
||||
&& EVP_DigestUpdate(ctx, in, in_len) == 1
|
||||
&& EVP_DigestSqueeze(ctx, out, out_len) == 1);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
shake_xof_2(EVP_MD_CTX *ctx, const EVP_MD *md, const uint8_t *in1, size_t in1_len,
|
||||
const uint8_t *in2, size_t in2_len, uint8_t *out, size_t out_len)
|
||||
{
|
||||
return EVP_DigestInit_ex2(ctx, md, NULL)
|
||||
&& EVP_DigestUpdate(ctx, in1, in1_len)
|
||||
&& EVP_DigestUpdate(ctx, in2, in2_len)
|
||||
&& EVP_DigestSqueeze(ctx, out, out_len);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
shake_xof_3(EVP_MD_CTX *ctx, const EVP_MD *md, const uint8_t *in1, size_t in1_len,
|
||||
const uint8_t *in2, size_t in2_len,
|
||||
const uint8_t *in3, size_t in3_len, uint8_t *out, size_t out_len)
|
||||
{
|
||||
return EVP_DigestInit_ex2(ctx, md, NULL)
|
||||
&& EVP_DigestUpdate(ctx, in1, in1_len)
|
||||
&& EVP_DigestUpdate(ctx, in2, in2_len)
|
||||
&& EVP_DigestUpdate(ctx, in3, in3_len)
|
||||
&& EVP_DigestSqueeze(ctx, out, out_len);
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/core_dispatch.h>
|
||||
#include <openssl/core_names.h>
|
||||
#include <openssl/err.h>
|
||||
#include <openssl/params.h>
|
||||
#include <openssl/proverr.h>
|
||||
#include <openssl/rand.h>
|
||||
#include "ml_dsa_key.h"
|
||||
#include "ml_dsa_matrix.h"
|
||||
#include "ml_dsa_hash.h"
|
||||
#include "internal/encoder.h"
|
||||
|
||||
const ML_DSA_PARAMS *ossl_ml_dsa_key_params(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params;
|
||||
}
|
||||
|
||||
/* Returns the seed data or NULL if there is no seed */
|
||||
const uint8_t *ossl_ml_dsa_key_get_seed(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->seed;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_get_prov_flags(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->prov_flags;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_set_prekey(ML_DSA_KEY *key, int flags_set, int flags_clr,
|
||||
const uint8_t *seed, size_t seed_len,
|
||||
const uint8_t *sk, size_t sk_len)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
if (key == NULL
|
||||
|| key->pub_encoding != NULL
|
||||
|| key->priv_encoding != NULL
|
||||
|| (sk != NULL && sk_len != key->params->sk_len)
|
||||
|| (seed != NULL && seed_len != ML_DSA_SEED_BYTES)
|
||||
|| key->seed != NULL)
|
||||
return 0;
|
||||
|
||||
if (sk != NULL
|
||||
&& (key->priv_encoding = OPENSSL_memdup(sk, sk_len)) == NULL)
|
||||
goto end;
|
||||
if (seed != NULL
|
||||
&& (key->seed = OPENSSL_memdup(seed, seed_len)) == NULL)
|
||||
goto end;
|
||||
key->prov_flags |= flags_set;
|
||||
key->prov_flags &= ~flags_clr;
|
||||
ret = 1;
|
||||
|
||||
end:
|
||||
if (!ret) {
|
||||
OPENSSL_free(key->priv_encoding);
|
||||
OPENSSL_free(key->seed);
|
||||
key->priv_encoding = key->seed = NULL;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Create a new ML_DSA_KEY object
|
||||
*
|
||||
* @param libctx A OSSL_LIB_CTX object used for fetching algorithms.
|
||||
* @param propq The property query used for fetching algorithms
|
||||
* @param alg The algorithm name associated with the key type
|
||||
* @returns The new ML_DSA_KEY object on success, or NULL on malloc failure
|
||||
*/
|
||||
ML_DSA_KEY *ossl_ml_dsa_key_new(OSSL_LIB_CTX *libctx, const char *propq,
|
||||
int evp_type)
|
||||
{
|
||||
ML_DSA_KEY *ret;
|
||||
const ML_DSA_PARAMS *params = ossl_ml_dsa_params_get(evp_type);
|
||||
|
||||
if (params == NULL)
|
||||
return NULL;
|
||||
|
||||
ret = OPENSSL_zalloc(sizeof(*ret));
|
||||
if (ret != NULL) {
|
||||
ret->libctx = libctx;
|
||||
ret->params = params;
|
||||
ret->prov_flags = ML_DSA_KEY_PROV_FLAGS_DEFAULT;
|
||||
ret->shake128_md = EVP_MD_fetch(libctx, "SHAKE-128", propq);
|
||||
ret->shake256_md = EVP_MD_fetch(libctx, "SHAKE-256", propq);
|
||||
if (ret->shake128_md == NULL || ret->shake256_md == NULL)
|
||||
goto err;
|
||||
}
|
||||
return ret;
|
||||
err:
|
||||
ossl_ml_dsa_key_free(ret);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_pub_alloc(ML_DSA_KEY *key)
|
||||
{
|
||||
if (key->t1.poly != NULL)
|
||||
return 0;
|
||||
return vector_alloc(&key->t1, key->params->k);
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_priv_alloc(ML_DSA_KEY *key)
|
||||
{
|
||||
size_t k = key->params->k, l = key->params->l;
|
||||
POLY *poly;
|
||||
|
||||
if (key->s1.poly != NULL)
|
||||
return 0;
|
||||
if (!vector_alloc(&key->s1, l + 2 * k))
|
||||
return 0;
|
||||
|
||||
poly = key->s1.poly;
|
||||
key->s1.num_poly = l;
|
||||
vector_init(&key->s2, poly + l, k);
|
||||
vector_init(&key->t0, poly + l + k, k);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Destroy an ML_DSA_KEY object
|
||||
*/
|
||||
void ossl_ml_dsa_key_free(ML_DSA_KEY *key)
|
||||
{
|
||||
if (key == NULL)
|
||||
return;
|
||||
|
||||
EVP_MD_free(key->shake128_md);
|
||||
EVP_MD_free(key->shake256_md);
|
||||
ossl_ml_dsa_key_reset(key);
|
||||
OPENSSL_free(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Factory reset an ML_DSA_KEY object
|
||||
*/
|
||||
void ossl_ml_dsa_key_reset(ML_DSA_KEY *key)
|
||||
{
|
||||
/*
|
||||
* The allocation for |s1.poly| subsumes those for |s2| and |t0|, which we
|
||||
* must not access after |s1|'s poly is freed.
|
||||
*/
|
||||
if (key->s1.poly != NULL) {
|
||||
vector_zero(&key->s1);
|
||||
vector_zero(&key->s2);
|
||||
vector_zero(&key->t0);
|
||||
vector_free(&key->s1);
|
||||
key->s2.poly = NULL;
|
||||
key->t0.poly = NULL;
|
||||
}
|
||||
/* The |t1| vector is public and allocated separately */
|
||||
vector_free(&key->t1);
|
||||
OPENSSL_cleanse(key->K, sizeof(key->K));
|
||||
OPENSSL_free(key->pub_encoding);
|
||||
key->pub_encoding = NULL;
|
||||
if (key->priv_encoding != NULL)
|
||||
OPENSSL_clear_free(key->priv_encoding, key->params->sk_len);
|
||||
key->priv_encoding = NULL;
|
||||
if (key->seed != NULL)
|
||||
OPENSSL_clear_free(key->seed, ML_DSA_SEED_BYTES);
|
||||
key->seed = NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Duplicate a key
|
||||
*
|
||||
* @param src A ML_DSA_KEY object to copy
|
||||
* @param selection to select public and/or private components. Selecting the
|
||||
* private key will also select the public key
|
||||
* @returns The duplicated key, or NULL on failure.
|
||||
*/
|
||||
ML_DSA_KEY *ossl_ml_dsa_key_dup(const ML_DSA_KEY *src, int selection)
|
||||
{
|
||||
ML_DSA_KEY *ret = NULL;
|
||||
|
||||
if (src == NULL)
|
||||
return NULL;
|
||||
|
||||
/* Prekeys with just a seed or private key are not dupable */
|
||||
if (src->pub_encoding == NULL
|
||||
&& (src->priv_encoding != NULL || src->seed != NULL))
|
||||
return NULL;
|
||||
|
||||
ret = OPENSSL_zalloc(sizeof(*ret));
|
||||
if (ret != NULL) {
|
||||
ret->libctx = src->libctx;
|
||||
ret->params = src->params;
|
||||
ret->prov_flags = src->prov_flags;
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) {
|
||||
if (src->pub_encoding != NULL) {
|
||||
/* The public components are present if the private key is present */
|
||||
memcpy(ret->rho, src->rho, sizeof(src->rho));
|
||||
memcpy(ret->tr, src->tr, sizeof(src->tr));
|
||||
if (src->t1.poly != NULL) {
|
||||
if (!ossl_ml_dsa_key_pub_alloc(ret))
|
||||
goto err;
|
||||
vector_copy(&ret->t1, &src->t1);
|
||||
}
|
||||
if ((ret->pub_encoding = OPENSSL_memdup(src->pub_encoding,
|
||||
src->params->pk_len)) == NULL)
|
||||
goto err;
|
||||
}
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) {
|
||||
if (src->priv_encoding != NULL) {
|
||||
memcpy(ret->K, src->K, sizeof(src->K));
|
||||
if (src->s1.poly != NULL) {
|
||||
if (!ossl_ml_dsa_key_priv_alloc(ret))
|
||||
goto err;
|
||||
vector_copy(&ret->s1, &src->s1);
|
||||
vector_copy(&ret->s2, &src->s2);
|
||||
vector_copy(&ret->t0, &src->t0);
|
||||
}
|
||||
if ((ret->priv_encoding =
|
||||
OPENSSL_memdup(src->priv_encoding,
|
||||
src->params->sk_len)) == NULL)
|
||||
goto err;
|
||||
}
|
||||
if (src->seed != NULL
|
||||
&& (ret->seed = OPENSSL_memdup(src->seed,
|
||||
ML_DSA_SEED_BYTES)) == NULL)
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
EVP_MD_up_ref(src->shake128_md);
|
||||
EVP_MD_up_ref(src->shake256_md);
|
||||
ret->shake128_md = src->shake128_md;
|
||||
ret->shake256_md = src->shake256_md;
|
||||
}
|
||||
return ret;
|
||||
err:
|
||||
ossl_ml_dsa_key_free(ret);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Are 2 keys equal?
|
||||
*
|
||||
* To be equal the keys must have matching public or private key data and
|
||||
* contain the same parameters.
|
||||
* (Note that in OpenSSL that the private key always has a public key component).
|
||||
*
|
||||
* @param key1 A ML_DSA_KEY object
|
||||
* @param key2 A ML_DSA_KEY object
|
||||
* @param selection to select public and/or private component comparison.
|
||||
* @returns 1 if the keys are equal otherwise it returns 0.
|
||||
*/
|
||||
int ossl_ml_dsa_key_equal(const ML_DSA_KEY *key1, const ML_DSA_KEY *key2,
|
||||
int selection)
|
||||
{
|
||||
int key_checked = 0;
|
||||
|
||||
if (key1->params != key2->params)
|
||||
return 0;
|
||||
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) {
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) {
|
||||
if (key1->pub_encoding != NULL && key2->pub_encoding != NULL) {
|
||||
if (memcmp(key1->pub_encoding, key2->pub_encoding,
|
||||
key1->params->pk_len) != 0)
|
||||
return 0;
|
||||
key_checked = 1;
|
||||
}
|
||||
}
|
||||
if (!key_checked
|
||||
&& (selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) {
|
||||
if (key1->priv_encoding != NULL && key2->priv_encoding != NULL) {
|
||||
if (memcmp(key1->priv_encoding, key2->priv_encoding,
|
||||
key1->params->sk_len) != 0)
|
||||
return 0;
|
||||
key_checked = 1;
|
||||
}
|
||||
}
|
||||
return key_checked;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_has(const ML_DSA_KEY *key, int selection)
|
||||
{
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) {
|
||||
/* Note that the public key always exists if there is a private key */
|
||||
if (ossl_ml_dsa_key_get_pub(key) == NULL)
|
||||
return 0; /* No public key */
|
||||
if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0
|
||||
&& ossl_ml_dsa_key_get_priv(key) == NULL)
|
||||
return 0; /* No private key */
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Given a key containing private key values for rho, s1 & s2
|
||||
* generate the public value t and return the compressed values t1, t0.
|
||||
*
|
||||
* @param key A private key containing params, rh0, s1 & s2.
|
||||
* @param md_ctx A EVP_MD_CTX used for sampling.
|
||||
* @param t1 The returned polynomial encoding of the 10 MSB of each coefficient
|
||||
* of the uncompressed public key polynomial t.
|
||||
* @param t0 The returned polynomial encoding of the 13 LSB of each coefficient
|
||||
* of the uncompressed public key polynomial t.
|
||||
* @returns 1 on success, or 0 on failure.
|
||||
*/
|
||||
static int public_from_private(const ML_DSA_KEY *key, EVP_MD_CTX *md_ctx,
|
||||
VECTOR *t1, VECTOR *t0)
|
||||
{
|
||||
const ML_DSA_PARAMS *params = key->params;
|
||||
uint32_t k = params->k, l = params->l;
|
||||
POLY *polys;
|
||||
MATRIX a_ntt;
|
||||
VECTOR s1_ntt;
|
||||
VECTOR t;
|
||||
|
||||
polys = OPENSSL_malloc(sizeof(*polys) * (k + l + k * l));
|
||||
if (polys == NULL)
|
||||
return 0;
|
||||
|
||||
vector_init(&t, polys, k);
|
||||
vector_init(&s1_ntt, t.poly + k, l);
|
||||
matrix_init(&a_ntt, s1_ntt.poly + l, k, l);
|
||||
|
||||
/* Using rho generate A' = A in NTT form */
|
||||
if (!matrix_expand_A(md_ctx, key->shake128_md, key->rho, &a_ntt))
|
||||
goto err;
|
||||
|
||||
/* t = NTT_inv(A' * NTT(s1)) + s2 */
|
||||
vector_copy(&s1_ntt, &key->s1);
|
||||
vector_ntt(&s1_ntt);
|
||||
|
||||
matrix_mult_vector(&a_ntt, &s1_ntt, &t);
|
||||
vector_ntt_inverse(&t);
|
||||
vector_add(&t, &key->s2, &t);
|
||||
|
||||
/* Compress t */
|
||||
vector_power2_round(&t, t1, t0);
|
||||
|
||||
/* Zeroize secret */
|
||||
vector_zero(&s1_ntt);
|
||||
err:
|
||||
OPENSSL_free(polys);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_public_from_private(ML_DSA_KEY *key)
|
||||
{
|
||||
int ret = 0;
|
||||
VECTOR t0;
|
||||
EVP_MD_CTX *md_ctx = NULL;
|
||||
|
||||
if (!vector_alloc(&t0, key->params->k)) /* t0 is already in the private key */
|
||||
return 0;
|
||||
ret = ((md_ctx = EVP_MD_CTX_new())!= NULL)
|
||||
&& ossl_ml_dsa_key_pub_alloc(key) /* allocate space for t1 */
|
||||
&& public_from_private(key, md_ctx, &key->t1, &t0)
|
||||
&& vector_equal(&t0, &key->t0) /* compare the generated t0 to the expected */
|
||||
&& ossl_ml_dsa_pk_encode(key)
|
||||
&& shake_xof(md_ctx, key->shake256_md,
|
||||
key->pub_encoding, key->params->pk_len,
|
||||
key->tr, sizeof(key->tr));
|
||||
vector_free(&t0);
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_key_pairwise_check(const ML_DSA_KEY *key)
|
||||
{
|
||||
int ret = 0;
|
||||
VECTOR t1, t0;
|
||||
POLY *polys = NULL;
|
||||
uint32_t k = key->params->k;
|
||||
EVP_MD_CTX *md_ctx = NULL;
|
||||
|
||||
if (key->pub_encoding == NULL || key->priv_encoding == 0)
|
||||
return 0;
|
||||
|
||||
polys = OPENSSL_malloc(sizeof(*polys) * (2 * k));
|
||||
if (polys == NULL)
|
||||
return 0;
|
||||
md_ctx = EVP_MD_CTX_new();
|
||||
if (md_ctx == NULL)
|
||||
goto err;
|
||||
|
||||
vector_init(&t1, polys, k);
|
||||
vector_init(&t0, polys + k, k);
|
||||
if (!public_from_private(key, md_ctx, &t1, &t0))
|
||||
goto err;
|
||||
|
||||
ret = vector_equal(&t1, &key->t1) && vector_equal(&t0, &key->t0);
|
||||
err:
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
OPENSSL_free(polys);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Generate a public-private key pair from a seed.
|
||||
* See FIPS 204, Algorithm 6 ML-DSA.KeyGen_internal().
|
||||
*
|
||||
* @param out The generated key (which contains params on input)
|
||||
*
|
||||
* @returns 1 on success or 0 on failure.
|
||||
*/
|
||||
static int keygen_internal(ML_DSA_KEY *out)
|
||||
{
|
||||
int ret = 0;
|
||||
uint8_t augmented_seed[ML_DSA_SEED_BYTES + 2];
|
||||
uint8_t expanded_seed[ML_DSA_RHO_BYTES + ML_DSA_PRIV_SEED_BYTES + ML_DSA_K_BYTES];
|
||||
const uint8_t *const rho = expanded_seed; /* p = Public Random Seed */
|
||||
const uint8_t *const priv_seed = expanded_seed + ML_DSA_RHO_BYTES;
|
||||
const uint8_t *const K = priv_seed + ML_DSA_PRIV_SEED_BYTES;
|
||||
const ML_DSA_PARAMS *params = out->params;
|
||||
EVP_MD_CTX *md_ctx = NULL;
|
||||
|
||||
if (out->seed == NULL
|
||||
|| (md_ctx = EVP_MD_CTX_new()) == NULL
|
||||
|| !ossl_ml_dsa_key_pub_alloc(out)
|
||||
|| !ossl_ml_dsa_key_priv_alloc(out))
|
||||
goto err;
|
||||
|
||||
/* augmented_seed = seed || k || l */
|
||||
memcpy(augmented_seed, out->seed, ML_DSA_SEED_BYTES);
|
||||
augmented_seed[ML_DSA_SEED_BYTES] = (uint8_t)params->k;
|
||||
augmented_seed[ML_DSA_SEED_BYTES + 1] = (uint8_t)params->l;
|
||||
/* Expand the seed into p[32], p'[64], K[32] */
|
||||
if (!shake_xof(md_ctx, out->shake256_md, augmented_seed, sizeof(augmented_seed),
|
||||
expanded_seed, sizeof(expanded_seed)))
|
||||
goto err;
|
||||
|
||||
memcpy(out->rho, rho, sizeof(out->rho));
|
||||
memcpy(out->K, K, sizeof(out->K));
|
||||
|
||||
ret = vector_expand_S(md_ctx, out->shake256_md, params->eta, priv_seed, &out->s1, &out->s2)
|
||||
&& public_from_private(out, md_ctx, &out->t1, &out->t0)
|
||||
&& ossl_ml_dsa_pk_encode(out)
|
||||
&& shake_xof(md_ctx, out->shake256_md, out->pub_encoding, out->params->pk_len,
|
||||
out->tr, sizeof(out->tr))
|
||||
&& ossl_ml_dsa_sk_encode(out);
|
||||
|
||||
err:
|
||||
if (out->seed != NULL && (out->prov_flags & ML_DSA_KEY_RETAIN_SEED) == 0) {
|
||||
OPENSSL_clear_free(out->seed, ML_DSA_SEED_BYTES);
|
||||
out->seed = NULL;
|
||||
}
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
OPENSSL_cleanse(augmented_seed, sizeof(augmented_seed));
|
||||
OPENSSL_cleanse(expanded_seed, sizeof(expanded_seed));
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ossl_ml_dsa_generate_key(ML_DSA_KEY *out)
|
||||
{
|
||||
size_t seed_len = ML_DSA_SEED_BYTES;
|
||||
uint8_t *sk;
|
||||
int ret;
|
||||
|
||||
if (out->seed == NULL) {
|
||||
if ((out->seed = OPENSSL_malloc(seed_len)) == NULL)
|
||||
return 0;
|
||||
if (RAND_priv_bytes_ex(out->libctx, out->seed, seed_len, 0) <= 0) {
|
||||
OPENSSL_free(out->seed);
|
||||
out->seed = NULL;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
/* We're generating from a seed, drop private prekey encoding */
|
||||
sk = out->priv_encoding;
|
||||
out->priv_encoding = NULL;
|
||||
if (sk == NULL) {
|
||||
ret = keygen_internal(out);
|
||||
} else {
|
||||
if ((ret = keygen_internal(out)) != 0
|
||||
&& memcmp(out->priv_encoding, sk, out->params->sk_len) != 0) {
|
||||
ret = 0;
|
||||
ossl_ml_dsa_key_reset(out);
|
||||
ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY,
|
||||
"explicit %s private key does not match seed",
|
||||
out->params->alg);
|
||||
}
|
||||
OPENSSL_free(sk);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief This is used when a ML DSA key is used for an operation.
|
||||
* This checks that the algorithm is the same (i.e. uses the same parameters)
|
||||
*
|
||||
* @param key A ML_DSA key to use for an operation.
|
||||
* @param evp_type The algorithm nid associated with an operation
|
||||
*
|
||||
* @returns 1 if the algorithm matches, or 0 otherwise.
|
||||
*/
|
||||
|
||||
int ossl_ml_dsa_key_matches(const ML_DSA_KEY *key, int evp_type)
|
||||
{
|
||||
return (key->params->evp_type == evp_type);
|
||||
}
|
||||
|
||||
/* Returns the public key data or NULL if there is no public key */
|
||||
const uint8_t *ossl_ml_dsa_key_get_pub(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->pub_encoding;
|
||||
}
|
||||
|
||||
/* Returns the encoded public key size */
|
||||
size_t ossl_ml_dsa_key_get_pub_len(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params->pk_len;
|
||||
}
|
||||
|
||||
size_t ossl_ml_dsa_key_get_collision_strength_bits(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params->bit_strength;
|
||||
}
|
||||
|
||||
/* Returns the private key data or NULL if there is no private key */
|
||||
const uint8_t *ossl_ml_dsa_key_get_priv(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->priv_encoding;
|
||||
}
|
||||
|
||||
size_t ossl_ml_dsa_key_get_priv_len(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params->sk_len;
|
||||
}
|
||||
|
||||
size_t ossl_ml_dsa_key_get_sig_len(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params->sig_len;
|
||||
}
|
||||
|
||||
OSSL_LIB_CTX *ossl_ml_dsa_key_get0_libctx(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key != NULL ? key->libctx : NULL;
|
||||
}
|
||||
|
||||
const char *ossl_ml_dsa_key_get_name(const ML_DSA_KEY *key)
|
||||
{
|
||||
return key->params->alg;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/e_os2.h>
|
||||
#include "ml_dsa_local.h"
|
||||
#include "ml_dsa_vector.h"
|
||||
|
||||
/* NOTE - any changes to this struct may require updates to ossl_ml_dsa_dup() */
|
||||
struct ml_dsa_key_st {
|
||||
OSSL_LIB_CTX *libctx;
|
||||
const ML_DSA_PARAMS *params;
|
||||
|
||||
EVP_MD *shake128_md;
|
||||
EVP_MD *shake256_md;
|
||||
|
||||
uint8_t rho[ML_DSA_RHO_BYTES]; /* public random seed */
|
||||
uint8_t tr[ML_DSA_TR_BYTES]; /* Pre-cached public key Hash */
|
||||
uint8_t K[ML_DSA_K_BYTES]; /* Private random seed for signing */
|
||||
|
||||
/*
|
||||
* The encoded public and private keys, these are non NULL if the key
|
||||
* components are generated or loaded.
|
||||
*
|
||||
* For keys that are decoded, but not yet loaded or imported into the
|
||||
* provider, the pub_encoding is NULL, while the seed or priv_encoding
|
||||
* is not NULL.
|
||||
*/
|
||||
uint8_t *pub_encoding;
|
||||
uint8_t *priv_encoding;
|
||||
uint8_t *seed;
|
||||
int prov_flags;
|
||||
|
||||
/*
|
||||
* t1 is the Polynomial encoding of the 10 MSB of each coefficient of the
|
||||
* uncompressed public key polynomial t. This is saved as part of the
|
||||
* public key. It is column vector of K polynomials.
|
||||
* (There are 23 bits in q-modulus.. i.e 10 bits = 23 - 13)
|
||||
* t1->poly is allocated.
|
||||
*/
|
||||
VECTOR t1;
|
||||
/*
|
||||
* t0 is the Polynomial encoding of the 13 LSB of each coefficient of the
|
||||
* uncompressed public key polynomial t. This is saved as part of the
|
||||
* private key. It is column vector of K polynomials.
|
||||
*/
|
||||
VECTOR t0;
|
||||
VECTOR s2; /* private secret of size K with short coefficients (-4..4) or (-2..2) */
|
||||
VECTOR s1; /* private secret of size L with short coefficients (-4..4) or (-2..2) */
|
||||
/* The s1->poly block is allocated and has space for s2 and t0 also */
|
||||
};
|
||||
@@ -0,0 +1,175 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "ml_dsa_local.h"
|
||||
|
||||
/* Key Compression related functions (Rounding & hints) */
|
||||
|
||||
/**
|
||||
* @brief Decompose r into (r1, r0) such that r == r1 * 2^13 + r0 mod q
|
||||
* See FIPS 204, Algorithm 35, Power2Round()
|
||||
*
|
||||
* Note: that this code is more complex than the FIPS 204 spec since it keeps
|
||||
* r0 as a positive number
|
||||
*
|
||||
* r mod +- 2^13 is defined as having a range of -4095..4096
|
||||
*
|
||||
* i.e for r = 0..4096 r1 = 0 and r0 = 0..4096
|
||||
* at r = 4097..8191 r1 = 1 and r0 = -4095..0
|
||||
* (but since r0 is kept positive it effectively adds q and then reduces by q if needed)
|
||||
* Similarly for the range r = 8192..8192+4096 r1=1 and r0=0..4096
|
||||
* & 12289..16383 r1=2 and r0=-4095..0
|
||||
*
|
||||
* @param r is in the range 0..q-1
|
||||
* @param r1 The returned top 10 MSB (i.e it ranges from 0..1023)
|
||||
* @param r0 The remainder in the range (0..4096 or q-4095..q-1)
|
||||
* So r0 has an effective range of 8192 (i.e. 13 bits).
|
||||
*/
|
||||
void ossl_ml_dsa_key_compress_power2_round(uint32_t r, uint32_t *r1, uint32_t *r0)
|
||||
{
|
||||
unsigned int mask;
|
||||
uint32_t r0_adjusted, r1_adjusted;
|
||||
|
||||
*r1 = r >> ML_DSA_D_BITS; /* top 13 bits */
|
||||
*r0 = r - (*r1 << ML_DSA_D_BITS); /* The remainder mod q */
|
||||
|
||||
r0_adjusted = mod_sub(*r0, 1 << ML_DSA_D_BITS);
|
||||
r1_adjusted = *r1 + 1;
|
||||
|
||||
/* Mask is set iff r0 > (2^(dropped_bits))/2. */
|
||||
mask = constant_time_lt((uint32_t)(1 << (ML_DSA_D_BITS - 1)), *r0);
|
||||
/* r0 = mask ? r0_adjusted : r0 */
|
||||
*r0 = constant_time_select_int(mask, r0_adjusted, *r0);
|
||||
/* r1 = mask ? r1_adjusted : r1 */
|
||||
*r1 = constant_time_select_int(mask, r1_adjusted, *r1);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief return the r1 component of Decomposing r into (r1, r0) such that
|
||||
* r == r1 * (2 * gamma2) + r0 mod q
|
||||
* See FIPS 204, Algorithm 37, HighBits()
|
||||
*
|
||||
* @param r A value to decompose in the range (0..q-1)
|
||||
* @param gamma2 Depending on the algorithm gamma2 is either (q-1)/32 or (q-1)/88
|
||||
* @returns r1 (The high order bits)
|
||||
*/
|
||||
uint32_t ossl_ml_dsa_key_compress_high_bits(uint32_t r, uint32_t gamma2)
|
||||
{
|
||||
int32_t r1 = (r + 127) >> 7;
|
||||
|
||||
if (gamma2 == ML_DSA_GAMMA2_Q_MINUS1_DIV32) {
|
||||
r1 = (r1 * 1025 + (1 << 21)) >> 22;
|
||||
r1 &= 15; /* mod 16 */
|
||||
return r1;
|
||||
} else {
|
||||
r1 = (r1 * 11275 + (1 << 23)) >> 24;
|
||||
r1 ^= ((43 - r1) >> 31) & r1;
|
||||
return r1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Decomposes r into (r1, r0) such that r == r1 * (2*gamma2) + r0 mod q.
|
||||
* See FIPS 204, Algorithm 36, Decompose()
|
||||
*
|
||||
* @param r A value to decompose in the range (0..q-1)
|
||||
* @param gamma2 Depending on the algorithm gamma2 is either (q-1)/32 or (q-1)/88
|
||||
* @param r1 The returned high order bits
|
||||
* @param r0 The returned low order bits
|
||||
*/
|
||||
void ossl_ml_dsa_key_compress_decompose(uint32_t r, uint32_t gamma2,
|
||||
uint32_t *r1, int32_t *r0)
|
||||
{
|
||||
*r1 = ossl_ml_dsa_key_compress_high_bits(r, gamma2);
|
||||
|
||||
*r0 = r - *r1 * 2 * (int32_t)gamma2;
|
||||
*r0 -= (((int32_t)ML_DSA_Q_MINUS1_DIV2 - *r0) >> 31) & (int32_t)ML_DSA_Q;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief return the r0 component of Decomposing r into (r1, r0) such that
|
||||
* r == r1 * (2 * gamma2) + r0 mod q
|
||||
* See FIPS 204, Algorithm 38, LowBits()
|
||||
*
|
||||
* @param r A value to decompose in the range (0..q-1)
|
||||
* @param gamma2 Depending on the algorithm gamma2 is either (q-1)/32 or (q-1)/88
|
||||
* @param r0 The returned low order bits
|
||||
*/
|
||||
int32_t ossl_ml_dsa_key_compress_low_bits(uint32_t r, uint32_t gamma2)
|
||||
{
|
||||
uint32_t r1;
|
||||
int32_t r0;
|
||||
|
||||
ossl_ml_dsa_key_compress_decompose(r, gamma2, &r1, &r0);
|
||||
return r0;
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Computes hint bit indicating whether adding z to r alters the high
|
||||
* bits of r
|
||||
* See FIPS 204, Algorithm 39, MakeHint().
|
||||
*
|
||||
* In the spec this takes two arguments, z and r, and is called with
|
||||
* z = -ct0
|
||||
* r = w - cs2 + ct0
|
||||
*
|
||||
* It then computes HighBits (algorithm 37) of z and z+r.
|
||||
* But z + r is just w - cs2, so this takes three arguments and saves an addition.
|
||||
*
|
||||
* @params ct0 A polynomial c (with coefficients of (-1,0,1)) multiplied by the
|
||||
* polynomial vector t0 (which encodes the least significant bits of each coefficient of the
|
||||
uncompressed public-key polynomial t)
|
||||
* @params cs2 A polynomial c (with coefficients of (-1,0,1)) multiplied by s2 (a secret polynomial)
|
||||
* @params gamma2 Depending on the algorithm gamma2 is either (q-1)/32 or (q-1)/88
|
||||
* @params w (A * y)
|
||||
* @returns The hint bit.
|
||||
*/
|
||||
int32_t ossl_ml_dsa_key_compress_make_hint(uint32_t ct0, uint32_t cs2,
|
||||
uint32_t gamma2, uint32_t w)
|
||||
{
|
||||
uint32_t r_plus_z = mod_sub(w, cs2);
|
||||
uint32_t r = reduce_once(r_plus_z + ct0);
|
||||
|
||||
return ossl_ml_dsa_key_compress_high_bits(r, gamma2)
|
||||
!= ossl_ml_dsa_key_compress_high_bits(r_plus_z, gamma2);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Returns the high bits of |r| adjusted according to hint |h|.
|
||||
* FIPS 204, Algorithm 40, UseHint().
|
||||
* This is not constant time.
|
||||
*
|
||||
* @param hint The hint bit which is either 0 or 1
|
||||
* @param r A value to decompose in the range (0..q-1)
|
||||
* @param gamma2 Depending on the algorithm gamma2 is either (q-1)/32 or (q-1)/88
|
||||
*
|
||||
* @returns The adjusted high bits or r.
|
||||
*/
|
||||
uint32_t ossl_ml_dsa_key_compress_use_hint(uint32_t hint, uint32_t r,
|
||||
uint32_t gamma2)
|
||||
{
|
||||
uint32_t r1;
|
||||
int32_t r0;
|
||||
|
||||
ossl_ml_dsa_key_compress_decompose(r, gamma2, &r1, &r0);
|
||||
|
||||
if (hint == 0)
|
||||
return r1;
|
||||
|
||||
if (gamma2 == ((ML_DSA_Q - 1) / 32)) {
|
||||
/* m = 16, thus |mod m| in the spec turns into |& 15| */
|
||||
return r0 > 0 ? (r1 + 1) & 15 : (r1 - 1) & 15;
|
||||
} else {
|
||||
/* m = 44 if gamma2 = ((q - 1) / 88) */
|
||||
if (r0 > 0)
|
||||
return (r1 == 43) ? 0 : r1 + 1;
|
||||
else
|
||||
return (r1 == 0) ? 43 : r1 - 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#ifndef OSSL_CRYPTO_ML_DSA_LOCAL_H
|
||||
# define OSSL_CRYPTO_ML_DSA_LOCAL_H
|
||||
|
||||
# include "crypto/ml_dsa.h"
|
||||
# include "internal/constant_time.h"
|
||||
# include "internal/packet.h"
|
||||
|
||||
/* The following constants are shared by ML-DSA-44, ML-DSA-65 & ML-DSA-87 */
|
||||
# define ML_DSA_Q 8380417 /* The modulus is 23 bits (2^23 - 2^13 + 1) */
|
||||
# define ML_DSA_Q_MINUS1_DIV2 ((ML_DSA_Q - 1) / 2)
|
||||
|
||||
# define ML_DSA_Q_BITS 23
|
||||
# define ML_DSA_Q_INV 58728449 /* q^-1 satisfies: q^-1 * q = 1 mod 2^32 */
|
||||
# define ML_DSA_Q_NEG_INV 4236238847 /* Inverse of -q modulo 2^32 */
|
||||
# define ML_DSA_DEGREE_INV_MONTGOMERY 41978 /* Inverse of 256 mod q, in Montgomery form. */
|
||||
|
||||
# define ML_DSA_D_BITS 13 /* The number of bits dropped from the public vector t */
|
||||
# define ML_DSA_NUM_POLY_COEFFICIENTS 256 /* The number of coefficients in the polynomials */
|
||||
# define ML_DSA_RHO_BYTES 32 /* p = Public Random Seed */
|
||||
# define ML_DSA_PRIV_SEED_BYTES 64 /* p' = Private random seed */
|
||||
# define ML_DSA_K_BYTES 32 /* K = Private random seed for signing */
|
||||
# define ML_DSA_TR_BYTES 64 /* Size of the Hash of the public key used for signing */
|
||||
# define ML_DSA_MU_BYTES 64 /* Size of the Hash for the message representative */
|
||||
# define ML_DSA_RHO_PRIME_BYTES 64 /* private random seed size */
|
||||
|
||||
/*
|
||||
* There is special case code related to encoding/decoding that tests the
|
||||
* for the following values.
|
||||
*/
|
||||
/*
|
||||
* The possible value for eta - If a new value is added, then all code
|
||||
* that accesses ML_DSA_ETA_4 would need to be modified.
|
||||
*/
|
||||
# define ML_DSA_ETA_4 4
|
||||
# define ML_DSA_ETA_2 2
|
||||
/*
|
||||
* The possible values of gamma1 - If a new value is added, then all code
|
||||
* that accesses ML_DSA_GAMMA1_TWO_POWER_19 would need to be modified.
|
||||
*/
|
||||
# define ML_DSA_GAMMA1_TWO_POWER_19 (1 << 19)
|
||||
# define ML_DSA_GAMMA1_TWO_POWER_17 (1 << 17)
|
||||
/*
|
||||
* The possible values for gamma2 - If a new value is added, then all code
|
||||
* that accesses ML_DSA_GAMMA2_Q_MINUS1_DIV32 would need to be modified.
|
||||
*/
|
||||
# define ML_DSA_GAMMA2_Q_MINUS1_DIV32 ((ML_DSA_Q - 1) / 32)
|
||||
# define ML_DSA_GAMMA2_Q_MINUS1_DIV88 ((ML_DSA_Q - 1) / 88)
|
||||
|
||||
typedef struct poly_st POLY;
|
||||
typedef struct vector_st VECTOR;
|
||||
typedef struct matrix_st MATRIX;
|
||||
typedef struct ml_dsa_sig_st ML_DSA_SIG;
|
||||
|
||||
int ossl_ml_dsa_matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md,
|
||||
const uint8_t *rho, MATRIX *out);
|
||||
int ossl_ml_dsa_vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta,
|
||||
const uint8_t *seed, VECTOR *s1, VECTOR *s2);
|
||||
void ossl_ml_dsa_matrix_mult_vector(const MATRIX *matrix_kl, const VECTOR *vl,
|
||||
VECTOR *vk);
|
||||
int ossl_ml_dsa_poly_expand_mask(POLY *out, const uint8_t *seed, size_t seed_len,
|
||||
uint32_t gamma1,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md);
|
||||
int ossl_ml_dsa_poly_sample_in_ball(POLY *out_c, const uint8_t *seed, int seed_len,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md,
|
||||
uint32_t tau);
|
||||
|
||||
void ossl_ml_dsa_poly_ntt(POLY *s);
|
||||
void ossl_ml_dsa_poly_ntt_inverse(POLY *s);
|
||||
void ossl_ml_dsa_poly_ntt_mult(const POLY *lhs, const POLY *rhs, POLY *out);
|
||||
|
||||
void ossl_ml_dsa_key_compress_power2_round(uint32_t r, uint32_t *r1, uint32_t *r0);
|
||||
uint32_t ossl_ml_dsa_key_compress_high_bits(uint32_t r, uint32_t gamma2);
|
||||
void ossl_ml_dsa_key_compress_decompose(uint32_t r, uint32_t gamma2,
|
||||
uint32_t *r1, int32_t *r0);
|
||||
void ossl_ml_dsa_key_compress_decompose(uint32_t r, uint32_t gamma2,
|
||||
uint32_t *r1, int32_t *r0);
|
||||
int32_t ossl_ml_dsa_key_compress_low_bits(uint32_t r, uint32_t gamma2);
|
||||
int32_t ossl_ml_dsa_key_compress_make_hint(uint32_t ct0, uint32_t cs2,
|
||||
uint32_t gamma2, uint32_t w);
|
||||
uint32_t ossl_ml_dsa_key_compress_use_hint(uint32_t hint, uint32_t r,
|
||||
uint32_t gamma2);
|
||||
|
||||
int ossl_ml_dsa_pk_encode(ML_DSA_KEY *key);
|
||||
int ossl_ml_dsa_sk_encode(ML_DSA_KEY *key);
|
||||
|
||||
int ossl_ml_dsa_sig_encode(const ML_DSA_SIG *sig, const ML_DSA_PARAMS *params,
|
||||
uint8_t *out);
|
||||
int ossl_ml_dsa_sig_decode(ML_DSA_SIG *sig, const uint8_t *in, size_t in_len,
|
||||
const ML_DSA_PARAMS *params);
|
||||
int ossl_ml_dsa_w1_encode(const VECTOR *w1, uint32_t gamma2,
|
||||
uint8_t *out, size_t out_len);
|
||||
int ossl_ml_dsa_poly_decode_expand_mask(POLY *out,
|
||||
const uint8_t *in, size_t in_len,
|
||||
uint32_t gamma1);
|
||||
|
||||
/*
|
||||
* @brief Reduces x mod q in constant time
|
||||
* i.e. return x < q ? x : x - q;
|
||||
*
|
||||
* @param x Where x is assumed to be in the range 0 <= x < 2*q
|
||||
* @returns the difference in the range 0..q-1
|
||||
*/
|
||||
static ossl_inline ossl_unused uint32_t reduce_once(uint32_t x)
|
||||
{
|
||||
return constant_time_select_32(constant_time_lt_32(x, ML_DSA_Q), x, x - ML_DSA_Q);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Calculate The positive value of (a-b) mod q in constant time.
|
||||
*
|
||||
* a - b mod q gives a value in the range -(q-1)..(q-1)
|
||||
* By adding q we get a range of 1..(2q-1).
|
||||
* Reducing this once then gives the range 0..q-1
|
||||
*
|
||||
* @param a The minuend assumed to be in the range 0..q-1
|
||||
* @param b The subtracthend assumed to be in the range 0..q-1.
|
||||
* @returns The value (q + a - b) mod q
|
||||
*/
|
||||
static ossl_inline ossl_unused uint32_t mod_sub(uint32_t a, uint32_t b)
|
||||
{
|
||||
return reduce_once(ML_DSA_Q + a - b);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Returns the absolute value in constant time.
|
||||
* i.e. return is_positive(x) ? x : -x;
|
||||
*/
|
||||
static ossl_inline ossl_unused uint32_t abs_signed(uint32_t x)
|
||||
{
|
||||
return constant_time_select_32(constant_time_lt_32(x, 0x80000000), x, 0u - x);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Returns the absolute value modulo q in constant time
|
||||
* i.e return x > (q - 1) / 2 ? q - x : x;
|
||||
*/
|
||||
static ossl_inline ossl_unused uint32_t abs_mod_prime(uint32_t x)
|
||||
{
|
||||
return constant_time_select_32(constant_time_lt_32(ML_DSA_Q_MINUS1_DIV2, x),
|
||||
ML_DSA_Q - x, x);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Returns the maximum of two values in constant time.
|
||||
* i.e return x < y ? y : x;
|
||||
*/
|
||||
static ossl_inline ossl_unused uint32_t maximum(uint32_t x, uint32_t y)
|
||||
{
|
||||
return constant_time_select_int(constant_time_lt(x, y), y, x);
|
||||
}
|
||||
|
||||
#endif /* OSSL_CRYPTO_ML_DSA_LOCAL_H */
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "ml_dsa_local.h"
|
||||
#include "ml_dsa_vector.h"
|
||||
#include "ml_dsa_matrix.h"
|
||||
|
||||
/*
|
||||
* Matrix multiply of a k*l matrix of polynomials by a 1 * l vector of
|
||||
* polynomials to produce a 1 * k vector of polynomial results.
|
||||
* i.e. t = a * s
|
||||
*
|
||||
* @param a A k * l matrix of polynomials in NTT form
|
||||
* @param s A 1 * l vector of polynomials in NTT form
|
||||
* @param t 1 * k vector of polynomial results in NTT form
|
||||
*/
|
||||
void ossl_ml_dsa_matrix_mult_vector(const MATRIX *a, const VECTOR *s,
|
||||
VECTOR *t)
|
||||
{
|
||||
size_t i, j;
|
||||
POLY *poly = a->m_poly;
|
||||
|
||||
vector_zero(t);
|
||||
|
||||
for (i = 0; i < a->k; i++) {
|
||||
for (j = 0; j < a->l; j++) {
|
||||
POLY product;
|
||||
|
||||
ossl_ml_dsa_poly_ntt_mult(poly++, &s->poly[j], &product);
|
||||
poly_add(&product, &t->poly[i], &t->poly[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
/* A 'k' by 'l' Matrix object ('k' rows and 'l' columns) containing polynomial scalars */
|
||||
struct matrix_st {
|
||||
POLY *m_poly;
|
||||
size_t k, l;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Initialize a Matrix object.
|
||||
*
|
||||
* @param m The matrix object.
|
||||
* @param polys A preallocated array of k * l polynomial blocks. |m| does not
|
||||
* own/free this.
|
||||
* @param k The number of rows
|
||||
* @param l The number of columns
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
matrix_init(MATRIX *m, POLY *polys, size_t k, size_t l)
|
||||
{
|
||||
m->k = k;
|
||||
m->l = l;
|
||||
m->m_poly = polys;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
matrix_mult_vector(const MATRIX *a, const VECTOR *s, VECTOR *t)
|
||||
{
|
||||
ossl_ml_dsa_matrix_mult_vector(a, s, t);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md, const uint8_t *rho,
|
||||
MATRIX *out)
|
||||
{
|
||||
return ossl_ml_dsa_matrix_expand_A(g_ctx, md, rho, out);
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "ml_dsa_local.h"
|
||||
#include "ml_dsa_poly.h"
|
||||
|
||||
/*
|
||||
* This file has multiple parts required for fast matrix multiplication,
|
||||
* 1) NTT (See https://eprint.iacr.org/2024/585.pdf)
|
||||
* NTT and NTT inverse transformations are Discrete Fourier Transforms in a
|
||||
* polynomial ring. Fast-Fourier Transformations can then be applied to make
|
||||
* multiplications n log(n). This uses the symmetry of the transformation to
|
||||
* reduce computations.
|
||||
*
|
||||
* 2) Montgomery multiplication
|
||||
* The multiplication of a.b mod q requires division by q which is a slow operation.
|
||||
*
|
||||
* When many multiplications mod q are required montgomery multiplication
|
||||
* can be used. This requires a number R > q such that R & q are coprime
|
||||
* (i.e. GCD(R, q) = 1), so that division happens using R instead of q.
|
||||
* If r is a power of 2 then this division can be done as a bit shift.
|
||||
*
|
||||
* Given that q = 2^23 - 2^13 + 1
|
||||
* We can chose a Montgomery multiplier of R = 2^32.
|
||||
*
|
||||
* To transform |a| into Montgomery form |m| we use
|
||||
* m = a mod q * ((2^32)*(2^32) mod q)
|
||||
* which is then Montgomery reduced, removing the excess factor of R = 2^32.
|
||||
*/
|
||||
|
||||
/*
|
||||
* The table in FIPS 204 Appendix B uses the following formula
|
||||
* zeta[k]= 1753^bitrev(k) mod q for (k = 1..255) (The first value is not used).
|
||||
*
|
||||
* As this implementation uses montgomery form with a multiplier of 2^32.
|
||||
* The values need to be transformed i.e.
|
||||
*
|
||||
* zetasMontgomery[k] = reduce_montgomery(zeta[k] * (2^32 * 2^32 mod(q)))
|
||||
* reduce_montgomery() is defined below..
|
||||
*/
|
||||
static const uint32_t zetas_montgomery[256] = {
|
||||
4193792, 25847, 5771523, 7861508, 237124, 7602457, 7504169, 466468,
|
||||
1826347, 2353451, 8021166, 6288512, 3119733, 5495562, 3111497, 2680103,
|
||||
2725464, 1024112, 7300517, 3585928, 7830929, 7260833, 2619752, 6271868,
|
||||
6262231, 4520680, 6980856, 5102745, 1757237, 8360995, 4010497, 280005,
|
||||
2706023, 95776, 3077325, 3530437, 6718724, 4788269, 5842901, 3915439,
|
||||
4519302, 5336701, 3574422, 5512770, 3539968, 8079950, 2348700, 7841118,
|
||||
6681150, 6736599, 3505694, 4558682, 3507263, 6239768, 6779997, 3699596,
|
||||
811944, 531354, 954230, 3881043, 3900724, 5823537, 2071892, 5582638,
|
||||
4450022, 6851714, 4702672, 5339162, 6927966, 3475950, 2176455, 6795196,
|
||||
7122806, 1939314, 4296819, 7380215, 5190273, 5223087, 4747489, 126922,
|
||||
3412210, 7396998, 2147896, 2715295, 5412772, 4686924, 7969390, 5903370,
|
||||
7709315, 7151892, 8357436, 7072248, 7998430, 1349076, 1852771, 6949987,
|
||||
5037034, 264944, 508951, 3097992, 44288, 7280319, 904516, 3958618,
|
||||
4656075, 8371839, 1653064, 5130689, 2389356, 8169440, 759969, 7063561,
|
||||
189548, 4827145, 3159746, 6529015, 5971092, 8202977, 1315589, 1341330,
|
||||
1285669, 6795489, 7567685, 6940675, 5361315, 4499357, 4751448, 3839961,
|
||||
2091667, 3407706, 2316500, 3817976, 5037939, 2244091, 5933984, 4817955,
|
||||
266997, 2434439, 7144689, 3513181, 4860065, 4621053, 7183191, 5187039,
|
||||
900702, 1859098, 909542, 819034, 495491, 6767243, 8337157, 7857917,
|
||||
7725090, 5257975, 2031748, 3207046, 4823422, 7855319, 7611795, 4784579,
|
||||
342297, 286988, 5942594, 4108315, 3437287, 5038140, 1735879, 203044,
|
||||
2842341, 2691481, 5790267, 1265009, 4055324, 1247620, 2486353, 1595974,
|
||||
4613401, 1250494, 2635921, 4832145, 5386378, 1869119, 1903435, 7329447,
|
||||
7047359, 1237275, 5062207, 6950192, 7929317, 1312455, 3306115, 6417775,
|
||||
7100756, 1917081, 5834105, 7005614, 1500165, 777191, 2235880, 3406031,
|
||||
7838005, 5548557, 6709241, 6533464, 5796124, 4656147, 594136, 4603424,
|
||||
6366809, 2432395, 2454455, 8215696, 1957272, 3369112, 185531, 7173032,
|
||||
5196991, 162844, 1616392, 3014001, 810149, 1652634, 4686184, 6581310,
|
||||
5341501, 3523897, 3866901, 269760, 2213111, 7404533, 1717735, 472078,
|
||||
7953734, 1723600, 6577327, 1910376, 6712985, 7276084, 8119771, 4546524,
|
||||
5441381, 6144432, 7959518, 6094090, 183443, 7403526, 1612842, 4834730,
|
||||
7826001, 3919660, 8332111, 7018208, 3937738, 1400424, 7534263, 1976782
|
||||
};
|
||||
|
||||
/*
|
||||
* @brief When multiplying 2 numbers mod q that are in montgomery form, the
|
||||
* product mod q needs to be multiplied by 2^-32 to be in montgomery form.
|
||||
* See FIPS 204, Algorithm 49, MontgomeryReduce()
|
||||
* Note it is slightly different due to the input range being positive
|
||||
*
|
||||
* @param a is the result of a multiply of 2 numbers in montgomery form,
|
||||
* in the range 0...(2^32)*q
|
||||
* @returns The Montgomery form of 'a' with multiplier 2^32 in the range 0..q-1
|
||||
* The result is congruent to x * 2^-32 mod q
|
||||
*/
|
||||
static uint32_t reduce_montgomery(uint64_t a)
|
||||
{
|
||||
uint64_t t = (uint32_t)a * (uint32_t)ML_DSA_Q_NEG_INV; /* a * -qinv */
|
||||
uint64_t b = a + t * ML_DSA_Q; /* a - t * q */
|
||||
uint32_t c = b >> 32; /* /2^32 = 0..2q */
|
||||
|
||||
return reduce_once(c); /* 0..q */
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Multiply two polynomials in the number theoretically transformed state.
|
||||
* See FIPS 204, Algorithm 45, MultiplyNTT()
|
||||
* This function has been modified to use montgomery multiplication
|
||||
*
|
||||
* @param lhs A polynomial multiplicand
|
||||
* @param rhs A polynomial multiplier
|
||||
* @param out The returned result of the polynomial multiply
|
||||
*/
|
||||
void ossl_ml_dsa_poly_ntt_mult(const POLY *lhs, const POLY *rhs, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] =
|
||||
reduce_montgomery((uint64_t)lhs->coeff[i] * (uint64_t)rhs->coeff[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* In place number theoretic transform of a given polynomial.
|
||||
*
|
||||
* See FIPS 204, Algorithm 41, NTT()
|
||||
* This function uses montgomery multiplication.
|
||||
*
|
||||
* @param p a polynomial that is used as the input, that is replaced with
|
||||
* the NTT of the polynomial
|
||||
*/
|
||||
void ossl_ml_dsa_poly_ntt(POLY *p)
|
||||
{
|
||||
int i, j, k;
|
||||
int step;
|
||||
int offset = ML_DSA_NUM_POLY_COEFFICIENTS;
|
||||
|
||||
/* Step: 1, 2, 4, 8, ..., 128 */
|
||||
for (step = 1; step < ML_DSA_NUM_POLY_COEFFICIENTS; step <<= 1) {
|
||||
k = 0;
|
||||
offset >>= 1; /* Offset: 128, 64, 32, 16, ..., 1 */
|
||||
for (i = 0; i < step; i++) {
|
||||
const uint32_t z_step_root = zetas_montgomery[step + i];
|
||||
|
||||
for (j = k; j < k + offset; j++) {
|
||||
uint32_t w_even = p->coeff[j];
|
||||
uint32_t t_odd =
|
||||
reduce_montgomery((uint64_t)z_step_root
|
||||
* (uint64_t)p->coeff[j + offset]);
|
||||
|
||||
p->coeff[j] = reduce_once(w_even + t_odd);
|
||||
p->coeff[j + offset] = mod_sub(w_even, t_odd);
|
||||
}
|
||||
k += 2 * offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief In place inverse number theoretic transform of a given polynomial.
|
||||
* See FIPS 204, Algorithm 42, NTT^-1()
|
||||
*
|
||||
* @param p a polynomial that is used as the input, that is overwritten with
|
||||
* the inverse of the NTT.
|
||||
*/
|
||||
void ossl_ml_dsa_poly_ntt_inverse(POLY *p)
|
||||
{
|
||||
/*
|
||||
* Step: 128, 64, 32, 16, ..., 1
|
||||
* Offset: 1, 2, 4, 8, ..., 128
|
||||
*/
|
||||
int i, j, k, offset, step = ML_DSA_NUM_POLY_COEFFICIENTS;
|
||||
/*
|
||||
* The multiplicative inverse of 256 mod q, in Montgomery form is
|
||||
* ((256^-1 mod q) * ((2^32 * 2^32) mod q)) mod q = (8347681 * 2365951) mod 8380417
|
||||
*/
|
||||
static const uint32_t inverse_degree_montgomery = 41978;
|
||||
|
||||
for (offset = 1; offset < ML_DSA_NUM_POLY_COEFFICIENTS; offset <<= 1) {
|
||||
step >>= 1;
|
||||
k = 0;
|
||||
for (i = 0; i < step; i++) {
|
||||
const uint32_t step_root =
|
||||
ML_DSA_Q - zetas_montgomery[step + (step - 1 - i)];
|
||||
|
||||
for (j = k; j < k + offset; j++) {
|
||||
uint32_t even = p->coeff[j];
|
||||
uint32_t odd = p->coeff[j + offset];
|
||||
|
||||
p->coeff[j] = reduce_once(odd + even);
|
||||
p->coeff[j + offset] =
|
||||
reduce_montgomery((uint64_t)step_root
|
||||
* (uint64_t)(ML_DSA_Q + even - odd));
|
||||
}
|
||||
k += 2 * offset;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
p->coeff[i] = reduce_montgomery((uint64_t)p->coeff[i] *
|
||||
(uint64_t)inverse_degree_montgomery);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <openssl/evp.h>
|
||||
#include "ml_dsa_local.h"
|
||||
|
||||
/* See FIPS 204 Section 4 Table 1 & Table 2 */
|
||||
#define ML_DSA_44_TAU 39
|
||||
#define ML_DSA_44_LAMBDA 128
|
||||
#define ML_DSA_44_K 4
|
||||
#define ML_DSA_44_L 4
|
||||
#define ML_DSA_44_ETA ML_DSA_ETA_2
|
||||
#define ML_DSA_44_BETA 78
|
||||
#define ML_DSA_44_OMEGA 80
|
||||
#define ML_DSA_44_SECURITY_CATEGORY 2
|
||||
|
||||
/* See FIPS 204 Section 4 Table 1 & Table 2 */
|
||||
#define ML_DSA_65_TAU 49
|
||||
#define ML_DSA_65_LAMBDA 192
|
||||
#define ML_DSA_65_K 6
|
||||
#define ML_DSA_65_L 5
|
||||
#define ML_DSA_65_ETA ML_DSA_ETA_4
|
||||
#define ML_DSA_65_BETA 196
|
||||
#define ML_DSA_65_OMEGA 55
|
||||
#define ML_DSA_65_SECURITY_CATEGORY 3
|
||||
|
||||
/* See FIPS 204 Section 4 Table 1 & Table 2 */
|
||||
#define ML_DSA_87_TAU 60
|
||||
#define ML_DSA_87_LAMBDA 256
|
||||
#define ML_DSA_87_K 8
|
||||
#define ML_DSA_87_L 7
|
||||
#define ML_DSA_87_ETA ML_DSA_ETA_2
|
||||
#define ML_DSA_87_BETA 120
|
||||
#define ML_DSA_87_OMEGA 75
|
||||
#define ML_DSA_87_SECURITY_CATEGORY 5
|
||||
|
||||
static const ML_DSA_PARAMS ml_dsa_params[] = {
|
||||
{ "ML-DSA-44",
|
||||
EVP_PKEY_ML_DSA_44,
|
||||
ML_DSA_44_TAU,
|
||||
ML_DSA_44_LAMBDA,
|
||||
ML_DSA_GAMMA1_TWO_POWER_17,
|
||||
ML_DSA_GAMMA2_Q_MINUS1_DIV88,
|
||||
ML_DSA_44_K,
|
||||
ML_DSA_44_L,
|
||||
ML_DSA_44_ETA,
|
||||
ML_DSA_44_BETA,
|
||||
ML_DSA_44_OMEGA,
|
||||
ML_DSA_44_SECURITY_CATEGORY,
|
||||
ML_DSA_44_PRIV_LEN,
|
||||
ML_DSA_44_PUB_LEN,
|
||||
ML_DSA_44_SIG_LEN
|
||||
},
|
||||
{ "ML-DSA-65",
|
||||
EVP_PKEY_ML_DSA_65,
|
||||
ML_DSA_65_TAU,
|
||||
ML_DSA_65_LAMBDA,
|
||||
ML_DSA_GAMMA1_TWO_POWER_19,
|
||||
ML_DSA_GAMMA2_Q_MINUS1_DIV32,
|
||||
ML_DSA_65_K,
|
||||
ML_DSA_65_L,
|
||||
ML_DSA_65_ETA,
|
||||
ML_DSA_65_BETA,
|
||||
ML_DSA_65_OMEGA,
|
||||
ML_DSA_65_SECURITY_CATEGORY,
|
||||
ML_DSA_65_PRIV_LEN,
|
||||
ML_DSA_65_PUB_LEN,
|
||||
ML_DSA_65_SIG_LEN
|
||||
},
|
||||
{ "ML-DSA-87",
|
||||
EVP_PKEY_ML_DSA_87,
|
||||
ML_DSA_87_TAU,
|
||||
ML_DSA_87_LAMBDA,
|
||||
ML_DSA_GAMMA1_TWO_POWER_19,
|
||||
ML_DSA_GAMMA2_Q_MINUS1_DIV32,
|
||||
ML_DSA_87_K,
|
||||
ML_DSA_87_L,
|
||||
ML_DSA_87_ETA,
|
||||
ML_DSA_87_BETA,
|
||||
ML_DSA_87_OMEGA,
|
||||
ML_DSA_87_SECURITY_CATEGORY,
|
||||
ML_DSA_87_PRIV_LEN,
|
||||
ML_DSA_87_PUB_LEN,
|
||||
ML_DSA_87_SIG_LEN
|
||||
},
|
||||
{NULL},
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A getter to convert an algorithm name into a ML_DSA_PARAMS object
|
||||
*/
|
||||
const ML_DSA_PARAMS *ossl_ml_dsa_params_get(int evp_type)
|
||||
{
|
||||
const ML_DSA_PARAMS *p;
|
||||
|
||||
for (p = ml_dsa_params; p->alg != NULL; ++p) {
|
||||
if (p->evp_type == evp_type)
|
||||
return p;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
#include <openssl/crypto.h>
|
||||
|
||||
#define ML_DSA_NUM_POLY_COEFFICIENTS 256
|
||||
|
||||
/* Polynomial object with 256 coefficients. The coefficients are unsigned 32 bits */
|
||||
struct poly_st {
|
||||
uint32_t coeff[ML_DSA_NUM_POLY_COEFFICIENTS];
|
||||
};
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_zero(POLY *p)
|
||||
{
|
||||
memset(p->coeff, 0, sizeof(*p));
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Polynomial addition.
|
||||
*
|
||||
* @param lhs A polynomial with coefficients in the range (0..q-1)
|
||||
* @param rhs A polynomial with coefficients in the range (0..q-1) to add
|
||||
* to the 'lhs'.
|
||||
* @param out The returned addition result with the coefficients all in the
|
||||
* range 0..q-1
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
poly_add(const POLY *lhs, const POLY *rhs, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = reduce_once(lhs->coeff[i] + rhs->coeff[i]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Polynomial subtraction.
|
||||
*
|
||||
* @param lhs A polynomial with coefficients in the range (0..q-1)
|
||||
* @param rhs A polynomial with coefficients in the range (0..q-1) to subtract
|
||||
* from the 'lhs'.
|
||||
* @param out The returned subtraction result with the coefficients all in the
|
||||
* range 0..q-1
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
poly_sub(const POLY *lhs, const POLY *rhs, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = mod_sub(lhs->coeff[i], rhs->coeff[i]);
|
||||
}
|
||||
|
||||
/* @returns 1 if the polynomials are equal, or 0 otherwise */
|
||||
static ossl_inline ossl_unused int
|
||||
poly_equal(const POLY *a, const POLY *b)
|
||||
{
|
||||
return CRYPTO_memcmp(a, b, sizeof(*a)) == 0;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_ntt(POLY *p)
|
||||
{
|
||||
ossl_ml_dsa_poly_ntt(p);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
poly_sample_in_ball_ntt(POLY *out, const uint8_t *seed, int seed_len,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md, uint32_t tau)
|
||||
{
|
||||
if (!ossl_ml_dsa_poly_sample_in_ball(out, seed, seed_len, h_ctx, md, tau))
|
||||
return 0;
|
||||
poly_ntt(out);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
poly_expand_mask(POLY *out, const uint8_t *seed, size_t seed_len,
|
||||
uint32_t gamma1, EVP_MD_CTX *h_ctx, const EVP_MD *md)
|
||||
{
|
||||
return ossl_ml_dsa_poly_expand_mask(out, seed, seed_len, gamma1, h_ctx, md);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Decompose the coefficients of a polynomial into (r1, r0) such that
|
||||
* coeff[i] == t1[i] * 2^13 + t0[i] mod q
|
||||
* See FIPS 204, Algorithm 35, Power2Round()
|
||||
*
|
||||
* @param t A polynomial containing coefficients in the range 0..q-1
|
||||
* @param t1 The returned polynomial containing coefficients that represent
|
||||
* the top 10 MSB of each coefficient in t (i.e each ranging from 0..1023)
|
||||
* @param t0 The remainder coefficients of t in the range (0..4096 or q-4095..q-1)
|
||||
* Each t0 coefficient has an effective range of 8192 (i.e. 13 bits).
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
poly_power2_round(const POLY *t, POLY *t1, POLY *t0)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
ossl_ml_dsa_key_compress_power2_round(t->coeff[i],
|
||||
t1->coeff + i, t0->coeff + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_scale_power2_round(POLY *in, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = (in->coeff[i] << ML_DSA_D_BITS);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_high_bits(const POLY *in, uint32_t gamma2, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = ossl_ml_dsa_key_compress_high_bits(in->coeff[i], gamma2);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_low_bits(const POLY *in, uint32_t gamma2, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = ossl_ml_dsa_key_compress_low_bits(in->coeff[i], gamma2);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_make_hint(const POLY *ct0, const POLY *cs2, const POLY *w, uint32_t gamma2,
|
||||
POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = ossl_ml_dsa_key_compress_make_hint(ct0->coeff[i],
|
||||
cs2->coeff[i],
|
||||
gamma2, w->coeff[i]);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_use_hint(const POLY *h, const POLY *r, uint32_t gamma2, POLY *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++)
|
||||
out->coeff[i] = ossl_ml_dsa_key_compress_use_hint(h->coeff[i],
|
||||
r->coeff[i], gamma2);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_max(const POLY *p, uint32_t *mx)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++) {
|
||||
uint32_t c = p->coeff[i];
|
||||
uint32_t abs = abs_mod_prime(c);
|
||||
|
||||
*mx = maximum(*mx, abs);
|
||||
}
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
poly_max_signed(const POLY *p, uint32_t *mx)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ML_DSA_NUM_POLY_COEFFICIENTS; i++) {
|
||||
uint32_t c = p->coeff[i];
|
||||
uint32_t abs = abs_signed(c);
|
||||
|
||||
*mx = maximum(*mx, abs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/byteorder.h>
|
||||
#include "ml_dsa_local.h"
|
||||
#include "ml_dsa_vector.h"
|
||||
#include "ml_dsa_matrix.h"
|
||||
#include "ml_dsa_hash.h"
|
||||
#include "internal/sha3.h"
|
||||
#include "internal/packet.h"
|
||||
|
||||
#define SHAKE128_BLOCKSIZE SHA3_BLOCKSIZE(128)
|
||||
#define SHAKE256_BLOCKSIZE SHA3_BLOCKSIZE(256)
|
||||
|
||||
/*
|
||||
* This is a constant time version of n % 5
|
||||
* Note that 0xFFFF / 5 = 0x3333, 2 is added to make an over-estimate of 1/5
|
||||
* and then we divide by (0xFFFF + 1)
|
||||
*/
|
||||
#define MOD5(n) ((n) - 5 * (0x3335 * (n) >> 16))
|
||||
|
||||
#if SHAKE128_BLOCKSIZE % 3 != 0
|
||||
# error "rej_ntt_poly() requires SHAKE128_BLOCKSIZE to be a multiple of 3"
|
||||
#endif
|
||||
|
||||
typedef int (COEFF_FROM_NIBBLE_FUNC)(uint32_t nibble, uint32_t *out);
|
||||
|
||||
static COEFF_FROM_NIBBLE_FUNC coeff_from_nibble_4;
|
||||
static COEFF_FROM_NIBBLE_FUNC coeff_from_nibble_2;
|
||||
|
||||
/**
|
||||
* @brief Combine 3 bytes to form an coefficient.
|
||||
* See FIPS 204, Algorithm 14, CoeffFromThreeBytes()
|
||||
*
|
||||
* This is not constant time as it is used to generate the matrix A which is public.
|
||||
*
|
||||
* @param s A byte array of 3 uniformly distributed bytes.
|
||||
* @param out The returned coefficient in the range 0..q-1.
|
||||
* @returns 1 if the value is less than q or 0 otherwise.
|
||||
* This is used for rejection sampling.
|
||||
*/
|
||||
static ossl_inline int coeff_from_three_bytes(const uint8_t *s, uint32_t *out)
|
||||
{
|
||||
/* Zero out the top bit of the 3rd byte to get a value in the range 0..2^23-1) */
|
||||
*out = (uint32_t)s[0] | ((uint32_t)s[1] << 8) | (((uint32_t)s[2] & 0x7f) << 16);
|
||||
return *out < ML_DSA_Q;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Generate a value in the range (q-4..0..4)
|
||||
* See FIPS 204, Algorithm 15, CoeffFromHalfByte() where eta = 4
|
||||
* Note the FIPS 204 code uses the range -4..4 (whereas this code adds q to the
|
||||
* negative numbers).
|
||||
*
|
||||
* @param nibble A value in the range 0..15
|
||||
* @param out The returned value if the range (q-4)..0..4 if nibble is < 9
|
||||
* @returns 1 nibble was in range, or 0 if the nibble was rejected.
|
||||
*/
|
||||
static ossl_inline int coeff_from_nibble_4(uint32_t nibble, uint32_t *out)
|
||||
{
|
||||
/*
|
||||
* This is not constant time but will not leak any important info since
|
||||
* the value is either chosen or thrown away.
|
||||
*/
|
||||
if (value_barrier_32(nibble < 9)) {
|
||||
*out = mod_sub(4, nibble);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Generate a value in the range (q-2..0..2)
|
||||
* See FIPS 204, Algorithm 15, CoeffFromHalfByte() where eta = 2
|
||||
* Note the FIPS 204 code uses the range -2..2 (whereas this code adds q to the
|
||||
* negative numbers).
|
||||
*
|
||||
* @param nibble A value in the range 0..15
|
||||
* @param out The returned value if the range (q-2)..0..2 if nibble is < 15
|
||||
* @returns 1 nibble was in range, or 0 if the nibble was rejected.
|
||||
*/
|
||||
static ossl_inline int coeff_from_nibble_2(uint32_t nibble, uint32_t *out)
|
||||
{
|
||||
if (value_barrier_32(nibble < 15)) {
|
||||
*out = mod_sub(2, MOD5(nibble));
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Use a seed value to generate a polynomial with coefficients in the
|
||||
* range of 0..q-1 using rejection sampling.
|
||||
* SHAKE128 is used to absorb the seed, and then sequences of 3 sample bytes are
|
||||
* squeezed to try to produce coefficients.
|
||||
* The SHAKE128 stream is used to get uniformly distributed elements.
|
||||
* This algorithm is used for matrix expansion and only operates on public inputs.
|
||||
*
|
||||
* See FIPS 204, Algorithm 30, RejNTTPoly()
|
||||
*
|
||||
* @param g_ctx A EVP_MD_CTX object used for sampling the seed.
|
||||
* @param md A pre-fetched SHAKE128 object.
|
||||
* @param seed The seed to use for sampling.
|
||||
* @param seed_len The size of |seed|
|
||||
* @param out The returned polynomial with coefficients in the range of
|
||||
* 0..q-1. This range is required for NTT.
|
||||
* @returns 1 if the polynomial was successfully generated, or 0 if any of the
|
||||
* digest operations failed.
|
||||
*/
|
||||
static int rej_ntt_poly(EVP_MD_CTX *g_ctx, const EVP_MD *md,
|
||||
const uint8_t *seed, size_t seed_len, POLY *out)
|
||||
{
|
||||
int j = 0;
|
||||
uint8_t blocks[SHAKE128_BLOCKSIZE], *b, *end = blocks + sizeof(blocks);
|
||||
|
||||
/*
|
||||
* Instead of just squeezing 3 bytes at a time, we grab a whole block
|
||||
* Note that the shake128 blocksize of 168 is divisible by 3.
|
||||
*/
|
||||
if (!shake_xof(g_ctx, md, seed, seed_len, blocks, sizeof(blocks)))
|
||||
return 0;
|
||||
|
||||
while (1) {
|
||||
for (b = blocks; b < end; b += 3) {
|
||||
if (coeff_from_three_bytes(b, &(out->coeff[j]))) {
|
||||
if (++j >= ML_DSA_NUM_POLY_COEFFICIENTS)
|
||||
return 1; /* finished */
|
||||
}
|
||||
}
|
||||
if (!EVP_DigestSqueeze(g_ctx, blocks, sizeof(blocks)))
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Use a seed value to generate a polynomial with coefficients in the
|
||||
* range of ((q-eta)..0..eta) using rejection sampling. eta is either 2 or 4.
|
||||
* SHAKE256 is used to absorb the seed, and then samples are squeezed.
|
||||
* See FIPS 204, Algorithm 31, RejBoundedPoly()
|
||||
*
|
||||
* @param h_ctx A EVP_MD_CTX object context used to sample the seed.
|
||||
* @param md A pre-fetched SHAKE256 object.
|
||||
* @param coef_from_nibble A function that is dependent on eta, which takes a
|
||||
* nibble and tries to see if it is in the correct range.
|
||||
* @param seed The seed to use for sampling.
|
||||
* @param seed_len The size of |seed|
|
||||
* @param out The returned polynomial with coefficients in the range of
|
||||
* ((q-eta)..0..eta)
|
||||
* @returns 1 if the polynomial was successfully generated, or 0 if any of the
|
||||
* digest operations failed.
|
||||
*/
|
||||
static int rej_bounded_poly(EVP_MD_CTX *h_ctx, const EVP_MD *md,
|
||||
COEFF_FROM_NIBBLE_FUNC *coef_from_nibble,
|
||||
const uint8_t *seed, size_t seed_len, POLY *out)
|
||||
{
|
||||
int j = 0;
|
||||
uint32_t z0, z1;
|
||||
uint8_t blocks[SHAKE256_BLOCKSIZE], *b, *end = blocks + sizeof(blocks);
|
||||
|
||||
/* Instead of just squeezing 1 byte at a time, we grab a whole block */
|
||||
if (!shake_xof(h_ctx, md, seed, seed_len, blocks, sizeof(blocks)))
|
||||
return 0;
|
||||
|
||||
while (1) {
|
||||
for (b = blocks; b < end; b++) {
|
||||
z0 = *b & 0x0F; /* lower nibble of byte */
|
||||
z1 = *b >> 4; /* high nibble of byte */
|
||||
|
||||
if (coef_from_nibble(z0, &out->coeff[j])
|
||||
&& ++j >= ML_DSA_NUM_POLY_COEFFICIENTS)
|
||||
return 1;
|
||||
if (coef_from_nibble(z1, &out->coeff[j])
|
||||
&& ++j >= ML_DSA_NUM_POLY_COEFFICIENTS)
|
||||
return 1;
|
||||
}
|
||||
if (!EVP_DigestSqueeze(h_ctx, blocks, sizeof(blocks)))
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Generate a k * l matrix that has uniformly distributed polynomial
|
||||
* elements using rejection sampling.
|
||||
* See FIPS 204, Algorithm 32, ExpandA()
|
||||
*
|
||||
* @param g_ctx A EVP_MD_CTX context used for rejection sampling
|
||||
* seed values generated from the seed rho.
|
||||
* @param md A pre-fetched SHAKE128 object
|
||||
* @param rho A 32 byte seed to generated the matrix from.
|
||||
* @param out The generated k * l matrix of polynomials with coefficients
|
||||
* in the range of 0..q-1.
|
||||
* @returns 1 if the matrix was generated, or 0 on error.
|
||||
*/
|
||||
int ossl_ml_dsa_matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md,
|
||||
const uint8_t *rho, MATRIX *out)
|
||||
{
|
||||
int ret = 0;
|
||||
size_t i, j;
|
||||
uint8_t derived_seed[ML_DSA_RHO_BYTES + 2];
|
||||
POLY *poly = out->m_poly;
|
||||
|
||||
/* The seed used for each matrix element is rho + column_index + row_index */
|
||||
memcpy(derived_seed, rho, ML_DSA_RHO_BYTES);
|
||||
|
||||
for (i = 0; i < out->k; i++) {
|
||||
for (j = 0; j < out->l; j++) {
|
||||
derived_seed[ML_DSA_RHO_BYTES + 1] = (uint8_t)i;
|
||||
derived_seed[ML_DSA_RHO_BYTES] = (uint8_t)j;
|
||||
/* Generate the polynomial for each matrix element using a unique seed */
|
||||
if (!rej_ntt_poly(g_ctx, md, derived_seed, sizeof(derived_seed), poly++))
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
ret = 1;
|
||||
err:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Generates 2 vectors using rejection sampling whose polynomial
|
||||
* coefficients are in the interval [q-eta..0..eta]
|
||||
*
|
||||
* See FIPS 204, Algorithm 33, ExpandS().
|
||||
* Note that in FIPS 204 the range -eta..eta is used.
|
||||
*
|
||||
* @param h_ctx A EVP_MD_CTX context to use to sample the seed.
|
||||
* @param md A pre-fetched SHAKE256 object.
|
||||
* @param eta Is either 2 or 4, and determines the range of the coefficients for
|
||||
* s1 and s2.
|
||||
* @param seed A 64 byte seed to use for sampling.
|
||||
* @param s1 A 1 * l column vector containing polynomials with coefficients in
|
||||
* the range (q-eta)..0..eta
|
||||
* @param s2 A 1 * k column vector containing polynomials with coefficients in
|
||||
* the range (q-eta)..0..eta
|
||||
* @returns 1 if s1 and s2 were successfully generated, or 0 otherwise.
|
||||
*/
|
||||
int ossl_ml_dsa_vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta,
|
||||
const uint8_t *seed, VECTOR *s1, VECTOR *s2)
|
||||
{
|
||||
int ret = 0;
|
||||
size_t i;
|
||||
size_t l = s1->num_poly;
|
||||
size_t k = s2->num_poly;
|
||||
uint8_t derived_seed[ML_DSA_PRIV_SEED_BYTES + 2];
|
||||
COEFF_FROM_NIBBLE_FUNC *coef_from_nibble_fn;
|
||||
|
||||
coef_from_nibble_fn = (eta == ML_DSA_ETA_4) ? coeff_from_nibble_4 : coeff_from_nibble_2;
|
||||
|
||||
/*
|
||||
* Each polynomial generated uses a unique seed that consists of
|
||||
* seed + counter (where the counter is 2 bytes starting at 0)
|
||||
*/
|
||||
memcpy(derived_seed, seed, ML_DSA_PRIV_SEED_BYTES);
|
||||
derived_seed[ML_DSA_PRIV_SEED_BYTES] = 0;
|
||||
derived_seed[ML_DSA_PRIV_SEED_BYTES + 1] = 0;
|
||||
|
||||
for (i = 0; i < l; i++) {
|
||||
if (!rej_bounded_poly(h_ctx, md, coef_from_nibble_fn,
|
||||
derived_seed, sizeof(derived_seed), &s1->poly[i]))
|
||||
goto err;
|
||||
++derived_seed[ML_DSA_PRIV_SEED_BYTES];
|
||||
}
|
||||
for (i = 0; i < k; i++) {
|
||||
if (!rej_bounded_poly(h_ctx, md, coef_from_nibble_fn,
|
||||
derived_seed, sizeof(derived_seed), &s2->poly[i]))
|
||||
goto err;
|
||||
++derived_seed[ML_DSA_PRIV_SEED_BYTES];
|
||||
}
|
||||
ret = 1;
|
||||
err:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* See FIPS 204, Algorithm 34, ExpandMask(), Step 4 & 5 */
|
||||
int ossl_ml_dsa_poly_expand_mask(POLY *out, const uint8_t *seed, size_t seed_len,
|
||||
uint32_t gamma1,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md)
|
||||
{
|
||||
uint8_t buf[32 * 20];
|
||||
size_t buf_len = 32 * (gamma1 == ML_DSA_GAMMA1_TWO_POWER_19 ? 20 : 18);
|
||||
|
||||
return shake_xof(h_ctx, md, seed, seed_len, buf, buf_len)
|
||||
&& ossl_ml_dsa_poly_decode_expand_mask(out, buf, buf_len, gamma1);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Sample a polynomial with coefficients in the range {-1..1}.
|
||||
* The number of non zero values (hamming weight) is given by tau
|
||||
*
|
||||
* See FIPS 204, Algorithm 29, SampleInBall()
|
||||
* This function is assumed to not be constant time.
|
||||
* The algorithm is based on Durstenfeld's version of the Fisher-Yates shuffle.
|
||||
*
|
||||
* Note that the coefficients returned by this implementation are positive
|
||||
* i.e one of q-1, 0, or 1.
|
||||
*
|
||||
* @param tau is the number of +1 or -1's in the polynomial 'out_c' (39, 49 or 60)
|
||||
* that is less than or equal to 64
|
||||
*/
|
||||
int ossl_ml_dsa_poly_sample_in_ball(POLY *out_c, const uint8_t *seed, int seed_len,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md,
|
||||
uint32_t tau)
|
||||
{
|
||||
uint8_t block[SHAKE256_BLOCKSIZE];
|
||||
uint64_t signs;
|
||||
int offset = 8;
|
||||
size_t end;
|
||||
|
||||
/*
|
||||
* Rather than squeeze 8 bytes followed by lots of 1 byte squeezes
|
||||
* the SHAKE blocksize is squeezed each time and buffered into 'block'.
|
||||
*/
|
||||
if (!shake_xof(h_ctx, md, seed, seed_len, block, sizeof(block)))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* grab the first 64 bits - since tau < 64
|
||||
* Each bit gives a +1 or -1 value.
|
||||
*/
|
||||
OPENSSL_load_u64_le(&signs, block);
|
||||
|
||||
poly_zero(out_c);
|
||||
|
||||
/* Loop tau times */
|
||||
for (end = 256 - tau; end < 256; end++) {
|
||||
size_t index; /* index is a random offset to write +1 or -1 */
|
||||
|
||||
/* rejection sample in {0..end} to choose an index to place -1 or 1 into */
|
||||
for (;;) {
|
||||
if (offset == sizeof(block)) {
|
||||
/* squeeze another block if the bytes from block have been used */
|
||||
if (!EVP_DigestSqueeze(h_ctx, block, sizeof(block)))
|
||||
return 0;
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
index = block[offset++];
|
||||
if (index <= end)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* In-place swap the coefficient we are about to replace to the end so
|
||||
* we don't lose any values that have been already written.
|
||||
*/
|
||||
out_c->coeff[end] = out_c->coeff[index];
|
||||
/* set the random coefficient value to either 1 or q-1 */
|
||||
out_c->coeff[index] = mod_sub(1, 2 * (signs & 1));
|
||||
signs >>= 1; /* grab the next random bit */
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/core_dispatch.h>
|
||||
#include <openssl/core_names.h>
|
||||
#include <openssl/params.h>
|
||||
#include <openssl/rand.h>
|
||||
#include "ml_dsa_local.h"
|
||||
#include "ml_dsa_key.h"
|
||||
#include "ml_dsa_matrix.h"
|
||||
#include "ml_dsa_sign.h"
|
||||
#include "ml_dsa_hash.h"
|
||||
|
||||
#define ML_DSA_MAX_LAMBDA 256 /* bit strength for ML-DSA-87 */
|
||||
|
||||
/*
|
||||
* @brief Initialize a Signature object by pointing all of its objects to
|
||||
* preallocated blocks. The values passed for hint, z and
|
||||
* c_tilde values are not owned/freed by the |sig| object.
|
||||
*
|
||||
* @param sig The ML_DSA_SIG to initialize.
|
||||
* @param hint A preallocated array of |k| polynomial blocks
|
||||
* @param k The number of |hint| polynomials
|
||||
* @param z A preallocated array of |l| polynomial blocks
|
||||
* @param l The number of |z| polynomials
|
||||
* @param c_tilde A preallocated buffer
|
||||
* @param c_tilde_len The size of |c_tilde|
|
||||
*/
|
||||
static void signature_init(ML_DSA_SIG *sig,
|
||||
POLY *hint, uint32_t k, POLY *z, uint32_t l,
|
||||
uint8_t *c_tilde, size_t c_tilde_len)
|
||||
{
|
||||
vector_init(&sig->z, z, l);
|
||||
vector_init(&sig->hint, hint, k);
|
||||
sig->c_tilde = c_tilde;
|
||||
sig->c_tilde_len = c_tilde_len;
|
||||
}
|
||||
|
||||
/*
|
||||
* FIPS 204, Algorithm 7, ML-DSA.Sign_internal()
|
||||
* @returns 1 on success and 0 on failure.
|
||||
*/
|
||||
static int ml_dsa_sign_internal(const ML_DSA_KEY *priv, int msg_is_mu,
|
||||
const uint8_t *encoded_msg,
|
||||
size_t encoded_msg_len,
|
||||
const uint8_t *rnd, size_t rnd_len,
|
||||
uint8_t *out_sig)
|
||||
{
|
||||
int ret = 0;
|
||||
const ML_DSA_PARAMS *params = priv->params;
|
||||
EVP_MD_CTX *md_ctx = NULL;
|
||||
uint32_t k = params->k, l = params->l;
|
||||
uint32_t gamma1 = params->gamma1, gamma2 = params->gamma2;
|
||||
uint8_t *alloc = NULL, *w1_encoded;
|
||||
size_t alloc_len, w1_encoded_len;
|
||||
size_t num_polys_sig_k = 2 * k;
|
||||
size_t num_polys_k = 5 * k;
|
||||
size_t num_polys_l = 3 * l;
|
||||
size_t num_polys_k_by_l = k * l;
|
||||
POLY *polys = NULL, *p, *c_ntt;
|
||||
VECTOR s1_ntt, s2_ntt, t0_ntt, w, w1, cs1, cs2, y;
|
||||
MATRIX a_ntt;
|
||||
ML_DSA_SIG sig;
|
||||
uint8_t mu[ML_DSA_MU_BYTES], *mu_ptr = mu;
|
||||
const size_t mu_len = sizeof(mu);
|
||||
uint8_t rho_prime[ML_DSA_RHO_PRIME_BYTES];
|
||||
uint8_t c_tilde[ML_DSA_MAX_LAMBDA / 4];
|
||||
size_t c_tilde_len = params->bit_strength >> 2;
|
||||
size_t kappa;
|
||||
|
||||
/*
|
||||
* Allocate a single blob for most of the variable size temporary variables.
|
||||
* Mostly used for VECTOR POLYNOMIALS (every POLY is 1K).
|
||||
*/
|
||||
w1_encoded_len = k * (gamma2 == ML_DSA_GAMMA2_Q_MINUS1_DIV88 ? 192 : 128);
|
||||
alloc_len = w1_encoded_len
|
||||
+ sizeof(*polys) * (1 + num_polys_k + num_polys_l
|
||||
+ num_polys_k_by_l + num_polys_sig_k);
|
||||
alloc = OPENSSL_malloc(alloc_len);
|
||||
if (alloc == NULL)
|
||||
return 0;
|
||||
md_ctx = EVP_MD_CTX_new();
|
||||
if (md_ctx == NULL)
|
||||
goto err;
|
||||
|
||||
w1_encoded = alloc;
|
||||
/* Init the temp vectors to point to the allocated polys blob */
|
||||
p = (POLY *)(w1_encoded + w1_encoded_len);
|
||||
c_ntt = p++;
|
||||
matrix_init(&a_ntt, p, k, l);
|
||||
p += num_polys_k_by_l;
|
||||
vector_init(&s2_ntt, p, k);
|
||||
vector_init(&t0_ntt, s2_ntt.poly + k, k);
|
||||
vector_init(&w, t0_ntt.poly + k, k);
|
||||
vector_init(&w1, w.poly + k, k);
|
||||
vector_init(&cs2, w1.poly + k, k);
|
||||
p += num_polys_k;
|
||||
vector_init(&s1_ntt, p, l);
|
||||
vector_init(&y, p + l, l);
|
||||
vector_init(&cs1, p + 2 * l, l);
|
||||
p += num_polys_l;
|
||||
signature_init(&sig, p, k, p + k, l, c_tilde, c_tilde_len);
|
||||
/* End of the allocated blob setup */
|
||||
|
||||
if (!matrix_expand_A(md_ctx, priv->shake128_md, priv->rho, &a_ntt))
|
||||
goto err;
|
||||
if (msg_is_mu) {
|
||||
if (encoded_msg_len != mu_len)
|
||||
goto err;
|
||||
mu_ptr = (uint8_t *)encoded_msg;
|
||||
} else {
|
||||
if (!shake_xof_2(md_ctx, priv->shake256_md, priv->tr, sizeof(priv->tr),
|
||||
encoded_msg, encoded_msg_len, mu_ptr, mu_len))
|
||||
goto err;
|
||||
}
|
||||
if (!shake_xof_3(md_ctx, priv->shake256_md, priv->K, sizeof(priv->K),
|
||||
rnd, rnd_len, mu_ptr, mu_len,
|
||||
rho_prime, sizeof(rho_prime)))
|
||||
goto err;
|
||||
|
||||
vector_copy(&s1_ntt, &priv->s1);
|
||||
vector_ntt(&s1_ntt);
|
||||
vector_copy(&s2_ntt, &priv->s2);
|
||||
vector_ntt(&s2_ntt);
|
||||
vector_copy(&t0_ntt, &priv->t0);
|
||||
vector_ntt(&t0_ntt);
|
||||
|
||||
/*
|
||||
* kappa must not exceed 2^16. But the probability of it
|
||||
* exceeding even 1000 iterations is vanishingly small.
|
||||
*/
|
||||
for (kappa = 0; ; kappa += l) {
|
||||
VECTOR *y_ntt = &cs1;
|
||||
VECTOR *r0 = &w1;
|
||||
VECTOR *ct0 = &w1;
|
||||
uint32_t z_max, r0_max, ct0_max, h_ones;
|
||||
|
||||
vector_expand_mask(&y, rho_prime, sizeof(rho_prime), kappa,
|
||||
gamma1, md_ctx, priv->shake256_md);
|
||||
vector_copy(y_ntt, &y);
|
||||
vector_ntt(y_ntt);
|
||||
|
||||
matrix_mult_vector(&a_ntt, y_ntt, &w);
|
||||
vector_ntt_inverse(&w);
|
||||
|
||||
vector_high_bits(&w, gamma2, &w1);
|
||||
ossl_ml_dsa_w1_encode(&w1, gamma2, w1_encoded, w1_encoded_len);
|
||||
|
||||
if (!shake_xof_2(md_ctx, priv->shake256_md, mu_ptr, mu_len,
|
||||
w1_encoded, w1_encoded_len, c_tilde, c_tilde_len))
|
||||
break;
|
||||
|
||||
if (!poly_sample_in_ball_ntt(c_ntt, c_tilde, c_tilde_len,
|
||||
md_ctx, priv->shake256_md, params->tau))
|
||||
break;
|
||||
|
||||
vector_mult_scalar(&s1_ntt, c_ntt, &cs1);
|
||||
vector_ntt_inverse(&cs1);
|
||||
vector_mult_scalar(&s2_ntt, c_ntt, &cs2);
|
||||
vector_ntt_inverse(&cs2);
|
||||
|
||||
vector_add(&y, &cs1, &sig.z);
|
||||
|
||||
/* r0 = lowbits(w - cs2) */
|
||||
vector_sub(&w, &cs2, r0);
|
||||
vector_low_bits(r0, gamma2, r0);
|
||||
|
||||
/*
|
||||
* Leaking that the signature is rejected is fine as the next attempt at a
|
||||
* signature will be (indistinguishable from) independent of this one.
|
||||
*/
|
||||
z_max = vector_max(&sig.z);
|
||||
r0_max = vector_max_signed(r0);
|
||||
if (value_barrier_32(constant_time_ge(z_max, gamma1 - params->beta)
|
||||
| constant_time_ge(r0_max, gamma2 - params->beta)))
|
||||
continue;
|
||||
|
||||
vector_mult_scalar(&t0_ntt, c_ntt, ct0);
|
||||
vector_ntt_inverse(ct0);
|
||||
vector_make_hint(ct0, &cs2, &w, gamma2, &sig.hint);
|
||||
|
||||
ct0_max = vector_max(ct0);
|
||||
h_ones = vector_count_ones(&sig.hint);
|
||||
/* Same reasoning applies to the leak as above */
|
||||
if (value_barrier_32(constant_time_ge(ct0_max, gamma2)
|
||||
| constant_time_lt(params->omega, h_ones)))
|
||||
continue;
|
||||
ret = ossl_ml_dsa_sig_encode(&sig, params, out_sig);
|
||||
break;
|
||||
}
|
||||
err:
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
OPENSSL_clear_free(alloc, alloc_len);
|
||||
OPENSSL_cleanse(rho_prime, sizeof(rho_prime));
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* See FIPS 204, Algorithm 8, ML-DSA.Verify_internal().
|
||||
*/
|
||||
static int ml_dsa_verify_internal(const ML_DSA_KEY *pub, int msg_is_mu,
|
||||
const uint8_t *msg_enc, size_t msg_enc_len,
|
||||
const uint8_t *sig_enc, size_t sig_enc_len)
|
||||
{
|
||||
int ret = 0;
|
||||
uint8_t *alloc = NULL, *w1_encoded;
|
||||
POLY *polys = NULL, *p, *c_ntt;
|
||||
MATRIX a_ntt;
|
||||
VECTOR az_ntt, ct1_ntt, *z_ntt, *w1, *w_approx;
|
||||
ML_DSA_SIG sig;
|
||||
const ML_DSA_PARAMS *params = pub->params;
|
||||
uint32_t k = pub->params->k;
|
||||
uint32_t l = pub->params->l;
|
||||
uint32_t gamma2 = params->gamma2;
|
||||
size_t w1_encoded_len;
|
||||
size_t num_polys_sig = k + l;
|
||||
size_t num_polys_k = 2 * k;
|
||||
size_t num_polys_l = 1 * l;
|
||||
size_t num_polys_k_by_l = k * l;
|
||||
uint8_t mu[ML_DSA_MU_BYTES], *mu_ptr = mu;
|
||||
const size_t mu_len = sizeof(mu);
|
||||
uint8_t c_tilde[ML_DSA_MAX_LAMBDA / 4];
|
||||
uint8_t c_tilde_sig[ML_DSA_MAX_LAMBDA / 4];
|
||||
EVP_MD_CTX *md_ctx = NULL;
|
||||
size_t c_tilde_len = params->bit_strength >> 2;
|
||||
uint32_t z_max;
|
||||
|
||||
/* Allocate space for all the POLYNOMIALS used by temporary VECTORS */
|
||||
w1_encoded_len = k * (gamma2 == ML_DSA_GAMMA2_Q_MINUS1_DIV88 ? 192 : 128);
|
||||
alloc = OPENSSL_malloc(w1_encoded_len
|
||||
+ sizeof(*polys) * (1 + num_polys_k
|
||||
+ num_polys_l
|
||||
+ num_polys_k_by_l
|
||||
+ num_polys_sig));
|
||||
if (alloc == NULL)
|
||||
return 0;
|
||||
md_ctx = EVP_MD_CTX_new();
|
||||
if (md_ctx == NULL)
|
||||
goto err;
|
||||
|
||||
w1_encoded = alloc;
|
||||
/* Init the temp vectors to point to the allocated polys blob */
|
||||
p = (POLY *)(w1_encoded + w1_encoded_len);
|
||||
c_ntt = p++;
|
||||
matrix_init(&a_ntt, p, k, l);
|
||||
p += num_polys_k_by_l;
|
||||
signature_init(&sig, p, k, p + k, l, c_tilde_sig, c_tilde_len);
|
||||
p += num_polys_sig;
|
||||
vector_init(&az_ntt, p, k);
|
||||
vector_init(&ct1_ntt, p + k, k);
|
||||
|
||||
if (!ossl_ml_dsa_sig_decode(&sig, sig_enc, sig_enc_len, pub->params)
|
||||
|| !matrix_expand_A(md_ctx, pub->shake128_md, pub->rho, &a_ntt))
|
||||
goto err;
|
||||
if (msg_is_mu) {
|
||||
if (msg_enc_len != mu_len)
|
||||
goto err;
|
||||
mu_ptr = (uint8_t *)msg_enc;
|
||||
} else {
|
||||
if (!shake_xof_2(md_ctx, pub->shake256_md, pub->tr, sizeof(pub->tr),
|
||||
msg_enc, msg_enc_len, mu_ptr, mu_len))
|
||||
goto err;
|
||||
}
|
||||
/* Compute verifiers challenge c_ntt = NTT(SampleInBall(c_tilde) */
|
||||
if (!poly_sample_in_ball_ntt(c_ntt, c_tilde_sig, c_tilde_len,
|
||||
md_ctx, pub->shake256_md, params->tau))
|
||||
goto err;
|
||||
|
||||
/* ct1_ntt = NTT(c) * NTT(t1 * 2^d) */
|
||||
vector_scale_power2_round_ntt(&pub->t1, &ct1_ntt);
|
||||
vector_mult_scalar(&ct1_ntt, c_ntt, &ct1_ntt);
|
||||
|
||||
/* compute z_max early in order to reuse sig.z */
|
||||
z_max = vector_max(&sig.z);
|
||||
|
||||
/* w_approx = NTT_inverse(A * NTT(z) - ct1_ntt) */
|
||||
z_ntt = &sig.z;
|
||||
vector_ntt(z_ntt);
|
||||
matrix_mult_vector(&a_ntt, z_ntt, &az_ntt);
|
||||
w_approx = &az_ntt;
|
||||
vector_sub(&az_ntt, &ct1_ntt, w_approx);
|
||||
vector_ntt_inverse(w_approx);
|
||||
|
||||
/* compute w1_encoded */
|
||||
w1 = w_approx;
|
||||
vector_use_hint(&sig.hint, w_approx, gamma2, w1);
|
||||
ossl_ml_dsa_w1_encode(w1, gamma2, w1_encoded, w1_encoded_len);
|
||||
|
||||
if (!shake_xof_3(md_ctx, pub->shake256_md, mu_ptr, mu_len,
|
||||
w1_encoded, w1_encoded_len, NULL, 0, c_tilde, c_tilde_len))
|
||||
goto err;
|
||||
|
||||
ret = (z_max < (uint32_t)(params->gamma1 - params->beta))
|
||||
&& memcmp(c_tilde, sig.c_tilde, c_tilde_len) == 0;
|
||||
err:
|
||||
OPENSSL_free(alloc);
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Encode a message
|
||||
* See FIPS 204 Algorithm 2 Step 10 (and algorithm 3 Step 5).
|
||||
*
|
||||
* ML_DSA pure signatures are encoded as M' = 00 || ctx_len || ctx || msg
|
||||
* Where ctx is the empty string by default and ctx_len <= 255.
|
||||
*
|
||||
* Note this code could be shared with SLH_DSA
|
||||
*
|
||||
* @param msg A message to encode
|
||||
* @param msg_len The size of |msg|
|
||||
* @param ctx An optional context to add to the message encoding.
|
||||
* @param ctx_len The size of |ctx|. It must be in the range 0..255
|
||||
* @param encode Use the Pure signature encoding if this is 1, and dont encode
|
||||
* if this value is 0.
|
||||
* @param tmp A small buffer that may be used if the message is small.
|
||||
* @param tmp_len The size of |tmp|
|
||||
* @param out_len The size of the returned encoded buffer.
|
||||
* @returns A buffer containing the encoded message. If the passed in
|
||||
* |tmp| buffer is big enough to hold the encoded message then it returns |tmp|
|
||||
* otherwise it allocates memory which must be freed by the caller. If |encode|
|
||||
* is 0 then it returns |msg|. NULL is returned if there is a failure.
|
||||
*/
|
||||
static uint8_t *msg_encode(const uint8_t *msg, size_t msg_len,
|
||||
const uint8_t *ctx, size_t ctx_len, int encode,
|
||||
uint8_t *tmp, size_t tmp_len, size_t *out_len)
|
||||
{
|
||||
uint8_t *encoded = NULL;
|
||||
size_t encoded_len;
|
||||
|
||||
if (encode == 0) {
|
||||
/* Raw message */
|
||||
*out_len = msg_len;
|
||||
return (uint8_t *)msg;
|
||||
}
|
||||
if (ctx_len > ML_DSA_MAX_CONTEXT_STRING_LEN)
|
||||
return NULL;
|
||||
|
||||
/* Pure encoding */
|
||||
encoded_len = 1 + 1 + ctx_len + msg_len;
|
||||
*out_len = encoded_len;
|
||||
if (encoded_len <= tmp_len) {
|
||||
encoded = tmp;
|
||||
} else {
|
||||
encoded = OPENSSL_malloc(encoded_len);
|
||||
if (encoded == NULL)
|
||||
return NULL;
|
||||
}
|
||||
encoded[0] = 0;
|
||||
encoded[1] = (uint8_t)ctx_len;
|
||||
memcpy(&encoded[2], ctx, ctx_len);
|
||||
memcpy(&encoded[2 + ctx_len], msg, msg_len);
|
||||
return encoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* See FIPS 204 Section 5.2 Algorithm 2 ML-DSA.Sign()
|
||||
*
|
||||
* @returns 1 on success, or 0 on error.
|
||||
*/
|
||||
int ossl_ml_dsa_sign(const ML_DSA_KEY *priv, int msg_is_mu,
|
||||
const uint8_t *msg, size_t msg_len,
|
||||
const uint8_t *context, size_t context_len,
|
||||
const uint8_t *rand, size_t rand_len, int encode,
|
||||
unsigned char *sig, size_t *sig_len, size_t sig_size)
|
||||
{
|
||||
int ret = 1;
|
||||
uint8_t m_tmp[1024], *m = m_tmp, *alloced_m = NULL;
|
||||
size_t m_len = 0;
|
||||
|
||||
if (ossl_ml_dsa_key_get_priv(priv) == NULL)
|
||||
return 0;
|
||||
if (sig != NULL) {
|
||||
if (sig_size < priv->params->sig_len)
|
||||
return 0;
|
||||
if (msg_is_mu) {
|
||||
m = (uint8_t *)msg;
|
||||
m_len = msg_len;
|
||||
} else {
|
||||
m = msg_encode(msg, msg_len, context, context_len, encode,
|
||||
m_tmp, sizeof(m_tmp), &m_len);
|
||||
if (m == NULL)
|
||||
return 0;
|
||||
if (m != msg && m != m_tmp)
|
||||
alloced_m = m;
|
||||
}
|
||||
ret = ml_dsa_sign_internal(priv, msg_is_mu, m, m_len, rand, rand_len, sig);
|
||||
OPENSSL_free(alloced_m);
|
||||
}
|
||||
if (sig_len != NULL)
|
||||
*sig_len = priv->params->sig_len;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* See FIPS 203 Section 5.3 Algorithm 3 ML-DSA.Verify()
|
||||
* @returns 1 on success, or 0 on error.
|
||||
*/
|
||||
int ossl_ml_dsa_verify(const ML_DSA_KEY *pub, int msg_is_mu,
|
||||
const uint8_t *msg, size_t msg_len,
|
||||
const uint8_t *context, size_t context_len, int encode,
|
||||
const uint8_t *sig, size_t sig_len)
|
||||
{
|
||||
uint8_t *m, *alloced_m = NULL;
|
||||
size_t m_len;
|
||||
uint8_t m_tmp[1024];
|
||||
int ret = 0;
|
||||
|
||||
if (ossl_ml_dsa_key_get_pub(pub) == NULL)
|
||||
return 0;
|
||||
|
||||
if (msg_is_mu) {
|
||||
m = (uint8_t *)msg;
|
||||
m_len = msg_len;
|
||||
} else {
|
||||
m = msg_encode(msg, msg_len, context, context_len, encode,
|
||||
m_tmp, sizeof(m_tmp), &m_len);
|
||||
if (m == NULL)
|
||||
return 0;
|
||||
if (m != msg && m != m_tmp)
|
||||
alloced_m = m;
|
||||
}
|
||||
|
||||
ret = ml_dsa_verify_internal(pub, msg_is_mu, m, m_len, sig, sig_len);
|
||||
OPENSSL_free(alloced_m);
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
struct ml_dsa_sig_st {
|
||||
VECTOR z;
|
||||
VECTOR hint;
|
||||
uint8_t *c_tilde;
|
||||
size_t c_tilde_len;
|
||||
};
|
||||
@@ -0,0 +1,257 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include "ml_dsa_poly.h"
|
||||
|
||||
struct vector_st {
|
||||
POLY *poly;
|
||||
size_t num_poly;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Initialize a Vector object.
|
||||
*
|
||||
* @param v The vector to initialize.
|
||||
* @param polys Preallocated storage for an array of Polynomials blocks. |v|
|
||||
* does not own/free this.
|
||||
* @param num_polys The number of |polys| blocks (k or l)
|
||||
*/
|
||||
static ossl_inline ossl_unused
|
||||
void vector_init(VECTOR *v, POLY *polys, size_t num_polys)
|
||||
{
|
||||
v->poly = polys;
|
||||
v->num_poly = num_polys;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused
|
||||
int vector_alloc(VECTOR *v, size_t num_polys)
|
||||
{
|
||||
v->poly = OPENSSL_malloc(num_polys * sizeof(POLY));
|
||||
if (v->poly == NULL)
|
||||
return 0;
|
||||
v->num_poly = num_polys;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused
|
||||
void vector_free(VECTOR *v)
|
||||
{
|
||||
OPENSSL_free(v->poly);
|
||||
v->poly = NULL;
|
||||
v->num_poly = 0;
|
||||
}
|
||||
|
||||
/* @brief zeroize a vectors polynomial coefficients */
|
||||
static ossl_inline ossl_unused
|
||||
void vector_zero(VECTOR *va)
|
||||
{
|
||||
if (va->poly != NULL)
|
||||
memset(va->poly, 0, va->num_poly * sizeof(va->poly[0]));
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief copy a vector
|
||||
* The assumption is that |dst| has already been initialized
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
vector_copy(VECTOR *dst, const VECTOR *src)
|
||||
{
|
||||
assert(dst->num_poly == src->num_poly);
|
||||
memcpy(dst->poly, src->poly, src->num_poly * sizeof(src->poly[0]));
|
||||
}
|
||||
|
||||
/* @brief return 1 if 2 vectors are equal, or 0 otherwise */
|
||||
static ossl_inline ossl_unused int
|
||||
vector_equal(const VECTOR *a, const VECTOR *b)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
if (a->num_poly != b->num_poly)
|
||||
return 0;
|
||||
for (i = 0; i < a->num_poly; ++i) {
|
||||
if (!poly_equal(a->poly + i, b->poly + i))
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* @brief add 2 vectors */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_add(const VECTOR *lhs, const VECTOR *rhs, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < lhs->num_poly; i++)
|
||||
poly_add(lhs->poly + i, rhs->poly + i, out->poly + i);
|
||||
}
|
||||
|
||||
/* @brief subtract 2 vectors */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_sub(const VECTOR *lhs, const VECTOR *rhs, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < lhs->num_poly; i++)
|
||||
poly_sub(lhs->poly + i, rhs->poly + i, out->poly + i);
|
||||
}
|
||||
|
||||
/* @brief convert a vector in place into NTT form */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_ntt(VECTOR *va)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < va->num_poly; i++)
|
||||
ossl_ml_dsa_poly_ntt(va->poly + i);
|
||||
}
|
||||
|
||||
/* @brief convert a vector in place into inverse NTT form */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_ntt_inverse(VECTOR *va)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < va->num_poly; i++)
|
||||
ossl_ml_dsa_poly_ntt_inverse(va->poly + i);
|
||||
}
|
||||
|
||||
/* @brief multiply a vector by a SCALAR polynomial */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_mult_scalar(const VECTOR *lhs, const POLY *rhs, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < lhs->num_poly; i++)
|
||||
ossl_ml_dsa_poly_ntt_mult(lhs->poly + i, rhs, out->poly + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused int
|
||||
vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta,
|
||||
const uint8_t *seed, VECTOR *s1, VECTOR *s2)
|
||||
{
|
||||
return ossl_ml_dsa_vector_expand_S(h_ctx, md, eta, seed, s1, s2);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
vector_expand_mask(VECTOR *out, const uint8_t *rho_prime, size_t rho_prime_len,
|
||||
uint32_t kappa, uint32_t gamma1,
|
||||
EVP_MD_CTX *h_ctx, const EVP_MD *md)
|
||||
{
|
||||
size_t i;
|
||||
uint8_t derived_seed[ML_DSA_RHO_PRIME_BYTES + 2];
|
||||
|
||||
memcpy(derived_seed, rho_prime, ML_DSA_RHO_PRIME_BYTES);
|
||||
|
||||
for (i = 0; i < out->num_poly; i++) {
|
||||
size_t index = kappa + i;
|
||||
|
||||
derived_seed[ML_DSA_RHO_PRIME_BYTES] = index & 0xFF;
|
||||
derived_seed[ML_DSA_RHO_PRIME_BYTES + 1] = (index >> 8) & 0xFF;
|
||||
poly_expand_mask(out->poly + i, derived_seed, sizeof(derived_seed),
|
||||
gamma1, h_ctx, md);
|
||||
}
|
||||
}
|
||||
|
||||
/* Scale back previously rounded value */
|
||||
static ossl_inline ossl_unused void
|
||||
vector_scale_power2_round_ntt(const VECTOR *in, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < in->num_poly; i++)
|
||||
poly_scale_power2_round(in->poly + i, out->poly + i);
|
||||
vector_ntt(out);
|
||||
}
|
||||
|
||||
/*
|
||||
* @brief Decompose all polynomial coefficients of a vector into (t1, t0) such
|
||||
* that coeff[i] == t1[i] * 2^13 + t0[i] mod q.
|
||||
* See FIPS 204, Algorithm 35, Power2Round()
|
||||
*/
|
||||
static ossl_inline ossl_unused void
|
||||
vector_power2_round(const VECTOR *t, VECTOR *t1, VECTOR *t0)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < t->num_poly; i++)
|
||||
poly_power2_round(t->poly + i, t1->poly + i, t0->poly + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
vector_high_bits(const VECTOR *in, uint32_t gamma2, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < out->num_poly; i++)
|
||||
poly_high_bits(in->poly + i, gamma2, out->poly + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
vector_low_bits(const VECTOR *in, uint32_t gamma2, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < out->num_poly; i++)
|
||||
poly_low_bits(in->poly + i, gamma2, out->poly + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused uint32_t
|
||||
vector_max(const VECTOR *v)
|
||||
{
|
||||
size_t i;
|
||||
uint32_t mx = 0;
|
||||
|
||||
for (i = 0; i < v->num_poly; i++)
|
||||
poly_max(v->poly + i, &mx);
|
||||
return mx;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused uint32_t
|
||||
vector_max_signed(const VECTOR *v)
|
||||
{
|
||||
size_t i;
|
||||
uint32_t mx = 0;
|
||||
|
||||
for (i = 0; i < v->num_poly; i++)
|
||||
poly_max_signed(v->poly + i, &mx);
|
||||
return mx;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused size_t
|
||||
vector_count_ones(const VECTOR *v)
|
||||
{
|
||||
int j;
|
||||
size_t i, count = 0;
|
||||
|
||||
for (i = 0; i < v->num_poly; i++)
|
||||
for (j = 0; j < ML_DSA_NUM_POLY_COEFFICIENTS; j++)
|
||||
count += v->poly[i].coeff[j];
|
||||
return count;
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
vector_make_hint(const VECTOR *ct0, const VECTOR *cs2, const VECTOR *w,
|
||||
uint32_t gamma2, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < out->num_poly; i++)
|
||||
poly_make_hint(ct0->poly + i, cs2->poly + i, w->poly + i, gamma2,
|
||||
out->poly + i);
|
||||
}
|
||||
|
||||
static ossl_inline ossl_unused void
|
||||
vector_use_hint(const VECTOR *h, const VECTOR *r, uint32_t gamma2, VECTOR *out)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < out->num_poly; i++)
|
||||
poly_use_hint(h->poly + i, r->poly + i, gamma2, out->poly + i);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
LIBS = ../../libcrypto
|
||||
|
||||
IF[{- !$disabled{'ml-kem'} -}]
|
||||
SOURCE[../../libcrypto]=ml_kem.c
|
||||
SOURCE[../../providers/libfips.a]=ml_kem.c
|
||||
ENDIF
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,975 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Jerry Shih <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
# - RISC-V Vector GCM/GMAC extension ('Zvkg')
|
||||
# - RISC-V Vector AES Block Cipher extension ('Zvkned')
|
||||
# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)
|
||||
|
||||
# Reference: https://github.com/riscv/riscv-crypto/issues/192#issuecomment-1270447575
|
||||
#
|
||||
# Assume we have 12 GCM blocks and we try to parallelize GCM computation for 4 blocks.
|
||||
# Tag = M0*H^12 + M1*H^11 + M2*H^10 + M3*H^9 +
|
||||
# M4*H^8 + M5*H^7 + M6*H^6 + M7*H^5 +
|
||||
# M8*H^4 + M9*H^3 + M10*H^2 + M11*H^1
|
||||
# We could rewrite the formula into:
|
||||
# T0 = 0
|
||||
# T1 = (T0+M1)*H^4 T2 = (T0+M2)*H^4 T3 = (T0+M3)*H^4 T4 = (T0+M4)*H^4
|
||||
# T5 = (T1+M5)*H^4 T6 = (T2+M6)*H^4 T7 = (T3+M7)*H^4 T8 = (T4+M8)*H^4
|
||||
# T9 = (T5+M9)*H^4 T10 = (T6+M10)*H^3 T11 = (T7+M11)*H^2 T12 = (T8+M12)*H^1
|
||||
#
|
||||
# We will multiply with [H^4, H^4, H^4, H^4] in each steps except the last iteration.
|
||||
# The last iteration will multiply with [H^4, H^3, H^2, H^1].
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
{
|
||||
my ($INP, $OUTP, $LEN, $KEYP, $IVP, $XIP) = ("a0", "a1", "a2", "a3", "a4", "a5");
|
||||
my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3");
|
||||
my ($PADDING_LEN32) = ("t4");
|
||||
my ($LEN32) = ("t5");
|
||||
my ($CTR) = ("t6");
|
||||
my ($FULL_BLOCK_LEN32) = ("a6");
|
||||
my ($ORIGINAL_LEN32) = ("a7");
|
||||
my ($PROCESSED_LEN) = ("a0");
|
||||
my ($CTR_MASK) = ("v0");
|
||||
my ($INPUT_PADDING_MASK) = ("v0");
|
||||
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
|
||||
$V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
|
||||
$V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
|
||||
$V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map("v$_",(0..31));
|
||||
|
||||
# Do aes-128 enc.
|
||||
sub aes_128_cipher_body {
|
||||
my $code=<<___;
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesz_vs $V28, $V1]}
|
||||
@{[vaesem_vs $V28, $V2]}
|
||||
@{[vaesem_vs $V28, $V3]}
|
||||
@{[vaesem_vs $V28, $V4]}
|
||||
@{[vaesem_vs $V28, $V5]}
|
||||
@{[vaesem_vs $V28, $V6]}
|
||||
@{[vaesem_vs $V28, $V7]}
|
||||
@{[vaesem_vs $V28, $V8]}
|
||||
@{[vaesem_vs $V28, $V9]}
|
||||
@{[vaesem_vs $V28, $V10]}
|
||||
@{[vaesef_vs $V28, $V11]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Do aes-192 enc.
|
||||
sub aes_192_cipher_body {
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# Load key 4
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 48
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesz_vs $V28, $V1]}
|
||||
@{[vaesem_vs $V28, $V2]}
|
||||
@{[vaesem_vs $V28, $V3]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 8
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 112
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V4]}
|
||||
@{[vaesem_vs $V28, $V5]}
|
||||
@{[vaesem_vs $V28, $V6]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 13
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 192
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V7]}
|
||||
@{[vaesem_vs $V28, $V8]}
|
||||
@{[vaesem_vs $V28, $V9]}
|
||||
@{[vaesem_vs $V28, $V10]}
|
||||
@{[vaesef_vs $V28, $V11]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Do aes-256 enc.
|
||||
sub aes_256_cipher_body {
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# Load key 3
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 32
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesz_vs $V28, $V1]}
|
||||
@{[vaesem_vs $V28, $V2]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 6
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 80
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V3]}
|
||||
@{[vaesem_vs $V28, $V4]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 9
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 128
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V5]}
|
||||
@{[vaesem_vs $V28, $V6]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 12
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 176
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V7]}
|
||||
@{[vaesem_vs $V28, $V8]}
|
||||
@{[vaesem_vs $V28, $V11]}
|
||||
# Load key 15
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
addi $TMP_REG, $KEYP, 224
|
||||
@{[vle32_v $V11, $TMP_REG]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vaesem_vs $V28, $V9]}
|
||||
@{[vaesem_vs $V28, $V10]}
|
||||
@{[vaesef_vs $V28, $V11]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
sub handle_padding_in_first_round {
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
bnez $PADDING_LEN32, 1f
|
||||
|
||||
## without padding
|
||||
# Store ciphertext/plaintext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
j 2f
|
||||
|
||||
## with padding
|
||||
1:
|
||||
# Store ciphertext/plaintext using mask
|
||||
@{[vse32_v $V28, $OUTP, $INPUT_PADDING_MASK]}
|
||||
|
||||
# Fill zero for the padding blocks
|
||||
@{[vsetvli "zero", $PADDING_LEN32, "e32", "m4", "tu", "ma"]}
|
||||
@{[vmv_v_i $V28, 0]}
|
||||
|
||||
# We have used mask register for `INPUT_PADDING_MASK` before. We need to
|
||||
# setup the ctr mask back.
|
||||
# ctr mask : [000100010001....]
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
|
||||
li $TMP_REG, 0b10001000
|
||||
@{[vmv_v_x $CTR_MASK, $TMP_REG]}
|
||||
2:
|
||||
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
|
||||
# Do aes-128 enc for first round.
|
||||
sub aes_128_first_round {
|
||||
my $PTR_OFFSET_REG = shift;
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# Load all 11 aes round keys to v1-v11 registers.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V2, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V3, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V4, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V5, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V6, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V7, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V8, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V9, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V10, $KEYP]}
|
||||
addi $KEYP, $KEYP, 16
|
||||
@{[vle32_v $V11, $KEYP]}
|
||||
|
||||
# We already have the ciphertext/plaintext and ctr data for the first round.
|
||||
@{[aes_128_cipher_body]}
|
||||
|
||||
# Compute AES ctr result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
@{[handle_padding_in_first_round $TMP_REG]}
|
||||
|
||||
add $INP, $INP, $PTR_OFFSET_REG
|
||||
add $OUTP, $OUTP, $PTR_OFFSET_REG
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Do aes-192 enc for first round.
|
||||
sub aes_192_first_round {
|
||||
my $PTR_OFFSET_REG = shift;
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# We run out of 32 vector registers, so we just preserve some round keys
|
||||
# and load the remaining round keys inside the aes body.
|
||||
# We keep the round keys for:
|
||||
# 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys.
|
||||
# The following keys will be loaded in the aes body:
|
||||
# 4, 8 and 13th keys.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
# key 1
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
# key 2
|
||||
addi $TMP_REG, $KEYP, 16
|
||||
@{[vle32_v $V2, $TMP_REG]}
|
||||
# key 3
|
||||
addi $TMP_REG, $KEYP, 32
|
||||
@{[vle32_v $V3, $TMP_REG]}
|
||||
# key 5
|
||||
addi $TMP_REG, $KEYP, 64
|
||||
@{[vle32_v $V4, $TMP_REG]}
|
||||
# key 6
|
||||
addi $TMP_REG, $KEYP, 80
|
||||
@{[vle32_v $V5, $TMP_REG]}
|
||||
# key 7
|
||||
addi $TMP_REG, $KEYP, 96
|
||||
@{[vle32_v $V6, $TMP_REG]}
|
||||
# key 9
|
||||
addi $TMP_REG, $KEYP, 128
|
||||
@{[vle32_v $V7, $TMP_REG]}
|
||||
# key 10
|
||||
addi $TMP_REG, $KEYP, 144
|
||||
@{[vle32_v $V8, $TMP_REG]}
|
||||
# key 11
|
||||
addi $TMP_REG, $KEYP, 160
|
||||
@{[vle32_v $V9, $TMP_REG]}
|
||||
# key 12
|
||||
addi $TMP_REG, $KEYP, 176
|
||||
@{[vle32_v $V10, $TMP_REG]}
|
||||
|
||||
# We already have the ciphertext/plaintext and ctr data for the first round.
|
||||
@{[aes_192_cipher_body $TMP_REG]}
|
||||
|
||||
# Compute AES ctr result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
@{[handle_padding_in_first_round $TMP_REG]}
|
||||
|
||||
add $INP, $INP, $PTR_OFFSET_REG
|
||||
add $OUTP, $OUTP, $PTR_OFFSET_REG
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Do aes-256 enc for first round.
|
||||
sub aes_256_first_round {
|
||||
my $PTR_OFFSET_REG = shift;
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# We run out of 32 vector registers, so we just preserve some round keys
|
||||
# and load the remaining round keys inside the aes body.
|
||||
# We keep the round keys for:
|
||||
# 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys.
|
||||
# The following keys will be loaded in the aes body:
|
||||
# 3, 6, 9, 12 and 15th keys.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
# key 1
|
||||
@{[vle32_v $V1, $KEYP]}
|
||||
# key 2
|
||||
addi $TMP_REG, $KEYP, 16
|
||||
@{[vle32_v $V2, $TMP_REG]}
|
||||
# key 4
|
||||
addi $TMP_REG, $KEYP, 48
|
||||
@{[vle32_v $V3, $TMP_REG]}
|
||||
# key 5
|
||||
addi $TMP_REG, $KEYP, 64
|
||||
@{[vle32_v $V4, $TMP_REG]}
|
||||
# key 7
|
||||
addi $TMP_REG, $KEYP, 96
|
||||
@{[vle32_v $V5, $TMP_REG]}
|
||||
# key 8
|
||||
addi $TMP_REG, $KEYP, 112
|
||||
@{[vle32_v $V6, $TMP_REG]}
|
||||
# key 10
|
||||
addi $TMP_REG, $KEYP, 144
|
||||
@{[vle32_v $V7, $TMP_REG]}
|
||||
# key 11
|
||||
addi $TMP_REG, $KEYP, 160
|
||||
@{[vle32_v $V8, $TMP_REG]}
|
||||
# key 13
|
||||
addi $TMP_REG, $KEYP, 192
|
||||
@{[vle32_v $V9, $TMP_REG]}
|
||||
# key 14
|
||||
addi $TMP_REG, $KEYP, 208
|
||||
@{[vle32_v $V10, $TMP_REG]}
|
||||
|
||||
# We already have the ciphertext/plaintext and ctr data for the first round.
|
||||
@{[aes_256_cipher_body $TMP_REG]}
|
||||
|
||||
# Compute AES ctr result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
@{[handle_padding_in_first_round $TMP_REG]}
|
||||
|
||||
add $INP, $INP, $PTR_OFFSET_REG
|
||||
add $OUTP, $OUTP, $PTR_OFFSET_REG
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
sub aes_gcm_init {
|
||||
my $code=<<___;
|
||||
# Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle
|
||||
# the multiple AES-GCM blocks at the same time within `LMUL=4` register.
|
||||
# The AES-GCM's SEW is e32 and EGW is 128 bits.
|
||||
# FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4
|
||||
# = (VLEN*4)/32
|
||||
# We could get the block_num using the VL value of `vsetvli with e32, m4`.
|
||||
@{[vsetvli $FULL_BLOCK_LEN32, "zero", "e32", "m4", "ta", "ma"]}
|
||||
# If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the
|
||||
# zero padding data to make sure we could always handle FULL_BLOCK_LEN32
|
||||
# blocks for all iterations.
|
||||
|
||||
## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm
|
||||
## block number in a LMUL=4 register group.
|
||||
## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4))
|
||||
## = (VLEN/32)
|
||||
## We could use vsetvli with `e32, m1` to compute the `n` number.
|
||||
@{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]}
|
||||
|
||||
# The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2).
|
||||
addi $T1, $XIP, 32
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V31, $T1]}
|
||||
|
||||
# Compute the H^n
|
||||
li $T1, 1
|
||||
1:
|
||||
@{[vgmul_vv $V31, $V31]}
|
||||
slli $T1, $T1, 1
|
||||
bltu $T1, $T0, 1b
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V16, 0]}
|
||||
@{[vaesz_vs $V16, $V31]}
|
||||
|
||||
#### Load plaintext into v24 and handle padding. We also load the init tag
|
||||
#### data into v20 and prepare the AES ctr input data into v12 and v28.
|
||||
@{[vmv_v_i $V20, 0]}
|
||||
|
||||
## Prepare the AES ctr input data into v12.
|
||||
# Setup ctr input mask.
|
||||
# ctr mask : [000100010001....]
|
||||
# Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use
|
||||
# `FULL_BLOCK_LEN32` here.
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
|
||||
li $T0, 0b10001000
|
||||
@{[vmv_v_x $CTR_MASK, $T0]}
|
||||
# Load IV.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V31, $IVP]}
|
||||
# Convert the big-endian counter into little-endian.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]}
|
||||
@{[vrev8_v $V31, $V31, $CTR_MASK]}
|
||||
# Splat the `single block of IV` to v12
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V12, 0]}
|
||||
@{[vaesz_vs $V12, $V31]}
|
||||
# Prepare the ctr counter into v8
|
||||
# v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...]
|
||||
@{[viota_m $V8, $CTR_MASK, $CTR_MASK]}
|
||||
# Merge IV and ctr counter into v12.
|
||||
# v12:[x, x, x, count+0, x, x, x, count+1, ...]
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
|
||||
@{[vadd_vv $V12, $V12, $V8, $CTR_MASK]}
|
||||
|
||||
li $PADDING_LEN32, 0
|
||||
# Get the SEW32 size in the first round.
|
||||
# If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then
|
||||
# we will have the leading padding zero.
|
||||
addi $T0, $FULL_BLOCK_LEN32, -1
|
||||
and $T0, $T0, $LEN32
|
||||
beqz $T0, 1f
|
||||
|
||||
## with padding
|
||||
sub $LEN32, $LEN32, $T0
|
||||
sub $PADDING_LEN32, $FULL_BLOCK_LEN32, $T0
|
||||
# padding block size
|
||||
srli $T1, $PADDING_LEN32, 2
|
||||
# padding byte size
|
||||
slli $T2, $PADDING_LEN32, 2
|
||||
|
||||
# Adjust the ctr counter to make the counter start from `counter+0` for the
|
||||
# first non-padding block.
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
|
||||
@{[vsub_vx $V12, $V12, $T1, $CTR_MASK]}
|
||||
# Prepare the AES ctr input into v28.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vmv_v_v $V28, $V12]}
|
||||
@{[vrev8_v $V28, $V28, $CTR_MASK]}
|
||||
|
||||
# Prepare the mask for input loading in the first round. We use
|
||||
# `VL=FULL_BLOCK_LEN32` with the mask in the first round.
|
||||
# Adjust input ptr.
|
||||
sub $INP, $INP, $T2
|
||||
# Adjust output ptr.
|
||||
sub $OUTP, $OUTP, $T2
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e16", "m2", "ta", "ma"]}
|
||||
@{[vid_v $V2]}
|
||||
# We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead.
|
||||
# The original code is:
|
||||
# vmsgeu.vx $INPUT_PADDING_MASK, $V2, $PADDING_LEN32
|
||||
addi $T0, $PADDING_LEN32, -1
|
||||
@{[vmsgtu_vx $INPUT_PADDING_MASK, $V2, $T0]}
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vmv_v_i $V24, 0]}
|
||||
# Load the input for length FULL_BLOCK_LEN32 with mask.
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
|
||||
@{[vle32_v $V24, $INP, $INPUT_PADDING_MASK]}
|
||||
|
||||
# Load the init `Xi` data to v20 with preceding zero padding.
|
||||
# Adjust Xi ptr.
|
||||
sub $T0, $XIP, $T2
|
||||
# Load for length `zero-padding-e32-length + 4`.
|
||||
addi $T1, $PADDING_LEN32, 4
|
||||
@{[vsetvli "zero", $T1, "e32", "m4", "tu", "mu"]}
|
||||
@{[vle32_v $V20, $T0, $INPUT_PADDING_MASK]}
|
||||
j 2f
|
||||
|
||||
1:
|
||||
## without padding
|
||||
sub $LEN32, $LEN32, $FULL_BLOCK_LEN32
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
@{[vle32_v $V24, $INP]}
|
||||
|
||||
# Load the init Xi data to v20.
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "tu", "ma"]}
|
||||
@{[vle32_v $V20, $XIP]}
|
||||
|
||||
# Prepare the AES ctr input into v28.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
|
||||
@{[vmv_v_v $V28, $V12]}
|
||||
@{[vrev8_v $V28, $V28, $CTR_MASK]}
|
||||
2:
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
sub prepare_input_and_ctr {
|
||||
my $PTR_OFFSET_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
|
||||
# Increase ctr in v12.
|
||||
@{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
|
||||
sub $LEN32, $LEN32, $FULL_BLOCK_LEN32
|
||||
# Load plaintext into v24
|
||||
@{[vsetvli "zero", "zero", "e32", "m4", "ta", "ma"]}
|
||||
@{[vle32_v $V24, $INP]}
|
||||
# Prepare the AES ctr input into v28.
|
||||
# The ctr data uses big-endian form.
|
||||
@{[vmv_v_v $V28, $V12]}
|
||||
add $INP, $INP, $PTR_OFFSET_REG
|
||||
@{[vsetvli "zero", "zero", "e32", "m4", "ta", "mu"]}
|
||||
@{[vrev8_v $V28, $V28, $CTR_MASK]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Store the current CTR back to IV buffer.
|
||||
sub store_current_ctr {
|
||||
my $code=<<___;
|
||||
@{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
|
||||
# Update current ctr value to v12
|
||||
@{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
|
||||
# Convert ctr to big-endian counter.
|
||||
@{[vrev8_v $V12, $V12, $CTR_MASK]}
|
||||
@{[vse32_v $V12, $IVP, $CTR_MASK]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
# Compute the final tag into v0 from the partial tag v20.
|
||||
sub compute_final_tag {
|
||||
my $TMP_REG = shift;
|
||||
|
||||
my $code=<<___;
|
||||
# The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2).
|
||||
# Load H to v1
|
||||
addi $TMP_REG, $XIP, 32
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vle32_v $V1, $TMP_REG]}
|
||||
# Multiply H for each partial tag and XOR them together.
|
||||
# Handle 1st partial tag
|
||||
@{[vmv_v_v $V0, $V20]}
|
||||
@{[vgmul_vv $V0, $V1]}
|
||||
# Handle 2nd to N-th partial tags
|
||||
li $TMP_REG, 4
|
||||
1:
|
||||
@{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
|
||||
@{[vslidedown_vx $V4, $V20, $TMP_REG]}
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vghsh_vv $V0, $V1, $V4]}
|
||||
addi $TMP_REG, $TMP_REG, 4
|
||||
blt $TMP_REG, $FULL_BLOCK_LEN32, 1b
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in,
|
||||
# unsigned char *out, size_t len,
|
||||
# const void *key,
|
||||
# unsigned char ivec[16], u64 *Xi);
|
||||
{
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
|
||||
.type rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,\@function
|
||||
rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt:
|
||||
srli $T0, $LEN, 4
|
||||
beqz $T0, .Lenc_end
|
||||
slli $LEN32, $T0, 2
|
||||
|
||||
mv $ORIGINAL_LEN32, $LEN32
|
||||
|
||||
@{[aes_gcm_init]}
|
||||
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEYP)
|
||||
li $T1, 14
|
||||
li $T2, 12
|
||||
li $T3, 10
|
||||
|
||||
beq $T0, $T1, aes_gcm_enc_blocks_256
|
||||
beq $T0, $T2, aes_gcm_enc_blocks_192
|
||||
beq $T0, $T3, aes_gcm_enc_blocks_128
|
||||
|
||||
.Lenc_end:
|
||||
li $PROCESSED_LEN, 0
|
||||
ret
|
||||
|
||||
.size rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_enc_blocks_128:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_128_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Lenc_blocks_128:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Lenc_blocks_128_end
|
||||
@{[vghsh_vv $V20, $V16, $V28]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_128_cipher_body]}
|
||||
|
||||
# Compute AES ctr ciphertext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store ciphertext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Lenc_blocks_128
|
||||
.Lenc_blocks_128_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V28]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_enc_blocks_192:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_192_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Lenc_blocks_192:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Lenc_blocks_192_end
|
||||
@{[vghsh_vv $V20, $V16, $V28]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_192_cipher_body $T1]}
|
||||
|
||||
# Compute AES ctr ciphertext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store ciphertext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Lenc_blocks_192
|
||||
.Lenc_blocks_192_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V28]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_enc_blocks_256:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_256_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Lenc_blocks_256:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Lenc_blocks_256_end
|
||||
@{[vghsh_vv $V20, $V16, $V28]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_256_cipher_body $T1]}
|
||||
|
||||
# Compute AES ctr ciphertext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store ciphertext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Lenc_blocks_256
|
||||
.Lenc_blocks_256_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V28]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256
|
||||
___
|
||||
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in,
|
||||
# unsigned char *out, size_t len,
|
||||
# const void *key,
|
||||
# unsigned char ivec[16], u64 *Xi);
|
||||
{
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
|
||||
.type rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,\@function
|
||||
rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt:
|
||||
srli $T0, $LEN, 4
|
||||
beqz $T0, .Ldec_end
|
||||
slli $LEN32, $T0, 2
|
||||
|
||||
mv $ORIGINAL_LEN32, $LEN32
|
||||
|
||||
@{[aes_gcm_init]}
|
||||
|
||||
# Load number of rounds
|
||||
lwu $T0, 240($KEYP)
|
||||
li $T1, 14
|
||||
li $T2, 12
|
||||
li $T3, 10
|
||||
|
||||
beq $T0, $T1, aes_gcm_dec_blocks_256
|
||||
beq $T0, $T2, aes_gcm_dec_blocks_192
|
||||
beq $T0, $T3, aes_gcm_dec_blocks_128
|
||||
|
||||
.Ldec_end:
|
||||
li $PROCESSED_LEN, 0
|
||||
ret
|
||||
.size rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_dec_blocks_128:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_128_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Ldec_blocks_128:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Ldec_blocks_256_end
|
||||
@{[vghsh_vv $V20, $V16, $V24]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_128_cipher_body]}
|
||||
|
||||
# Compute AES ctr plaintext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store plaintext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Ldec_blocks_128
|
||||
.Ldec_blocks_128_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V24]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_dec_blocks_192:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_192_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Ldec_blocks_192:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Ldec_blocks_192_end
|
||||
@{[vghsh_vv $V20, $V16, $V24]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_192_cipher_body $T1]}
|
||||
|
||||
# Compute AES ctr plaintext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store plaintext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Ldec_blocks_192
|
||||
.Ldec_blocks_192_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V24]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192
|
||||
___
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
aes_gcm_dec_blocks_256:
|
||||
srli $CTR, $FULL_BLOCK_LEN32, 2
|
||||
slli $T0, $FULL_BLOCK_LEN32, 2
|
||||
|
||||
@{[aes_256_first_round $T0, $T1]}
|
||||
|
||||
@{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
|
||||
|
||||
.Ldec_blocks_256:
|
||||
# Compute the partial tags.
|
||||
# The partial tags will multiply with [H^n, H^n, ..., H^n]
|
||||
# [tag0, tag1, ...] =
|
||||
# ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
|
||||
# We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
|
||||
beqz $LEN32, .Ldec_blocks_256_end
|
||||
@{[vghsh_vv $V20, $V16, $V24]}
|
||||
|
||||
@{[prepare_input_and_ctr $T0]}
|
||||
|
||||
@{[aes_256_cipher_body $T1]}
|
||||
|
||||
# Compute AES ctr plaintext result.
|
||||
@{[vxor_vv $V28, $V28, $V24]}
|
||||
|
||||
# Store plaintext
|
||||
@{[vse32_v $V28, $OUTP]}
|
||||
add $OUTP, $OUTP, $T0
|
||||
|
||||
j .Ldec_blocks_256
|
||||
.Ldec_blocks_256_end:
|
||||
|
||||
# Add ciphertext into partial tag
|
||||
@{[vxor_vv $V20, $V20, $V24]}
|
||||
|
||||
@{[store_current_ctr]}
|
||||
|
||||
@{[compute_final_tag $T1]}
|
||||
|
||||
# Save the final tag
|
||||
@{[vse32_v $V0, $XIP]}
|
||||
|
||||
# return the processed size.
|
||||
slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
|
||||
ret
|
||||
.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256
|
||||
___
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,378 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Christoph Müllner <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
# - RISC-V Vector Carryless Multiplication extension ('Zvbc')
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void gcm_init_rv64i_zvkb_zvbc(u128 Htable[16], const u64 H[2]);
|
||||
#
|
||||
# input: H: 128-bit H - secret parameter E(K, 0^128)
|
||||
# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zvkb_zvbc and
|
||||
# gcm_ghash_rv64i_zvkb_zvbc
|
||||
{
|
||||
my ($Htable,$H,$TMP0,$TMP1,$TMP2) = ("a0","a1","t0","t1","t2");
|
||||
my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zvkb_zvbc
|
||||
.type gcm_init_rv64i_zvkb_zvbc,\@function
|
||||
gcm_init_rv64i_zvkb_zvbc:
|
||||
# Load/store data in reverse order.
|
||||
# This is needed as a part of endianness swap.
|
||||
add $H, $H, 8
|
||||
li $TMP0, -8
|
||||
li $TMP1, 63
|
||||
la $TMP2, Lpolymod
|
||||
|
||||
@{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu
|
||||
|
||||
@{[vlse64_v $V1, $H, $TMP0]} # vlse64.v v1, (a1), t0
|
||||
@{[vle64_v $V2, $TMP2]} # vle64.v v2, (t2)
|
||||
|
||||
# Shift one left and get the carry bits.
|
||||
@{[vsrl_vx $V3, $V1, $TMP1]} # vsrl.vx v3, v1, t1
|
||||
@{[vsll_vi $V1, $V1, 1]} # vsll.vi v1, v1, 1
|
||||
|
||||
# Use the fact that the polynomial degree is no more than 128,
|
||||
# i.e. only the LSB of the upper half could be set.
|
||||
# Thanks to this we don't need to do the full reduction here.
|
||||
# Instead simply subtract the reduction polynomial.
|
||||
# This idea was taken from x86 ghash implementation in OpenSSL.
|
||||
@{[vslideup_vi $V4, $V3, 1]} # vslideup.vi v4, v3, 1
|
||||
@{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
|
||||
|
||||
@{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2
|
||||
@{[vor_vv_v0t $V1, $V1, $V4]} # vor.vv v1, v1, v4, v0.t
|
||||
|
||||
# Need to set the mask to 3, if the carry bit is set.
|
||||
@{[vmv_v_v $V0, $V3]} # vmv.v.v v0, v3
|
||||
@{[vmv_v_i $V3, 0]} # vmv.v.i v3, 0
|
||||
@{[vmerge_vim $V3, $V3, 3]} # vmerge.vim v3, v3, 3, v0
|
||||
@{[vmv_v_v $V0, $V3]} # vmv.v.v v0, v3
|
||||
|
||||
@{[vxor_vv_v0t $V1, $V1, $V2]} # vxor.vv v1, v1, v2, v0.t
|
||||
|
||||
@{[vse64_v $V1, $Htable]} # vse64.v v1, (a0)
|
||||
ret
|
||||
.size gcm_init_rv64i_zvkb_zvbc,.-gcm_init_rv64i_zvkb_zvbc
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_gmult_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16]);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: preprocessed H
|
||||
# output: Xi: next hash value Xi = (Xi * H mod f)
|
||||
{
|
||||
my ($Xi,$Htable,$TMP0,$TMP1,$TMP2,$TMP3,$TMP4) = ("a0","a1","t0","t1","t2","t3","t4");
|
||||
my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6");
|
||||
|
||||
$code .= <<___;
|
||||
.text
|
||||
.p2align 3
|
||||
.globl gcm_gmult_rv64i_zvkb_zvbc
|
||||
.type gcm_gmult_rv64i_zvkb_zvbc,\@function
|
||||
gcm_gmult_rv64i_zvkb_zvbc:
|
||||
ld $TMP0, ($Htable)
|
||||
ld $TMP1, 8($Htable)
|
||||
li $TMP2, 63
|
||||
la $TMP3, Lpolymod
|
||||
ld $TMP3, 8($TMP3)
|
||||
|
||||
# Load/store data in reverse order.
|
||||
# This is needed as a part of endianness swap.
|
||||
add $Xi, $Xi, 8
|
||||
li $TMP4, -8
|
||||
|
||||
@{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu
|
||||
|
||||
@{[vlse64_v $V5, $Xi, $TMP4]} # vlse64.v v5, (a0), t4
|
||||
@{[vrev8_v $V5, $V5]} # vrev8.v v5, v5
|
||||
|
||||
# Multiplication
|
||||
|
||||
# Do two 64x64 multiplications in one go to save some time
|
||||
# and simplify things.
|
||||
|
||||
# A = a1a0 (t1, t0)
|
||||
# B = b1b0 (v5)
|
||||
# C = c1c0 (256 bit)
|
||||
# c1 = a1b1 + (a0b1)h + (a1b0)h
|
||||
# c0 = a0b0 + (a0b1)l + (a1b0)h
|
||||
|
||||
# v1 = (a0b1)l,(a0b0)l
|
||||
@{[vclmul_vx $V1, $V5, $TMP0]} # vclmul.vx v1, v5, t0
|
||||
# v3 = (a0b1)h,(a0b0)h
|
||||
@{[vclmulh_vx $V3, $V5, $TMP0]} # vclmulh.vx v3, v5, t0
|
||||
|
||||
# v4 = (a1b1)l,(a1b0)l
|
||||
@{[vclmul_vx $V4, $V5, $TMP1]} # vclmul.vx v4, v5, t1
|
||||
# v2 = (a1b1)h,(a1b0)h
|
||||
@{[vclmulh_vx $V2, $V5, $TMP1]} # vclmulh.vx v2, v5, t1
|
||||
|
||||
# Is there a better way to do this?
|
||||
# Would need to swap the order of elements within a vector register.
|
||||
@{[vslideup_vi $V5, $V3, 1]} # vslideup.vi v5, v3, 1
|
||||
@{[vslideup_vi $V6, $V4, 1]} # vslideup.vi v6, v4, 1
|
||||
@{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
|
||||
@{[vslidedown_vi $V4, $V4, 1]} # vslidedown.vi v4, v4, 1
|
||||
|
||||
@{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
|
||||
# v2 += (a0b1)h
|
||||
@{[vxor_vv_v0t $V2, $V2, $V3]} # vxor.vv v2, v2, v3, v0.t
|
||||
# v2 += (a1b1)l
|
||||
@{[vxor_vv_v0t $V2, $V2, $V4]} # vxor.vv v2, v2, v4, v0.t
|
||||
|
||||
@{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2
|
||||
# v1 += (a0b0)h,0
|
||||
@{[vxor_vv_v0t $V1, $V1, $V5]} # vxor.vv v1, v1, v5, v0.t
|
||||
# v1 += (a1b0)l,0
|
||||
@{[vxor_vv_v0t $V1, $V1, $V6]} # vxor.vv v1, v1, v6, v0.t
|
||||
|
||||
# Now the 256bit product should be stored in (v2,v1)
|
||||
# v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l
|
||||
# v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l
|
||||
|
||||
# Reduction
|
||||
# Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0]
|
||||
# This is a slight variation of the Gueron's Montgomery reduction.
|
||||
# The difference being the order of some operations has been changed,
|
||||
# to make a better use of vclmul(h) instructions.
|
||||
|
||||
# First step:
|
||||
# c1 += (c0 * P)l
|
||||
# vmv.v.i v0, 2
|
||||
@{[vslideup_vi_v0t $V3, $V1, 1]} # vslideup.vi v3, v1, 1, v0.t
|
||||
@{[vclmul_vx_v0t $V3, $V3, $TMP3]} # vclmul.vx v3, v3, t3, v0.t
|
||||
@{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
|
||||
|
||||
# Second step:
|
||||
# D = d1,d0 is final result
|
||||
# We want:
|
||||
# m1 = c1 + (c1 * P)h
|
||||
# m0 = (c1 * P)l + (c0 * P)h + c0
|
||||
# d1 = c3 + m1
|
||||
# d0 = c2 + m0
|
||||
|
||||
#v3 = (c1 * P)l, 0
|
||||
@{[vclmul_vx_v0t $V3, $V1, $TMP3]} # vclmul.vx v3, v1, t3, v0.t
|
||||
#v4 = (c1 * P)h, (c0 * P)h
|
||||
@{[vclmulh_vx $V4, $V1, $TMP3]} # vclmulh.vx v4, v1, t3
|
||||
|
||||
@{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
|
||||
@{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
|
||||
|
||||
@{[vxor_vv $V1, $V1, $V4]} # vxor.vv v1, v1, v4
|
||||
@{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
|
||||
|
||||
# XOR in the upper upper part of the product
|
||||
@{[vxor_vv $V2, $V2, $V1]} # vxor.vv v2, v2, v1
|
||||
|
||||
@{[vrev8_v $V2, $V2]} # vrev8.v v2, v2
|
||||
@{[vsse64_v $V2, $Xi, $TMP4]} # vsse64.v v2, (a0), t4
|
||||
ret
|
||||
.size gcm_gmult_rv64i_zvkb_zvbc,.-gcm_gmult_rv64i_zvkb_zvbc
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_ghash_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16],
|
||||
# const u8 *inp, size_t len);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: preprocessed H
|
||||
# inp: pointer to input data
|
||||
# len: length of input data in bytes (multiple of block size)
|
||||
# output: Xi: Xi+1 (next hash value Xi)
|
||||
{
|
||||
my ($Xi,$Htable,$inp,$len,$TMP0,$TMP1,$TMP2,$TMP3,$M8,$TMP5,$TMP6) = ("a0","a1","a2","a3","t0","t1","t2","t3","t4","t5","t6");
|
||||
my ($V0,$V1,$V2,$V3,$V4,$V5,$V6,$Vinp) = ("v0","v1","v2","v3","v4","v5","v6","v7");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_ghash_rv64i_zvkb_zvbc
|
||||
.type gcm_ghash_rv64i_zvkb_zvbc,\@function
|
||||
gcm_ghash_rv64i_zvkb_zvbc:
|
||||
ld $TMP0, ($Htable)
|
||||
ld $TMP1, 8($Htable)
|
||||
li $TMP2, 63
|
||||
la $TMP3, Lpolymod
|
||||
ld $TMP3, 8($TMP3)
|
||||
|
||||
# Load/store data in reverse order.
|
||||
# This is needed as a part of endianness swap.
|
||||
add $Xi, $Xi, 8
|
||||
add $inp, $inp, 8
|
||||
li $M8, -8
|
||||
|
||||
@{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu
|
||||
|
||||
@{[vlse64_v $V5, $Xi, $M8]} # vlse64.v v5, (a0), t4
|
||||
|
||||
Lstep:
|
||||
# Read input data
|
||||
@{[vlse64_v $Vinp, $inp, $M8]} # vle64.v v0, (a2)
|
||||
add $inp, $inp, 16
|
||||
add $len, $len, -16
|
||||
# XOR them into Xi
|
||||
@{[vxor_vv $V5, $V5, $Vinp]} # vxor.vv v0, v0, v1
|
||||
|
||||
@{[vrev8_v $V5, $V5]} # vrev8.v v5, v5
|
||||
|
||||
# Multiplication
|
||||
|
||||
# Do two 64x64 multiplications in one go to save some time
|
||||
# and simplify things.
|
||||
|
||||
# A = a1a0 (t1, t0)
|
||||
# B = b1b0 (v5)
|
||||
# C = c1c0 (256 bit)
|
||||
# c1 = a1b1 + (a0b1)h + (a1b0)h
|
||||
# c0 = a0b0 + (a0b1)l + (a1b0)h
|
||||
|
||||
# v1 = (a0b1)l,(a0b0)l
|
||||
@{[vclmul_vx $V1, $V5, $TMP0]} # vclmul.vx v1, v5, t0
|
||||
# v3 = (a0b1)h,(a0b0)h
|
||||
@{[vclmulh_vx $V3, $V5, $TMP0]} # vclmulh.vx v3, v5, t0
|
||||
|
||||
# v4 = (a1b1)l,(a1b0)l
|
||||
@{[vclmul_vx $V4, $V5, $TMP1]} # vclmul.vx v4, v5, t1
|
||||
# v2 = (a1b1)h,(a1b0)h
|
||||
@{[vclmulh_vx $V2, $V5, $TMP1]} # vclmulh.vx v2, v5, t1
|
||||
|
||||
# Is there a better way to do this?
|
||||
# Would need to swap the order of elements within a vector register.
|
||||
@{[vslideup_vi $V5, $V3, 1]} # vslideup.vi v5, v3, 1
|
||||
@{[vslideup_vi $V6, $V4, 1]} # vslideup.vi v6, v4, 1
|
||||
@{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
|
||||
@{[vslidedown_vi $V4, $V4, 1]} # vslidedown.vi v4, v4, 1
|
||||
|
||||
@{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
|
||||
# v2 += (a0b1)h
|
||||
@{[vxor_vv_v0t $V2, $V2, $V3]} # vxor.vv v2, v2, v3, v0.t
|
||||
# v2 += (a1b1)l
|
||||
@{[vxor_vv_v0t $V2, $V2, $V4]} # vxor.vv v2, v2, v4, v0.t
|
||||
|
||||
@{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2
|
||||
# v1 += (a0b0)h,0
|
||||
@{[vxor_vv_v0t $V1, $V1, $V5]} # vxor.vv v1, v1, v5, v0.t
|
||||
# v1 += (a1b0)l,0
|
||||
@{[vxor_vv_v0t $V1, $V1, $V6]} # vxor.vv v1, v1, v6, v0.t
|
||||
|
||||
# Now the 256bit product should be stored in (v2,v1)
|
||||
# v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l
|
||||
# v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l
|
||||
|
||||
# Reduction
|
||||
# Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0]
|
||||
# This is a slight variation of the Gueron's Montgomery reduction.
|
||||
# The difference being the order of some operations has been changed,
|
||||
# to make a better use of vclmul(h) instructions.
|
||||
|
||||
# First step:
|
||||
# c1 += (c0 * P)l
|
||||
# vmv.v.i v0, 2
|
||||
@{[vslideup_vi_v0t $V3, $V1, 1]} # vslideup.vi v3, v1, 1, v0.t
|
||||
@{[vclmul_vx_v0t $V3, $V3, $TMP3]} # vclmul.vx v3, v3, t3, v0.t
|
||||
@{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
|
||||
|
||||
# Second step:
|
||||
# D = d1,d0 is final result
|
||||
# We want:
|
||||
# m1 = c1 + (c1 * P)h
|
||||
# m0 = (c1 * P)l + (c0 * P)h + c0
|
||||
# d1 = c3 + m1
|
||||
# d0 = c2 + m0
|
||||
|
||||
#v3 = (c1 * P)l, 0
|
||||
@{[vclmul_vx_v0t $V3, $V1, $TMP3]} # vclmul.vx v3, v1, t3, v0.t
|
||||
#v4 = (c1 * P)h, (c0 * P)h
|
||||
@{[vclmulh_vx $V4, $V1, $TMP3]} # vclmulh.vx v4, v1, t3
|
||||
|
||||
@{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
|
||||
@{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
|
||||
|
||||
@{[vxor_vv $V1, $V1, $V4]} # vxor.vv v1, v1, v4
|
||||
@{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
|
||||
|
||||
# XOR in the upper upper part of the product
|
||||
@{[vxor_vv $V2, $V2, $V1]} # vxor.vv v2, v2, v1
|
||||
|
||||
@{[vrev8_v $V5, $V2]} # vrev8.v v2, v2
|
||||
|
||||
bnez $len, Lstep
|
||||
|
||||
@{[vsse64_v $V5, $Xi, $M8]} # vsse64.v v2, (a0), t4
|
||||
ret
|
||||
.size gcm_ghash_rv64i_zvkb_zvbc,.-gcm_ghash_rv64i_zvkb_zvbc
|
||||
___
|
||||
}
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 4
|
||||
Lpolymod:
|
||||
.dword 0x0000000000000001
|
||||
.dword 0xc200000000000000
|
||||
.size Lpolymod,.-Lpolymod
|
||||
___
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,169 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Christoph Müllner <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector GCM/GMAC extension ('Zvkg')
|
||||
#
|
||||
# Optional:
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void gcm_init_rv64i_zvkg(u128 Htable[16], const u64 H[2]);
|
||||
# void gcm_init_rv64i_zvkg_zvkb(u128 Htable[16], const u64 H[2]);
|
||||
#
|
||||
# input: H: 128-bit H - secret parameter E(K, 0^128)
|
||||
# output: Htable: Copy of secret parameter (in normalized byte order)
|
||||
#
|
||||
# All callers of this function revert the byte-order unconditionally
|
||||
# on little-endian machines. So we need to revert the byte-order back.
|
||||
{
|
||||
my ($Htable,$H,$VAL0,$VAL1,$TMP0) = ("a0","a1","a2","a3","t0");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zvkg
|
||||
.type gcm_init_rv64i_zvkg,\@function
|
||||
gcm_init_rv64i_zvkg:
|
||||
ld $VAL0, 0($H)
|
||||
ld $VAL1, 8($H)
|
||||
@{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]}
|
||||
@{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]}
|
||||
ret
|
||||
.size gcm_init_rv64i_zvkg,.-gcm_init_rv64i_zvkg
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($Htable,$H,$V0) = ("a0","a1","v0");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zvkg_zvkb
|
||||
.type gcm_init_rv64i_zvkg_zvkb,\@function
|
||||
gcm_init_rv64i_zvkg_zvkb:
|
||||
@{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, ta, ma
|
||||
@{[vle64_v $V0, $H]} # vle64.v v0, (a1)
|
||||
@{[vrev8_v $V0, $V0]} # vrev8.v v0, v0
|
||||
@{[vse64_v $V0, $Htable]} # vse64.v v0, (a0)
|
||||
ret
|
||||
.size gcm_init_rv64i_zvkg_zvkb,.-gcm_init_rv64i_zvkg_zvkb
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_gmult_rv64i_zvkg(u64 Xi[2], const u128 Htable[16]);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: copy of H
|
||||
# output: Xi: next hash value Xi
|
||||
{
|
||||
my ($Xi,$Htable) = ("a0","a1");
|
||||
my ($VD,$VS2) = ("v1","v2");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_gmult_rv64i_zvkg
|
||||
.type gcm_gmult_rv64i_zvkg,\@function
|
||||
gcm_gmult_rv64i_zvkg:
|
||||
@{[vsetivli__x0_4_e32_m1_tu_mu]}
|
||||
@{[vle32_v $VS2, $Htable]}
|
||||
@{[vle32_v $VD, $Xi]}
|
||||
@{[vgmul_vv $VD, $VS2]}
|
||||
@{[vse32_v $VD, $Xi]}
|
||||
ret
|
||||
.size gcm_gmult_rv64i_zvkg,.-gcm_gmult_rv64i_zvkg
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16],
|
||||
# const u8 *inp, size_t len);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: copy of H
|
||||
# inp: pointer to input data
|
||||
# len: length of input data in bytes (multiple of block size)
|
||||
# output: Xi: Xi+1 (next hash value Xi)
|
||||
{
|
||||
my ($Xi,$Htable,$inp,$len) = ("a0","a1","a2","a3");
|
||||
my ($vXi,$vH,$vinp,$Vzero) = ("v1","v2","v3","v4");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_ghash_rv64i_zvkg
|
||||
.type gcm_ghash_rv64i_zvkg,\@function
|
||||
gcm_ghash_rv64i_zvkg:
|
||||
@{[vsetivli__x0_4_e32_m1_tu_mu]}
|
||||
@{[vle32_v $vH, $Htable]}
|
||||
@{[vle32_v $vXi, $Xi]}
|
||||
|
||||
Lstep:
|
||||
@{[vle32_v $vinp, $inp]}
|
||||
add $inp, $inp, 16
|
||||
add $len, $len, -16
|
||||
@{[vghsh_vv $vXi, $vH, $vinp]}
|
||||
bnez $len, Lstep
|
||||
|
||||
@{[vse32_v $vXi, $Xi]}
|
||||
ret
|
||||
|
||||
.size gcm_ghash_rv64i_zvkg,.-gcm_ghash_rv64i_zvkg
|
||||
___
|
||||
}
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,428 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Christoph Müllner <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
################################################################################
|
||||
# void gcm_init_rv64i_zbc(u128 Htable[16], const u64 H[2]);
|
||||
# void gcm_init_rv64i_zbc__zbb(u128 Htable[16], const u64 H[2]);
|
||||
# void gcm_init_rv64i_zbc__zbkb(u128 Htable[16], const u64 H[2]);
|
||||
#
|
||||
# input: H: 128-bit H - secret parameter E(K, 0^128)
|
||||
# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zbc* and
|
||||
# gcm_ghash_rv64i_zbc*
|
||||
#
|
||||
# All callers of this function revert the byte-order unconditionally
|
||||
# on little-endian machines. So we need to revert the byte-order back.
|
||||
# Additionally we reverse the bits of each byte.
|
||||
|
||||
{
|
||||
my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zbc
|
||||
.type gcm_init_rv64i_zbc,\@function
|
||||
gcm_init_rv64i_zbc:
|
||||
ld $VAL0,0($H)
|
||||
ld $VAL1,8($H)
|
||||
@{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]}
|
||||
@{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]}
|
||||
@{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]}
|
||||
@{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]}
|
||||
ret
|
||||
.size gcm_init_rv64i_zbc,.-gcm_init_rv64i_zbc
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zbc__zbb
|
||||
.type gcm_init_rv64i_zbc__zbb,\@function
|
||||
gcm_init_rv64i_zbc__zbb:
|
||||
ld $VAL0,0($H)
|
||||
ld $VAL1,8($H)
|
||||
@{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]}
|
||||
@{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]}
|
||||
@{[rev8 $VAL0, $VAL0]}
|
||||
@{[rev8 $VAL1, $VAL1]}
|
||||
sd $VAL0,0($Htable)
|
||||
sd $VAL1,8($Htable)
|
||||
ret
|
||||
.size gcm_init_rv64i_zbc__zbb,.-gcm_init_rv64i_zbc__zbb
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($Htable,$H,$TMP0,$TMP1) = ("a0","a1","t0","t1");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_init_rv64i_zbc__zbkb
|
||||
.type gcm_init_rv64i_zbc__zbkb,\@function
|
||||
gcm_init_rv64i_zbc__zbkb:
|
||||
ld $TMP0,0($H)
|
||||
ld $TMP1,8($H)
|
||||
@{[brev8 $TMP0, $TMP0]}
|
||||
@{[brev8 $TMP1, $TMP1]}
|
||||
@{[rev8 $TMP0, $TMP0]}
|
||||
@{[rev8 $TMP1, $TMP1]}
|
||||
sd $TMP0,0($Htable)
|
||||
sd $TMP1,8($Htable)
|
||||
ret
|
||||
.size gcm_init_rv64i_zbc__zbkb,.-gcm_init_rv64i_zbc__zbkb
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_gmult_rv64i_zbc(u64 Xi[2], const u128 Htable[16]);
|
||||
# void gcm_gmult_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16]);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: copy of H
|
||||
# output: Xi: next hash value Xi
|
||||
#
|
||||
# Compute GMULT (Xi*H mod f) using the Zbc (clmul) and Zbb (basic bit manip)
|
||||
# extensions. Using the no-Karatsuba approach and clmul for the final reduction.
|
||||
# This results in an implementation with minimized number of instructions.
|
||||
# HW with clmul latencies higher than 2 cycles might observe a performance
|
||||
# improvement with Karatsuba. HW with clmul latencies higher than 6 cycles
|
||||
# might observe a performance improvement with additionally converting the
|
||||
# reduction to shift&xor. For a full discussion of this estimates see
|
||||
# https://github.com/riscv/riscv-crypto/blob/master/doc/supp/gcm-mode-cmul.adoc
|
||||
{
|
||||
my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
|
||||
my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_gmult_rv64i_zbc
|
||||
.type gcm_gmult_rv64i_zbc,\@function
|
||||
gcm_gmult_rv64i_zbc:
|
||||
# Load Xi and bit-reverse it
|
||||
ld $x0, 0($Xi)
|
||||
ld $x1, 8($Xi)
|
||||
@{[brev8_rv64i $x0, $z0, $z1, $z2]}
|
||||
@{[brev8_rv64i $x1, $z0, $z1, $z2]}
|
||||
|
||||
# Load the key (already bit-reversed)
|
||||
ld $y0, 0($Htable)
|
||||
ld $y1, 8($Htable)
|
||||
|
||||
# Load the reduction constant
|
||||
la $polymod, Lpolymod
|
||||
lbu $polymod, 0($polymod)
|
||||
|
||||
# Multiplication (without Karatsuba)
|
||||
@{[clmulh $z3, $x1, $y1]}
|
||||
@{[clmul $z2, $x1, $y1]}
|
||||
@{[clmulh $t1, $x0, $y1]}
|
||||
@{[clmul $z1, $x0, $y1]}
|
||||
xor $z2, $z2, $t1
|
||||
@{[clmulh $t1, $x1, $y0]}
|
||||
@{[clmul $t0, $x1, $y0]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $x0, $y0]}
|
||||
@{[clmul $z0, $x0, $y0]}
|
||||
xor $z1, $z1, $t1
|
||||
|
||||
# Reduction with clmul
|
||||
@{[clmulh $t1, $z3, $polymod]}
|
||||
@{[clmul $t0, $z3, $polymod]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $z2, $polymod]}
|
||||
@{[clmul $t0, $z2, $polymod]}
|
||||
xor $x1, $z1, $t1
|
||||
xor $x0, $z0, $t0
|
||||
|
||||
# Bit-reverse Xi back and store it
|
||||
@{[brev8_rv64i $x0, $z0, $z1, $z2]}
|
||||
@{[brev8_rv64i $x1, $z0, $z1, $z2]}
|
||||
sd $x0, 0($Xi)
|
||||
sd $x1, 8($Xi)
|
||||
ret
|
||||
.size gcm_gmult_rv64i_zbc,.-gcm_gmult_rv64i_zbc
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
|
||||
my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_gmult_rv64i_zbc__zbkb
|
||||
.type gcm_gmult_rv64i_zbc__zbkb,\@function
|
||||
gcm_gmult_rv64i_zbc__zbkb:
|
||||
# Load Xi and bit-reverse it
|
||||
ld $x0, 0($Xi)
|
||||
ld $x1, 8($Xi)
|
||||
@{[brev8 $x0, $x0]}
|
||||
@{[brev8 $x1, $x1]}
|
||||
|
||||
# Load the key (already bit-reversed)
|
||||
ld $y0, 0($Htable)
|
||||
ld $y1, 8($Htable)
|
||||
|
||||
# Load the reduction constant
|
||||
la $polymod, Lpolymod
|
||||
lbu $polymod, 0($polymod)
|
||||
|
||||
# Multiplication (without Karatsuba)
|
||||
@{[clmulh $z3, $x1, $y1]}
|
||||
@{[clmul $z2, $x1, $y1]}
|
||||
@{[clmulh $t1, $x0, $y1]}
|
||||
@{[clmul $z1, $x0, $y1]}
|
||||
xor $z2, $z2, $t1
|
||||
@{[clmulh $t1, $x1, $y0]}
|
||||
@{[clmul $t0, $x1, $y0]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $x0, $y0]}
|
||||
@{[clmul $z0, $x0, $y0]}
|
||||
xor $z1, $z1, $t1
|
||||
|
||||
# Reduction with clmul
|
||||
@{[clmulh $t1, $z3, $polymod]}
|
||||
@{[clmul $t0, $z3, $polymod]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $z2, $polymod]}
|
||||
@{[clmul $t0, $z2, $polymod]}
|
||||
xor $x1, $z1, $t1
|
||||
xor $x0, $z0, $t0
|
||||
|
||||
# Bit-reverse Xi back and store it
|
||||
@{[brev8 $x0, $x0]}
|
||||
@{[brev8 $x1, $x1]}
|
||||
sd $x0, 0($Xi)
|
||||
sd $x1, 8($Xi)
|
||||
ret
|
||||
.size gcm_gmult_rv64i_zbc__zbkb,.-gcm_gmult_rv64i_zbc__zbkb
|
||||
___
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void gcm_ghash_rv64i_zbc(u64 Xi[2], const u128 Htable[16],
|
||||
# const u8 *inp, size_t len);
|
||||
# void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16],
|
||||
# const u8 *inp, size_t len);
|
||||
#
|
||||
# input: Xi: current hash value
|
||||
# Htable: copy of H
|
||||
# inp: pointer to input data
|
||||
# len: length of input data in bytes (multiple of block size)
|
||||
# output: Xi: Xi+1 (next hash value Xi)
|
||||
{
|
||||
my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
|
||||
my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_ghash_rv64i_zbc
|
||||
.type gcm_ghash_rv64i_zbc,\@function
|
||||
gcm_ghash_rv64i_zbc:
|
||||
# Load Xi and bit-reverse it
|
||||
ld $x0, 0($Xi)
|
||||
ld $x1, 8($Xi)
|
||||
@{[brev8_rv64i $x0, $z0, $z1, $z2]}
|
||||
@{[brev8_rv64i $x1, $z0, $z1, $z2]}
|
||||
|
||||
# Load the key (already bit-reversed)
|
||||
ld $y0, 0($Htable)
|
||||
ld $y1, 8($Htable)
|
||||
|
||||
# Load the reduction constant
|
||||
la $polymod, Lpolymod
|
||||
lbu $polymod, 0($polymod)
|
||||
|
||||
Lstep:
|
||||
# Load the input data, bit-reverse them, and XOR them with Xi
|
||||
ld $t0, 0($inp)
|
||||
ld $t1, 8($inp)
|
||||
add $inp, $inp, 16
|
||||
add $len, $len, -16
|
||||
@{[brev8_rv64i $t0, $z0, $z1, $z2]}
|
||||
@{[brev8_rv64i $t1, $z0, $z1, $z2]}
|
||||
xor $x0, $x0, $t0
|
||||
xor $x1, $x1, $t1
|
||||
|
||||
# Multiplication (without Karatsuba)
|
||||
@{[clmulh $z3, $x1, $y1]}
|
||||
@{[clmul $z2, $x1, $y1]}
|
||||
@{[clmulh $t1, $x0, $y1]}
|
||||
@{[clmul $z1, $x0, $y1]}
|
||||
xor $z2, $z2, $t1
|
||||
@{[clmulh $t1, $x1, $y0]}
|
||||
@{[clmul $t0, $x1, $y0]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $x0, $y0]}
|
||||
@{[clmul $z0, $x0, $y0]}
|
||||
xor $z1, $z1, $t1
|
||||
|
||||
# Reduction with clmul
|
||||
@{[clmulh $t1, $z3, $polymod]}
|
||||
@{[clmul $t0, $z3, $polymod]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $z2, $polymod]}
|
||||
@{[clmul $t0, $z2, $polymod]}
|
||||
xor $x1, $z1, $t1
|
||||
xor $x0, $z0, $t0
|
||||
|
||||
# Iterate over all blocks
|
||||
bnez $len, Lstep
|
||||
|
||||
# Bit-reverse final Xi back and store it
|
||||
@{[brev8_rv64i $x0, $z0, $z1, $z2]}
|
||||
@{[brev8_rv64i $x1, $z0, $z1, $z2]}
|
||||
sd $x0, 0($Xi)
|
||||
sd $x1, 8($Xi)
|
||||
ret
|
||||
.size gcm_ghash_rv64i_zbc,.-gcm_ghash_rv64i_zbc
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
|
||||
my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
.globl gcm_ghash_rv64i_zbc__zbkb
|
||||
.type gcm_ghash_rv64i_zbc__zbkb,\@function
|
||||
gcm_ghash_rv64i_zbc__zbkb:
|
||||
# Load Xi and bit-reverse it
|
||||
ld $x0, 0($Xi)
|
||||
ld $x1, 8($Xi)
|
||||
@{[brev8 $x0, $x0]}
|
||||
@{[brev8 $x1, $x1]}
|
||||
|
||||
# Load the key (already bit-reversed)
|
||||
ld $y0, 0($Htable)
|
||||
ld $y1, 8($Htable)
|
||||
|
||||
# Load the reduction constant
|
||||
la $polymod, Lpolymod
|
||||
lbu $polymod, 0($polymod)
|
||||
|
||||
Lstep_zkbk:
|
||||
# Load the input data, bit-reverse them, and XOR them with Xi
|
||||
ld $t0, 0($inp)
|
||||
ld $t1, 8($inp)
|
||||
add $inp, $inp, 16
|
||||
add $len, $len, -16
|
||||
@{[brev8 $t0, $t0]}
|
||||
@{[brev8 $t1, $t1]}
|
||||
xor $x0, $x0, $t0
|
||||
xor $x1, $x1, $t1
|
||||
|
||||
# Multiplication (without Karatsuba)
|
||||
@{[clmulh $z3, $x1, $y1]}
|
||||
@{[clmul $z2, $x1, $y1]}
|
||||
@{[clmulh $t1, $x0, $y1]}
|
||||
@{[clmul $z1, $x0, $y1]}
|
||||
xor $z2, $z2, $t1
|
||||
@{[clmulh $t1, $x1, $y0]}
|
||||
@{[clmul $t0, $x1, $y0]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $x0, $y0]}
|
||||
@{[clmul $z0, $x0, $y0]}
|
||||
xor $z1, $z1, $t1
|
||||
|
||||
# Reduction with clmul
|
||||
@{[clmulh $t1, $z3, $polymod]}
|
||||
@{[clmul $t0, $z3, $polymod]}
|
||||
xor $z2, $z2, $t1
|
||||
xor $z1, $z1, $t0
|
||||
@{[clmulh $t1, $z2, $polymod]}
|
||||
@{[clmul $t0, $z2, $polymod]}
|
||||
xor $x1, $z1, $t1
|
||||
xor $x0, $z0, $t0
|
||||
|
||||
# Iterate over all blocks
|
||||
bnez $len, Lstep_zkbk
|
||||
|
||||
# Bit-reverse final Xi back and store it
|
||||
@{[brev8 $x0, $x0]}
|
||||
@{[brev8 $x1, $x1]}
|
||||
sd $x0, 0($Xi)
|
||||
sd $x1, 8($Xi)
|
||||
ret
|
||||
.size gcm_ghash_rv64i_zbc__zbkb,.-gcm_ghash_rv64i_zbc__zbkb
|
||||
___
|
||||
}
|
||||
|
||||
$code .= <<___;
|
||||
.p2align 3
|
||||
Lbrev8_const:
|
||||
.dword 0xAAAAAAAAAAAAAAAA
|
||||
.dword 0xCCCCCCCCCCCCCCCC
|
||||
.dword 0xF0F0F0F0F0F0F0F0
|
||||
.size Lbrev8_const,.-Lbrev8_const
|
||||
|
||||
Lpolymod:
|
||||
.byte 0x87
|
||||
.size Lpolymod,.-Lpolymod
|
||||
___
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,199 @@
|
||||
/*
|
||||
* Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <openssl/crypto.h>
|
||||
#include "internal/endian.h"
|
||||
#include "crypto/modes.h"
|
||||
|
||||
#ifndef STRICT_ALIGNMENT
|
||||
# ifdef __GNUC__
|
||||
typedef u64 u64_a1 __attribute((__aligned__(1)));
|
||||
# else
|
||||
typedef u64 u64_a1;
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int ossl_crypto_xts128gb_encrypt(const XTS128_CONTEXT *ctx,
|
||||
const unsigned char iv[16],
|
||||
const unsigned char *inp, unsigned char *out,
|
||||
size_t len, int enc)
|
||||
{
|
||||
DECLARE_IS_ENDIAN;
|
||||
union {
|
||||
u64 u[2];
|
||||
u32 d[4];
|
||||
u8 c[16];
|
||||
} tweak, scratch;
|
||||
unsigned int i;
|
||||
|
||||
if (len < 16)
|
||||
return -1;
|
||||
|
||||
memcpy(tweak.c, iv, 16);
|
||||
|
||||
(*ctx->block2) (tweak.c, tweak.c, ctx->key2);
|
||||
|
||||
if (!enc && (len % 16))
|
||||
len -= 16;
|
||||
|
||||
while (len >= 16) {
|
||||
#if defined(STRICT_ALIGNMENT)
|
||||
memcpy(scratch.c, inp, 16);
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
#else
|
||||
scratch.u[0] = ((u64_a1 *)inp)[0] ^ tweak.u[0];
|
||||
scratch.u[1] = ((u64_a1 *)inp)[1] ^ tweak.u[1];
|
||||
#endif
|
||||
(*ctx->block1) (scratch.c, scratch.c, ctx->key1);
|
||||
#if defined(STRICT_ALIGNMENT)
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
memcpy(out, scratch.c, 16);
|
||||
#else
|
||||
((u64_a1 *)out)[0] = scratch.u[0] ^= tweak.u[0];
|
||||
((u64_a1 *)out)[1] = scratch.u[1] ^= tweak.u[1];
|
||||
#endif
|
||||
inp += 16;
|
||||
out += 16;
|
||||
len -= 16;
|
||||
|
||||
if (len == 0)
|
||||
return 0;
|
||||
|
||||
if (IS_LITTLE_ENDIAN) {
|
||||
u8 res;
|
||||
u64 hi, lo;
|
||||
#ifdef BSWAP8
|
||||
hi = BSWAP8(tweak.u[0]);
|
||||
lo = BSWAP8(tweak.u[1]);
|
||||
#else
|
||||
u8 *p = tweak.c;
|
||||
|
||||
hi = (u64)GETU32(p) << 32 | GETU32(p + 4);
|
||||
lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
|
||||
#endif
|
||||
res = (u8)lo & 1;
|
||||
tweak.u[0] = (lo >> 1) | (hi << 63);
|
||||
tweak.u[1] = hi >> 1;
|
||||
if (res)
|
||||
tweak.c[15] ^= 0xe1;
|
||||
#ifdef BSWAP8
|
||||
hi = BSWAP8(tweak.u[0]);
|
||||
lo = BSWAP8(tweak.u[1]);
|
||||
#else
|
||||
p = tweak.c;
|
||||
|
||||
hi = (u64)GETU32(p) << 32 | GETU32(p + 4);
|
||||
lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
|
||||
#endif
|
||||
tweak.u[0] = lo;
|
||||
tweak.u[1] = hi;
|
||||
} else {
|
||||
u8 carry, res;
|
||||
carry = 0;
|
||||
for (i = 0; i < 16; ++i) {
|
||||
res = (tweak.c[i] << 7) & 0x80;
|
||||
tweak.c[i] = ((tweak.c[i] >> 1) + carry) & 0xff;
|
||||
carry = res;
|
||||
}
|
||||
if (res)
|
||||
tweak.c[0] ^= 0xe1;
|
||||
}
|
||||
}
|
||||
if (enc) {
|
||||
for (i = 0; i < len; ++i) {
|
||||
u8 c = inp[i];
|
||||
out[i] = scratch.c[i];
|
||||
scratch.c[i] = c;
|
||||
}
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
(*ctx->block1) (scratch.c, scratch.c, ctx->key1);
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
memcpy(out - 16, scratch.c, 16);
|
||||
} else {
|
||||
union {
|
||||
u64 u[2];
|
||||
u8 c[16];
|
||||
} tweak1;
|
||||
|
||||
if (IS_LITTLE_ENDIAN) {
|
||||
u8 res;
|
||||
u64 hi, lo;
|
||||
#ifdef BSWAP8
|
||||
hi = BSWAP8(tweak.u[0]);
|
||||
lo = BSWAP8(tweak.u[1]);
|
||||
#else
|
||||
u8 *p = tweak.c;
|
||||
|
||||
hi = (u64)GETU32(p) << 32 | GETU32(p + 4);
|
||||
lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
|
||||
#endif
|
||||
res = (u8)lo & 1;
|
||||
tweak1.u[0] = (lo >> 1) | (hi << 63);
|
||||
tweak1.u[1] = hi >> 1;
|
||||
if (res)
|
||||
tweak1.c[15] ^= 0xe1;
|
||||
#ifdef BSWAP8
|
||||
hi = BSWAP8(tweak1.u[0]);
|
||||
lo = BSWAP8(tweak1.u[1]);
|
||||
#else
|
||||
p = tweak1.c;
|
||||
|
||||
hi = (u64)GETU32(p) << 32 | GETU32(p + 4);
|
||||
lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
|
||||
#endif
|
||||
tweak1.u[0] = lo;
|
||||
tweak1.u[1] = hi;
|
||||
} else {
|
||||
u8 carry, res;
|
||||
carry = 0;
|
||||
for (i = 0; i < 16; ++i) {
|
||||
res = (tweak.c[i] << 7) & 0x80;
|
||||
tweak1.c[i] = ((tweak.c[i] >> 1) + carry) & 0xff;
|
||||
carry = res;
|
||||
}
|
||||
if (res)
|
||||
tweak1.c[0] ^= 0xe1;
|
||||
}
|
||||
#if defined(STRICT_ALIGNMENT)
|
||||
memcpy(scratch.c, inp, 16);
|
||||
scratch.u[0] ^= tweak1.u[0];
|
||||
scratch.u[1] ^= tweak1.u[1];
|
||||
#else
|
||||
scratch.u[0] = ((u64_a1 *)inp)[0] ^ tweak1.u[0];
|
||||
scratch.u[1] = ((u64_a1 *)inp)[1] ^ tweak1.u[1];
|
||||
#endif
|
||||
(*ctx->block1) (scratch.c, scratch.c, ctx->key1);
|
||||
scratch.u[0] ^= tweak1.u[0];
|
||||
scratch.u[1] ^= tweak1.u[1];
|
||||
|
||||
for (i = 0; i < len; ++i) {
|
||||
u8 c = inp[16 + i];
|
||||
out[16 + i] = scratch.c[i];
|
||||
scratch.c[i] = c;
|
||||
}
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
(*ctx->block1) (scratch.c, scratch.c, ctx->key1);
|
||||
#if defined(STRICT_ALIGNMENT)
|
||||
scratch.u[0] ^= tweak.u[0];
|
||||
scratch.u[1] ^= tweak.u[1];
|
||||
memcpy(out, scratch.c, 16);
|
||||
#else
|
||||
((u64_a1 *)out)[0] = scratch.u[0] ^ tweak.u[0];
|
||||
((u64_a1 *)out)[1] = scratch.u[1] ^ tweak.u[1];
|
||||
#endif
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* {- join("\n * ", @autowarntext) -}
|
||||
*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
{-
|
||||
use OpenSSL::paramnames qw(produce_decoder);
|
||||
-}
|
||||
|
||||
#include "internal/e_os.h"
|
||||
#include "internal/param_names.h"
|
||||
#include <string.h>
|
||||
|
||||
/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */
|
||||
{- produce_decoder(); -}
|
||||
/* End of TRIE */
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,81 @@
|
||||
#include "internal/quic_vlint.h"
|
||||
#include "internal/e_os.h"
|
||||
|
||||
#ifndef OPENSSL_NO_QUIC
|
||||
|
||||
void ossl_quic_vlint_encode_n(uint8_t *buf, uint64_t v, int n)
|
||||
{
|
||||
if (n == 1) {
|
||||
buf[0] = (uint8_t)v;
|
||||
} else if (n == 2) {
|
||||
buf[0] = (uint8_t)(0x40 | ((v >> 8) & 0x3F));
|
||||
buf[1] = (uint8_t)v;
|
||||
} else if (n == 4) {
|
||||
buf[0] = (uint8_t)(0x80 | ((v >> 24) & 0x3F));
|
||||
buf[1] = (uint8_t)(v >> 16);
|
||||
buf[2] = (uint8_t)(v >> 8);
|
||||
buf[3] = (uint8_t)v;
|
||||
} else {
|
||||
buf[0] = (uint8_t)(0xC0 | ((v >> 56) & 0x3F));
|
||||
buf[1] = (uint8_t)(v >> 48);
|
||||
buf[2] = (uint8_t)(v >> 40);
|
||||
buf[3] = (uint8_t)(v >> 32);
|
||||
buf[4] = (uint8_t)(v >> 24);
|
||||
buf[5] = (uint8_t)(v >> 16);
|
||||
buf[6] = (uint8_t)(v >> 8);
|
||||
buf[7] = (uint8_t)v;
|
||||
}
|
||||
}
|
||||
|
||||
void ossl_quic_vlint_encode(uint8_t *buf, uint64_t v)
|
||||
{
|
||||
ossl_quic_vlint_encode_n(buf, v, ossl_quic_vlint_encode_len(v));
|
||||
}
|
||||
|
||||
uint64_t ossl_quic_vlint_decode_unchecked(const unsigned char *buf)
|
||||
{
|
||||
uint8_t first_byte = buf[0];
|
||||
size_t sz = ossl_quic_vlint_decode_len(first_byte);
|
||||
|
||||
if (sz == 1)
|
||||
return first_byte & 0x3F;
|
||||
|
||||
if (sz == 2)
|
||||
return ((uint64_t)(first_byte & 0x3F) << 8)
|
||||
| buf[1];
|
||||
|
||||
if (sz == 4)
|
||||
return ((uint64_t)(first_byte & 0x3F) << 24)
|
||||
| ((uint64_t)buf[1] << 16)
|
||||
| ((uint64_t)buf[2] << 8)
|
||||
| buf[3];
|
||||
|
||||
return ((uint64_t)(first_byte & 0x3F) << 56)
|
||||
| ((uint64_t)buf[1] << 48)
|
||||
| ((uint64_t)buf[2] << 40)
|
||||
| ((uint64_t)buf[3] << 32)
|
||||
| ((uint64_t)buf[4] << 24)
|
||||
| ((uint64_t)buf[5] << 16)
|
||||
| ((uint64_t)buf[6] << 8)
|
||||
| buf[7];
|
||||
}
|
||||
|
||||
int ossl_quic_vlint_decode(const unsigned char *buf, size_t buf_len, uint64_t *v)
|
||||
{
|
||||
size_t dec_len;
|
||||
uint64_t x;
|
||||
|
||||
if (buf_len < 1)
|
||||
return 0;
|
||||
|
||||
dec_len = ossl_quic_vlint_decode_len(buf[0]);
|
||||
if (buf_len < dec_len)
|
||||
return 0;
|
||||
|
||||
x = ossl_quic_vlint_decode_unchecked(buf);
|
||||
|
||||
*v = x;
|
||||
return dec_len;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include "crypto/rand.h"
|
||||
#include "internal/common.h"
|
||||
|
||||
/*
|
||||
* Implementation an optimal random integer in a range function.
|
||||
*
|
||||
* Essentially it boils down to incrementally generating a fixed point
|
||||
* number on the interval [0, 1) and multiplying this number by the upper
|
||||
* range limit. Once it is certain what the fractional part contributes to
|
||||
* the integral part of the product, the algorithm has produced a definitive
|
||||
* result.
|
||||
*
|
||||
* Refer: https://github.com/apple/swift/pull/39143 for a fuller description
|
||||
* of the algorithm.
|
||||
*/
|
||||
uint32_t ossl_rand_uniform_uint32(OSSL_LIB_CTX *ctx, uint32_t upper, int *err)
|
||||
{
|
||||
uint32_t i, f; /* integer and fractional parts */
|
||||
uint32_t f2, rand; /* extra fractional part and random material */
|
||||
uint64_t prod; /* temporary holding double width product */
|
||||
const int max_followup_iterations = 10;
|
||||
int j;
|
||||
|
||||
if (!ossl_assert(upper > 0)) {
|
||||
*err = 0;
|
||||
return 0;
|
||||
}
|
||||
if (ossl_unlikely(upper == 1))
|
||||
return 0;
|
||||
|
||||
/* Get 32 bits of entropy */
|
||||
if (RAND_bytes_ex(ctx, (unsigned char *)&rand, sizeof(rand), 0) <= 0) {
|
||||
*err = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* We are generating a fixed point number on the interval [0, 1).
|
||||
* Multiplying this by the range gives us a number on [0, upper).
|
||||
* The high word of the multiplication result represents the integral
|
||||
* part we want. The lower word is the fractional part. We can early exit if
|
||||
* if the fractional part is small enough that no carry from the next lower
|
||||
* word can cause an overflow and carry into the integer part. This
|
||||
* happens when the fractional part is bounded by 2^32 - upper which
|
||||
* can be simplified to just -upper (as an unsigned integer).
|
||||
*/
|
||||
prod = (uint64_t)upper * rand;
|
||||
i = prod >> 32;
|
||||
f = prod & 0xffffffff;
|
||||
if (ossl_likely(f <= 1 + ~upper)) /* 1+~upper == -upper but compilers whine */
|
||||
return i;
|
||||
|
||||
/*
|
||||
* We're in the position where the carry from the next word *might* cause
|
||||
* a carry to the integral part. The process here is to generate the next
|
||||
* word, multiply it by the range and add that to the current word. If
|
||||
* it overflows, the carry propagates to the integer part (return i+1).
|
||||
* If it can no longer overflow regardless of further lower order bits,
|
||||
* we are done (return i). If there is still a chance of overflow, we
|
||||
* repeat the process with the next lower word.
|
||||
*
|
||||
* Each *bit* of randomness has a probability of one half of terminating
|
||||
* this process, so each each word beyond the first has a probability
|
||||
* of 2^-32 of not terminating the process. That is, we're extremely
|
||||
* likely to stop very rapidly.
|
||||
*/
|
||||
for (j = 0; j < max_followup_iterations; j++) {
|
||||
if (RAND_bytes_ex(ctx, (unsigned char *)&rand, sizeof(rand), 0) <= 0) {
|
||||
*err = 1;
|
||||
return 0;
|
||||
}
|
||||
prod = (uint64_t)upper * rand;
|
||||
f2 = prod >> 32;
|
||||
f += f2;
|
||||
/* On overflow, add the carry to our result */
|
||||
if (f < f2)
|
||||
return i + 1;
|
||||
/* For not all 1 bits, there is no carry so return the result */
|
||||
if (ossl_likely(f != 0xffffffff))
|
||||
return i;
|
||||
/* setup for the next word of randomness */
|
||||
f = prod & 0xffffffff;
|
||||
}
|
||||
/*
|
||||
* If we get here, we've consumed 32 * max_followup_iterations + 32 bits
|
||||
* with no firm decision, this gives a bias with probability < 2^-(32*n),
|
||||
* which is likely acceptable.
|
||||
*/
|
||||
return i;
|
||||
}
|
||||
|
||||
uint32_t ossl_rand_range_uint32(OSSL_LIB_CTX *ctx, uint32_t lower, uint32_t upper,
|
||||
int *err)
|
||||
{
|
||||
if (!ossl_assert(lower < upper)) {
|
||||
*err = 1;
|
||||
return 0;
|
||||
}
|
||||
return lower + ossl_rand_uniform_uint32(ctx, upper - lower, err);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#ifndef OPENSSL_RCU_INTERNAL_H
|
||||
# define OPENSSL_RCU_INTERNAL_H
|
||||
# pragma once
|
||||
|
||||
struct rcu_qp;
|
||||
|
||||
struct rcu_cb_item {
|
||||
rcu_cb_fn fn;
|
||||
void *data;
|
||||
struct rcu_cb_item *next;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,105 @@
|
||||
#! /usr/bin/env perl
|
||||
# Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
{
|
||||
my ($in_a,$in_b,$len,$x,$temp1,$temp2) = ('a0','a1','a2','t0','t1','t2');
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl CRYPTO_memcmp
|
||||
.type CRYPTO_memcmp,\@function
|
||||
CRYPTO_memcmp:
|
||||
li $x,0
|
||||
beqz $len,2f # len == 0
|
||||
1:
|
||||
lbu $temp1,0($in_a)
|
||||
lbu $temp2,0($in_b)
|
||||
addi $in_a,$in_a,1
|
||||
addi $in_b,$in_b,1
|
||||
addi $len,$len,-1
|
||||
xor $temp1,$temp1,$temp2
|
||||
or $x,$x,$temp1
|
||||
bgtz $len,1b
|
||||
2:
|
||||
mv a0,$x
|
||||
ret
|
||||
___
|
||||
}
|
||||
{
|
||||
my ($ptr,$len,$temp1,$temp2) = ('a0','a1','t0','t1');
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# void OPENSSL_cleanse(void *ptr, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl OPENSSL_cleanse
|
||||
.type OPENSSL_cleanse,\@function
|
||||
OPENSSL_cleanse:
|
||||
beqz $len,2f # len == 0, return
|
||||
srli $temp1,$len,4
|
||||
bnez $temp1,3f # len > 15
|
||||
|
||||
1: # Store <= 15 individual bytes
|
||||
sb x0,0($ptr)
|
||||
addi $ptr,$ptr,1
|
||||
addi $len,$len,-1
|
||||
bnez $len,1b
|
||||
2:
|
||||
ret
|
||||
|
||||
3: # Store individual bytes until we are aligned
|
||||
andi $temp1,$ptr,0x3
|
||||
beqz $temp1,4f
|
||||
sb x0,0($ptr)
|
||||
addi $ptr,$ptr,1
|
||||
addi $len,$len,-1
|
||||
j 3b
|
||||
|
||||
4: # Store aligned words
|
||||
li $temp2,4
|
||||
4:
|
||||
sw x0,0($ptr)
|
||||
addi $ptr,$ptr,4
|
||||
addi $len,$len,-4
|
||||
bge $len,$temp2,4b # if len>=4 loop
|
||||
bnez $len,1b # if len<4 and len != 0, store remaining bytes
|
||||
ret
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($ret) = ('a0');
|
||||
$code .= <<___;
|
||||
################################################################################
|
||||
# size_t riscv_vlen_asm(void)
|
||||
# Return VLEN (i.e. the length of a vector register in bits).
|
||||
.p2align 3
|
||||
.globl riscv_vlen_asm
|
||||
.type riscv_vlen_asm,\@function
|
||||
riscv_vlen_asm:
|
||||
csrr $ret, vlenb
|
||||
slli $ret, $ret, 3
|
||||
ret
|
||||
.size riscv_vlen_asm,.-riscv_vlen_asm
|
||||
___
|
||||
}
|
||||
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,105 @@
|
||||
#! /usr/bin/env perl
|
||||
# Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
# this file except in compliance with the License. You can obtain a copy
|
||||
# in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
{
|
||||
my ($in_a,$in_b,$len,$x,$temp1,$temp2) = ('a0','a1','a2','t0','t1','t2');
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl CRYPTO_memcmp
|
||||
.type CRYPTO_memcmp,\@function
|
||||
CRYPTO_memcmp:
|
||||
li $x,0
|
||||
beqz $len,2f # len == 0
|
||||
1:
|
||||
lbu $temp1,0($in_a)
|
||||
lbu $temp2,0($in_b)
|
||||
addi $in_a,$in_a,1
|
||||
addi $in_b,$in_b,1
|
||||
addi $len,$len,-1
|
||||
xor $temp1,$temp1,$temp2
|
||||
or $x,$x,$temp1
|
||||
bgtz $len,1b
|
||||
2:
|
||||
mv a0,$x
|
||||
ret
|
||||
___
|
||||
}
|
||||
{
|
||||
my ($ptr,$len,$temp1,$temp2) = ('a0','a1','t0','t1');
|
||||
$code.=<<___;
|
||||
################################################################################
|
||||
# void OPENSSL_cleanse(void *ptr, size_t len)
|
||||
################################################################################
|
||||
.text
|
||||
.balign 16
|
||||
.globl OPENSSL_cleanse
|
||||
.type OPENSSL_cleanse,\@function
|
||||
OPENSSL_cleanse:
|
||||
beqz $len,2f # len == 0, return
|
||||
srli $temp1,$len,4
|
||||
bnez $temp1,3f # len > 15
|
||||
|
||||
1: # Store <= 15 individual bytes
|
||||
sb x0,0($ptr)
|
||||
addi $ptr,$ptr,1
|
||||
addi $len,$len,-1
|
||||
bnez $len,1b
|
||||
2:
|
||||
ret
|
||||
|
||||
3: # Store individual bytes until we are aligned
|
||||
andi $temp1,$ptr,0x7
|
||||
beqz $temp1,4f
|
||||
sb x0,0($ptr)
|
||||
addi $ptr,$ptr,1
|
||||
addi $len,$len,-1
|
||||
j 3b
|
||||
|
||||
4: # Store aligned dwords
|
||||
li $temp2,8
|
||||
4:
|
||||
sd x0,0($ptr)
|
||||
addi $ptr,$ptr,8
|
||||
addi $len,$len,-8
|
||||
bge $len,$temp2,4b # if len>=8 loop
|
||||
bnez $len,1b # if len<8 and len != 0, store remaining bytes
|
||||
ret
|
||||
___
|
||||
}
|
||||
|
||||
{
|
||||
my ($ret) = ('a0');
|
||||
$code .= <<___;
|
||||
################################################################################
|
||||
# size_t riscv_vlen_asm(void)
|
||||
# Return VLEN (i.e. the length of a vector register in bits).
|
||||
.p2align 3
|
||||
.globl riscv_vlen_asm
|
||||
.type riscv_vlen_asm,\@function
|
||||
riscv_vlen_asm:
|
||||
csrr $ret, vlenb
|
||||
slli $ret, $ret, 3
|
||||
ret
|
||||
.size riscv_vlen_asm,.-riscv_vlen_asm
|
||||
___
|
||||
}
|
||||
|
||||
print $code;
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
#include <stdint.h>
|
||||
#include <openssl/crypto.h>
|
||||
#include "internal/cryptlib.h"
|
||||
|
||||
#define OPENSSL_RISCVCAP_IMPL
|
||||
#include "crypto/riscv_arch.h"
|
||||
|
||||
#ifdef OSSL_RISCV_HWPROBE
|
||||
# include <unistd.h>
|
||||
# include <sys/syscall.h>
|
||||
# include <asm/hwprobe.h>
|
||||
# include <sys/auxv.h>
|
||||
#endif
|
||||
|
||||
extern size_t riscv_vlen_asm(void);
|
||||
|
||||
static void parse_env(const char *envstr);
|
||||
static void strtoupper(char *str);
|
||||
|
||||
static size_t vlen = 0;
|
||||
|
||||
#ifdef OSSL_RISCV_HWPROBE
|
||||
unsigned int OPENSSL_riscv_hwcap_P = 0;
|
||||
#endif
|
||||
|
||||
uint32_t OPENSSL_rdtsc(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void strtoupper(char *str)
|
||||
{
|
||||
for (char *x = str; *x; ++x)
|
||||
*x = toupper((unsigned char)*x);
|
||||
}
|
||||
|
||||
/* parse_env() parses a RISC-V architecture string. An example of such a string
|
||||
* is "rv64gc_zba_zbb_zbc_zbs". Currently, the rv64gc part is ignored
|
||||
* and we simply search for "_[extension]" in the arch string to see if we
|
||||
* should enable a given extension.
|
||||
*/
|
||||
#define BUFLEN 256
|
||||
static void parse_env(const char *envstr)
|
||||
{
|
||||
char envstrupper[BUFLEN];
|
||||
char buf[BUFLEN];
|
||||
|
||||
/* Convert env str to all uppercase */
|
||||
OPENSSL_strlcpy(envstrupper, envstr, sizeof(envstrupper));
|
||||
strtoupper(envstrupper);
|
||||
|
||||
for (size_t i = 0; i < kRISCVNumCaps; ++i) {
|
||||
/* Prefix capability with underscore in preparation for search */
|
||||
BIO_snprintf(buf, BUFLEN, "_%s", RISCV_capabilities[i].name);
|
||||
if (strstr(envstrupper, buf) != NULL) {
|
||||
/* Match, set relevant bit in OPENSSL_riscvcap_P[] */
|
||||
OPENSSL_riscvcap_P[RISCV_capabilities[i].index] |=
|
||||
(1 << RISCV_capabilities[i].bit_offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef OSSL_RISCV_HWPROBE
|
||||
static long riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
|
||||
size_t cpu_count, unsigned long *cpus,
|
||||
unsigned int flags)
|
||||
{
|
||||
return syscall(__NR_riscv_hwprobe, pairs, pair_count, cpu_count, cpus, flags);
|
||||
}
|
||||
|
||||
static void hwprobe_to_cap(void)
|
||||
{
|
||||
long ret;
|
||||
struct riscv_hwprobe pairs[OSSL_RISCV_HWPROBE_PAIR_COUNT] = {
|
||||
OSSL_RISCV_HWPROBE_PAIR_CONTENT
|
||||
};
|
||||
|
||||
ret = riscv_hwprobe(pairs, OSSL_RISCV_HWPROBE_PAIR_COUNT, 0, NULL, 0);
|
||||
/* if hwprobe syscall does not exist, ret would be -ENOSYS */
|
||||
if (ret == 0) {
|
||||
for (size_t i = 0; i < kRISCVNumCaps; ++i) {
|
||||
for (size_t j = 0; j != OSSL_RISCV_HWPROBE_PAIR_COUNT; ++j) {
|
||||
if (pairs[j].key == RISCV_capabilities[i].hwprobe_key
|
||||
&& (pairs[j].value & RISCV_capabilities[i].hwprobe_value)
|
||||
!= 0)
|
||||
if (!IS_IN_DEPEND_VECTOR(RISCV_capabilities[i].bit_offset) || VECTOR_CAPABLE)
|
||||
/* Match, set relevant bit in OPENSSL_riscvcap_P[] */
|
||||
OPENSSL_riscvcap_P[RISCV_capabilities[i].index] |=
|
||||
(1 << RISCV_capabilities[i].bit_offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif /* OSSL_RISCV_HWPROBE */
|
||||
|
||||
size_t riscv_vlen(void)
|
||||
{
|
||||
return vlen;
|
||||
}
|
||||
|
||||
# if defined(__GNUC__) && __GNUC__>=2
|
||||
__attribute__ ((constructor))
|
||||
# endif
|
||||
void OPENSSL_cpuid_setup(void)
|
||||
{
|
||||
char *e;
|
||||
static int trigger = 0;
|
||||
|
||||
if (trigger != 0)
|
||||
return;
|
||||
trigger = 1;
|
||||
|
||||
if ((e = getenv("OPENSSL_riscvcap"))) {
|
||||
parse_env(e);
|
||||
}
|
||||
#ifdef OSSL_RISCV_HWPROBE
|
||||
else {
|
||||
OPENSSL_riscv_hwcap_P = getauxval(AT_HWCAP);
|
||||
hwprobe_to_cap();
|
||||
}
|
||||
#endif
|
||||
|
||||
if (RISCV_HAS_V()) {
|
||||
vlen = riscv_vlen_asm();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Christoph Müllner <[email protected]>
|
||||
# Copyright (c) 2023, Phoebe Chen <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# The generated code of this file depends on the following RISC-V extensions:
|
||||
# - RV64I
|
||||
# - RISC-V Vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
# - RISC-V Vector SHA-2 Secure Hash extension ('Zvknha' or 'Zvknhb')
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
|
||||
$V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
|
||||
$V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
|
||||
$V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map("v$_",(0..31));
|
||||
|
||||
my $K256 = "K256";
|
||||
|
||||
# Function arguments
|
||||
my ($H, $INP, $LEN, $KT, $H2, $INDEX_PATTERN) = ("a0", "a1", "a2", "a3", "t3", "t4");
|
||||
|
||||
sub sha_256_load_constant {
|
||||
my $code=<<___;
|
||||
la $KT, $K256 # Load round constants K256
|
||||
@{[vle32_v $V10, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V11, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V12, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V13, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V14, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V15, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V16, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V17, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V18, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V19, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V20, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V21, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V22, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V23, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V24, $KT]}
|
||||
addi $KT, $KT, 16
|
||||
@{[vle32_v $V25, $KT]}
|
||||
___
|
||||
|
||||
return $code;
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# void sha256_block_data_order_zvkb_zvknha_or_zvknhb(void *c, const void *p, size_t len)
|
||||
$code .= <<___;
|
||||
.p2align 2
|
||||
.globl sha256_block_data_order_zvkb_zvknha_or_zvknhb
|
||||
.type sha256_block_data_order_zvkb_zvknha_or_zvknhb,\@function
|
||||
sha256_block_data_order_zvkb_zvknha_or_zvknhb:
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
|
||||
@{[sha_256_load_constant]}
|
||||
|
||||
# H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c}
|
||||
# The dst vtype is e32m1 and the index vtype is e8mf4.
|
||||
# We use index-load with the following index pattern at v26.
|
||||
# i8 index:
|
||||
# 20, 16, 4, 0
|
||||
# Instead of setting the i8 index, we could use a single 32bit
|
||||
# little-endian value to cover the 4xi8 index.
|
||||
# i32 value:
|
||||
# 0x 00 04 10 14
|
||||
li $INDEX_PATTERN, 0x00041014
|
||||
@{[vsetivli "zero", 1, "e32", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_x $V26, $INDEX_PATTERN]}
|
||||
|
||||
addi $H2, $H, 8
|
||||
|
||||
# Use index-load to get {f,e,b,a},{h,g,d,c}
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
@{[vluxei8_v $V6, $H, $V26]}
|
||||
@{[vluxei8_v $V7, $H2, $V26]}
|
||||
|
||||
# Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling.
|
||||
# The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking.
|
||||
@{[vsetivli "zero", 1, "e8", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_i $V0, 0x01]}
|
||||
|
||||
@{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
|
||||
|
||||
L_round_loop:
|
||||
# Decrement length by 1
|
||||
add $LEN, $LEN, -1
|
||||
|
||||
# Keep the current state as we need it later: H' = H+{a',b',c',...,h'}.
|
||||
@{[vmv_v_v $V30, $V6]}
|
||||
@{[vmv_v_v $V31, $V7]}
|
||||
|
||||
# Load the 512-bits of the message block in v1-v4 and perform
|
||||
# an endian swap on each 4 bytes element.
|
||||
@{[vle32_v $V1, $INP]}
|
||||
@{[vrev8_v $V1, $V1]}
|
||||
add $INP, $INP, 16
|
||||
@{[vle32_v $V2, $INP]}
|
||||
@{[vrev8_v $V2, $V2]}
|
||||
add $INP, $INP, 16
|
||||
@{[vle32_v $V3, $INP]}
|
||||
@{[vrev8_v $V3, $V3]}
|
||||
add $INP, $INP, 16
|
||||
@{[vle32_v $V4, $INP]}
|
||||
@{[vrev8_v $V4, $V4]}
|
||||
add $INP, $INP, 16
|
||||
|
||||
# Quad-round 0 (+0, Wt from oldest to newest in v1->v2->v3->v4)
|
||||
@{[vadd_vv $V5, $V10, $V1]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V3, $V2, $V0]}
|
||||
@{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[19:16]
|
||||
|
||||
# Quad-round 1 (+1, v2->v3->v4->v1)
|
||||
@{[vadd_vv $V5, $V11, $V2]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V4, $V3, $V0]}
|
||||
@{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[23:20]
|
||||
|
||||
# Quad-round 2 (+2, v3->v4->v1->v2)
|
||||
@{[vadd_vv $V5, $V12, $V3]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V1, $V4, $V0]}
|
||||
@{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[27:24]
|
||||
|
||||
# Quad-round 3 (+3, v4->v1->v2->v3)
|
||||
@{[vadd_vv $V5, $V13, $V4]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V2, $V1, $V0]}
|
||||
@{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[31:28]
|
||||
|
||||
# Quad-round 4 (+0, v1->v2->v3->v4)
|
||||
@{[vadd_vv $V5, $V14, $V1]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V3, $V2, $V0]}
|
||||
@{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[35:32]
|
||||
|
||||
# Quad-round 5 (+1, v2->v3->v4->v1)
|
||||
@{[vadd_vv $V5, $V15, $V2]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V4, $V3, $V0]}
|
||||
@{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[39:36]
|
||||
|
||||
# Quad-round 6 (+2, v3->v4->v1->v2)
|
||||
@{[vadd_vv $V5, $V16, $V3]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V1, $V4, $V0]}
|
||||
@{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[43:40]
|
||||
|
||||
# Quad-round 7 (+3, v4->v1->v2->v3)
|
||||
@{[vadd_vv $V5, $V17, $V4]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V2, $V1, $V0]}
|
||||
@{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[47:44]
|
||||
|
||||
# Quad-round 8 (+0, v1->v2->v3->v4)
|
||||
@{[vadd_vv $V5, $V18, $V1]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V3, $V2, $V0]}
|
||||
@{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[51:48]
|
||||
|
||||
# Quad-round 9 (+1, v2->v3->v4->v1)
|
||||
@{[vadd_vv $V5, $V19, $V2]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V4, $V3, $V0]}
|
||||
@{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[55:52]
|
||||
|
||||
# Quad-round 10 (+2, v3->v4->v1->v2)
|
||||
@{[vadd_vv $V5, $V20, $V3]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V1, $V4, $V0]}
|
||||
@{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[59:56]
|
||||
|
||||
# Quad-round 11 (+3, v4->v1->v2->v3)
|
||||
@{[vadd_vv $V5, $V21, $V4]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
@{[vmerge_vvm $V5, $V2, $V1, $V0]}
|
||||
@{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[63:60]
|
||||
|
||||
# Quad-round 12 (+0, v1->v2->v3->v4)
|
||||
# Note that we stop generating new message schedule words (Wt, v1-13)
|
||||
# as we already generated all the words we end up consuming (i.e., W[63:60]).
|
||||
@{[vadd_vv $V5, $V22, $V1]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
|
||||
# Quad-round 13 (+1, v2->v3->v4->v1)
|
||||
@{[vadd_vv $V5, $V23, $V2]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
|
||||
# Quad-round 14 (+2, v3->v4->v1->v2)
|
||||
@{[vadd_vv $V5, $V24, $V3]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
|
||||
# Quad-round 15 (+3, v4->v1->v2->v3)
|
||||
@{[vadd_vv $V5, $V25, $V4]}
|
||||
@{[vsha2cl_vv $V7, $V6, $V5]}
|
||||
@{[vsha2ch_vv $V6, $V7, $V5]}
|
||||
|
||||
# H' = H+{a',b',c',...,h'}
|
||||
@{[vadd_vv $V6, $V30, $V6]}
|
||||
@{[vadd_vv $V7, $V31, $V7]}
|
||||
bnez $LEN, L_round_loop
|
||||
|
||||
# Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}.
|
||||
@{[vsuxei8_v $V6, $H, $V26]}
|
||||
@{[vsuxei8_v $V7, $H2, $V26]}
|
||||
|
||||
ret
|
||||
.size sha256_block_data_order_zvkb_zvknha_or_zvknhb,.-sha256_block_data_order_zvkb_zvknha_or_zvknhb
|
||||
|
||||
.p2align 2
|
||||
.type $K256,\@object
|
||||
$K256:
|
||||
.word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
|
||||
.word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
|
||||
.word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
|
||||
.word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
|
||||
.word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
|
||||
.word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
|
||||
.word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
|
||||
.word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
|
||||
.word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
|
||||
.word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
|
||||
.word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
|
||||
.word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
|
||||
.word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
|
||||
.word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
|
||||
.word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
|
||||
.word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
|
||||
.size $K256,.-$K256
|
||||
___
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,264 @@
|
||||
#! /usr/bin/env perl
|
||||
# This file is dual-licensed, meaning that you can use it under your
|
||||
# choice of either of the following two licenses:
|
||||
#
|
||||
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License 2.0 (the "License"). You can obtain
|
||||
# a copy in the file LICENSE in the source distribution or at
|
||||
# https://www.openssl.org/source/license.html
|
||||
#
|
||||
# or
|
||||
#
|
||||
# Copyright (c) 2023, Christoph Müllner <[email protected]>
|
||||
# Copyright (c) 2023, Phoebe Chen <[email protected]>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# The generated code of this file depends on the following RISC-V extensions:
|
||||
# - RV64I
|
||||
# - RISC-V vector ('V') with VLEN >= 128
|
||||
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
|
||||
# - RISC-V Vector SHA-2 Secure Hash extension ('Zvknhb')
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin";
|
||||
use lib "$Bin/../../perlasm";
|
||||
use riscv;
|
||||
|
||||
# $output is the last argument if it looks like a file (it has an extension)
|
||||
# $flavour is the first argument if it doesn't look like a file
|
||||
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
||||
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
||||
|
||||
$output and open STDOUT,">$output";
|
||||
|
||||
my $code=<<___;
|
||||
.text
|
||||
___
|
||||
|
||||
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
|
||||
$V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
|
||||
$V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
|
||||
$V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
|
||||
) = map("v$_",(0..31));
|
||||
|
||||
my $K512 = "K512";
|
||||
|
||||
# Function arguments
|
||||
my ($H, $INP, $LEN, $KT, $H2, $INDEX_PATTERN) = ("a0", "a1", "a2", "a3", "t3", "t4");
|
||||
|
||||
################################################################################
|
||||
# void sha512_block_data_order_zvkb_zvknhb(void *c, const void *p, size_t len)
|
||||
$code .= <<___;
|
||||
.p2align 2
|
||||
.globl sha512_block_data_order_zvkb_zvknhb
|
||||
.type sha512_block_data_order_zvkb_zvknhb,\@function
|
||||
sha512_block_data_order_zvkb_zvknhb:
|
||||
@{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]}
|
||||
|
||||
# H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c}
|
||||
# The dst vtype is e64m2 and the index vtype is e8mf4.
|
||||
# We use index-load with the following index pattern at v1.
|
||||
# i8 index:
|
||||
# 40, 32, 8, 0
|
||||
# Instead of setting the i8 index, we could use a single 32bit
|
||||
# little-endian value to cover the 4xi8 index.
|
||||
# i32 value:
|
||||
# 0x 00 08 20 28
|
||||
li $INDEX_PATTERN, 0x00082028
|
||||
@{[vsetivli "zero", 1, "e32", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_x $V1, $INDEX_PATTERN]}
|
||||
|
||||
addi $H2, $H, 16
|
||||
|
||||
# Use index-load to get {f,e,b,a},{h,g,d,c}
|
||||
@{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]}
|
||||
@{[vluxei8_v $V22, $H, $V1]}
|
||||
@{[vluxei8_v $V24, $H2, $V1]}
|
||||
|
||||
# Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling.
|
||||
# The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking.
|
||||
@{[vsetivli "zero", 1, "e8", "m1", "ta", "ma"]}
|
||||
@{[vmv_v_i $V0, 0x01]}
|
||||
|
||||
@{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]}
|
||||
|
||||
L_round_loop:
|
||||
# Load round constants K512
|
||||
la $KT, $K512
|
||||
|
||||
# Decrement length by 1
|
||||
addi $LEN, $LEN, -1
|
||||
|
||||
# Keep the current state as we need it later: H' = H+{a',b',c',...,h'}.
|
||||
@{[vmv_v_v $V26, $V22]}
|
||||
@{[vmv_v_v $V28, $V24]}
|
||||
|
||||
# Load the 1024-bits of the message block in v10-v16 and perform the endian
|
||||
# swap.
|
||||
@{[vle64_v $V10, $INP]}
|
||||
@{[vrev8_v $V10, $V10]}
|
||||
addi $INP, $INP, 32
|
||||
@{[vle64_v $V12, $INP]}
|
||||
@{[vrev8_v $V12, $V12]}
|
||||
addi $INP, $INP, 32
|
||||
@{[vle64_v $V14, $INP]}
|
||||
@{[vrev8_v $V14, $V14]}
|
||||
addi $INP, $INP, 32
|
||||
@{[vle64_v $V16, $INP]}
|
||||
@{[vrev8_v $V16, $V16]}
|
||||
addi $INP, $INP, 32
|
||||
|
||||
.rept 4
|
||||
# Quad-round 0 (+0, v10->v12->v14->v16)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V10]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
@{[vmerge_vvm $V18, $V14, $V12, $V0]}
|
||||
@{[vsha2ms_vv $V10, $V18, $V16]}
|
||||
|
||||
# Quad-round 1 (+1, v12->v14->v16->v10)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V12]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
@{[vmerge_vvm $V18, $V16, $V14, $V0]}
|
||||
@{[vsha2ms_vv $V12, $V18, $V10]}
|
||||
|
||||
# Quad-round 2 (+2, v14->v16->v10->v12)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V14]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
@{[vmerge_vvm $V18, $V10, $V16, $V0]}
|
||||
@{[vsha2ms_vv $V14, $V18, $V12]}
|
||||
|
||||
# Quad-round 3 (+3, v16->v10->v12->v14)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V16]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
@{[vmerge_vvm $V18, $V12, $V10, $V0]}
|
||||
@{[vsha2ms_vv $V16, $V18, $V14]}
|
||||
.endr
|
||||
|
||||
# Quad-round 16 (+0, v10->v12->v14->v16)
|
||||
# Note that we stop generating new message schedule words (Wt, v10-16)
|
||||
# as we already generated all the words we end up consuming (i.e., W[79:76]).
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V10]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
|
||||
# Quad-round 17 (+1, v12->v14->v16->v10)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V12]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
|
||||
# Quad-round 18 (+2, v14->v16->v10->v12)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
addi $KT, $KT, 32
|
||||
@{[vadd_vv $V18, $V20, $V14]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
|
||||
# Quad-round 19 (+3, v16->v10->v12->v14)
|
||||
@{[vle64_v $V20, ($KT)]}
|
||||
# No t1 increment needed.
|
||||
@{[vadd_vv $V18, $V20, $V16]}
|
||||
@{[vsha2cl_vv $V24, $V22, $V18]}
|
||||
@{[vsha2ch_vv $V22, $V24, $V18]}
|
||||
|
||||
# H' = H+{a',b',c',...,h'}
|
||||
@{[vadd_vv $V22, $V26, $V22]}
|
||||
@{[vadd_vv $V24, $V28, $V24]}
|
||||
bnez $LEN, L_round_loop
|
||||
|
||||
# Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}.
|
||||
@{[vsuxei8_v $V22, ($H), $V1]}
|
||||
@{[vsuxei8_v $V24, ($H2), $V1]}
|
||||
|
||||
ret
|
||||
.size sha512_block_data_order_zvkb_zvknhb,.-sha512_block_data_order_zvkb_zvknhb
|
||||
|
||||
.p2align 3
|
||||
.type $K512,\@object
|
||||
$K512:
|
||||
.dword 0x428a2f98d728ae22, 0x7137449123ef65cd
|
||||
.dword 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc
|
||||
.dword 0x3956c25bf348b538, 0x59f111f1b605d019
|
||||
.dword 0x923f82a4af194f9b, 0xab1c5ed5da6d8118
|
||||
.dword 0xd807aa98a3030242, 0x12835b0145706fbe
|
||||
.dword 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2
|
||||
.dword 0x72be5d74f27b896f, 0x80deb1fe3b1696b1
|
||||
.dword 0x9bdc06a725c71235, 0xc19bf174cf692694
|
||||
.dword 0xe49b69c19ef14ad2, 0xefbe4786384f25e3
|
||||
.dword 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65
|
||||
.dword 0x2de92c6f592b0275, 0x4a7484aa6ea6e483
|
||||
.dword 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5
|
||||
.dword 0x983e5152ee66dfab, 0xa831c66d2db43210
|
||||
.dword 0xb00327c898fb213f, 0xbf597fc7beef0ee4
|
||||
.dword 0xc6e00bf33da88fc2, 0xd5a79147930aa725
|
||||
.dword 0x06ca6351e003826f, 0x142929670a0e6e70
|
||||
.dword 0x27b70a8546d22ffc, 0x2e1b21385c26c926
|
||||
.dword 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df
|
||||
.dword 0x650a73548baf63de, 0x766a0abb3c77b2a8
|
||||
.dword 0x81c2c92e47edaee6, 0x92722c851482353b
|
||||
.dword 0xa2bfe8a14cf10364, 0xa81a664bbc423001
|
||||
.dword 0xc24b8b70d0f89791, 0xc76c51a30654be30
|
||||
.dword 0xd192e819d6ef5218, 0xd69906245565a910
|
||||
.dword 0xf40e35855771202a, 0x106aa07032bbd1b8
|
||||
.dword 0x19a4c116b8d2d0c8, 0x1e376c085141ab53
|
||||
.dword 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8
|
||||
.dword 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb
|
||||
.dword 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3
|
||||
.dword 0x748f82ee5defb2fc, 0x78a5636f43172f60
|
||||
.dword 0x84c87814a1f0ab72, 0x8cc702081a6439ec
|
||||
.dword 0x90befffa23631e28, 0xa4506cebde82bde9
|
||||
.dword 0xbef9a3f7b2c67915, 0xc67178f2e372532b
|
||||
.dword 0xca273eceea26619c, 0xd186b8c721c0c207
|
||||
.dword 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178
|
||||
.dword 0x06f067aa72176fba, 0x0a637dc5a2c898a6
|
||||
.dword 0x113f9804bef90dae, 0x1b710b35131c471b
|
||||
.dword 0x28db77f523047d84, 0x32caab7b40c72493
|
||||
.dword 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c
|
||||
.dword 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a
|
||||
.dword 0x5fcb6fab3ad6faec, 0x6c44198c4a475817
|
||||
.size $K512,.-$K512
|
||||
___
|
||||
|
||||
print $code;
|
||||
|
||||
close STDOUT or die "error closing STDOUT: $!";
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <openssl/opensslconf.h>
|
||||
#include <openssl/sha.h>
|
||||
#include "crypto/riscv_arch.h"
|
||||
|
||||
void sha256_block_data_order_zvkb_zvknha_or_zvknhb(void *ctx, const void *in,
|
||||
size_t num);
|
||||
void sha256_block_data_order_c(void *ctx, const void *in, size_t num);
|
||||
void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num);
|
||||
|
||||
void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num)
|
||||
{
|
||||
if (RISCV_HAS_ZVKB() && (RISCV_HAS_ZVKNHA() || RISCV_HAS_ZVKNHB()) &&
|
||||
riscv_vlen() >= 128) {
|
||||
sha256_block_data_order_zvkb_zvknha_or_zvknhb(ctx, in, num);
|
||||
} else {
|
||||
sha256_block_data_order_c(ctx, in, num);
|
||||
}
|
||||
}
|
||||
|
||||
void sha512_block_data_order_zvkb_zvknhb(void *ctx, const void *in, size_t num);
|
||||
void sha512_block_data_order_c(void *ctx, const void *in, size_t num);
|
||||
void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num);
|
||||
|
||||
void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num)
|
||||
{
|
||||
if (RISCV_HAS_ZVKB_AND_ZVKNHB() && riscv_vlen() >= 128) {
|
||||
sha512_block_data_order_zvkb_zvknhb(ctx, in, num);
|
||||
} else {
|
||||
sha512_block_data_order_c(ctx, in, num);
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
|
||||
#include <openssl/crypto.h>
|
||||
#include "internal/e_os.h"
|
||||
|
||||
/* system-specific variants defining OSSL_sleep() */
|
||||
#if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__)
|
||||
|
||||
# if defined(OPENSSL_USE_USLEEP) \
|
||||
|| defined(__DJGPP__) \
|
||||
|| (defined(__TANDEM) && defined(_REENTRANT))
|
||||
|
||||
/*
|
||||
* usleep() was made obsolete by POSIX.1-2008, and nanosleep()
|
||||
* should be used instead. However, nanosleep() isn't implemented
|
||||
* on the platforms given above, so we still use it for those.
|
||||
* Also, OPENSSL_USE_USLEEP can be defined to enable the use of
|
||||
* usleep, if it turns out that nanosleep() is unavailable.
|
||||
*/
|
||||
|
||||
# include <unistd.h>
|
||||
void OSSL_sleep(uint64_t millis)
|
||||
{
|
||||
unsigned int s = (unsigned int)(millis / 1000);
|
||||
unsigned int us = (unsigned int)((millis % 1000) * 1000);
|
||||
|
||||
if (s > 0)
|
||||
sleep(s);
|
||||
/*
|
||||
* On NonStop with the PUT thread model, thread context switch is
|
||||
* cooperative, with usleep() being a "natural" context switch point.
|
||||
* We avoid checking us > 0 here, to allow that context switch to
|
||||
* happen.
|
||||
*/
|
||||
usleep(us);
|
||||
}
|
||||
|
||||
# elif defined(__TANDEM) && !defined(_REENTRANT)
|
||||
|
||||
# include <cextdecs.h(PROCESS_DELAY_)>
|
||||
void OSSL_sleep(uint64_t millis)
|
||||
{
|
||||
/* HPNS does not support usleep for non threaded apps */
|
||||
PROCESS_DELAY_(millis * 1000);
|
||||
}
|
||||
|
||||
# else
|
||||
|
||||
/* nanosleep is defined by POSIX.1-2001 */
|
||||
# include <time.h>
|
||||
void OSSL_sleep(uint64_t millis)
|
||||
{
|
||||
struct timespec ts;
|
||||
|
||||
ts.tv_sec = (long int) (millis / 1000);
|
||||
ts.tv_nsec = (long int) (millis % 1000) * 1000000ul;
|
||||
nanosleep(&ts, NULL);
|
||||
}
|
||||
|
||||
# endif
|
||||
#elif defined(_WIN32) && !defined(OPENSSL_SYS_UEFI)
|
||||
# include <windows.h>
|
||||
|
||||
void OSSL_sleep(uint64_t millis)
|
||||
{
|
||||
/*
|
||||
* Windows' Sleep() takes a DWORD argument, which is smaller than
|
||||
* a uint64_t, so we need to limit it to 49 days, which should be enough.
|
||||
*/
|
||||
DWORD limited_millis = (DWORD)-1;
|
||||
|
||||
if (millis < limited_millis)
|
||||
limited_millis = (DWORD)millis;
|
||||
Sleep(limited_millis);
|
||||
}
|
||||
|
||||
#else
|
||||
/* Fallback to a busy wait */
|
||||
# include "internal/time.h"
|
||||
|
||||
static void ossl_sleep_secs(uint64_t secs)
|
||||
{
|
||||
/*
|
||||
* sleep() takes an unsigned int argument, which is smaller than
|
||||
* a uint64_t, so it needs to be limited to 136 years which
|
||||
* should be enough even for Sleeping Beauty.
|
||||
*/
|
||||
unsigned int limited_secs = UINT_MAX;
|
||||
|
||||
if (secs < limited_secs)
|
||||
limited_secs = (unsigned int)secs;
|
||||
sleep(limited_secs);
|
||||
}
|
||||
|
||||
static void ossl_sleep_millis(uint64_t millis)
|
||||
{
|
||||
const OSSL_TIME finish
|
||||
= ossl_time_add(ossl_time_now(), ossl_ms2time(millis));
|
||||
|
||||
while (ossl_time_compare(ossl_time_now(), finish) < 0)
|
||||
/* busy wait */ ;
|
||||
}
|
||||
|
||||
void OSSL_sleep(uint64_t millis)
|
||||
{
|
||||
ossl_sleep_secs(millis / 1000);
|
||||
ossl_sleep_millis(millis % 1000);
|
||||
}
|
||||
#endif /* defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) */
|
||||
@@ -0,0 +1,9 @@
|
||||
LIBS=../../libcrypto
|
||||
|
||||
$COMMON=slh_adrs.c slh_dsa.c slh_dsa_hash_ctx.c slh_dsa_key.c slh_fors.c slh_hash.c \
|
||||
slh_hypertree.c slh_params.c slh_wots.c slh_xmss.c
|
||||
|
||||
IF[{- !$disabled{'slh-dsa'} -}]
|
||||
SOURCE[../../libcrypto]=$COMMON
|
||||
SOURCE[../../providers/libfips.a]=$COMMON
|
||||
ENDIF
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License 2.0 (the "License"). You may not use
|
||||
* this file except in compliance with the License. You can obtain a copy
|
||||
* in the file LICENSE in the source distribution or at
|
||||
* https://www.openssl.org/source/license.html
|
||||
*/
|
||||
#include <string.h>
|
||||
#include <openssl/byteorder.h>
|
||||
#include "slh_adrs.h"
|
||||
|
||||
/* See FIPS 205 - Section 4.3 Table 1 Uncompressed Addresses */
|
||||
#define SLH_ADRS_OFF_LAYER_ADR 0
|
||||
#define SLH_ADRS_OFF_TREE_ADR 4
|
||||
#define SLH_ADRS_OFF_TYPE 16
|
||||
#define SLH_ADRS_OFF_KEYPAIR_ADDR 20
|
||||
#define SLH_ADRS_OFF_CHAIN_ADDR 24
|
||||
#define SLH_ADRS_OFF_HASH_ADDR 28
|
||||
#define SLH_ADRS_OFF_TREE_INDEX SLH_ADRS_OFF_HASH_ADDR
|
||||
#define SLH_ADRS_SIZE_TYPE 4
|
||||
/* Number of bytes after type to clear */
|
||||
#define SLH_ADRS_SIZE_TYPECLEAR SLH_ADRS_SIZE - (SLH_ADRS_OFF_TYPE + SLH_ADRS_SIZE_TYPE)
|
||||
#define SLH_ADRS_SIZE_KEYPAIR_ADDR 4
|
||||
|
||||
/* See FIPS 205 - Section 11.2 Table 3 Compressed Addresses */
|
||||
#define SLH_ADRSC_OFF_LAYER_ADR 0
|
||||
#define SLH_ADRSC_OFF_TREE_ADR 1
|
||||
#define SLH_ADRSC_OFF_TYPE 9
|
||||
#define SLH_ADRSC_OFF_KEYPAIR_ADDR 10
|
||||
#define SLH_ADRSC_OFF_CHAIN_ADDR 14
|
||||
#define SLH_ADRSC_OFF_HASH_ADDR 18
|
||||
#define SLH_ADRSC_OFF_TREE_INDEX SLH_ADRSC_OFF_HASH_ADDR
|
||||
#define SLH_ADRSC_SIZE_TYPE 1
|
||||
#define SLH_ADRSC_SIZE_TYPECLEAR SLH_ADRS_SIZE_TYPECLEAR
|
||||
#define SLH_ADRSC_SIZE_KEYPAIR_ADDR SLH_ADRS_SIZE_KEYPAIR_ADDR
|
||||
|
||||
#define slh_adrs_set_tree_height slh_adrs_set_chain_address
|
||||
#define slh_adrs_set_tree_index slh_adrs_set_hash_address
|
||||
|
||||
#define slh_adrsc_set_tree_height slh_adrsc_set_chain_address
|
||||
#define slh_adrsc_set_tree_index slh_adrsc_set_hash_address
|
||||
|
||||
static OSSL_SLH_ADRS_FUNC_set_layer_address slh_adrs_set_layer_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_tree_address slh_adrs_set_tree_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_type_and_clear slh_adrs_set_type_and_clear;
|
||||
static OSSL_SLH_ADRS_FUNC_set_keypair_address slh_adrs_set_keypair_address;
|
||||
static OSSL_SLH_ADRS_FUNC_copy_keypair_address slh_adrs_copy_keypair_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_chain_address slh_adrs_set_chain_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_hash_address slh_adrs_set_hash_address;
|
||||
static OSSL_SLH_ADRS_FUNC_zero slh_adrs_zero;
|
||||
static OSSL_SLH_ADRS_FUNC_copy slh_adrs_copy;
|
||||
|
||||
static OSSL_SLH_ADRS_FUNC_set_layer_address slh_adrsc_set_layer_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_tree_address slh_adrsc_set_tree_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_type_and_clear slh_adrsc_set_type_and_clear;
|
||||
static OSSL_SLH_ADRS_FUNC_set_keypair_address slh_adrsc_set_keypair_address;
|
||||
static OSSL_SLH_ADRS_FUNC_copy_keypair_address slh_adrsc_copy_keypair_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_chain_address slh_adrsc_set_chain_address;
|
||||
static OSSL_SLH_ADRS_FUNC_set_hash_address slh_adrsc_set_hash_address;
|
||||
static OSSL_SLH_ADRS_FUNC_zero slh_adrsc_zero;
|
||||
static OSSL_SLH_ADRS_FUNC_copy slh_adrsc_copy;
|
||||
|
||||
/*
|
||||
* The non compressed versions of the ADRS functions use 32 bytes
|
||||
* This is only used by SHAKE.
|
||||
*/
|
||||
static void slh_adrs_set_layer_address(uint8_t *adrs, uint32_t layer)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrs + SLH_ADRS_OFF_LAYER_ADR, layer);
|
||||
}
|
||||
static void slh_adrs_set_tree_address(uint8_t *adrs, uint64_t address)
|
||||
{
|
||||
/*
|
||||
* There are 12 bytes reserved for this - but the largest number
|
||||
* used by the parameter sets is only 64 bits. Because this is BE the
|
||||
* first 4 of the 12 bytes will be zeros. This assumes that the 4 bytes
|
||||
* are zero initially.
|
||||
*/
|
||||
OPENSSL_store_u64_be(adrs + SLH_ADRS_OFF_TREE_ADR + 4, address);
|
||||
}
|
||||
static void slh_adrs_set_type_and_clear(uint8_t *adrs, uint32_t type)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrs + SLH_ADRS_OFF_TYPE, type);
|
||||
memset(adrs + SLH_ADRS_OFF_TYPE + SLH_ADRS_SIZE_TYPE, 0, SLH_ADRS_SIZE_TYPECLEAR);
|
||||
}
|
||||
static void slh_adrs_set_keypair_address(uint8_t *adrs, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrs + SLH_ADRS_OFF_KEYPAIR_ADDR, in);
|
||||
}
|
||||
static void slh_adrs_copy_keypair_address(uint8_t *dst, const uint8_t *src)
|
||||
{
|
||||
memcpy(dst + SLH_ADRS_OFF_KEYPAIR_ADDR, src + SLH_ADRS_OFF_KEYPAIR_ADDR,
|
||||
SLH_ADRS_SIZE_KEYPAIR_ADDR);
|
||||
}
|
||||
static void slh_adrs_set_chain_address(uint8_t *adrs, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrs + SLH_ADRS_OFF_CHAIN_ADDR, in);
|
||||
}
|
||||
static void slh_adrs_set_hash_address(uint8_t *adrs, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrs + SLH_ADRS_OFF_HASH_ADDR, in);
|
||||
}
|
||||
static void slh_adrs_zero(uint8_t *adrs)
|
||||
{
|
||||
memset(adrs, 0, SLH_ADRS_SIZE);
|
||||
}
|
||||
static void slh_adrs_copy(uint8_t *dst, const uint8_t *src)
|
||||
{
|
||||
memcpy(dst, src, SLH_ADRS_SIZE);
|
||||
}
|
||||
|
||||
/* Compressed versions of ADRS functions See Table 3 */
|
||||
static void slh_adrsc_set_layer_address(uint8_t *adrsc, uint32_t layer)
|
||||
{
|
||||
adrsc[SLH_ADRSC_OFF_LAYER_ADR] = (uint8_t)layer;
|
||||
}
|
||||
static void slh_adrsc_set_tree_address(uint8_t *adrsc, uint64_t in)
|
||||
{
|
||||
OPENSSL_store_u64_be(adrsc + SLH_ADRSC_OFF_TREE_ADR, in);
|
||||
}
|
||||
static void slh_adrsc_set_type_and_clear(uint8_t *adrsc, uint32_t type)
|
||||
{
|
||||
adrsc[SLH_ADRSC_OFF_TYPE] = (uint8_t)type;
|
||||
memset(adrsc + SLH_ADRSC_OFF_TYPE + SLH_ADRSC_SIZE_TYPE, 0, SLH_ADRSC_SIZE_TYPECLEAR);
|
||||
}
|
||||
static void slh_adrsc_set_keypair_address(uint8_t *adrsc, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrsc + SLH_ADRSC_OFF_KEYPAIR_ADDR, in);
|
||||
}
|
||||
static void slh_adrsc_copy_keypair_address(uint8_t *dst, const uint8_t *src)
|
||||
{
|
||||
memcpy(dst + SLH_ADRSC_OFF_KEYPAIR_ADDR, src + SLH_ADRSC_OFF_KEYPAIR_ADDR,
|
||||
SLH_ADRSC_SIZE_KEYPAIR_ADDR);
|
||||
}
|
||||
static void slh_adrsc_set_chain_address(uint8_t *adrsc, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrsc + SLH_ADRSC_OFF_CHAIN_ADDR, in);
|
||||
}
|
||||
static void slh_adrsc_set_hash_address(uint8_t *adrsc, uint32_t in)
|
||||
{
|
||||
OPENSSL_store_u32_be(adrsc + SLH_ADRSC_OFF_HASH_ADDR, in);
|
||||
}
|
||||
static void slh_adrsc_zero(uint8_t *adrsc)
|
||||
{
|
||||
memset(adrsc, 0, SLH_ADRSC_SIZE);
|
||||
}
|
||||
static void slh_adrsc_copy(uint8_t *dst, const uint8_t *src)
|
||||
{
|
||||
memcpy(dst, src, SLH_ADRSC_SIZE);
|
||||
}
|
||||
|
||||
const SLH_ADRS_FUNC *ossl_slh_get_adrs_fn(int is_compressed)
|
||||
{
|
||||
static const SLH_ADRS_FUNC methods[] = {
|
||||
{
|
||||
slh_adrs_set_layer_address,
|
||||
slh_adrs_set_tree_address,
|
||||
slh_adrs_set_type_and_clear,
|
||||
slh_adrs_set_keypair_address,
|
||||
slh_adrs_copy_keypair_address,
|
||||
slh_adrs_set_chain_address,
|
||||
slh_adrs_set_tree_height,
|
||||
slh_adrs_set_hash_address,
|
||||
slh_adrs_set_tree_index,
|
||||
slh_adrs_zero,
|
||||
slh_adrs_copy,
|
||||
},
|
||||
{
|
||||
slh_adrsc_set_layer_address,
|
||||
slh_adrsc_set_tree_address,
|
||||
slh_adrsc_set_type_and_clear,
|
||||
slh_adrsc_set_keypair_address,
|
||||
slh_adrsc_copy_keypair_address,
|
||||
slh_adrsc_set_chain_address,
|
||||
slh_adrsc_set_tree_height,
|
||||
slh_adrsc_set_hash_address,
|
||||
slh_adrsc_set_tree_index,
|
||||
slh_adrsc_zero,
|
||||
slh_adrsc_copy,
|
||||
}
|
||||
};
|
||||
return &methods[is_compressed == 0 ? 0 : 1];
|
||||
}
|
||||
Loaded 100 of 1095 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user