mirror of
https://github.com/FEX-Emu/linux.git
synced 2024-12-23 18:07:03 +00:00
crypto: ccp - Use a single queue for proper ordering of tfm requests
Move to a single queue to serialize requests within a tfm. When testing using IPSec with a large number of network connections the per cpu tfm queuing logic was not working properly. Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Tom Lendacky
Herbert Xu
parent
c11baa02c5
commit
bc3854476f
@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
|
||||
static LIST_HEAD(hash_algs);
|
||||
static LIST_HEAD(cipher_algs);
|
||||
|
||||
/* For any tfm, requests for that tfm on the same CPU must be returned
|
||||
* in the order received. With multiple queues available, the CCP can
|
||||
* process more than one cmd at a time. Therefore we must maintain
|
||||
* a cmd list to insure the proper ordering of requests on a given tfm/cpu
|
||||
* combination.
|
||||
/* For any tfm, requests for that tfm must be returned on the order
|
||||
* received. With multiple queues available, the CCP can process more
|
||||
* than one cmd at a time. Therefore we must maintain a cmd list to insure
|
||||
* the proper ordering of requests on a given tfm.
|
||||
*/
|
||||
struct ccp_crypto_cpu_queue {
|
||||
struct ccp_crypto_queue {
|
||||
struct list_head cmds;
|
||||
struct list_head *backlog;
|
||||
unsigned int cmd_count;
|
||||
};
|
||||
#define CCP_CRYPTO_MAX_QLEN 50
|
||||
#define CCP_CRYPTO_MAX_QLEN 100
|
||||
|
||||
struct ccp_crypto_percpu_queue {
|
||||
struct ccp_crypto_cpu_queue __percpu *cpu_queue;
|
||||
};
|
||||
static struct ccp_crypto_percpu_queue req_queue;
|
||||
static struct ccp_crypto_queue req_queue;
|
||||
static spinlock_t req_queue_lock;
|
||||
|
||||
struct ccp_crypto_cmd {
|
||||
struct list_head entry;
|
||||
@ -71,8 +68,6 @@ struct ccp_crypto_cmd {
|
||||
|
||||
/* Used for held command processing to determine state */
|
||||
int ret;
|
||||
|
||||
int cpu;
|
||||
};
|
||||
|
||||
struct ccp_crypto_cpu {
|
||||
@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err)
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* ccp_crypto_cmd_complete must be called while running on the appropriate
|
||||
* cpu and the caller must have done a get_cpu to disable preemption
|
||||
*/
|
||||
static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
|
||||
struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
|
||||
{
|
||||
struct ccp_crypto_cpu_queue *cpu_queue;
|
||||
struct ccp_crypto_cmd *held = NULL, *tmp;
|
||||
unsigned long flags;
|
||||
|
||||
*backlog = NULL;
|
||||
|
||||
cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
|
||||
spin_lock_irqsave(&req_queue_lock, flags);
|
||||
|
||||
/* Held cmds will be after the current cmd in the queue so start
|
||||
* searching for a cmd with a matching tfm for submission.
|
||||
*/
|
||||
tmp = crypto_cmd;
|
||||
list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) {
|
||||
list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
|
||||
if (crypto_cmd->tfm != tmp->tfm)
|
||||
continue;
|
||||
held = tmp;
|
||||
@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
|
||||
* Because cmds can be executed from any point in the cmd list
|
||||
* special precautions have to be taken when handling the backlog.
|
||||
*/
|
||||
if (cpu_queue->backlog != &cpu_queue->cmds) {
|
||||
if (req_queue.backlog != &req_queue.cmds) {
|
||||
/* Skip over this cmd if it is the next backlog cmd */
|
||||
if (cpu_queue->backlog == &crypto_cmd->entry)
|
||||
cpu_queue->backlog = crypto_cmd->entry.next;
|
||||
if (req_queue.backlog == &crypto_cmd->entry)
|
||||
req_queue.backlog = crypto_cmd->entry.next;
|
||||
|
||||
*backlog = container_of(cpu_queue->backlog,
|
||||
*backlog = container_of(req_queue.backlog,
|
||||
struct ccp_crypto_cmd, entry);
|
||||
cpu_queue->backlog = cpu_queue->backlog->next;
|
||||
req_queue.backlog = req_queue.backlog->next;
|
||||
|
||||
/* Skip over this cmd if it is now the next backlog cmd */
|
||||
if (cpu_queue->backlog == &crypto_cmd->entry)
|
||||
cpu_queue->backlog = crypto_cmd->entry.next;
|
||||
if (req_queue.backlog == &crypto_cmd->entry)
|
||||
req_queue.backlog = crypto_cmd->entry.next;
|
||||
}
|
||||
|
||||
/* Remove the cmd entry from the list of cmds */
|
||||
cpu_queue->cmd_count--;
|
||||
req_queue.cmd_count--;
|
||||
list_del(&crypto_cmd->entry);
|
||||
|
||||
spin_unlock_irqrestore(&req_queue_lock, flags);
|
||||
|
||||
return held;
|
||||
}
|
||||
|
||||
static void ccp_crypto_complete_on_cpu(struct work_struct *work)
|
||||
static void ccp_crypto_complete(void *data, int err)
|
||||
{
|
||||
struct ccp_crypto_cpu *cpu_work =
|
||||
container_of(work, struct ccp_crypto_cpu, work);
|
||||
struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd;
|
||||
struct ccp_crypto_cmd *crypto_cmd = data;
|
||||
struct ccp_crypto_cmd *held, *next, *backlog;
|
||||
struct crypto_async_request *req = crypto_cmd->req;
|
||||
struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
|
||||
int cpu, ret;
|
||||
int ret;
|
||||
|
||||
cpu = get_cpu();
|
||||
|
||||
if (cpu_work->err == -EINPROGRESS) {
|
||||
if (err == -EINPROGRESS) {
|
||||
/* Only propogate the -EINPROGRESS if necessary */
|
||||
if (crypto_cmd->ret == -EBUSY) {
|
||||
crypto_cmd->ret = -EINPROGRESS;
|
||||
req->complete(req, -EINPROGRESS);
|
||||
}
|
||||
|
||||
goto e_cpu;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Operation has completed - update the queue before invoking
|
||||
@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
|
||||
req->complete(req, -EINPROGRESS);
|
||||
|
||||
/* Completion callbacks */
|
||||
ret = cpu_work->err;
|
||||
ret = err;
|
||||
if (ctx->complete)
|
||||
ret = ctx->complete(req, ret);
|
||||
req->complete(req, ret);
|
||||
@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
|
||||
}
|
||||
|
||||
kfree(crypto_cmd);
|
||||
|
||||
e_cpu:
|
||||
put_cpu();
|
||||
|
||||
complete(&cpu_work->completion);
|
||||
}
|
||||
|
||||
static void ccp_crypto_complete(void *data, int err)
|
||||
{
|
||||
struct ccp_crypto_cmd *crypto_cmd = data;
|
||||
struct ccp_crypto_cpu cpu_work;
|
||||
|
||||
INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu);
|
||||
init_completion(&cpu_work.completion);
|
||||
cpu_work.crypto_cmd = crypto_cmd;
|
||||
cpu_work.err = err;
|
||||
|
||||
schedule_work_on(crypto_cmd->cpu, &cpu_work.work);
|
||||
|
||||
/* Keep the completion call synchronous */
|
||||
wait_for_completion(&cpu_work.completion);
|
||||
}
|
||||
|
||||
static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
|
||||
{
|
||||
struct ccp_crypto_cpu_queue *cpu_queue;
|
||||
struct ccp_crypto_cmd *active = NULL, *tmp;
|
||||
int cpu, ret;
|
||||
unsigned long flags;
|
||||
int ret;
|
||||
|
||||
cpu = get_cpu();
|
||||
crypto_cmd->cpu = cpu;
|
||||
|
||||
cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
|
||||
spin_lock_irqsave(&req_queue_lock, flags);
|
||||
|
||||
/* Check if the cmd can/should be queued */
|
||||
if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
|
||||
if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
|
||||
ret = -EBUSY;
|
||||
if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
|
||||
goto e_cpu;
|
||||
goto e_lock;
|
||||
}
|
||||
|
||||
/* Look for an entry with the same tfm. If there is a cmd
|
||||
* with the same tfm in the list for this cpu then the current
|
||||
* cmd cannot be submitted to the CCP yet.
|
||||
* with the same tfm in the list then the current cmd cannot
|
||||
* be submitted to the CCP yet.
|
||||
*/
|
||||
list_for_each_entry(tmp, &cpu_queue->cmds, entry) {
|
||||
list_for_each_entry(tmp, &req_queue.cmds, entry) {
|
||||
if (crypto_cmd->tfm != tmp->tfm)
|
||||
continue;
|
||||
active = tmp;
|
||||
@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
|
||||
if (!active) {
|
||||
ret = ccp_enqueue_cmd(crypto_cmd->cmd);
|
||||
if (!ccp_crypto_success(ret))
|
||||
goto e_cpu;
|
||||
goto e_lock;
|
||||
}
|
||||
|
||||
if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
|
||||
if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
|
||||
ret = -EBUSY;
|
||||
if (cpu_queue->backlog == &cpu_queue->cmds)
|
||||
cpu_queue->backlog = &crypto_cmd->entry;
|
||||
if (req_queue.backlog == &req_queue.cmds)
|
||||
req_queue.backlog = &crypto_cmd->entry;
|
||||
}
|
||||
crypto_cmd->ret = ret;
|
||||
|
||||
cpu_queue->cmd_count++;
|
||||
list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds);
|
||||
req_queue.cmd_count++;
|
||||
list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
|
||||
|
||||
e_cpu:
|
||||
put_cpu();
|
||||
e_lock:
|
||||
spin_unlock_irqrestore(&req_queue_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -387,50 +352,18 @@ static void ccp_unregister_algs(void)
|
||||
}
|
||||
}
|
||||
|
||||
static int ccp_init_queues(void)
|
||||
{
|
||||
struct ccp_crypto_cpu_queue *cpu_queue;
|
||||
int cpu;
|
||||
|
||||
req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue);
|
||||
if (!req_queue.cpu_queue)
|
||||
return -ENOMEM;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
|
||||
INIT_LIST_HEAD(&cpu_queue->cmds);
|
||||
cpu_queue->backlog = &cpu_queue->cmds;
|
||||
cpu_queue->cmd_count = 0;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ccp_fini_queue(void)
|
||||
{
|
||||
struct ccp_crypto_cpu_queue *cpu_queue;
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
|
||||
BUG_ON(!list_empty(&cpu_queue->cmds));
|
||||
}
|
||||
free_percpu(req_queue.cpu_queue);
|
||||
}
|
||||
|
||||
static int ccp_crypto_init(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = ccp_init_queues();
|
||||
if (ret)
|
||||
return ret;
|
||||
spin_lock_init(&req_queue_lock);
|
||||
INIT_LIST_HEAD(&req_queue.cmds);
|
||||
req_queue.backlog = &req_queue.cmds;
|
||||
req_queue.cmd_count = 0;
|
||||
|
||||
ret = ccp_register_algs();
|
||||
if (ret) {
|
||||
if (ret)
|
||||
ccp_unregister_algs();
|
||||
ccp_fini_queue();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -438,7 +371,6 @@ static int ccp_crypto_init(void)
|
||||
static void ccp_crypto_exit(void)
|
||||
{
|
||||
ccp_unregister_algs();
|
||||
ccp_fini_queue();
|
||||
}
|
||||
|
||||
module_init(ccp_crypto_init);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user