!SQPOLL setups don't need acquire/release semantics on SQ ring updates

If we're not using SQPOLL, then we know there will be a system call
between the SQ ktail writes or khead reads. This means we don't need
acquire/release semantics on the SQ ring, so make them dependent on
how the ring was setup.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
2022-09-23 08:29:13 -06:00
parent ed764358d7
commit 4555346461
2 changed files with 9 additions and 3 deletions
+5 -2
View File
@@ -1240,12 +1240,15 @@ static inline int io_uring_wait_cqe(struct io_uring *ring,
static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring)
{
struct io_uring_sq *sq = &ring->sq;
unsigned int head = io_uring_smp_load_acquire(sq->khead);
unsigned int next = sq->sqe_tail + 1;
unsigned int head, next = sq->sqe_tail + 1;
int shift = 0;
if (ring->flags & IORING_SETUP_SQE128)
shift = 1;
if (!(ring->flags & IORING_SETUP_SQPOLL))
head = IO_URING_READ_ONCE(*sq->khead);
else
head = io_uring_smp_load_acquire(sq->khead);
if (next - head <= sq->ring_entries) {
struct io_uring_sqe *sqe;
+4 -1
View File
@@ -195,7 +195,10 @@ unsigned __io_uring_flush_sq(struct io_uring *ring)
/*
* Ensure kernel sees the SQE updates before the tail update.
*/
io_uring_smp_store_release(sq->ktail, tail);
if (!(ring->flags & IORING_SETUP_SQPOLL))
io_uring_smp_store_release(sq->ktail, tail);
else
IO_URING_WRITE_ONCE(*sq->ktail, tail);
}
/*
* This _may_ look problematic, as we're not supposed to be reading