diff --git a/src/include/liburing.h b/src/include/liburing.h index df748aad..66c5095d 100644 --- a/src/include/liburing.h +++ b/src/include/liburing.h @@ -43,7 +43,9 @@ extern "C" { struct io_uring_sq { unsigned *khead; unsigned *ktail; + // Deprecated: use `ring_mask` instead of `*kring_mask` unsigned *kring_mask; + // Deprecated: use `ring_entries` instead of `*kring_entries` unsigned *kring_entries; unsigned *kflags; unsigned *kdropped; @@ -56,13 +58,18 @@ struct io_uring_sq { size_t ring_sz; void *ring_ptr; - unsigned pad[4]; + unsigned ring_mask; + unsigned ring_entries; + + unsigned pad[2]; }; struct io_uring_cq { unsigned *khead; unsigned *ktail; + // Deprecated: use `ring_mask` instead of `*kring_mask` unsigned *kring_mask; + // Deprecated: use `ring_entries` instead of `*kring_entries` unsigned *kring_entries; unsigned *kflags; unsigned *koverflow; @@ -71,7 +78,10 @@ struct io_uring_cq { size_t ring_sz; void *ring_ptr; - unsigned pad[4]; + unsigned ring_mask; + unsigned ring_entries; + + unsigned pad[2]; }; struct io_uring { @@ -220,7 +230,7 @@ int __io_uring_get_cqe(struct io_uring *ring, */ \ for (head = *(ring)->cq.khead; \ (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \ - &(ring)->cq.cqes[io_uring_cqe_index(ring, head, *(ring)->cq.kring_mask)] : NULL)); \ + &(ring)->cq.cqes[io_uring_cqe_index(ring, head, (ring)->cq.ring_mask)] : NULL)); \ head++) \ /* @@ -1034,7 +1044,7 @@ static inline unsigned io_uring_sq_ready(const struct io_uring *ring) */ static inline unsigned io_uring_sq_space_left(const struct io_uring *ring) { - return *ring->sq.kring_entries - io_uring_sq_ready(ring); + return ring->sq.ring_entries - io_uring_sq_ready(ring); } /* @@ -1124,7 +1134,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe *cqe; int err = 0; unsigned available; - unsigned mask = *ring->cq.kring_mask; + unsigned mask = ring->cq.ring_mask; int shift = 0; if (ring->flags & IORING_SETUP_CQE32) @@ -1202,10 +1212,10 @@ static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) if (ring->flags & IORING_SETUP_SQE128) shift = 1; - if (next - head <= *sq->kring_entries) { + if (next - head <= sq->ring_entries) { struct io_uring_sqe *sqe; - sqe = &sq->sqes[(sq->sqe_tail & *sq->kring_mask) << shift]; + sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; sq->sqe_tail = next; return sqe; } diff --git a/src/queue.c b/src/queue.c index 72cc77b9..e43c4195 100644 --- a/src/queue.c +++ b/src/queue.c @@ -142,7 +142,7 @@ again: ready = io_uring_cq_ready(ring); if (ready) { unsigned head = *ring->cq.khead; - unsigned mask = *ring->cq.kring_mask; + unsigned mask = ring->cq.ring_mask; unsigned last; int i = 0; @@ -178,7 +178,7 @@ done: int __io_uring_flush_sq(struct io_uring *ring) { struct io_uring_sq *sq = &ring->sq; - const unsigned mask = *sq->kring_mask; + const unsigned mask = sq->ring_mask; unsigned ktail = *sq->ktail; unsigned to_submit = sq->sqe_tail - sq->sqe_head; diff --git a/src/setup.c b/src/setup.c index dec55d90..d811d81d 100644 --- a/src/setup.c +++ b/src/setup.c @@ -80,6 +80,11 @@ err: cq->cqes = cq->ring_ptr + p->cq_off.cqes; if (p->cq_off.flags) cq->kflags = cq->ring_ptr + p->cq_off.flags; + + sq->ring_mask = *sq->kring_mask; + sq->ring_entries = *sq->kring_entries; + cq->ring_mask = *cq->kring_mask; + cq->ring_entries = *cq->kring_entries; return 0; } @@ -119,7 +124,7 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring) len = sizeof(struct io_uring_sqe); if (ring->flags & IORING_SETUP_SQE128) len += 64; - len *= *ring->sq.kring_entries; + len *= ring->sq.ring_entries; ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK); if (ret < 0) return ret; @@ -182,7 +187,7 @@ __cold void io_uring_queue_exit(struct io_uring *ring) sqe_size = sizeof(struct io_uring_sqe); if (ring->flags & IORING_SETUP_SQE128) sqe_size += 64; - __sys_munmap(sq->sqes, sqe_size * *sq->kring_entries); + __sys_munmap(sq->sqes, sqe_size * sq->ring_entries); io_uring_unmap_rings(sq, cq); /* * Not strictly required, but frees up the slot we used now rather diff --git a/test/accept-reuse.c b/test/accept-reuse.c index 7822433b..66eded23 100644 --- a/test/accept-reuse.c +++ b/test/accept-reuse.c @@ -26,7 +26,7 @@ int submit_sqe(void) struct io_uring_sq *sq = &io_uring.sq; const unsigned tail = *sq->ktail; - sq->array[tail & *sq->kring_mask] = 0; + sq->array[tail & sq->ring_mask] = 0; io_uring_smp_store_release(sq->ktail, tail + 1); return sys_io_uring_enter(io_uring.ring_fd, 1, 0, 0, NULL); diff --git a/test/accept.c b/test/accept.c index f0d84f4f..b35ded4b 100644 --- a/test/accept.c +++ b/test/accept.c @@ -240,7 +240,7 @@ static void cause_overflow(struct io_uring *ring) { int i, ret; - for (i = 0; i < *ring->cq.kring_entries; i++) { + for (i = 0; i < ring->cq.ring_entries; i++) { struct io_uring_sqe *sqe = io_uring_get_sqe(ring); io_uring_prep_nop(sqe); diff --git a/test/io_uring_enter.c b/test/io_uring_enter.c index 429f6ef9..67cc8c58 100644 --- a/test/io_uring_enter.c +++ b/test/io_uring_enter.c @@ -194,7 +194,7 @@ int main(int argc, char **argv) perror("io_uring_queue_init"); exit(T_EXIT_FAIL); } - mask = *sq->kring_mask; + mask = sq->ring_mask; /* invalid flags */ status |= try_io_uring_enter(ring.ring_fd, 1, 0, ~0U, NULL, -EINVAL); @@ -209,7 +209,7 @@ int main(int argc, char **argv) status |= try_io_uring_enter(ring.ring_fd, 0, 0, 0, NULL, 0); /* fill the sq ring */ - sq_entries = *ring.sq.kring_entries; + sq_entries = ring.sq.ring_entries; submit_io(&ring, sq_entries); ret = __sys_io_uring_enter(ring.ring_fd, 0, sq_entries, IORING_ENTER_GETEVENTS, NULL); @@ -235,7 +235,7 @@ int main(int argc, char **argv) * Add an invalid index to the submission queue. This should * result in the dropped counter increasing. */ - index = *sq->kring_entries + 1; // invalid index + index = sq->ring_entries + 1; // invalid index dropped = *sq->kdropped; ktail = *sq->ktail; sq->array[ktail & mask] = index;