Merge branch 'refactor/cache-ring-size' of https://github.com/calebsander/liburing

* 'refactor/cache-ring-size' of https://github.com/calebsander/liburing:
  liburing: cache ring sizes and masks

src/include/liburing.h

@@ -43,7 +43,9 @@ extern "C" {
 struct io_uring_sq {
 	unsigned *khead;
 	unsigned *ktail;
+	// Deprecated: use `ring_mask` instead of `*kring_mask`
 	unsigned *kring_mask;
+	// Deprecated: use `ring_entries` instead of `*kring_entries`
 	unsigned *kring_entries;
 	unsigned *kflags;
 	unsigned *kdropped;
@@ -56,13 +58,18 @@ struct io_uring_sq {
 	size_t ring_sz;
 	void *ring_ptr;
 
-	unsigned pad[4];
+	unsigned ring_mask;
+	unsigned ring_entries;
+
+	unsigned pad[2];
 };
 
 struct io_uring_cq {
 	unsigned *khead;
 	unsigned *ktail;
+	// Deprecated: use `ring_mask` instead of `*kring_mask`
 	unsigned *kring_mask;
+	// Deprecated: use `ring_entries` instead of `*kring_entries`
 	unsigned *kring_entries;
 	unsigned *kflags;
 	unsigned *koverflow;
@@ -71,7 +78,10 @@ struct io_uring_cq {
 	size_t ring_sz;
 	void *ring_ptr;
 
-	unsigned pad[4];
+	unsigned ring_mask;
+	unsigned ring_entries;
+
+	unsigned pad[2];
 };
 
 struct io_uring {
@@ -220,7 +230,7 @@ int __io_uring_get_cqe(struct io_uring *ring,
 	 */								\
 	for (head = *(ring)->cq.khead;					\
 	     (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
-		&(ring)->cq.cqes[io_uring_cqe_index(ring, head, *(ring)->cq.kring_mask)] : NULL)); \
+		&(ring)->cq.cqes[io_uring_cqe_index(ring, head, (ring)->cq.ring_mask)] : NULL)); \
 	     head++)							\
 
 /*
@@ -1034,7 +1044,7 @@ static inline unsigned io_uring_sq_ready(const struct io_uring *ring)
  */
 static inline unsigned io_uring_sq_space_left(const struct io_uring *ring)
 {
-	return *ring->sq.kring_entries - io_uring_sq_ready(ring);
+	return ring->sq.ring_entries - io_uring_sq_ready(ring);
 }
 
 /*
@@ -1124,7 +1134,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
 	struct io_uring_cqe *cqe;
 	int err = 0;
 	unsigned available;
-	unsigned mask = *ring->cq.kring_mask;
+	unsigned mask = ring->cq.ring_mask;
 	int shift = 0;
 
 	if (ring->flags & IORING_SETUP_CQE32)
@@ -1202,10 +1212,10 @@ static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring)
 	if (ring->flags & IORING_SETUP_SQE128)
 		shift = 1;
 
-	if (next - head <= *sq->kring_entries) {
+	if (next - head <= sq->ring_entries) {
 		struct io_uring_sqe *sqe;
 
-		sqe = &sq->sqes[(sq->sqe_tail & *sq->kring_mask) << shift];
+		sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift];
 		sq->sqe_tail = next;
 		return sqe;
 	}

src/queue.c

@@ -142,7 +142,7 @@ again:
 	ready = io_uring_cq_ready(ring);
 	if (ready) {
 		unsigned head = *ring->cq.khead;
-		unsigned mask = *ring->cq.kring_mask;
+		unsigned mask = ring->cq.ring_mask;
 		unsigned last;
 		int i = 0;
 
@@ -178,7 +178,7 @@ done:
 int __io_uring_flush_sq(struct io_uring *ring)
 {
 	struct io_uring_sq *sq = &ring->sq;
-	const unsigned mask = *sq->kring_mask;
+	const unsigned mask = sq->ring_mask;
 	unsigned ktail = *sq->ktail;
 	unsigned to_submit = sq->sqe_tail - sq->sqe_head;
 

src/setup.c

@@ -80,6 +80,11 @@ err:
 	cq->cqes = cq->ring_ptr + p->cq_off.cqes;
 	if (p->cq_off.flags)
 		cq->kflags = cq->ring_ptr + p->cq_off.flags;
+
+	sq->ring_mask = *sq->kring_mask;
+	sq->ring_entries = *sq->kring_entries;
+	cq->ring_mask = *cq->kring_mask;
+	cq->ring_entries = *cq->kring_entries;
 	return 0;
 }
 
@@ -119,7 +124,7 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring)
 	len = sizeof(struct io_uring_sqe);
 	if (ring->flags & IORING_SETUP_SQE128)
 		len += 64;
-	len *= *ring->sq.kring_entries;
+	len *= ring->sq.ring_entries;
 	ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK);
 	if (ret < 0)
 		return ret;
@@ -182,7 +187,7 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
 	sqe_size = sizeof(struct io_uring_sqe);
 	if (ring->flags & IORING_SETUP_SQE128)
 		sqe_size += 64;
-	__sys_munmap(sq->sqes, sqe_size * *sq->kring_entries);
+	__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
 	io_uring_unmap_rings(sq, cq);
 	/*
 	 * Not strictly required, but frees up the slot we used now rather

test/accept-reuse.c

@@ -26,7 +26,7 @@ int submit_sqe(void)
 	struct io_uring_sq *sq = &io_uring.sq;
 	const unsigned tail = *sq->ktail;
 
-	sq->array[tail & *sq->kring_mask] = 0;
+	sq->array[tail & sq->ring_mask] = 0;
 	io_uring_smp_store_release(sq->ktail, tail + 1);
 
 	return sys_io_uring_enter(io_uring.ring_fd, 1, 0, 0, NULL);

test/accept.c

@@ -240,7 +240,7 @@ static void cause_overflow(struct io_uring *ring)
 {
 	int i, ret;
 
-	for (i = 0; i < *ring->cq.kring_entries; i++) {
+	for (i = 0; i < ring->cq.ring_entries; i++) {
 		struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
 
 		io_uring_prep_nop(sqe);

test/io_uring_enter.c

@@ -194,7 +194,7 @@ int main(int argc, char **argv)
 		perror("io_uring_queue_init");
 		exit(T_EXIT_FAIL);
 	}
-	mask = *sq->kring_mask;
+	mask = sq->ring_mask;
 
 	/* invalid flags */
 	status |= try_io_uring_enter(ring.ring_fd, 1, 0, ~0U, NULL, -EINVAL);
@@ -209,7 +209,7 @@ int main(int argc, char **argv)
 	status |= try_io_uring_enter(ring.ring_fd, 0, 0, 0, NULL, 0);
 
 	/* fill the sq ring */
-	sq_entries = *ring.sq.kring_entries;
+	sq_entries = ring.sq.ring_entries;
 	submit_io(&ring, sq_entries);
 	ret = __sys_io_uring_enter(ring.ring_fd, 0, sq_entries,
 					IORING_ENTER_GETEVENTS, NULL);
@@ -235,7 +235,7 @@ int main(int argc, char **argv)
 	 * Add an invalid index to the submission queue.  This should
 	 * result in the dropped counter increasing.
 	 */
-	index = *sq->kring_entries + 1; // invalid index
+	index = sq->ring_entries + 1; // invalid index
 	dropped = *sq->kdropped;
 	ktail = *sq->ktail;
 	sq->array[ktail & mask] = index;