third_party_liburing/test/buf-ring.c

/* SPDX-License-Identifier: MIT */
/*
 * Description: run various shared buffer ring sanity checks
 *
 */
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <sys/mman.h>

#include "liburing.h"
#include "helpers.h"

static int no_buf_ring;
static int pagesize;

/* test trying to register classic group when ring group exists */
static int test_mixed_reg2(int bgid)
{
	struct io_uring_buf_ring *br;
	struct io_uring_sqe *sqe;
	struct io_uring_cqe *cqe;
	struct io_uring ring;
	void *bufs;
	int ret;

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return 0;
	else if (ret != T_SETUP_OK)
		return 1;

	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
	if (!br) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	/* provide classic buffers, group 1 */
	bufs = malloc(8 * 1024);
	sqe = io_uring_get_sqe(&ring);
	io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0);
	io_uring_submit(&ring);
	ret = io_uring_wait_cqe(&ring, &cqe);
	if (ret) {
		fprintf(stderr, "wait_cqe %d\n", ret);
		return 1;
	}
	if (cqe->res != -EEXIST && cqe->res != -EINVAL) {
		fprintf(stderr, "cqe res %d\n", cqe->res);
		return 1;
	}
	io_uring_cqe_seen(&ring, cqe);

	io_uring_free_buf_ring(&ring, br, 32, bgid);
	io_uring_queue_exit(&ring);
	return 0;
}

/* test trying to register ring group when  classic group exists */
static int test_mixed_reg(int bgid)
{
	struct io_uring_buf_ring *br;
	struct io_uring_sqe *sqe;
	struct io_uring_cqe *cqe;
	struct io_uring ring;
	void *bufs;
	int ret;

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return 0;
	else if (ret != T_SETUP_OK)
		return 1;

	/* provide classic buffers, group 1 */
	bufs = malloc(8 * 1024);
	sqe = io_uring_get_sqe(&ring);
	io_uring_prep_provide_buffers(sqe, bufs, 1024, 8, bgid, 0);
	io_uring_submit(&ring);
	ret = io_uring_wait_cqe(&ring, &cqe);
	if (ret) {
		fprintf(stderr, "wait_cqe %d\n", ret);
		return 1;
	}
	if (cqe->res) {
		fprintf(stderr, "cqe res %d\n", cqe->res);
		return 1;
	}
	io_uring_cqe_seen(&ring, cqe);

	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
	if (br) {
		fprintf(stderr, "Buffer ring setup succeeded unexpectedly %d\n", ret);
		return 1;
	}

	io_uring_queue_exit(&ring);
	return 0;
}

static int test_double_reg_unreg(int bgid)
{
	struct io_uring_buf_reg reg = { };
	struct io_uring_buf_ring *br;
	struct io_uring ring;
	int ret;

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return 0;
	else if (ret != T_SETUP_OK)
		return 1;

	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
	if (!br) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	/* check that 2nd register with same bgid fails */
	reg.ring_addr = (unsigned long) br;
	reg.ring_entries = 32;
	reg.bgid = bgid;

	ret = io_uring_register_buf_ring(&ring, &reg, 0);
	if (ret != -EEXIST) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
	if (ret) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	ret = io_uring_unregister_buf_ring(&ring, bgid);
	if (ret != -EINVAL && ret != -ENOENT) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	io_uring_queue_exit(&ring);
	return 0;
}

static int test_reg_unreg(int bgid)
{
	struct io_uring_buf_ring *br;
	struct io_uring ring;
	int ret;

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return 0;
	else if (ret != T_SETUP_OK)
		return 1;

	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
	if (!br) {
		if (ret == -EINVAL) {
			no_buf_ring = 1;
			return 0;
		}
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return 1;
	}

	ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
	if (ret) {
		fprintf(stderr, "Buffer ring unregister failed %d\n", ret);
		return 1;
	}

	io_uring_queue_exit(&ring);
	return 0;
}

static int test_bad_reg(int bgid)
{
	struct io_uring ring;
	int ret;
	struct io_uring_buf_reg reg = { };

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return 0;
	else if (ret != T_SETUP_OK)
		return 1;

	reg.ring_addr = 4096;
	reg.ring_entries = 32;
	reg.bgid = bgid;

	ret = io_uring_register_buf_ring(&ring, &reg, 0);
	if (!ret)
		fprintf(stderr, "Buffer ring register worked unexpectedly\n");

	io_uring_queue_exit(&ring);
	return !ret;
}

static int test_full_page_reg(int bgid)
{
#if defined(__hppa__)
	return T_EXIT_SKIP;
#else
	struct io_uring ring;
	int ret;
	void *ptr;
	struct io_uring_buf_reg reg = { };
	int entries = pagesize / sizeof(struct io_uring_buf);

	ret = io_uring_queue_init(1, &ring, 0);
	if (ret) {
		fprintf(stderr, "queue init failed %d\n", ret);
		return T_EXIT_FAIL;
	}

	ret = posix_memalign(&ptr, pagesize, pagesize * 2);
	if (ret) {
		fprintf(stderr, "posix_memalign failed %d\n", ret);
		goto err;
	}

	ret = mprotect(ptr + pagesize, pagesize, PROT_NONE);
	if (ret) {
		fprintf(stderr, "mprotect failed %d\n", errno);
		goto err1;
	}

	reg.ring_addr = (unsigned long) ptr;
	reg.ring_entries = entries;
	reg.bgid = bgid;

	ret = io_uring_register_buf_ring(&ring, &reg, 0);
	if (ret)
		fprintf(stderr, "register buf ring failed %d\n", ret);

	if (mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_WRITE))
		fprintf(stderr, "reverting mprotect failed %d\n", errno);

err1:
	free(ptr);
err:
	io_uring_queue_exit(&ring);
	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
#endif
}

static int test_one_read(int fd, int bgid, struct io_uring *ring)
{
	int ret;
	struct io_uring_cqe *cqe;
	struct io_uring_sqe *sqe;

	sqe = io_uring_get_sqe(ring);
	if (!sqe) {
		fprintf(stderr, "get sqe failed\n");
		return -1;
	}

	io_uring_prep_read(sqe, fd, NULL, 1, 0);
	sqe->flags |= IOSQE_BUFFER_SELECT;
	sqe->buf_group = bgid;
	ret = io_uring_submit(ring);
	if (ret <= 0) {
		fprintf(stderr, "sqe submit failed: %d\n", ret);
		return -1;
	}

	ret = io_uring_wait_cqe(ring, &cqe);
	if (ret < 0) {
		fprintf(stderr, "wait completion %d\n", ret);
		return -1;
	}
	ret = cqe->res;
	io_uring_cqe_seen(ring, cqe);

	if (ret == -ENOBUFS)
		return ret;

	if (ret != 1) {
		fprintf(stderr, "read result %d\n", ret);
		return -1;
	}

	return cqe->flags >> 16;
}

static int test_running(int bgid, int entries, int loops, int use_mmap)
{
	int ring_mask = io_uring_buf_ring_mask(entries);
	struct io_uring_buf_ring *br;
	int ret, loop, idx, read_fd;
	struct io_uring ring;
	char buffer[8];
	bool *buffers;

	ret = t_create_ring(1, &ring, 0);
	if (ret == T_SETUP_SKIP)
		return T_EXIT_SKIP;
	else if (ret != T_SETUP_OK)
		return T_EXIT_FAIL;

	if (!use_mmap) {
		br = io_uring_setup_buf_ring(&ring, entries, bgid, 0, &ret);
		if (!br) {
			/* by now should have checked if this is supported or not */
			fprintf(stderr, "Buffer ring register failed %d\n", ret);
			return T_EXIT_FAIL;
		}
	} else {
		struct io_uring_buf_reg reg = {
			.ring_entries = entries,
			.bgid = bgid,
			.flags = IOU_PBUF_RING_MMAP,
		};
		size_t ring_size;
		off_t off;

		ret = io_uring_register_buf_ring(&ring, &reg, 0);
		if (ret) {
			if (ret == -EINVAL)
				return T_EXIT_SKIP;
			fprintf(stderr, "mmap ring register failed %d\n", ret);
			return T_EXIT_FAIL;
		}

		off = IORING_OFF_PBUF_RING |
			(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
		ring_size = sizeof(struct io_uring_buf) * entries;
		br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
				MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
		if (br == MAP_FAILED) {
			perror("mmap");
			return T_EXIT_FAIL;
		}
	}

	buffers = malloc(sizeof(bool) * entries);
	if (!buffers)
		return T_EXIT_SKIP;

	read_fd = open("/dev/zero", O_RDONLY);
	if (read_fd < 0)
		return T_EXIT_SKIP;

	for (loop = 0; loop < loops; loop++) {
		memset(buffers, 0, sizeof(bool) * entries);
		for (idx = 0; idx < entries; idx++)
			io_uring_buf_ring_add(br, buffer, sizeof(buffer), idx, ring_mask, idx);
		io_uring_buf_ring_advance(br, entries);

		for (idx = 0; idx < entries; idx++) {
			memset(buffer, 1, sizeof(buffer));
			ret = test_one_read(read_fd, bgid, &ring);
			if (ret < 0) {
				fprintf(stderr, "bad run %d/%d = %d\n", loop, idx, ret);
				return T_EXIT_FAIL;
			}
			if (buffers[ret]) {
				fprintf(stderr, "reused buffer %d/%d = %d!\n", loop, idx, ret);
				return T_EXIT_FAIL;
			}
			if (buffer[0] != 0) {
				fprintf(stderr, "unexpected read %d %d/%d = %d!\n",
						(int)buffer[0], loop, idx, ret);
				return T_EXIT_FAIL;
			}
			if (buffer[1] != 1) {
				fprintf(stderr, "unexpected spilled read %d %d/%d = %d!\n",
						(int)buffer[1], loop, idx, ret);
				return T_EXIT_FAIL;
			}
			buffers[ret] = true;
		}
		ret = test_one_read(read_fd, bgid, &ring);
		if (ret != -ENOBUFS) {
			fprintf(stderr, "expected enobufs run %d = %d\n", loop, ret);
			return T_EXIT_FAIL;
		}

	}

	ret = io_uring_unregister_buf_ring(&ring, bgid);
	if (ret) {
		fprintf(stderr, "Buffer ring register failed %d\n", ret);
		return T_EXIT_FAIL;
	}

	close(read_fd);
	io_uring_queue_exit(&ring);
	free(buffers);
	return T_EXIT_PASS;
}

int main(int argc, char *argv[])
{
	int bgids[] = { 1, 127, -1 };
	int entries[] = {1, 32768, 4096, -1 };
	int ret, i;

	if (argc > 1)
		return T_EXIT_SKIP;

	pagesize = getpagesize();

	for (i = 0; bgids[i] != -1; i++) {
		ret = test_reg_unreg(bgids[i]);
		if (ret) {
			fprintf(stderr, "test_reg_unreg failed\n");
			return T_EXIT_FAIL;
		}
		if (no_buf_ring)
			break;

		ret = test_bad_reg(bgids[i]);
		if (ret) {
			fprintf(stderr, "test_bad_reg failed\n");
			return T_EXIT_FAIL;
		}

		ret = test_double_reg_unreg(bgids[i]);
		if (ret) {
			fprintf(stderr, "test_double_reg_unreg failed\n");
			return T_EXIT_FAIL;
		}

		ret = test_mixed_reg(bgids[i]);
		if (ret) {
			fprintf(stderr, "test_mixed_reg failed\n");
			return T_EXIT_FAIL;
		}

		ret = test_mixed_reg2(bgids[i]);
		if (ret) {
			fprintf(stderr, "test_mixed_reg2 failed\n");
			return T_EXIT_FAIL;
		}

		ret = test_full_page_reg(bgids[i]);
		if (ret == T_EXIT_FAIL) {
			fprintf(stderr, "test_full_page_reg failed\n");
			return T_EXIT_FAIL;
		}
	}

	for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
		ret = test_running(2, entries[i], 3, 0);
		if (ret) {
			fprintf(stderr, "test_running(%d) failed\n", entries[i]);
			return T_EXIT_FAIL;
		}
	}

	for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
		ret = test_running(2, entries[i], 3, 1);
		if (ret == T_EXIT_SKIP) {
			break;
		} else if (ret != T_EXIT_PASS) {
			fprintf(stderr, "test_running(%d) mmap failed\n", entries[i]);
			return T_EXIT_FAIL;
		}
	}


	return T_EXIT_PASS;
}