openharmony/third_party_liburing
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_liburing.git synced 2026-07-01 06:41:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

!23 liburing升级2.7

Browse Source 
Merge pull request !23 from f房芷仪/1014
This commit is contained in:
openharmony_ci
2024-10-21 04:31:19 +00:00
committed by Gitee
parent f7e5bb0d5f 0ae77b95ef
commit edb796bef8
190 changed files with 17185 additions and 2580 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+23 -17
View File
@@ -26,7 +26,8 @@ jobs:
cxx_pkg: clang
cc: clang
cxx: clang++
extra_flags: -Wshorten-64-to-32
liburing_extra_flags: -Wshorten-64-to-32
extra_flags: -Wmissing-prototypes -Wstrict-prototypes -Wunreachable-code-loop-increment -Wunreachable-code -Wmissing-variable-declarations -Wextra-semi-stmt
# x86 (32-bit) gcc
- arch: i686
@@ -49,6 +50,13 @@ jobs:
cc: arm-linux-gnueabi-gcc
cxx: arm-linux-gnueabi-g++
# riscv64
- arch: riscv64
cc_pkg: gcc-riscv64-linux-gnu
cxx_pkg: g++-riscv64-linux-gnu
cc: riscv64-linux-gnu-gcc
cxx: riscv64-linux-gnu-g++
# powerpc64
- arch: powerpc64
cc_pkg: gcc-powerpc64-linux-gnu
@@ -84,23 +92,31 @@ jobs:
cc: mips-linux-gnu-gcc
cxx: mips-linux-gnu-g++
# hppa
- arch: hppa
cc_pkg: gcc-hppa-linux-gnu
cxx_pkg: g++-hppa-linux-gnu
cc: hppa-linux-gnu-gcc
cxx: hppa-linux-gnu-g++
env:
FLAGS: -g -O3 -Wall -Wextra -Werror
FLAGS: -g -O3 -Wall -Wextra -Werror -Wno-sign-compare ${{matrix.extra_flags}}
# Flags for building sources in src/ dir only.
LIBURING_CFLAGS: ${{matrix.extra_flags}}
LIBURING_CFLAGS: ${{matrix.liburing_extra_flags}}
steps:
- name: Checkout source
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Install Compilers
run: |
if [[ "${{matrix.cc_pkg}}" == "clang" ]]; then \
wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh; \
sudo bash /tmp/llvm.sh 16; \
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 400; \
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 400; \
sudo apt-get purge --auto-remove llvm python3-lldb-14 llvm-14 -y; \
sudo bash /tmp/llvm.sh 17; \
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 400; \
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 400; \
else \
sudo apt-get update -y; \
sudo apt-get install -y ${{matrix.cc_pkg}} ${{matrix.cxx_pkg}}; \
@@ -116,16 +132,6 @@ jobs:
./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}};
make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
- name: Build nolibc
run: |
if [[ "${{matrix.arch}}" == "x86_64" || "${{matrix.arch}}" == "i686" || "${{matrix.arch}}" == "aarch64" ]]; then \
make clean; \
./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}} --nolibc; \
make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS"; \
else \
echo "Skipping nolibc build, this arch doesn't support building liburing without libc"; \
fi;
- name: Test install command
run: |
sudo make install;
.github/workflows/shellcheck.yml
+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
steps:
- name: Checkout source
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Display shellcheck version
run: shellcheck --version
.gitignore
+9
View File
@@ -9,15 +9,23 @@
/src/liburing.a
/src/liburing.so*
/src/liburing-ffi.a
/src/liburing-ffi.so*
/src/include/liburing/compat.h
/src/include/liburing/io_uring_version.h
/examples/io_uring-close-test
/examples/io_uring-cp
/examples/io_uring-test
/examples/io_uring-udp
/examples/link-cp
/examples/napi-busy-poll-client
/examples/napi-busy-poll-server
/examples/ucontext-cp
/examples/poll-bench
/examples/proxy
/examples/send-zerocopy
/examples/rsrc-update-bench
/test/*.t
/test/*.dmesg
@@ -28,5 +36,6 @@ config-host.mak
config.log
liburing.pc
liburing-ffi.pc
cscope.out
CHANGELOG
+65
View File
@@ -1,3 +1,68 @@
liburing-2.7 release
- Man page updates
- Sync with kernel 6.10
- send/recv bundle support
- accept nowait and CQE_F_MORE
- Add and update test cases
- Fix io_uring_queue_init_mem() returning a value that was too small,
potentially causing memory corruption in userspace by overwriting
64 bytes beyond the returned value. Also add test case for that.
- Add 64-bit length variants of io_uring_prep_{m,f}advise()
- Add BIND/LISTEN support and helpers / man pages
- Add io_uring_enable_rings.3 man page
- Fix bug in io_uring_prep_read_multishot()
- Fixup bundle test cases
- Add fixed-hugepage test case
- Fix io_uring_prep_fixed_fd_install.3 man page
- Note 'len' == 0 requirement in io_uring_prep_send.3 man page
- Fix some test cases for skipping on older kernels
liburing-2.6 release
- Add getsockopt and setsockopt socket commands
- Add test cases to test/hardlink
- Man page fixes
- Add futex support, and test cases
- Add waitid support, and test cases
- Add read multishot, and test cases
- Add support for IORING_SETUP_NO_SQARRAY
- Use IORING_SETUP_NO_SQARRAY as the default
- Add support for IORING_OP_FIXED_FD_INSTALL
- Add io_uring_prep_fixed_fd_install() helper
- Support for napi busy polling
- Improve/add test cases
- Man page fixes
- Add sample 'proxy' example
liburing-2.5 release
- Add support for io_uring_prep_cmd_sock()
- Add support for application allocated ring memory, for placing rings
in huge mem. Available through io_uring_queue_init_mem().
- Add support for registered ring fds
- Various documentation updates
- Various fixes
liburing-2.4 release
- Add io_uring_{major,minor,check}_version() functions.
- Add IO_URING_{MAJOR,MINOR,CHECK}_VERSION() macros.
- FFI support (for non-C/C++ languages integration).
- Add io_uring_prep_msg_ring_cqe_flags() function.
- Deprecate --nolibc configure option.
- CONFIG_NOLIBC is always enabled on x86-64, x86, and aarch64.
- Add support for IORING_REGISTER_USE_REGISTERED_RING and use if available.
- Add io_uring_close_ring_fd() function.
- Add io_uring_prep_msg_ring_fd_alloc function.
- Add io_uring_free_buf_ring() and io_uring_setup_buf_ring() functions.
- Ensure that io_uring_prep_accept_direct(), io_uring_prep_openat_direct(),
io_uring_prep_openat2_direct(), io_uring_prep_msg_ring_fd(), and
io_uring_prep_socket_direct() factor in being called with
IORING_FILE_INDEX_ALLOC for allocating a direct descriptor.
- Add io_uring_prep_sendto() function.
- Add io_uring_prep_cmd_sock() function.
liburing-2.3 release
- Support non-libc build for aarch64.
Makefile
+19 -7
View File
@@ -11,11 +11,11 @@ all:
@$(MAKE) -C test
@$(MAKE) -C examples
.PHONY: all install default clean test
.PHONY: FORCE cscope
library:
@$(MAKE) -C src
partcheck: all
@echo "make partcheck => TODO add tests with out kernel support"
.PHONY: all install default clean test library
.PHONY: FORCE cscope
runtests: all
@$(MAKE) -C test runtests
@@ -25,7 +25,7 @@ runtests-parallel: all
@$(MAKE) -C test runtests-parallel
config-host.mak: configure
@if [ ! -e "$@" ]; then \
+@if [ ! -e "$@" ]; then \
echo "Running configure ..."; \
./configure; \
else \
@@ -45,13 +45,14 @@ endif
-e "s%@VERSION@%$(VERSION)%g" \
$< >$@
install: $(NAME).pc
install: $(NAME).pc $(NAME)-ffi.pc
@$(MAKE) -C src install prefix=$(DESTDIR)$(prefix) \
includedir=$(DESTDIR)$(includedir) \
libdir=$(DESTDIR)$(libdir) \
libdevdir=$(DESTDIR)$(libdevdir) \
relativelibdir=$(relativelibdir)
$(INSTALL) -D -m 644 $(NAME).pc $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME).pc
$(INSTALL) -D -m 644 $(NAME)-ffi.pc $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME)-ffi.pc
$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man2
$(INSTALL) -m 644 man/*.2 $(DESTDIR)$(mandir)/man2
$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man3
@@ -59,11 +60,22 @@ install: $(NAME).pc
$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man7
$(INSTALL) -m 644 man/*.7 $(DESTDIR)$(mandir)/man7
uninstall:
@$(MAKE) -C src uninstall prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
@rm -f $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME).pc
@rm -f $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME)-ffi.pc
@rm -rf $(DESTDIR)$(mandir)/man2/io_uring*.2
@rm -rf $(DESTDIR)$(mandir)/man3/io_uring*.3
@rm -rf $(DESTDIR)$(mandir)/man7/io_uring*.7
install-tests:
@$(MAKE) -C test install prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
uninstall-tests:
@$(MAKE) -C test uninstall prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
clean:
@rm -f config-host.mak config-host.h cscope.out $(NAME).pc test/*.dmesg
@rm -f config-host.mak config-host.h cscope.out $(NAME).pc $(NAME)-ffi.pc test/*.dmesg
@$(MAKE) -C src clean
@$(MAKE) -C test clean
@$(MAKE) -C examples clean
Makefile.common
+1
View File
@@ -3,4 +3,5 @@ NAME=liburing
SPECFILE=$(TOP)/$(NAME).spec
VERSION=$(shell awk '/Version:/ { print $$2 }' $(SPECFILE))
VERSION_MAJOR=$(shell echo $(VERSION) | cut -d. -f1)
VERSION_MINOR=$(shell echo $(VERSION) | cut -d. -f2)
TAG = $(NAME)-$(VERSION)
README
+48
View File
@@ -47,6 +47,54 @@ the kernel io_uring support. Please note that this suite isn't expected to
pass on older kernels, and may even crash or hang older kernels!
Building liburing
-----------------
#
# Prepare build config (optional).
#
# --cc specifies the C compiler.
# --cxx specifies the C++ compiler.
#
./configure --cc=gcc --cxx=g++;
#
# Build liburing.
#
make -j$(nproc);
#
# Install liburing (headers, shared/static libs, and manpage).
#
sudo make install;
See './configure --help' for more information about build config options.
FFI support
-----------
By default, the build results in 4 lib files:
2 shared libs:
liburing.so
liburing-ffi.so
2 static libs:
liburing.a
liburing-ffi.a
Languages and applications that can't use 'static inline' functions in
liburing.h should use the FFI variants.
liburing's main public interface lives in liburing.h as 'static inline'
functions. Users wishing to consume liburing purely as a binary dependency
should link against liburing-ffi. It contains definitions for every 'static
inline' function.
License
-------
README.OpenSource
+2 -2
View File
@@ -3,9 +3,9 @@
"Name": "liburing",
"License": "MIT License",
"License File": "LICENSE",
"Version Number": "2.3",
"Version Number": "2.7",
"Owner": "mailto:maojingjing1@huawei.com",
"Upstream URL": "https://github.com/axboe/liburing.git",
"Upstream URL": "https://github.com/axboe/liburing/releases/tag/liburing-2.7",
"Description": "liburing provides helpers to setup and reardown io_uring instances, and also a simplified interface for applications that don't need (or want) to deal with the full kernel side implementation."
}
]
bundle.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@ohos/liburing",
"description": "liburing provides helpers to setup and teardown io_uring instances, and also a simplified interface for applications that don't need (or want) to deal with the full kernel side implementation.",
"version": "2.3",
"version": "2.7",
"license": "MIT License",
"publishAs": "code-segment",
"segment": {
configure
Vendored Executable → Regular
+145 -8
View File
@@ -10,7 +10,7 @@ for opt do
case "$opt" in
--help|-h) show_help=yes
;;
--prefix=*) prefix="$optarg"
--prefix=*) prefix="$(realpath -s $optarg)"
;;
--includedir=*) includedir="$optarg"
;;
@@ -26,7 +26,7 @@ for opt do
;;
--cxx=*) cxx="$optarg"
;;
--nolibc) liburing_nolibc="yes"
--use-libc) use_libc=yes
;;
*)
echo "ERROR: unknown option $opt"
@@ -75,7 +75,7 @@ Options: [defaults in brackets after descriptions]
--datadir=PATH install shared data in PATH [$datadir]
--cc=CMD use CMD as the C compiler
--cxx=CMD use CMD as the C++ compiler
--nolibc build liburing without libc
--use-libc use libc for liburing (useful for hardening)
EOF
exit 0
fi
@@ -115,7 +115,7 @@ print_config() {
}
# Default CFLAGS
CFLAGS="-D_GNU_SOURCE -include config-host.h"
CFLAGS="-D_GNU_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -include config-host.h"
BUILD_CFLAGS=""
# Print configure header at the top of $config_host_h
@@ -202,6 +202,15 @@ print_and_output_mak "relativelibdir" "$relativelibdir"
print_and_output_mak "mandir" "$mandir"
print_and_output_mak "datadir" "$datadir"
####################################################
# Check for correct compiler runtime library to link with
libgcc_link_flag="-lgcc"
if $cc -print-libgcc-file-name >/dev/null 2>&1; then
libgcc_link_flag="$($cc $CFLAGS $LDFLAGS -print-libgcc-file-name)"
fi
print_and_output_mak "libgcc_link_flag" "$libgcc_link_flag"
####################################################
##########################################
# check for compiler -Wstringop-overflow
stringop_overflow="no"
@@ -384,14 +393,87 @@ if compile_prog "" "" "nvme uring cmd"; then
fi
print_config "NVMe uring command support" "$nvme_uring_cmd"
##########################################
# Check futexv support
futexv="no"
cat > $TMPC << EOF
#include <linux/futex.h>
#include <unistd.h>
#include <string.h>
int main(void)
{
struct futex_waitv fw;
memset(&fw, FUTEX_32, sizeof(fw));
return sizeof(struct futex_waitv);
}
EOF
if compile_prog "" "" "futexv"; then
futexv="yes"
fi
print_config "futex waitv support" "$futexv"
##########################################
# Check idtype_t support
has_idtype_t="no"
cat > $TMPC << EOF
#include <sys/wait.h>
int main(void)
{
idtype_t v;
return 0;
}
EOF
if compile_prog "" "" "idtype_t"; then
has_idtype_t="yes"
fi
print_config "has_idtype_t" "$has_idtype_t"
#############################################################################
liburing_nolibc="no"
if test "$use_libc" != "yes"; then
#
# Currently, CONFIG_NOLIBC only supports x86-64, x86 (32-bit), aarch64 and riscv64.
#
cat > $TMPC << EOF
int main(void){
#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || (defined(__riscv) && __riscv_xlen == 64)
return 0;
#else
#error libc is needed
#endif
}
EOF
if compile_prog "" "" "nolibc"; then
liburing_nolibc="yes"
fi
fi
print_config "nolibc" "$liburing_nolibc";
#############################################################################
####################################################
# Most Android devices don't have sys/fanotify.h
has_fanotify="no"
cat > $TMPC << EOF
#include <sys/fanotify.h>
int main(void)
{
return 0;
}
EOF
if compile_prog "" "" "fanotify"; then
has_fanotify="yes"
fi
print_config "has_fanotify" "$has_fanotify"
####################################################
if test "$liburing_nolibc" = "yes"; then
output_sym "CONFIG_NOLIBC"
else
liburing_nolibc="no"
fi
print_config "liburing_nolibc" "$liburing_nolibc"
if test "$__kernel_rwf_t" = "yes"; then
output_sym "CONFIG_HAVE_KERNEL_RWF_T"
fi
@@ -422,12 +504,36 @@ fi
if test "$nvme_uring_cmd" = "yes"; then
output_sym "CONFIG_HAVE_NVME_URING"
fi
if test "$has_fanotify" = "yes"; then
output_sym "CONFIG_HAVE_FANOTIFY"
fi
if test "$futexv" = "yes"; then
output_sym "CONFIG_HAVE_FUTEXV"
fi
echo "CC=$cc" >> $config_host_mak
print_config "CC" "$cc"
echo "CXX=$cxx" >> $config_host_mak
print_config "CXX" "$cxx"
# generate io_uring_version.h
# Reset MAKEFLAGS
MAKEFLAGS=
MAKE_PRINT_VARS="include Makefile.common\nprint-%%: ; @echo \$(\$*)\n"
VERSION_MAJOR=$(printf "$MAKE_PRINT_VARS" | make -s --no-print-directory -f - print-VERSION_MAJOR)
VERSION_MINOR=$(printf "$MAKE_PRINT_VARS" | make -s --no-print-directory -f - print-VERSION_MINOR)
io_uring_version_h="src/include/liburing/io_uring_version.h"
cat > $io_uring_version_h << EOF
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_VERSION_H
#define LIBURING_VERSION_H
#define IO_URING_VERSION_MAJOR $VERSION_MAJOR
#define IO_URING_VERSION_MINOR $VERSION_MINOR
#endif
EOF
# generate compat.h
compat_h="src/include/liburing/compat.h"
cat > $compat_h << EOF
@@ -452,10 +558,15 @@ struct __kernel_timespec {
long long tv_nsec;
};
/* <linux/time_types.h> is not available, so it can't be included */
#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
EOF
else
cat >> $compat_h << EOF
#include <linux/time_types.h>
/* <linux/time_types.h> is included above and not needed again */
#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
EOF
fi
@@ -481,7 +592,33 @@ cat >> $compat_h << EOF
EOF
fi
if test "$futexv" != "yes"; then
cat >> $compat_h << EOF
#include <inttypes.h>
#define FUTEX_32 2
#define FUTEX_WAITV_MAX 128
struct futex_waitv {
uint64_t val;
uint64_t uaddr;
uint32_t flags;
uint32_t __reserved;
};
EOF
fi
if test "$has_idtype_t" != "yes"; then
cat >> $compat_h << EOF
typedef enum
{
P_ALL, /* Wait for any child. */
P_PID, /* Wait for specified process. */
P_PGID /* Wait for members of process group. */
} idtype_t;
EOF
fi
cat >> $compat_h << EOF
#endif
EOF
debian/changelog
Vendored
+11
View File
@@ -1,3 +1,14 @@
liburing (2.2-1) stable; urgency=low
* Update to 2.2
* Bump up so version to 2
* Drop liburing1-udeb
* Package using dh instead of using dh_* helpers manually
* Add linux header dependency to liburing-dev
* Bump up debhelper-compact level to 13
-- Kefu Chai <tchaikov@gmail.com> Sun, 16 Oct 2022 16:30:48 +0800
liburing (0.7-1) stable; urgency=low
* Update to 0.7
debian/compat
Vendored
-1
View File
@@ -1 +0,0 @@
9
debian/control
Vendored
+7 -16
View File
@@ -2,13 +2,14 @@ Source: liburing
Section: libs
Priority: optional
Maintainer: Liu Changcheng <changcheng.liu@intel.com>
Build-Depends: debhelper (>=9)
Build-Depends:
debhelper-compat (= 13)
Standards-Version: 4.1.4
Homepage: https://git.kernel.dk/cgit/liburing/tree/README
Vcs-Git: https://git.kernel.dk/liburing
Vcs-Browser: https://git.kernel.dk/cgit/liburing/
Package: liburing1
Package: liburing2
Architecture: linux-any
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
@@ -21,24 +22,14 @@ Description: userspace library for using io_uring
.
This package contains the shared library.
Package: liburing1-udeb
Package-Type: udeb
Section: debian-installer
Architecture: linux-any
Depends: ${misc:Depends}, ${shlibs:Depends},
Description: userspace library for using io_uring
io_uring is kernel feature to improve development
The newese Linux IO interface, io_uring could improve
system performance a lot. liburing is the userpace
library to use io_uring feature.
.
This package contains the udeb shared library.
Package: liburing-dev
Section: libdevel
Architecture: linux-any
Multi-Arch: same
Depends: ${misc:Depends}, liburing1 (= ${binary:Version}),
Depends:
${misc:Depends},
liburing2 (= ${binary:Version}),
linux-libc-dev (>= 5.1)
Description: userspace library for using io_uring
io_uring is kernel feature to improve development
The newese Linux IO interface, io_uring could improve
debian/liburing-dev.manpages
Vendored
+5 -6
View File
@@ -1,6 +1,5 @@
man/io_uring_setup.2
man/io_uring_enter.2
man/io_uring_register.2
man/io_uring_queue_exit.3
man/io_uring_queue_init.3
man/io_uring_get_sqe.3
usr/share/man/man2/io_uring_*.2
usr/share/man/man3/io_uring_*.3
usr/share/man/man7/io_uring.7
usr/share/man/man3/IO_URING_*.3
usr/share/man/man3/__io_uring_*.3
debian/liburing1-udeb.install
Vendored
-1
View File
@@ -1 +0,0 @@
lib/*/lib*.so.*
debian/liburing1.install
Vendored
-1
View File
@@ -1 +0,0 @@
lib/*/lib*.so.*
debian/liburing1.symbols
Vendored
-32
View File
@@ -1,32 +0,0 @@
liburing.so.1 liburing1 #MINVER#
(symver)LIBURING_0.1 0.1-1
io_uring_get_sqe@LIBURING_0.1 0.1-1
io_uring_queue_exit@LIBURING_0.1 0.1-1
io_uring_queue_init@LIBURING_0.1 0.1-1
io_uring_queue_mmap@LIBURING_0.1 0.1-1
io_uring_register_buffers@LIBURING_0.1 0.1-1
io_uring_register_eventfd@LIBURING_0.1 0.1-1
io_uring_register_eventfd_async@LIBURING_0.6 0.6-1
io_uring_register_files@LIBURING_0.1 0.1-1
io_uring_submit@LIBURING_0.1 0.1-1
io_uring_submit_and_wait@LIBURING_0.1 0.1-1
io_uring_unregister_buffers@LIBURING_0.1 0.1-1
io_uring_unregister_files@LIBURING_0.1 0.1-1
(symver)LIBURING_0.2 0.2-1
__io_uring_get_cqe@LIBURING_0.2 0.2-1
io_uring_queue_init_params@LIBURING_0.2 0.2-1
io_uring_register_files_update@LIBURING_0.2 0.2-1
io_uring_peek_batch_cqe@LIBURING_0.2 0.2-1
io_uring_wait_cqe_timeout@LIBURING_0.2 0.2-1
io_uring_wait_cqes@LIBURING_0.2 0.2-1
(symver)LIBURING_0.3 0.3-1
(symver)LIBURING_0.4 0.4-1
(symver)LIBURING_0.5 0.5-1
(symver)LIBURING_0.6 0.6-1
(symver)LIBURING_0.7 0.7-1
io_uring_get_probe@LIBURING_0.4 0.4-1
io_uring_get_probe_ring@LIBURING_0.4 0.4-1
io_uring_register_personality@LIBURING_0.4 0.4-1
io_uring_register_probe@LIBURING_0.4 0.4-1
io_uring_ring_dontfork@LIBURING_0.4 0.4-1
io_uring_unregister_personality@LIBURING_0.4 0.4-1
debian/liburing2.install
Vendored
+1
View File
@@ -0,0 +1 @@
usr/lib/*/lib*.so.*
debian/liburing2.symbols
Vendored
+56
View File
@@ -0,0 +1,56 @@
liburing.so.2 liburing2 #MINVER# [47/1887]
LIBURING_2.0@LIBURING_2.0 0.7-1
LIBURING_2.1@LIBURING_2.1 0.7-1
LIBURING_2.2@LIBURING_2.2 0.7-1
LIBURING_2.3@LIBURING_2.3 0.7-1
__io_uring_get_cqe@LIBURING_2.0 0.7-1
__io_uring_sqring_wait@LIBURING_2.0 0.7-1
io_uring_enter2@LIBURING_2.3 0.7-1
io_uring_enter@LIBURING_2.3 0.7-1
io_uring_free_probe@LIBURING_2.0 0.7-1
io_uring_get_events@LIBURING_2.3 0.7-1
io_uring_get_probe@LIBURING_2.0 0.7-1
io_uring_get_probe_ring@LIBURING_2.0 0.7-1
io_uring_get_sqe@LIBURING_2.0 0.7-1
io_uring_mlock_size@LIBURING_2.1 0.7-1
io_uring_mlock_size_params@LIBURING_2.1 0.7-1
io_uring_peek_batch_cqe@LIBURING_2.0 0.7-1
io_uring_queue_exit@LIBURING_2.0 0.7-1
io_uring_queue_init@LIBURING_2.0 0.7-1
io_uring_queue_init_params@LIBURING_2.0 0.7-1
io_uring_queue_mmap@LIBURING_2.0 0.7-1
io_uring_register@LIBURING_2.3 0.7-1
io_uring_register_buf_ring@LIBURING_2.2 0.7-1
io_uring_register_buffers@LIBURING_2.0 0.7-1
io_uring_register_buffers_sparse@LIBURING_2.2 0.7-1
io_uring_register_buffers_tags@LIBURING_2.1 0.7-1
io_uring_register_buffers_update_tag@LIBURING_2.1 0.7-1
io_uring_register_eventfd@LIBURING_2.0 0.7-1
io_uring_register_eventfd_async@LIBURING_2.0 0.7-1
io_uring_register_file_alloc_range@LIBURING_2.3 0.7-1
io_uring_register_files@LIBURING_2.0 0.7-1
io_uring_register_files_sparse@LIBURING_2.2 0.7-1
io_uring_register_files_tags@LIBURING_2.1 0.7-1
io_uring_register_files_update@LIBURING_2.0 0.7-1
io_uring_register_files_update_tag@LIBURING_2.1 0.7-1
io_uring_register_iowq_aff@LIBURING_2.1 0.7-1
io_uring_register_iowq_max_workers@LIBURING_2.1 0.7-1
io_uring_register_personality@LIBURING_2.0 0.7-1
io_uring_register_probe@LIBURING_2.0 0.7-1
io_uring_register_ring_fd@LIBURING_2.2 0.7-1
io_uring_register_sync_cancel@LIBURING_2.3 0.7-1
io_uring_ring_dontfork@LIBURING_2.0 0.7-1
io_uring_setup@LIBURING_2.3 0.7-1
io_uring_submit@LIBURING_2.0 0.7-1
io_uring_submit_and_get_events@LIBURING_2.3 0.7-1
io_uring_submit_and_wait@LIBURING_2.0 0.7-1
io_uring_submit_and_wait_timeout@LIBURING_2.2 0.7-1
io_uring_unregister_buf_ring@LIBURING_2.2 0.7-1
io_uring_unregister_buffers@LIBURING_2.0 0.7-1
io_uring_unregister_eventfd@LIBURING_2.0 0.7-1
io_uring_unregister_files@LIBURING_2.0 0.7-1
io_uring_unregister_iowq_aff@LIBURING_2.1 0.7-1
io_uring_unregister_personality@LIBURING_2.0 0.7-1
io_uring_unregister_ring_fd@LIBURING_2.2 0.7-1
io_uring_wait_cqe_timeout@LIBURING_2.0 0.7-1
io_uring_wait_cqes@LIBURING_2.0 0.7-1
debian/rules
Vendored Executable → Regular
+14 -66
View File
@@ -5,77 +5,25 @@
DEB_BUILD_MAINT_OPTIONS = hardening=+bindnow
DEB_CFLAGS_MAINT_PREPEND = -Wall
DEB_BUILD_OPTIONS += nocheck
include /usr/share/dpkg/default.mk
include /usr/share/dpkg/buildtools.mk
export CC
%:
dh $@ --parallel
lib := liburing1
libdbg := $(lib)-dbg
libudeb := $(lib)-udeb
libdev := liburing-dev
build-indep:
build-arch:
dh_testdir
$(MAKE) CPPFLAGS="$(CPPFLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
build: build-indep build-arch
clean:
dh_testdir
dh_testroot
$(MAKE) clean
dh_clean
check-arch: build-arch
dh_testdir
override_dh_auto_configure:
./configure \
--prefix=/usr \
--includedir=/usr/include \
--datadir=/usr/share \
--mandir=/usr/share/man \
--libdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
--libdevdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
--cc=$(CC)
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
$(MAKE) CPPFLAGS="$(CPPFLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
partcheck
$(MAKE) runtests
endif
install-arch: check-arch
dh_testdir
dh_testroot
dh_clean
dh_installdirs
$(MAKE) install \
DESTDIR=$(CURDIR)/debian/tmp \
libdir=/lib/$(DEB_HOST_MULTIARCH) \
libdevdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
relativelibdir=/lib/$(DEB_HOST_MULTIARCH)/
binary: binary-indep binary-arch
binary-indep:
# Nothing to do.
binary-arch: install-arch
dh_testdir
dh_testroot
dh_install -a
dh_installdocs -a
dh_installexamples -a
dh_installman -a
dh_lintian -a
dh_link -a
dh_strip -a --ddeb-migration='$(libdbg) (<< 0.3)'
dh_compress -a
dh_fixperms -a
dh_makeshlibs -a --add-udeb '$(libudeb)'
dh_shlibdeps -a
dh_installdeb -a
dh_gencontrol -a
dh_md5sums -a
dh_builddeb -a
.PHONY: clean build-indep build-arch build
.PHONY: install-arch binary-indep binary-arch binary
examples/Makefile
+16 -4
View File
@@ -10,13 +10,21 @@ ifneq ($(MAKECMDGOALS),clean)
include ../config-host.mak
endif
LDFLAGS ?=
override LDFLAGS += -L../src/ -luring -lpthread
example_srcs := \
io_uring-close-test.c \
io_uring-cp.c \
io_uring-test.c \
io_uring-udp.c \
link-cp.c \
napi-busy-poll-client.c \
napi-busy-poll-server.c \
poll-bench.c \
send-zerocopy.c
send-zerocopy.c \
rsrc-update-bench.c \
proxy.c
all_targets :=
@@ -24,16 +32,20 @@ all_targets :=
ifdef CONFIG_HAVE_UCONTEXT
example_srcs += ucontext-cp.c
endif
all_targets += ucontext-cp
all_targets += ucontext-cp helpers.o
example_targets := $(patsubst %.c,%,$(patsubst %.cc,%,$(example_srcs)))
all_targets += $(example_targets)
helpers = helpers.o
all: $(example_targets)
%: %.c ../src/liburing.a
$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS)
helpers.o: helpers.c
$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ -c $<
%: %.c $(helpers) ../src/liburing.a
$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
clean:
@rm -f $(all_targets)
examples/helpers.c
+62
View File
@@ -0,0 +1,62 @@
/* SPDX-License-Identifier: MIT */
#include <fcntl.h>
#include <stdint.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>
#include "helpers.h"
int setup_listening_socket(int port, int ipv6)
{
struct sockaddr_in srv_addr = { };
struct sockaddr_in6 srv_addr6 = { };
int fd, enable, ret, domain;
if (ipv6)
domain = AF_INET6;
else
domain = AF_INET;
fd = socket(domain, SOCK_STREAM, 0);
if (fd == -1) {
perror("socket()");
return -1;
}
enable = 1;
ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
if (ret < 0) {
perror("setsockopt(SO_REUSEADDR)");
return -1;
}
if (ipv6) {
srv_addr6.sin6_family = AF_INET6;
srv_addr6.sin6_port = htons(port);
srv_addr6.sin6_addr = in6addr_any;
ret = bind(fd, (const struct sockaddr *)&srv_addr6, sizeof(srv_addr6));
} else {
srv_addr.sin_family = AF_INET;
srv_addr.sin_port = htons(port);
srv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
ret = bind(fd, (const struct sockaddr *)&srv_addr, sizeof(srv_addr));
}
if (ret < 0) {
perror("bind()");
return -1;
}
if (listen(fd, 1024) < 0) {
perror("listen()");
return -1;
}
return fd;
}
examples/helpers.h
+7
View File
@@ -0,0 +1,7 @@
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_EX_HELPERS_H
#define LIBURING_EX_HELPERS_H
int setup_listening_socket(int port, int ipv6);
#endif
examples/io_uring-close-test.c
+123
View File
@@ -0,0 +1,123 @@
/* SPDX-License-Identifier: MIT */
/*
* Simple app that demonstrates how to setup an io_uring interface, and use it
* via a registered ring fd, without leaving the original fd open.
*
* gcc -Wall -O2 -D_GNU_SOURCE -o io_uring-close-test io_uring-close-test.c -luring
*/
#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "liburing.h"
#define QD 4
int main(int argc, char *argv[])
{
struct io_uring ring;
int i, fd, ret, pending, done;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct iovec *iovecs;
struct stat sb;
ssize_t fsize;
off_t offset;
void *buf;
if (argc < 2) {
printf("%s: file\n", argv[0]);
return 1;
}
ret = io_uring_queue_init(QD, &ring, 0);
if (ret < 0) {
fprintf(stderr, "queue_init: %s\n", strerror(-ret));
return 1;
}
ret = io_uring_register_ring_fd(&ring);
if (ret < 0) {
fprintf(stderr, "register_ring_fd: %s\n", strerror(-ret));
return 1;
}
ret = io_uring_close_ring_fd(&ring);
if (ret < 0) {
fprintf(stderr, "close_ring_fd: %s\n", strerror(-ret));
return 1;
}
fd = open(argv[1], O_RDONLY);
if (fd < 0) {
perror("open");
return 1;
}
if (fstat(fd, &sb) < 0) {
perror("fstat");
return 1;
}
fsize = 0;
iovecs = calloc(QD, sizeof(struct iovec));
for (i = 0; i < QD; i++) {
if (posix_memalign(&buf, 4096, 4096))
return 1;
iovecs[i].iov_base = buf;
iovecs[i].iov_len = 4096;
fsize += 4096;
}
offset = 0;
i = 0;
do {
sqe = io_uring_get_sqe(&ring);
if (!sqe)
break;
io_uring_prep_readv(sqe, fd, &iovecs[i], 1, offset);
offset += iovecs[i].iov_len;
i++;
if (offset > sb.st_size)
break;
} while (1);
ret = io_uring_submit(&ring);
if (ret < 0) {
fprintf(stderr, "io_uring_submit: %s\n", strerror(-ret));
return 1;
} else if (ret != i) {
fprintf(stderr, "io_uring_submit submitted less %d\n", ret);
return 1;
}
done = 0;
pending = ret;
fsize = 0;
for (i = 0; i < pending; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret < 0) {
fprintf(stderr, "io_uring_wait_cqe: %s\n", strerror(-ret));
return 1;
}
done++;
ret = 0;
if (cqe->res != 4096 && cqe->res + fsize != sb.st_size) {
fprintf(stderr, "ret=%d, wanted 4096\n", cqe->res);
ret = 1;
}
fsize += cqe->res;
io_uring_cqe_seen(&ring, cqe);
if (ret)
break;
}
printf("Submitted=%d, completed=%d, bytes=%lu\n", pending, done,
(unsigned long) fsize);
close(fd);
io_uring_queue_exit(&ring);
return 0;
}
examples/io_uring-test.c
+1 -1
View File
@@ -69,7 +69,7 @@ int main(int argc, char *argv[])
io_uring_prep_readv(sqe, fd, &iovecs[i], 1, offset);
offset += iovecs[i].iov_len;
i++;
if (offset > sb.st_size)
if (offset >= sb.st_size)
break;
} while (1);
examples/io_uring-udp.c
+11 -3
View File
@@ -271,14 +271,22 @@ static int process_cqe_recv(struct ctx *ctx, struct io_uring_cqe *cqe,
}
if (ctx->verbose) {
struct sockaddr_in *addr = io_uring_recvmsg_name(o);
struct sockaddr_in6 *addr6 = (void *)addr;
char buff[INET6_ADDRSTRLEN + 1];
const char *name;
struct sockaddr_in *addr = io_uring_recvmsg_name(o);
void *paddr;
name = inet_ntop(ctx->af, addr, buff, sizeof(buff));
if (ctx->af == AF_INET6)
paddr = &addr6->sin6_addr;
else
paddr = &addr->sin_addr;
name = inet_ntop(ctx->af, paddr, buff, sizeof(buff));
if (!name)
name = "<INVALID>";
fprintf(stderr, "received %u bytes %d from %s:%d\n",
fprintf(stderr, "received %u bytes %d from [%s]:%d\n",
io_uring_recvmsg_payload_length(o, cqe->res, &ctx->msg),
o->namelen, name, (int)ntohs(addr->sin_port));
}
examples/napi-busy-poll-client.c
+509
View File
@@ -0,0 +1,509 @@
/* SPDX-License-Identifier: MIT */
/*
* Simple ping/pong client which can use the io_uring NAPI support.
*
* Needs to be run as root because it sets SCHED_FIFO scheduling class,
* but will work without that.
*
* Example:
*
* sudo examples/napi-busy-poll-client -a 192.168.2.2 -n100000 -p4444 \
* -b -t10 -u
*
* send and receive 100k packets, using NAPI.
*/
#include <ctype.h>
#include <errno.h>
#include <float.h>
#include <getopt.h>
#include <liburing.h>
#include <math.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#define MAXBUFLEN 100
#define PORTNOLEN 10
#define ADDRLEN 80
#define RINGSIZE 1024
#define printable(ch) (isprint((unsigned char)ch) ? ch : '#')
enum {
IOURING_RECV,
IOURING_SEND,
IOURING_RECVMSG,
IOURING_SENDMSG
};
struct ctx
{
struct io_uring ring;
union {
struct sockaddr_in6 saddr6;
struct sockaddr_in saddr;
};
int sockfd;
int buffer_len;
int num_pings;
bool napi_check;
union {
char buffer[MAXBUFLEN];
struct timespec ts;
};
int rtt_index;
double *rtt;
};
struct options
{
int num_pings;
__u32 timeout;
bool sq_poll;
bool defer_tw;
bool busy_loop;
bool prefer_busy_poll;
bool ipv6;
char port[PORTNOLEN];
char addr[ADDRLEN];
};
static struct option longopts[] =
{
{"address" , 1, NULL, 'a'},
{"busy" , 0, NULL, 'b'},
{"help" , 0, NULL, 'h'},
{"num_pings", 1, NULL, 'n'},
{"port" , 1, NULL, 'p'},
{"prefer" , 1, NULL, 'u'},
{"sqpoll" , 0, NULL, 's'},
{"timeout" , 1, NULL, 't'},
{NULL , 0, NULL, 0 }
};
static void printUsage(const char *name)
{
fprintf(stderr,
"Usage: %s [-l|--listen] [-a|--address ip_address] [-p|--port port-no] [-s|--sqpoll]"
" [-b|--busy] [-n|--num pings] [-t|--timeout busy-poll-timeout] [-u||--prefer] [-6] [-h|--help]\n"
"--address\n"
"-a : remote or local ipv6 address\n"
"--busy\n"
"-b : busy poll io_uring instead of blocking.\n"
"--num_pings\n"
"-n : number of pings\n"
"--port\n"
"-p : port\n"
"--sqpoll\n"
"-s : Configure io_uring to use SQPOLL thread\n"
"--timeout\n"
"-t : Configure NAPI busy poll timeout"
"--prefer\n"
"-u : prefer NAPI busy poll\n"
"-6 : use IPV6\n"
"--help\n"
"-h : Display this usage message\n\n",
name);
}
static void printError(const char *msg, int opt)
{
if (msg && opt)
fprintf(stderr, "%s (-%c)\n", msg, printable(opt));
}
static void setProcessScheduler(void)
{
struct sched_param param;
param.sched_priority = sched_get_priority_max(SCHED_FIFO);
if (sched_setscheduler(0, SCHED_FIFO, &param) < 0)
fprintf(stderr, "sched_setscheduler() failed: (%d) %s\n",
errno, strerror(errno));
}
static double diffTimespec(const struct timespec *time1, const struct timespec *time0)
{
return (time1->tv_sec - time0->tv_sec)
+ (time1->tv_nsec - time0->tv_nsec) / 1000000000.0;
}
static uint64_t encodeUserData(char type, int fd)
{
return (uint32_t)fd | ((uint64_t)type << 56);
}
static void decodeUserData(uint64_t data, char *type, int *fd)
{
*type = data >> 56;
*fd = data & 0xffffffffU;
}
static const char *opTypeToStr(char type)
{
const char *res;
switch (type) {
case IOURING_RECV:
res = "IOURING_RECV";
break;
case IOURING_SEND:
res = "IOURING_SEND";
break;
case IOURING_RECVMSG:
res = "IOURING_RECVMSG";
break;
case IOURING_SENDMSG:
res = "IOURING_SENDMSG";
break;
default:
res = "Unknown";
}
return res;
}
static void reportNapi(struct ctx *ctx)
{
unsigned int napi_id = 0;
socklen_t len = sizeof(napi_id);
getsockopt(ctx->sockfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);
if (napi_id)
printf(" napi id: %d\n", napi_id);
else
printf(" unassigned napi id\n");
ctx->napi_check = true;
}
static void sendPing(struct ctx *ctx)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
clock_gettime(CLOCK_REALTIME, (struct timespec *)ctx->buffer);
io_uring_prep_send(sqe, ctx->sockfd, ctx->buffer, sizeof(struct timespec), 0);
sqe->user_data = encodeUserData(IOURING_SEND, ctx->sockfd);
}
static void receivePing(struct ctx *ctx)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
io_uring_prep_recv(sqe, ctx->sockfd, ctx->buffer, MAXBUFLEN, 0);
sqe->user_data = encodeUserData(IOURING_RECV, ctx->sockfd);
}
static void recordRTT(struct ctx *ctx)
{
struct timespec startTs = ctx->ts;
// Send next ping.
sendPing(ctx);
// Store round-trip time.
ctx->rtt[ctx->rtt_index] = diffTimespec(&ctx->ts, &startTs);
ctx->rtt_index++;
}
static void printStats(struct ctx *ctx)
{
double minRTT = DBL_MAX;
double maxRTT = 0.0;
double avgRTT = 0.0;
double stddevRTT = 0.0;
// Calculate min, max, avg.
for (int i = 0; i < ctx->rtt_index; i++) {
if (ctx->rtt[i] < minRTT)
minRTT = ctx->rtt[i];
if (ctx->rtt[i] > maxRTT)
maxRTT = ctx->rtt[i];
avgRTT += ctx->rtt[i];
}
avgRTT /= ctx->rtt_index;
// Calculate stddev.
for (int i = 0; i < ctx->rtt_index; i++)
stddevRTT += fabs(ctx->rtt[i] - avgRTT);
stddevRTT /= ctx->rtt_index;
fprintf(stdout, " rtt(us) min/avg/max/mdev = %.3f/%.3f/%.3f/%.3f\n",
minRTT * 1000000, avgRTT * 1000000, maxRTT * 1000000, stddevRTT * 1000000);
}
static int completion(struct ctx *ctx, struct io_uring_cqe *cqe)
{
char type;
int fd;
int res = cqe->res;
decodeUserData(cqe->user_data, &type, &fd);
if (res < 0) {
fprintf(stderr, "unexpected %s failure: (%d) %s\n",
opTypeToStr(type), -res, strerror(-res));
return -1;
}
switch (type) {
case IOURING_SEND:
receivePing(ctx);
break;
case IOURING_RECV:
if (res != sizeof(struct timespec)) {
fprintf(stderr, "unexpected ping reply len: %d\n", res);
abort();
}
if (!ctx->napi_check) {
reportNapi(ctx);
sendPing(ctx);
} else {
recordRTT(ctx);
}
--ctx->num_pings;
break;
default:
fprintf(stderr, "unexpected %s completion\n",
opTypeToStr(type));
return -1;
break;
}
return 0;
}
int main(int argc, char *argv[])
{
struct ctx ctx;
struct options opt;
struct __kernel_timespec *tsPtr;
struct __kernel_timespec ts;
struct io_uring_params params;
struct io_uring_napi napi;
int flag, ret, af;
memset(&opt, 0, sizeof(struct options));
// Process flags.
while ((flag = getopt_long(argc, argv, ":hs:bua:n:p:t:6d:", longopts, NULL)) != -1) {
switch (flag) {
case 'a':
strcpy(opt.addr, optarg);
break;
case 'b':
opt.busy_loop = true;
break;
case 'h':
printUsage(argv[0]);
exit(0);
break;
case 'n':
opt.num_pings = atoi(optarg) + 1;
break;
case 'p':
strcpy(opt.port, optarg);
break;
case 's':
opt.sq_poll = !!atoi(optarg);
break;
case 't':
opt.timeout = atoi(optarg);
break;
case 'u':
opt.prefer_busy_poll = true;
break;
case '6':
opt.ipv6 = true;
break;
case 'd':
opt.defer_tw = !!atoi(optarg);
break;
case ':':
printError("Missing argument", optopt);
printUsage(argv[0]);
exit(-1);
break;
case '?':
printError("Unrecognized option", optopt);
printUsage(argv[0]);
exit(-1);
break;
default:
fprintf(stderr, "Fatal: Unexpected case in CmdLineProcessor switch()\n");
exit(-1);
break;
}
}
if (strlen(opt.addr) == 0) {
fprintf(stderr, "address option is mandatory\n");
printUsage(argv[0]);
exit(1);
}
if (opt.ipv6) {
af = AF_INET6;
ctx.saddr6.sin6_port = htons(atoi(opt.port));
ctx.saddr6.sin6_family = AF_INET6;
} else {
af = AF_INET;
ctx.saddr.sin_port = htons(atoi(opt.port));
ctx.saddr.sin_family = AF_INET;
}
if (opt.ipv6)
ret = inet_pton(af, opt.addr, &ctx.saddr6.sin6_addr);
else
ret = inet_pton(af, opt.addr, &ctx.saddr.sin_addr);
if (ret <= 0) {
fprintf(stderr, "inet_pton error for %s\n", optarg);
printUsage(argv[0]);
exit(1);
}
// Connect to server.
fprintf(stdout, "Connecting to %s... (port=%s) to send %d pings\n", opt.addr, opt.port, opt.num_pings - 1);
if ((ctx.sockfd = socket(af, SOCK_DGRAM, 0)) < 0) {
fprintf(stderr, "socket() failed: (%d) %s\n", errno, strerror(errno));
exit(1);
}
if (opt.ipv6)
ret = connect(ctx.sockfd, (struct sockaddr *)&ctx.saddr6, sizeof(struct sockaddr_in6));
else
ret = connect(ctx.sockfd, (struct sockaddr *)&ctx.saddr, sizeof(struct sockaddr_in));
if (ret < 0) {
fprintf(stderr, "connect() failed: (%d) %s\n", errno, strerror(errno));
exit(1);
}
// Setup ring.
memset(&params, 0, sizeof(params));
memset(&ts, 0, sizeof(ts));
memset(&napi, 0, sizeof(napi));
params.flags = IORING_SETUP_SINGLE_ISSUER;
if (opt.defer_tw) {
params.flags |= IORING_SETUP_DEFER_TASKRUN;
} else if (opt.sq_poll) {
params.flags = IORING_SETUP_SQPOLL;
params.sq_thread_idle = 50;
} else {
params.flags |= IORING_SETUP_COOP_TASKRUN;
}
ret = io_uring_queue_init_params(RINGSIZE, &ctx.ring, &params);
if (ret) {
fprintf(stderr, "io_uring_queue_init_params() failed: (%d) %s\n",
ret, strerror(-ret));
exit(1);
}
if (opt.timeout || opt.prefer_busy_poll) {
napi.prefer_busy_poll = opt.prefer_busy_poll;
napi.busy_poll_to = opt.timeout;
ret = io_uring_register_napi(&ctx.ring, &napi);
if (ret) {
fprintf(stderr, "io_uring_register_napi: %d\n", ret);
exit(1);
}
}
if (opt.busy_loop)
tsPtr = &ts;
else
tsPtr = NULL;
// Use realtime scheduler.
setProcessScheduler();
// Copy payload.
clock_gettime(CLOCK_REALTIME, &ctx.ts);
// Setup context.
ctx.napi_check = false;
ctx.buffer_len = sizeof(struct timespec);
ctx.num_pings = opt.num_pings;
ctx.rtt_index = 0;
ctx.rtt = (double *)malloc(sizeof(double) * opt.num_pings);
if (!ctx.rtt) {
fprintf(stderr, "Cannot allocate results array\n");
exit(1);
}
// Send initial message to get napi id.
sendPing(&ctx);
while (ctx.num_pings != 0) {
int res;
unsigned num_completed = 0;
unsigned head;
struct io_uring_cqe *cqe;
do {
res = io_uring_submit_and_wait_timeout(&ctx.ring, &cqe, 1, tsPtr, NULL);
if (res >= 0)
break;
else if (res == -ETIME)
continue;
fprintf(stderr, "submit_and_wait: %d\n", res);
exit(1);
} while (1);
io_uring_for_each_cqe(&ctx.ring, head, cqe) {
++num_completed;
if (completion(&ctx, cqe))
goto out;
}
if (num_completed)
io_uring_cq_advance(&ctx.ring, num_completed);
}
printStats(&ctx);
out:
// Clean up.
if (opt.timeout || opt.prefer_busy_poll) {
ret = io_uring_unregister_napi(&ctx.ring, &napi);
if (ret)
fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
if (opt.timeout != napi.busy_poll_to ||
opt.prefer_busy_poll != napi.prefer_busy_poll) {
fprintf(stderr, "Expected busy poll to = %d, got %d\n",
opt.timeout, napi.busy_poll_to);
fprintf(stderr, "Expected prefer busy poll = %d, got %d\n",
opt.prefer_busy_poll, napi.prefer_busy_poll);
}
} else {
ret = io_uring_unregister_napi(&ctx.ring, NULL);
if (ret)
fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
}
io_uring_queue_exit(&ctx.ring);
free(ctx.rtt);
close(ctx.sockfd);
return 0;
}
examples/napi-busy-poll-server.c
+450
View File
@@ -0,0 +1,450 @@
/* SPDX-License-Identifier: MIT */
/*
* Simple ping/pong backend which can use the io_uring NAPI support.
*
* Needs to be run as root because it sets SCHED_FIFO scheduling class,
* but will work without that.
*
* Example:
*
* sudo examples/napi-busy-poll-server -l -a 192.168.2.2 -n100000 \
* -p4444 -t10 -b -u
*
* will respond to 100k packages, using NAPI.
*/
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <liburing.h>
#include <math.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#define MAXBUFLEN 100
#define PORTNOLEN 10
#define ADDRLEN 80
#define RINGSIZE 1024
#define printable(ch) (isprint((unsigned char)ch) ? ch : '#')
enum {
IOURING_RECV,
IOURING_SEND,
IOURING_RECVMSG,
IOURING_SENDMSG
};
struct ctx
{
struct io_uring ring;
union {
struct sockaddr_in6 saddr6;
struct sockaddr_in saddr;
};
struct iovec iov;
struct msghdr msg;
int sockfd;
int buffer_len;
int num_pings;
bool napi_check;
union {
char buffer[MAXBUFLEN];
struct timespec ts;
};
};
struct options
{
int num_pings;
__u32 timeout;
bool listen;
bool defer_tw;
bool sq_poll;
bool busy_loop;
bool prefer_busy_poll;
bool ipv6;
char port[PORTNOLEN];
char addr[ADDRLEN];
};
static struct options opt;
static struct option longopts[] =
{
{"address" , 1, NULL, 'a'},
{"busy" , 0, NULL, 'b'},
{"help" , 0, NULL, 'h'},
{"listen" , 0, NULL, 'l'},
{"num_pings", 1, NULL, 'n'},
{"port" , 1, NULL, 'p'},
{"prefer" , 1, NULL, 'u'},
{"sqpoll" , 0, NULL, 's'},
{"timeout" , 1, NULL, 't'},
{NULL , 0, NULL, 0 }
};
static void printUsage(const char *name)
{
fprintf(stderr,
"Usage: %s [-l|--listen] [-a|--address ip_address] [-p|--port port-no] [-s|--sqpoll]"
" [-b|--busy] [-n|--num pings] [-t|--timeout busy-poll-timeout] [-u|--prefer] [-6] [-h|--help]\n"
" --listen\n"
"-l : Server mode\n"
"--address\n"
"-a : remote or local ipv6 address\n"
"--busy\n"
"-b : busy poll io_uring instead of blocking.\n"
"--num_pings\n"
"-n : number of pings\n"
"--port\n"
"-p : port\n"
"--sqpoll\n"
"-s : Configure io_uring to use SQPOLL thread\n"
"--timeout\n"
"-t : Configure NAPI busy poll timeout"
"--prefer\n"
"-u : prefer NAPI busy poll\n"
"-6 : use IPV6\n"
"--help\n"
"-h : Display this usage message\n\n",
name);
}
static void printError(const char *msg, int opt)
{
if (msg && opt)
fprintf(stderr, "%s (-%c)\n", msg, printable(opt));
}
static void setProcessScheduler(void)
{
struct sched_param param;
param.sched_priority = sched_get_priority_max(SCHED_FIFO);
if (sched_setscheduler(0, SCHED_FIFO, &param) < 0)
fprintf(stderr, "sched_setscheduler() failed: (%d) %s\n",
errno, strerror(errno));
}
static uint64_t encodeUserData(char type, int fd)
{
return (uint32_t)fd | ((__u64)type << 56);
}
static void decodeUserData(uint64_t data, char *type, int *fd)
{
*type = data >> 56;
*fd = data & 0xffffffffU;
}
static const char *opTypeToStr(char type)
{
const char *res;
switch (type) {
case IOURING_RECV:
res = "IOURING_RECV";
break;
case IOURING_SEND:
res = "IOURING_SEND";
break;
case IOURING_RECVMSG:
res = "IOURING_RECVMSG";
break;
case IOURING_SENDMSG:
res = "IOURING_SENDMSG";
break;
default:
res = "Unknown";
}
return res;
}
static void reportNapi(struct ctx *ctx)
{
unsigned int napi_id = 0;
socklen_t len = sizeof(napi_id);
getsockopt(ctx->sockfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);
if (napi_id)
printf(" napi id: %d\n", napi_id);
else
printf(" unassigned napi id\n");
ctx->napi_check = true;
}
static void sendPing(struct ctx *ctx)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
io_uring_prep_sendmsg(sqe, ctx->sockfd, &ctx->msg, 0);
sqe->user_data = encodeUserData(IOURING_SENDMSG, ctx->sockfd);
}
static void receivePing(struct ctx *ctx)
{
struct io_uring_sqe *sqe;
bzero(&ctx->msg, sizeof(struct msghdr));
if (opt.ipv6) {
ctx->msg.msg_name = &ctx->saddr6;
ctx->msg.msg_namelen = sizeof(struct sockaddr_in6);
} else {
ctx->msg.msg_name = &ctx->saddr;
ctx->msg.msg_namelen = sizeof(struct sockaddr_in);
}
ctx->iov.iov_base = ctx->buffer;
ctx->iov.iov_len = MAXBUFLEN;
ctx->msg.msg_iov = &ctx->iov;
ctx->msg.msg_iovlen = 1;
sqe = io_uring_get_sqe(&ctx->ring);
io_uring_prep_recvmsg(sqe, ctx->sockfd, &ctx->msg, 0);
sqe->user_data = encodeUserData(IOURING_RECVMSG, ctx->sockfd);
}
static void completion(struct ctx *ctx, struct io_uring_cqe *cqe)
{
char type;
int fd;
int res = cqe->res;
decodeUserData(cqe->user_data, &type, &fd);
if (res < 0) {
fprintf(stderr, "unexpected %s failure: (%d) %s\n",
opTypeToStr(type), -res, strerror(-res));
abort();
}
switch (type) {
case IOURING_SENDMSG:
receivePing(ctx);
--ctx->num_pings;
break;
case IOURING_RECVMSG:
ctx->iov.iov_len = res;
sendPing(ctx);
if (!ctx->napi_check)
reportNapi(ctx);
break;
default:
fprintf(stderr, "unexpected %s completion\n",
opTypeToStr(type));
abort();
break;
}
}
int main(int argc, char *argv[])
{
int flag;
struct ctx ctx;
struct __kernel_timespec *tsPtr;
struct __kernel_timespec ts;
struct io_uring_params params;
struct io_uring_napi napi;
int ret, af;
memset(&opt, 0, sizeof(struct options));
// Process flags.
while ((flag = getopt_long(argc, argv, ":lhs:bua:n:p:t:6d:", longopts, NULL)) != -1) {
switch (flag) {
case 'a':
strcpy(opt.addr, optarg);
break;
case 'b':
opt.busy_loop = true;
break;
case 'h':
printUsage(argv[0]);
exit(0);
break;
case 'l':
opt.listen = true;
break;
case 'n':
opt.num_pings = atoi(optarg) + 1;
break;
case 'p':
strcpy(opt.port, optarg);
break;
case 's':
opt.sq_poll = !!atoi(optarg);
break;
case 't':
opt.timeout = atoi(optarg);
break;
case 'u':
opt.prefer_busy_poll = true;
break;
case '6':
opt.ipv6 = true;
break;
case 'd':
opt.defer_tw = !!atoi(optarg);
break;
case ':':
printError("Missing argument", optopt);
printUsage(argv[0]);
exit(-1);
break;
case '?':
printError("Unrecognized option", optopt);
printUsage(argv[0]);
exit(-1);
break;
default:
fprintf(stderr, "Fatal: Unexpected case in CmdLineProcessor switch()\n");
exit(-1);
break;
}
}
if (strlen(opt.addr) == 0) {
fprintf(stderr, "address option is mandatory\n");
printUsage(argv[0]);
exit(1);
}
if (opt.ipv6) {
af = AF_INET6;
ctx.saddr6.sin6_port = htons(atoi(opt.port));
ctx.saddr6.sin6_family = AF_INET6;
} else {
af = AF_INET;
ctx.saddr.sin_port = htons(atoi(opt.port));
ctx.saddr.sin_family = AF_INET;
}
if (opt.ipv6)
ret = inet_pton(AF_INET6, opt.addr, &ctx.saddr6.sin6_addr);
else
ret = inet_pton(AF_INET, opt.addr, &ctx.saddr.sin_addr);
if (ret <= 0) {
fprintf(stderr, "inet_pton error for %s\n", optarg);
printUsage(argv[0]);
exit(1);
}
// Connect to server.
fprintf(stdout, "Listening %s : %s...\n", opt.addr, opt.port);
if ((ctx.sockfd = socket(af, SOCK_DGRAM, 0)) < 0) {
fprintf(stderr, "socket() failed: (%d) %s\n", errno, strerror(errno));
exit(1);
}
if (opt.ipv6)
ret = bind(ctx.sockfd, (struct sockaddr *)&ctx.saddr6, sizeof(struct sockaddr_in6));
else
ret = bind(ctx.sockfd, (struct sockaddr *)&ctx.saddr, sizeof(struct sockaddr_in));
if (ret < 0) {
fprintf(stderr, "bind() failed: (%d) %s\n", errno, strerror(errno));
exit(1);
}
// Setup ring.
memset(&params, 0, sizeof(params));
memset(&ts, 0, sizeof(ts));
memset(&napi, 0, sizeof(napi));
params.flags = IORING_SETUP_SINGLE_ISSUER;
if (opt.defer_tw) {
params.flags |= IORING_SETUP_DEFER_TASKRUN;
} else if (opt.sq_poll) {
params.flags = IORING_SETUP_SQPOLL;
params.sq_thread_idle = 50;
} else {
params.flags |= IORING_SETUP_COOP_TASKRUN;
}
ret = io_uring_queue_init_params(RINGSIZE, &ctx.ring, &params);
if (ret) {
fprintf(stderr, "io_uring_queue_init_params() failed: (%d) %s\n",
ret, strerror(-ret));
exit(1);
}
if (opt.timeout || opt.prefer_busy_poll) {
napi.prefer_busy_poll = opt.prefer_busy_poll;
napi.busy_poll_to = opt.timeout;
ret = io_uring_register_napi(&ctx.ring, &napi);
if (ret) {
fprintf(stderr, "io_uring_register_napi: %d\n", ret);
exit(1);
}
}
if (opt.busy_loop)
tsPtr = &ts;
else
tsPtr = NULL;
// Use realtime scheduler.
setProcessScheduler();
// Copy payload.
clock_gettime(CLOCK_REALTIME, &ctx.ts);
// Setup context.
ctx.napi_check = false;
ctx.buffer_len = sizeof(struct timespec);
ctx.num_pings = opt.num_pings;
// Receive initial message to get napi id.
receivePing(&ctx);
while (ctx.num_pings != 0) {
int res;
unsigned int num_completed = 0;
unsigned int head;
struct io_uring_cqe *cqe;
do {
res = io_uring_submit_and_wait_timeout(&ctx.ring, &cqe, 1, tsPtr, NULL);
if (res >= 0)
break;
else if (res == -ETIME)
continue;
fprintf(stderr, "submit_and_wait: %d\n", res);
exit(1);
} while (1);
io_uring_for_each_cqe(&ctx.ring, head, cqe) {
++num_completed;
completion(&ctx, cqe);
}
if (num_completed)
io_uring_cq_advance(&ctx.ring, num_completed);
}
// Clean up.
if (opt.timeout || opt.prefer_busy_poll) {
ret = io_uring_unregister_napi(&ctx.ring, &napi);
if (ret)
fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
}
io_uring_queue_exit(&ctx.ring);
close(ctx.sockfd);
return 0;
}
examples/proxy.c
+2461
View File
File diff suppressed because it is too large Load Diff
examples/proxy.h
+102
View File
@@ -0,0 +1,102 @@
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_PROXY_H
#define LIBURING_PROXY_H
#include <sys/time.h>
/*
* Generic opcode agnostic encoding to sqe/cqe->user_data
*/
struct userdata {
union {
struct {
uint16_t op_tid; /* 4 bits op, 12 bits tid */
uint16_t bid;
uint16_t fd;
};
uint64_t val;
};
};
#define OP_SHIFT (12)
#define TID_MASK ((1U << 12) - 1)
/*
* Packs the information that we will need at completion time into the
* sqe->user_data field, which is passed back in the completion in
* cqe->user_data. Some apps would need more space than this, and in fact
* I'd love to pack the requested IO size in here, and it's not uncommon to
* see apps use this field as just a cookie to either index a data structure
* at completion time, or even just put the pointer to the associated
* structure into this field.
*/
static inline void __encode_userdata(struct io_uring_sqe *sqe, int tid, int op,
int bid, int fd)
{
struct userdata ud = {
.op_tid = (op << OP_SHIFT) | tid,
.bid = bid,
.fd = fd
};
io_uring_sqe_set_data64(sqe, ud.val);
}
static inline uint64_t __raw_encode(int tid, int op, int bid, int fd)
{
struct userdata ud = {
.op_tid = (op << OP_SHIFT) | tid,
.bid = bid,
.fd = fd
};
return ud.val;
}
static inline int cqe_to_op(struct io_uring_cqe *cqe)
{
struct userdata ud = { .val = cqe->user_data };
return ud.op_tid >> OP_SHIFT;
}
static inline int cqe_to_bid(struct io_uring_cqe *cqe)
{
struct userdata ud = { .val = cqe->user_data };
return ud.bid;
}
static inline int cqe_to_fd(struct io_uring_cqe *cqe)
{
struct userdata ud = { .val = cqe->user_data };
return ud.fd;
}
static unsigned long long mtime_since(const struct timeval *s,
const struct timeval *e)
{
long long sec, usec;
sec = e->tv_sec - s->tv_sec;
usec = (e->tv_usec - s->tv_usec);
if (sec > 0 && usec < 0) {
sec--;
usec += 1000000;
}
sec *= 1000;
usec /= 1000;
return sec + usec;
}
static unsigned long long mtime_since_now(struct timeval *tv)
{
struct timeval end;
gettimeofday(&end, NULL);
return mtime_since(tv, &end);
}
#endif
examples/rsrc-update-bench.c
+100
View File
@@ -0,0 +1,100 @@
/* SPDX-License-Identifier: MIT */
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <poll.h>
#include <sys/time.h>
#include <sys/wait.h>
#include "liburing.h"
static unsigned long runtime_ms = 10000;
static unsigned long gettimeofday_ms(void)
{
struct timeval tv;
gettimeofday(&tv, NULL);
return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
}
int main(void)
{
unsigned long tstop;
unsigned long nr_reqs = 0;
struct io_uring_cqe *cqe;
struct io_uring_sqe *sqe;
struct io_uring ring;
int pipe1[2];
int ret, i, qd = 32;
int table_size = 128;
if (pipe(pipe1) != 0) {
perror("pipe");
return 1;
}
ret = io_uring_queue_init(1024, &ring, IORING_SETUP_SINGLE_ISSUER |
IORING_SETUP_DEFER_TASKRUN);
if (ret) {
fprintf(stderr, "io_uring_queue_init failed: %d\n", ret);
return 1;
}
ret = io_uring_register_ring_fd(&ring);
if (ret < 0) {
fprintf(stderr, "io_uring_register_ring_fd failed\n");
return 1;
}
ret = io_uring_register_files_sparse(&ring, table_size);
if (ret < 0) {
fprintf(stderr, "io_uring_register_files_sparse failed\n");
return 1;
}
for (i = 0; i < table_size; i++) {
ret = io_uring_register_files_update(&ring, i, pipe1, 1);
if (ret < 0) {
fprintf(stderr, "io_uring_register_files_update failed\n");
return 1;
}
}
srand(time(NULL));
tstop = gettimeofday_ms() + runtime_ms;
do {
int off = rand();
for (i = 0; i < qd; i++) {
sqe = io_uring_get_sqe(&ring);
int roff = (off + i) % table_size;
io_uring_prep_files_update(sqe, pipe1, 1, roff);
}
ret = io_uring_submit(&ring);
if (ret != qd) {
fprintf(stderr, "child: sqe submit failed: %d\n", ret);
return 1;
}
for (i = 0; i < qd; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret < 0) {
fprintf(stderr, "child: wait completion %d\n", ret);
break;
}
io_uring_cqe_seen(&ring, cqe);
nr_reqs++;
}
} while (gettimeofday_ms() < tstop);
fprintf(stderr, "max updates/s: %lu\n", nr_reqs * 1000UL / runtime_ms);
io_uring_queue_exit(&ring);
close(pipe1[0]);
close(pipe1[1]);
return 0;
}
examples/send-zerocopy.c
+380 -61
View File
@@ -5,15 +5,17 @@
#include <stdint.h>
#include <assert.h>
#include <errno.h>
#include <error.h>
#include <limits.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdbool.h>
#include <stdarg.h>
#include <string.h>
#include <pthread.h>
#include <poll.h>
#include <sched.h>
#include <arpa/inet.h>
#include <linux/errqueue.h>
#include <linux/if_packet.h>
#include <linux/ipv6.h>
#include <linux/socket.h>
@@ -35,27 +37,107 @@
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/mman.h>
#include <linux/mman.h>
#include <signal.h>
#include "liburing.h"
#define ZC_TAG 0xfffffffULL
#define MAX_SUBMIT_NR 512
#define MAX_THREADS 100
struct thread_data {
pthread_t thread;
void *ret;
int idx;
unsigned long long packets;
unsigned long long bytes;
unsigned long long dt_ms;
struct sockaddr_storage dst_addr;
int fd;
};
static bool cfg_reg_ringfd = true;
static bool cfg_fixed_files = 1;
static bool cfg_zc = 1;
static int cfg_nr_reqs = 8;
static bool cfg_fixed_buf = 1;
static bool cfg_hugetlb = 0;
static bool cfg_defer_taskrun = 0;
static int cfg_cpu = -1;
static bool cfg_rx = 0;
static unsigned cfg_nr_threads = 1;
static int cfg_family = PF_UNSPEC;
static int cfg_type = 0;
static int cfg_payload_len;
static int cfg_port = 8000;
static int cfg_runtime_ms = 4200;
static bool cfg_rx_poll = false;
static socklen_t cfg_alen;
static struct sockaddr_storage cfg_dst_addr;
static char *str_addr = NULL;
static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
static char payload_buf[IP_MAXPACKET] __attribute__((aligned(4096)));
static char *payload;
static struct thread_data threads[MAX_THREADS];
static pthread_barrier_t barrier;
static bool should_stop = false;
static void sigint_handler(__attribute__((__unused__)) int sig)
{
/* kill if should_stop can't unblock threads fast enough */
if (should_stop)
_exit(-1);
should_stop = true;
}
/*
* Implementation of error(3), prints an error message and exits.
*/
static void t_error(int status, int errnum, const char *format, ...)
{
va_list args;
va_start(args, format);
vfprintf(stderr, format, args);
if (errnum)
fprintf(stderr, ": %s", strerror(errnum));
fprintf(stderr, "\n");
va_end(args);
exit(status);
}
static void set_cpu_affinity(void)
{
cpu_set_t mask;
if (cfg_cpu == -1)
return;
CPU_ZERO(&mask);
CPU_SET(cfg_cpu, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask))
t_error(1, errno, "unable to pin cpu\n");
}
static void set_iowq_affinity(struct io_uring *ring)
{
cpu_set_t mask;
int ret;
if (cfg_cpu == -1)
return;
CPU_ZERO(&mask);
CPU_SET(cfg_cpu, &mask);
ret = io_uring_register_iowq_aff(ring, 1, &mask);
if (ret)
t_error(1, ret, "unabled to set io-wq affinity\n");
}
static unsigned long gettimeofday_ms(void)
{
@@ -68,7 +150,7 @@ static unsigned long gettimeofday_ms(void)
static void do_setsockopt(int fd, int level, int optname, int val)
{
if (setsockopt(fd, level, optname, &val, sizeof(val)))
error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
t_error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
}
static void setup_sockaddr(int domain, const char *str_addr,
@@ -76,42 +158,156 @@ static void setup_sockaddr(int domain, const char *str_addr,
{
struct sockaddr_in6 *addr6 = (void *) sockaddr;
struct sockaddr_in *addr4 = (void *) sockaddr;
int port = cfg_port;
switch (domain) {
case PF_INET:
memset(addr4, 0, sizeof(*addr4));
addr4->sin_family = AF_INET;
addr4->sin_port = htons(cfg_port);
addr4->sin_port = htons(port);
if (str_addr &&
inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
error(1, 0, "ipv4 parse error: %s", str_addr);
t_error(1, 0, "ipv4 parse error: %s", str_addr);
break;
case PF_INET6:
memset(addr6, 0, sizeof(*addr6));
addr6->sin6_family = AF_INET6;
addr6->sin6_port = htons(cfg_port);
addr6->sin6_port = htons(port);
if (str_addr &&
inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
error(1, 0, "ipv6 parse error: %s", str_addr);
t_error(1, 0, "ipv6 parse error: %s", str_addr);
break;
default:
error(1, 0, "illegal domain");
t_error(1, 0, "illegal domain");
}
}
static int do_setup_tx(int domain, int type, int protocol)
static int do_poll(int fd, int events)
{
int fd;
struct pollfd pfd;
int ret;
pfd.events = events;
pfd.revents = 0;
pfd.fd = fd;
ret = poll(&pfd, 1, -1);
if (ret == -1)
t_error(1, errno, "poll");
return ret && (pfd.revents & events);
}
/* Flush all outstanding bytes for the tcp receive queue */
static int do_flush_tcp(struct thread_data *td, int fd)
{
int ret;
/* MSG_TRUNC flushes up to len bytes */
ret = recv(fd, NULL, 1 << 21, MSG_TRUNC | MSG_DONTWAIT);
if (ret == -1 && errno == EAGAIN)
return 0;
if (ret == -1)
t_error(1, errno, "flush");
if (!ret)
return 1;
td->packets++;
td->bytes += ret;
return 0;
}
/* Flush all outstanding datagrams. Verify first few bytes of each. */
static int do_flush_datagram(struct thread_data *td, int fd)
{
long ret, off = 0;
char buf[64];
/* MSG_TRUNC will return full datagram length */
ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT | MSG_TRUNC);
if (ret == -1 && errno == EAGAIN)
return 0;
if (ret == -1)
t_error(1, errno, "recv");
if (ret != cfg_payload_len)
t_error(1, 0, "recv: ret=%u != %u", ret, cfg_payload_len);
if ((unsigned long) ret > sizeof(buf) - off)
ret = sizeof(buf) - off;
if (memcmp(buf + off, payload, ret))
t_error(1, 0, "recv: data mismatch");
td->packets++;
td->bytes += cfg_payload_len;
return 0;
}
static void do_setup_rx(int domain, int type, int protocol)
{
struct sockaddr_storage addr = {};
struct thread_data *td;
int listen_fd, fd;
unsigned int i;
fd = socket(domain, type, protocol);
if (fd == -1)
error(1, errno, "socket t");
t_error(1, errno, "socket r");
do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);
do_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, 1);
if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))
error(1, errno, "connect");
return fd;
setup_sockaddr(cfg_family, str_addr, &addr);
if (bind(fd, (void *)&addr, cfg_alen))
t_error(1, errno, "bind");
if (type != SOCK_STREAM) {
if (cfg_nr_threads != 1)
t_error(1, 0, "udp rx cant multithread");
threads[0].fd = fd;
return;
}
listen_fd = fd;
if (listen(listen_fd, cfg_nr_threads))
t_error(1, errno, "listen");
for (i = 0; i < cfg_nr_threads; i++) {
td = &threads[i];
fd = accept(listen_fd, NULL, NULL);
if (fd == -1)
t_error(1, errno, "accept");
td->fd = fd;
}
if (close(listen_fd))
t_error(1, errno, "close listen sock");
}
static void *do_rx(void *arg)
{
struct thread_data *td = arg;
const int cfg_receiver_wait_ms = 400;
uint64_t tstop;
int ret, fd = td->fd;
tstop = gettimeofday_ms() + cfg_runtime_ms + cfg_receiver_wait_ms;
do {
if (cfg_type == SOCK_STREAM)
ret = do_flush_tcp(td, fd);
else
ret = do_flush_datagram(td, fd);
if (ret)
break;
do_poll(fd, POLLIN);
} while (gettimeofday_ms() < tstop);
if (close(fd))
t_error(1, errno, "close");
pthread_exit(&td->ret);
return NULL;
}
static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
@@ -125,35 +321,47 @@ static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
ret = io_uring_wait_cqe(ring, &cqe);
if (ret)
error(1, ret, "wait cqe");
t_error(1, ret, "wait cqe");
return cqe;
}
static void do_tx(int domain, int type, int protocol)
static void do_tx(struct thread_data *td, int domain, int type, int protocol)
{
unsigned long packets = 0;
unsigned long bytes = 0;
const int notif_slack = 128;
struct io_uring ring;
struct iovec iov;
uint64_t tstop;
uint64_t tstart;
int i, fd, ret;
int compl_cqes = 0;
int ring_flags = IORING_SETUP_COOP_TASKRUN | IORING_SETUP_SINGLE_ISSUER;
unsigned loop = 0;
fd = do_setup_tx(domain, type, protocol);
if (cfg_defer_taskrun)
ring_flags |= IORING_SETUP_DEFER_TASKRUN;
ret = io_uring_queue_init(512, &ring, IORING_SETUP_COOP_TASKRUN);
fd = socket(domain, type, protocol);
if (fd == -1)
t_error(1, errno, "socket t");
if (connect(fd, (void *)&td->dst_addr, cfg_alen))
t_error(1, errno, "connect, idx %i", td->idx);
ret = io_uring_queue_init(512, &ring, ring_flags);
if (ret)
error(1, ret, "io_uring: queue init");
t_error(1, ret, "io_uring: queue init");
set_cpu_affinity();
set_iowq_affinity(&ring);
if (cfg_fixed_files) {
ret = io_uring_register_files(&ring, &fd, 1);
if (ret < 0)
error(1, ret, "io_uring: files registration");
t_error(1, ret, "io_uring: files registration");
}
if (cfg_reg_ringfd) {
ret = io_uring_register_ring_fd(&ring);
if (ret < 0)
error(1, ret, "io_uring: io_uring_register_ring_fd");
t_error(1, ret, "io_uring: io_uring_register_ring_fd");
}
iov.iov_base = payload;
@@ -161,9 +369,22 @@ static void do_tx(int domain, int type, int protocol)
ret = io_uring_register_buffers(&ring, &iov, 1);
if (ret)
error(1, ret, "io_uring: buffer registration");
t_error(1, ret, "io_uring: buffer registration");
tstop = gettimeofday_ms() + cfg_runtime_ms;
if (cfg_rx_poll) {
struct io_uring_sqe *sqe;
sqe = io_uring_get_sqe(&ring);
io_uring_prep_poll_add(sqe, fd, POLLIN);
ret = io_uring_submit(&ring);
if (ret != 1)
t_error(1, ret, "submit poll");
}
pthread_barrier_wait(&barrier);
tstart = gettimeofday_ms();
do {
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
@@ -191,16 +412,20 @@ static void do_tx(int domain, int type, int protocol)
}
}
ret = io_uring_submit(&ring);
if (cfg_defer_taskrun && compl_cqes >= notif_slack)
ret = io_uring_submit_and_get_events(&ring);
else
ret = io_uring_submit(&ring);
if (ret != cfg_nr_reqs)
error(1, ret, "submit");
t_error(1, ret, "submit");
for (i = 0; i < cfg_nr_reqs; i++) {
cqe = wait_cqe_fast(&ring);
if (cqe->flags & IORING_CQE_F_NOTIF) {
if (cqe->flags & IORING_CQE_F_MORE)
error(1, -EINVAL, "F_MORE notif");
t_error(1, -EINVAL, "F_MORE notif");
compl_cqes--;
i--;
io_uring_cqe_seen(&ring, cqe);
@@ -210,28 +435,27 @@ static void do_tx(int domain, int type, int protocol)
compl_cqes++;
if (cqe->res >= 0) {
packets++;
bytes += cqe->res;
td->packets++;
td->bytes += cqe->res;
} else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE ||
cqe->res == -ECONNRESET) {
fprintf(stderr, "Connection failure");
fprintf(stderr, "Connection failure\n");
goto out_fail;
} else if (cqe->res != -EAGAIN) {
error(1, cqe->res, "send failed");
t_error(1, cqe->res, "send failed");
}
io_uring_cqe_seen(&ring, cqe);
}
} while (gettimeofday_ms() < tstop);
if (should_stop)
break;
} while ((++loop % 16 != 0) || gettimeofday_ms() < tstart + cfg_runtime_ms);
td->dt_ms = gettimeofday_ms() - tstart;
out_fail:
shutdown(fd, SHUT_RDWR);
if (close(fd))
error(1, errno, "close");
fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
packets, bytes >> 20,
packets / (cfg_runtime_ms / 1000),
(bytes >> 20) / (cfg_runtime_ms / 1000));
t_error(1, errno, "close");
while (compl_cqes) {
struct io_uring_cqe *cqe = wait_cqe_fast(&ring);
@@ -242,47 +466,67 @@ out_fail:
io_uring_queue_exit(&ring);
}
static void do_test(int domain, int type, int protocol)
static void *do_test(void *arg)
{
int i;
struct thread_data *td = arg;
int protocol = 0;
for (i = 0; i < IP_MAXPACKET; i++)
payload[i] = 'a' + (i % 26);
setup_sockaddr(cfg_family, str_addr, &td->dst_addr);
do_tx(domain, type, protocol);
do_tx(td, cfg_family, cfg_type, protocol);
pthread_exit(&td->ret);
return NULL;
}
static void usage(const char *filepath)
{
error(1, 0, "Usage: %s [-n<N>] [-z<val>] [-s<payload size>] "
"(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
printf("Usage:\t%s <protocol> <ip-version> -D<addr> [options]\n", filepath);
printf("\t%s <protocol> <ip-version> -R [options]\n\n", filepath);
printf(" -4\t\tUse IPv4\n");
printf(" -6\t\tUse IPv4\n");
printf(" -D <address>\tDestination address\n");
printf(" -p <port>\tServer port to listen on/connect to\n");
printf(" -s <size>\tBytes per request\n");
printf(" -s <size>\tBytes per request\n");
printf(" -n <nr>\tNumber of parallel requests\n");
printf(" -z <mode>\tZerocopy mode, 0 to disable, enabled otherwise\n");
printf(" -b <mode>\tUse registered buffers\n");
printf(" -l <mode>\tUse huge pages\n");
printf(" -d\t\tUse defer taskrun\n");
printf(" -C <cpu>\tPin to the specified CPU\n");
printf(" -T <nr>\tNumber of threads to use for sending\n");
printf(" -R\t\tPlay the server role\n");
printf(" -t <seconds>\tTime in seconds\n");
}
static void parse_opts(int argc, char **argv)
{
const int max_payload_len = sizeof(payload) -
const int max_payload_len = IP_MAXPACKET -
sizeof(struct ipv6hdr) -
sizeof(struct tcphdr) -
40 /* max tcp options */;
int c;
char *daddr = NULL;
if (argc <= 1)
if (argc <= 1) {
usage(argv[0]);
exit(0);
}
cfg_payload_len = max_payload_len;
while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:k")) != -1) {
while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:l:dC:T:Ry")) != -1) {
switch (c) {
case '4':
if (cfg_family != PF_UNSPEC)
error(1, 0, "Pass one of -4 or -6");
t_error(1, 0, "Pass one of -4 or -6");
cfg_family = PF_INET;
cfg_alen = sizeof(struct sockaddr_in);
break;
case '6':
if (cfg_family != PF_UNSPEC)
error(1, 0, "Pass one of -4 or -6");
t_error(1, 0, "Pass one of -4 or -6");
cfg_family = PF_INET6;
cfg_alen = sizeof(struct sockaddr_in6);
break;
@@ -307,15 +551,35 @@ static void parse_opts(int argc, char **argv)
case 'b':
cfg_fixed_buf = strtoul(optarg, NULL, 0);
break;
case 'l':
cfg_hugetlb = strtoul(optarg, NULL, 0);
break;
case 'd':
cfg_defer_taskrun = 1;
break;
case 'C':
cfg_cpu = strtol(optarg, NULL, 0);
break;
case 'T':
cfg_nr_threads = strtol(optarg, NULL, 0);
if (cfg_nr_threads > MAX_THREADS)
t_error(1, 0, "too many threads\n");
break;
case 'R':
cfg_rx = 1;
break;
case 'y':
cfg_rx_poll = 1;
break;
}
}
if (cfg_nr_reqs > MAX_SUBMIT_NR)
error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
t_error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
if (cfg_payload_len > max_payload_len)
error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
t_error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
setup_sockaddr(cfg_family, daddr, &cfg_dst_addr);
str_addr = daddr;
if (optind != argc - 1)
usage(argv[0]);
@@ -323,17 +587,72 @@ static void parse_opts(int argc, char **argv)
int main(int argc, char **argv)
{
unsigned long long tsum = 0;
unsigned long long packets = 0, bytes = 0;
struct thread_data *td;
const char *cfg_test;
unsigned int i;
void *res;
parse_opts(argc, argv);
set_cpu_affinity();
payload = payload_buf;
if (cfg_hugetlb) {
payload = mmap(NULL, 2*1024*1024, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS,
-1, 0);
if (payload == MAP_FAILED) {
fprintf(stderr, "hugetlb alloc failed\n");
return 1;
}
}
cfg_test = argv[argc - 1];
if (!strcmp(cfg_test, "tcp"))
do_test(cfg_family, SOCK_STREAM, 0);
cfg_type = SOCK_STREAM;
else if (!strcmp(cfg_test, "udp"))
do_test(cfg_family, SOCK_DGRAM, 0);
cfg_type = SOCK_DGRAM;
else
error(1, 0, "unknown cfg_test %s", cfg_test);
t_error(1, 0, "unknown cfg_test %s", cfg_test);
pthread_barrier_init(&barrier, NULL, cfg_nr_threads);
for (i = 0; i < IP_MAXPACKET; i++)
payload[i] = 'a' + (i % 26);
for (i = 0; i < cfg_nr_threads; i++) {
td = &threads[i];
td->idx = i;
}
if (cfg_rx)
do_setup_rx(cfg_family, cfg_type, 0);
if (!cfg_rx)
signal(SIGINT, sigint_handler);
for (i = 0; i < cfg_nr_threads; i++)
pthread_create(&threads[i].thread, NULL,
!cfg_rx ? do_test : do_rx, &threads[i]);
for (i = 0; i < cfg_nr_threads; i++) {
td = &threads[i];
pthread_join(td->thread, &res);
packets += td->packets;
bytes += td->bytes;
tsum += td->dt_ms;
}
tsum = tsum / cfg_nr_threads;
if (!tsum) {
printf("The run is too short, can't gather stats\n");
} else {
printf("packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n",
packets, bytes >> 20,
packets * 1000 / tsum,
(bytes >> 20) * 1000 / tsum);
}
pthread_barrier_destroy(&barrier);
return 0;
}
examples/ucontext-cp.c
+2 -17
View File
@@ -68,23 +68,8 @@ DEFINE_AWAIT_OP(readv)
DEFINE_AWAIT_OP(writev)
#undef DEFINE_AWAIT_OP
int await_poll(async_context *pctx, int fd, short poll_mask) {
struct io_uring_sqe *sqe = io_uring_get_sqe(pctx->ring);
struct io_uring_cqe *cqe;
if (!sqe)
return -1;
io_uring_prep_poll_add(sqe, fd, poll_mask);
io_uring_sqe_set_data(sqe, pctx);
swapcontext(&pctx->ctx_fnew, &pctx->ctx_main);
io_uring_peek_cqe(pctx->ring, &cqe);
assert(cqe);
io_uring_cqe_seen(pctx->ring, cqe);
return cqe->res;
}
int await_delay(async_context *pctx, time_t seconds) {
static int await_delay(async_context *pctx, time_t seconds)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(pctx->ring);
struct io_uring_cqe *cqe;
struct __kernel_timespec ts = {
liburing-ffi.pc.in
+12
View File
@@ -0,0 +1,12 @@
prefix=@prefix@
exec_prefix=${prefix}
libdir=@libdir@
includedir=@includedir@
Name: @NAME@
Version: @VERSION@
Description: io_uring FFI library
URL: https://git.kernel.dk/cgit/liburing/
Libs: -L${libdir} -luring-ffi
Cflags: -I${includedir}
liburing.pc.in
+2 -2
View File
@@ -6,7 +6,7 @@ includedir=@includedir@
Name: @NAME@
Version: @VERSION@
Description: io_uring library
URL: http://git.kernel.dk/cgit/liburing/
URL: https://git.kernel.dk/cgit/liburing/
Libs: -L${libdir} -luring
Cflags: -I${includedir}
Cflags: -I${includedir} -D_GNU_SOURCE
liburing.spec
+2 -2
View File
@@ -1,5 +1,5 @@
Name: liburing
Version: 2.3
Version: 2.7
Release: 1%{?dist}
Summary: Linux-native io_uring I/O access library
License: (GPLv2 with exceptions and LGPLv2+) or MIT
@@ -27,7 +27,7 @@ for the Linux-native io_uring.
%build
%set_build_flags
./configure --prefix=%{_prefix} --libdir=/%{_libdir} --libdevdir=/%{_libdir} --mandir=%{_mandir} --includedir=%{_includedir}
./configure --prefix=%{_prefix} --libdir=%{_libdir} --libdevdir=%{_libdir} --mandir=%{_mandir} --includedir=%{_includedir}
%make_build
make-debs.sh
Executable → Regular
+7 -5
View File
@@ -13,16 +13,18 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
set -xe
# Create dir for build
base=${1:-/tmp/release}
codename=$(lsb_release -sc)
distro=unstable
releasedir=$base/$(lsb_release -si)/liburing
rm -rf $releasedir
mkdir -p $releasedir
HEAD=$(which head)
DCH=$(which dch)
src_dir=$(readlink -e `basename $0`)
liburing_dir=$(dirname $src_dir)
@@ -38,12 +40,12 @@ cd ${releasedir}/${outfile}
git clean -dxf
# Change changelog if it's needed
cur_ver=`head -l debian/changelog | sed -n -e 's/.* (\(.*\)) .*/\1/p'`
cur_ver=`$HEAD < debian/changelog | sed -n -e 's/.* (\(.*\)) .*/\1/p'`
if [ "$cur_ver" != "$version-1" ]; then
dch -D $codename --force-distribution -b -v "$version-1" "new version"
$DCH -D $distro --force-distribution -b -v "$version-1" "new version"
fi
# Create tar archieve
# Create tar archive
cd ../
tar cvzf ${outfile}.tar.gz ${outfile}
ln -s ${outfile}.tar.gz ${orgfile}.orig.tar.gz
src/Makefile
+48 -9
View File
@@ -8,23 +8,29 @@ libdevdir ?= $(prefix)/lib
LIBURING_CFLAGS ?=
CPPFLAGS ?=
override CPPFLAGS += -D_GNU_SOURCE \
-Iinclude/ -include ../config-host.h
CFLAGS ?= -g -O3 -Wall -Wextra -fno-stack-protector
override CFLAGS += -Wno-unused-parameter -Wno-sign-compare \
-Iinclude/ -include ../config-host.h \
-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
CFLAGS ?= -O3 -Wall -Wextra -fno-stack-protector
override CFLAGS += -Wno-unused-parameter \
-DLIBURING_INTERNAL \
$(LIBURING_CFLAGS)
SO_CFLAGS=-fPIC $(CFLAGS)
L_CFLAGS=$(CFLAGS)
LINK_FLAGS=
LINK_FLAGS=-Wl,-z,defs
LINK_FLAGS+=$(LDFLAGS)
ENABLE_SHARED ?= 1
soname=liburing.so.$(VERSION_MAJOR)
libname=liburing.so.$(VERSION)
ffi_soname=liburing-ffi.so.$(VERSION_MAJOR)
ffi_libname=liburing-ffi.so.$(VERSION)
all_targets += liburing.a
all_targets += liburing-ffi.a
ifeq ($(ENABLE_SHARED),1)
all_targets += $(libname)
all_targets += $(ffi_libname)
endif
include ../Makefile.quiet
@@ -35,18 +41,20 @@ endif
all: $(all_targets)
liburing_srcs := setup.c queue.c register.c syscall.c
liburing_srcs := setup.c queue.c register.c syscall.c version.c
ifeq ($(CONFIG_NOLIBC),y)
liburing_srcs += nolibc.c
override CFLAGS += -nostdlib -nodefaultlibs -ffreestanding
override CPPFLAGS += -nostdlib -nodefaultlibs -ffreestanding
override LINK_FLAGS += -nostdlib -nodefaultlibs
override CFLAGS += -nostdlib -nodefaultlibs -ffreestanding -fno-builtin -fno-stack-protector
override CPPFLAGS += -nostdlib -nodefaultlibs -ffreestanding -fno-builtin -fno-stack-protector
override LINK_FLAGS += -nostdlib -nodefaultlibs $(libgcc_link_flag)
endif
override CPPFLAGS += -MT "$@" -MMD -MP -MF "$@.d"
liburing_objs := $(patsubst %.c,%.ol,$(liburing_srcs))
liburing_sobjs := $(patsubst %.c,%.os,$(liburing_srcs))
liburing_ffi_objs := ffi.ol
liburing_ffi_sobjs := ffi.os
%.os: %.c
$(QUIET_CC)$(CC) $(CPPFLAGS) $(SO_CFLAGS) -c -o $@ $<
@@ -65,25 +73,56 @@ liburing.a: $(liburing_objs)
$(QUIET_AR)$(AR) r liburing.a $^
$(QUIET_RANLIB)$(RANLIB) liburing.a
liburing-ffi.a: $(liburing_objs) $(liburing_ffi_objs)
@rm -f liburing-ffi.a
$(QUIET_AR)$(AR) r liburing-ffi.a $^
$(QUIET_RANLIB)$(RANLIB) liburing-ffi.a
$(libname): $(liburing_sobjs) liburing.map
$(QUIET_CC)$(CC) $(SO_CFLAGS) -shared -Wl,--version-script=liburing.map -Wl,-soname=$(soname) -o $@ $(liburing_sobjs) $(LINK_FLAGS)
$(ffi_libname): $(liburing_ffi_objs) $(liburing_ffi_sobjs) $(liburing_sobjs) liburing-ffi.map
$(QUIET_CC)$(CC) $(SO_CFLAGS) -shared -Wl,--version-script=liburing-ffi.map -Wl,-soname=$(ffi_soname) -o $@ $(liburing_sobjs) $(liburing_ffi_sobjs) $(LINK_FLAGS)
install: $(all_targets)
install -D -m 644 include/liburing/io_uring.h $(includedir)/liburing/io_uring.h
install -D -m 644 include/liburing.h $(includedir)/liburing.h
install -D -m 644 include/liburing/compat.h $(includedir)/liburing/compat.h
install -D -m 644 include/liburing/barrier.h $(includedir)/liburing/barrier.h
install -D -m 644 include/liburing/io_uring_version.h $(includedir)/liburing/io_uring_version.h
install -D -m 644 liburing.a $(libdevdir)/liburing.a
install -D -m 644 liburing-ffi.a $(libdevdir)/liburing-ffi.a
ifeq ($(ENABLE_SHARED),1)
install -D -m 755 $(libname) $(libdir)/$(libname)
install -D -m 755 $(ffi_libname) $(libdir)/$(ffi_libname)
ln -sf $(libname) $(libdir)/$(soname)
ln -sf $(relativelibdir)$(libname) $(libdevdir)/liburing.so
ln -sf $(ffi_libname) $(libdir)/$(ffi_soname)
ln -sf $(relativelibdir)$(ffi_libname) $(libdevdir)/liburing-ffi.so
endif
uninstall:
@rm -f $(includedir)/liburing/io_uring.h
@rm -f $(includedir)/liburing.h
@rm -f $(includedir)/liburing/compat.h
@rm -f $(includedir)/liburing/barrier.h
@rm -f $(includedir)/liburing/io_uring_version.h
@rm -f $(libdevdir)/liburing.a
@rm -f $(libdevdir)/liburing-ffi.a
ifeq ($(ENABLE_SHARED),1)
@rm -f $(libdir)/$(libname)
@rm -f $(libdir)/$(ffi_libname)
@rm -f $(libdir)/$(soname)
@rm -f $(libdevdir)/liburing.so
@rm -f $(libdir)/$(ffi_soname)
@rm -f $(libdevdir)/liburing-ffi.so
endif
clean:
@rm -f $(all_targets) $(liburing_objs) $(liburing_sobjs) $(soname).new
@rm -f $(all_targets) $(liburing_objs) $(liburing_sobjs) $(liburing_ffi_objs) $(liburing_ffi_sobjs) $(soname).new
@rm -f *.so* *.a *.o *.d
@rm -f include/liburing/compat.h
@rm -f include/liburing/io_uring_version.h
@# When cleaning, we don't include ../config-host.mak,
@# so the nolibc objects are always skipped, clean them up!
src/arch/aarch64/lib.h
+1 -2
View File
@@ -4,7 +4,6 @@
#define LIBURING_ARCH_AARCH64_LIB_H
#include <elf.h>
#include <sys/auxv.h>
#include "../../syscall.h"
static inline long __get_page_size(void)
@@ -21,7 +20,7 @@ static inline long __get_page_size(void)
ssize_t x;
x = __sys_read(fd, buf, sizeof(buf));
if (x < sizeof(buf))
if (x < (long) sizeof(buf))
break;
if (buf[0] == AT_PAGESZ) {
src/arch/riscv64/lib.h
+48
View File
@@ -0,0 +1,48 @@
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_ARCH_RISCV64_LIB_H
#define LIBURING_ARCH_RISCV64_LIB_H
#include <elf.h>
#include <sys/auxv.h>
#include "../../syscall.h"
static inline long __get_page_size(void)
{
Elf64_Off buf[2];
long ret = 4096;
int fd;
fd = __sys_open("/proc/self/auxv", O_RDONLY, 0);
if (fd < 0)
return ret;
while (1) {
ssize_t x;
x = __sys_read(fd, buf, sizeof(buf));
if (x < (long) sizeof(buf))
break;
if (buf[0] == AT_PAGESZ) {
ret = buf[1];
break;
}
}
__sys_close(fd);
return ret;
}
static inline long get_page_size(void)
{
static long cache_val;
if (cache_val)
return cache_val;
cache_val = __get_page_size();
return cache_val;
}
#endif /* #ifndef LIBURING_ARCH_RISCV64_LIB_H */
src/arch/riscv64/syscall.h
+100
View File
@@ -0,0 +1,100 @@
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_ARCH_RISCV64_SYSCALL_H
#define LIBURING_ARCH_RISCV64_SYSCALL_H
#if defined(__riscv) && __riscv_xlen == 64
#define __do_syscallM(...) ({ \
__asm__ volatile ( \
"ecall" \
: "=r"(a0) \
: __VA_ARGS__ \
: "memory", "a1"); \
(long) a0; \
})
#define __do_syscallN(...) ({ \
__asm__ volatile ( \
"ecall" \
: "=r"(a0) \
: __VA_ARGS__ \
: "memory"); \
(long) a0; \
})
#define __do_syscall0(__n) ({ \
register long a7 __asm__("a7") = __n; \
register long a0 __asm__("a0"); \
\
__do_syscallM("r" (a7)); \
})
#define __do_syscall1(__n, __a) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
\
__do_syscallM("r" (a7), "0" (a0)); \
})
#define __do_syscall2(__n, __a, __b) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
register __typeof__(__b) a1 __asm__("a1") = __b; \
\
__do_syscallN("r" (a7), "0" (a0), "r" (a1)); \
})
#define __do_syscall3(__n, __a, __b, __c) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
register __typeof__(__b) a1 __asm__("a1") = __b; \
register __typeof__(__c) a2 __asm__("a2") = __c; \
\
__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2)); \
})
#define __do_syscall4(__n, __a, __b, __c, __d) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
register __typeof__(__b) a1 __asm__("a1") = __b; \
register __typeof__(__c) a2 __asm__("a2") = __c; \
register __typeof__(__d) a3 __asm__("a3") = __d; \
\
__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3));\
})
#define __do_syscall5(__n, __a, __b, __c, __d, __e) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
register __typeof__(__b) a1 __asm__("a1") = __b; \
register __typeof__(__c) a2 __asm__("a2") = __c; \
register __typeof__(__d) a3 __asm__("a3") = __d; \
register __typeof__(__e) a4 __asm__("a4") = __e; \
\
__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3), \
"r"(a4)); \
})
#define __do_syscall6(__n, __a, __b, __c, __d, __e, __f) ({ \
register long a7 __asm__("a7") = __n; \
register __typeof__(__a) a0 __asm__("a0") = __a; \
register __typeof__(__b) a1 __asm__("a1") = __b; \
register __typeof__(__c) a2 __asm__("a2") = __c; \
register __typeof__(__d) a3 __asm__("a3") = __d; \
register __typeof__(__e) a4 __asm__("a4") = __e; \
register __typeof__(__f) a5 __asm__("a5") = __f; \
\
__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3), \
"r" (a4), "r"(a5)); \
})
#include "../syscall-defs.h"
#else /* #if defined(__riscv) && __riscv_xlen == 64 */
#include "../generic/syscall.h"
#endif /* #if defined(__riscv) && __riscv_xlen == 64 */
#endif /* #ifndef LIBURING_ARCH_RISCV64_SYSCALL_H */
src/ffi.c
+15
View File
@@ -0,0 +1,15 @@
/* SPDX-License-Identifier: MIT */
#define IOURINGINLINE
#ifdef __clang__
// clang doesn't seem to particularly like that we're including a header that
// deliberately contains function definitions so we explicitly silence it
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmissing-prototypes"
#endif
#include "liburing.h"
#ifdef __clang__
#pragma clang diagnostic pop
#endif
src/include/liburing.h
+410 -177
View File
File diff suppressed because it is too large Load Diff
src/include/liburing/io_uring.h
+147 -25
View File
@@ -12,12 +12,11 @@
#include <linux/types.h>
/*
* this file is shared with liburing and that has to autodetect
* if linux/time_types.h is available
* if linux/time_types.h is available or not, it can
* define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
* if linux/time_types.h is not available
*/
#ifdef __KERNEL__
#define HAVE_LINUX_TIME_TYPES_H 1
#endif
#ifdef HAVE_LINUX_TIME_TYPES_H
#ifndef UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
#include <linux/time_types.h>
#endif
@@ -44,6 +43,10 @@ struct io_uring_sqe {
union {
__u64 addr; /* pointer to buffer or iovecs */
__u64 splice_off_in;
struct {
__u32 level;
__u32 optname;
};
};
__u32 len; /* buffer size or number of iovecs */
union {
@@ -66,6 +69,10 @@ struct io_uring_sqe {
__u32 xattr_flags;
__u32 msg_ring_flags;
__u32 uring_cmd_flags;
__u32 waitid_flags;
__u32 futex_flags;
__u32 install_fd_flags;
__u32 nop_flags;
};
__u64 user_data; /* data to be passed back at completion time */
/* pack this to avoid bogus arm OABI complaints */
@@ -80,6 +87,7 @@ struct io_uring_sqe {
union {
__s32 splice_fd_in;
__u32 file_index;
__u32 optlen;
struct {
__u16 addr_len;
__u16 __pad3[1];
@@ -90,6 +98,7 @@ struct io_uring_sqe {
__u64 addr3;
__u64 __pad2[1];
};
__u64 optval;
/*
* If the ring is initialized with IORING_SETUP_SQE128, then
* this field is used for 80 bytes of arbitrary command data
@@ -174,6 +183,23 @@ enum {
*/
#define IORING_SETUP_DEFER_TASKRUN (1U << 13)
/*
* Application provides ring memory
*/
#define IORING_SETUP_NO_MMAP (1U << 14)
/*
* Register the ring fd in itself for use with
* IORING_REGISTER_USE_REGISTERED_RING; return a registered fd index rather
* than an fd.
*/
#define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15)
/*
* Removes indirection through the SQ index array.
*/
#define IORING_SETUP_NO_SQARRAY (1U << 16)
enum io_uring_op {
IORING_OP_NOP,
IORING_OP_READV,
@@ -224,6 +250,15 @@ enum io_uring_op {
IORING_OP_URING_CMD,
IORING_OP_SEND_ZC,
IORING_OP_SENDMSG_ZC,
IORING_OP_READ_MULTISHOT,
IORING_OP_WAITID,
IORING_OP_FUTEX_WAIT,
IORING_OP_FUTEX_WAKE,
IORING_OP_FUTEX_WAITV,
IORING_OP_FIXED_FD_INSTALL,
IORING_OP_FTRUNCATE,
IORING_OP_BIND,
IORING_OP_LISTEN,
/* this goes last, obviously */
IORING_OP_LAST,
@@ -231,7 +266,7 @@ enum io_uring_op {
/*
* sqe->uring_cmd_flags
* IORING_URING_CMD_FIXED use registered buffer; pass thig flag
* IORING_URING_CMD_FIXED use registered buffer; pass this flag
* along with setting sqe->buf_index.
*/
#define IORING_URING_CMD_FIXED (1U << 0)
@@ -251,6 +286,7 @@ enum io_uring_op {
#define IORING_TIMEOUT_REALTIME (1U << 3)
#define IORING_LINK_TIMEOUT_UPDATE (1U << 4)
#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5)
#define IORING_TIMEOUT_MULTISHOT (1U << 6)
#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
/*
@@ -281,7 +317,7 @@ enum io_uring_op {
* ASYNC_CANCEL flags.
*
* IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key
* IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
* IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
* request 'user_data'
* IORING_ASYNC_CANCEL_ANY Match any request
* IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor
@@ -305,15 +341,44 @@ enum io_uring_op {
*
* IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in
* the buf_index field.
*
* IORING_SEND_ZC_REPORT_USAGE
* If set, SEND[MSG]_ZC should report
* the zerocopy usage in cqe.res
* for the IORING_CQE_F_NOTIF cqe.
* 0 is reported if zerocopy was actually possible.
* IORING_NOTIF_USAGE_ZC_COPIED if data was copied
* (at least partially).
*
* IORING_RECVSEND_BUNDLE Used with IOSQE_BUFFER_SELECT. If set, send wil
* grab as many buffers from the buffer group ID
* given and send them all. The completion result
* will be the number of buffers send, with the
* starting buffer ID in cqe->flags as per usual
* for provided buffer usage. The buffers will be
* contigious from the starting buffer ID.
*/
#define IORING_RECVSEND_POLL_FIRST (1U << 0)
#define IORING_RECV_MULTISHOT (1U << 1)
#define IORING_RECVSEND_FIXED_BUF (1U << 2)
#define IORING_SEND_ZC_REPORT_USAGE (1U << 3)
#define IORING_RECVSEND_BUNDLE (1U << 4)
/*
* cqe.res for IORING_CQE_F_NOTIF if
* IORING_SEND_ZC_REPORT_USAGE was requested
*
* It should be treated as a flag, all other
* bits of cqe.res should be treated as reserved!
*/
#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31)
/*
* accept flags stored in sqe->ioprio
*/
#define IORING_ACCEPT_MULTISHOT (1U << 0)
#define IORING_ACCEPT_DONTWAIT (1U << 1)
#define IORING_ACCEPT_POLL_FIRST (1U << 2)
/*
* IORING_OP_MSG_RING command types, stored in sqe->addr
@@ -330,12 +395,28 @@ enum {
* applicable for IORING_MSG_DATA, obviously.
*/
#define IORING_MSG_RING_CQE_SKIP (1U << 0)
/* Pass through the flags from sqe->file_index to cqe->flags */
#define IORING_MSG_RING_FLAGS_PASS (1U << 1)
/*
* IORING_OP_FIXED_FD_INSTALL flags (sqe->install_fd_flags)
*
* IORING_FIXED_FD_NO_CLOEXEC Don't mark the fd as O_CLOEXEC
*/
#define IORING_FIXED_FD_NO_CLOEXEC (1U << 0)
/*
* IORING_OP_NOP flags (sqe->nop_flags)
*
* IORING_NOP_INJECT_RESULT Inject result from sqe->result
*/
#define IORING_NOP_INJECT_RESULT (1U << 0)
/*
* IO completion data structure (Completion Queue Entry)
*/
struct io_uring_cqe {
__u64 user_data; /* sqe->data submission passed back */
__u64 user_data; /* sqe->user_data value passed back */
__s32 res; /* result code for this event */
__u32 flags;
@@ -370,6 +451,9 @@ enum {
#define IORING_OFF_SQ_RING 0ULL
#define IORING_OFF_CQ_RING 0x8000000ULL
#define IORING_OFF_SQES 0x10000000ULL
#define IORING_OFF_PBUF_RING 0x80000000ULL
#define IORING_OFF_PBUF_SHIFT 16
#define IORING_OFF_MMAP_MASK 0xf8000000ULL
/*
* Filled with the offset for mmap(2)
@@ -383,7 +467,7 @@ struct io_sqring_offsets {
__u32 dropped;
__u32 array;
__u32 resv1;
__u64 resv2;
__u64 user_addr;
};
/*
@@ -402,7 +486,7 @@ struct io_cqring_offsets {
__u32 cqes;
__u32 flags;
__u32 resv1;
__u64 resv2;
__u64 user_addr;
};
/*
@@ -453,6 +537,8 @@ struct io_uring_params {
#define IORING_FEAT_RSRC_TAGS (1U << 10)
#define IORING_FEAT_CQE_SKIP (1U << 11)
#define IORING_FEAT_LINKED_FILE (1U << 12)
#define IORING_FEAT_REG_REG_RING (1U << 13)
#define IORING_FEAT_RECVSEND_BUNDLE (1U << 14)
/*
* io_uring_register(2) opcodes and arguments
@@ -499,8 +585,18 @@ enum {
/* register a range of fixed file slots for automatic slot allocation */
IORING_REGISTER_FILE_ALLOC_RANGE = 25,
/* return status information for a buffer group */
IORING_REGISTER_PBUF_STATUS = 26,
/* set/clear busy poll settings */
IORING_REGISTER_NAPI = 27,
IORING_UNREGISTER_NAPI = 28,
/* this goes last */
IORING_REGISTER_LAST
IORING_REGISTER_LAST,
/* flag added to the opcode to use a registered ring fd */
IORING_REGISTER_USE_REGISTERED_RING = 1U << 31
};
/* io-wq worker categories */
@@ -545,19 +641,6 @@ struct io_uring_rsrc_update2 {
__u32 resv2;
};
struct io_uring_notification_slot {
__u64 tag;
__u64 resv[3];
};
struct io_uring_notification_register {
__u32 nr_slots;
__u32 resv;
__u64 resv2;
__u64 data;
__u64 resv3;
};
/* Skip updating fd indexes set to this value in the fd table */
#define IORING_REGISTER_FILES_SKIP (-2)
@@ -612,15 +695,44 @@ struct io_uring_buf_ring {
};
};
/*
* Flags for IORING_REGISTER_PBUF_RING.
*
* IOU_PBUF_RING_MMAP: If set, kernel will allocate the memory for the ring.
* The application must not set a ring_addr in struct
* io_uring_buf_reg, instead it must subsequently call
* mmap(2) with the offset set as:
* IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT)
* to get a virtual mapping for the ring.
*/
enum {
IOU_PBUF_RING_MMAP = 1,
};
/* argument for IORING_(UN)REGISTER_PBUF_RING */
struct io_uring_buf_reg {
__u64 ring_addr;
__u32 ring_entries;
__u16 bgid;
__u16 pad;
__u16 flags;
__u64 resv[3];
};
/* argument for IORING_REGISTER_PBUF_STATUS */
struct io_uring_buf_status {
__u32 buf_group; /* input */
__u32 head; /* output */
__u32 resv[8];
};
/* argument for IORING_(UN)REGISTER_NAPI */
struct io_uring_napi {
__u32 busy_poll_to;
__u8 prefer_busy_poll;
__u8 pad[3];
__u64 resv;
};
/*
* io_uring_restriction->opcode values
*/
@@ -675,6 +787,16 @@ struct io_uring_recvmsg_out {
__u32 flags;
};
/*
* Argument for IORING_OP_URING_CMD when file is a socket
*/
enum {
SOCKET_URING_OP_SIOCINQ = 0,
SOCKET_URING_OP_SIOCOUTQ,
SOCKET_URING_OP_GETSOCKOPT,
SOCKET_URING_OP_SETSOCKOPT,
};
#ifdef __cplusplus
}
#endif
src/int_flags.h
+2
View File
@@ -4,6 +4,8 @@
enum {
INT_FLAG_REG_RING = 1,
INT_FLAG_REG_REG_RING = 2,
INT_FLAG_APP_MEM = 4,
};
#endif
src/lib.h
+7 -16
View File
@@ -10,6 +10,8 @@
#include "arch/x86/lib.h"
#elif defined(__aarch64__)
#include "arch/aarch64/lib.h"
#elif defined(__riscv) && __riscv_xlen == 64
#include "arch/riscv64/lib.h"
#else
/*
* We don't have nolibc support for this arch. Must use libc!
@@ -37,25 +39,14 @@
#define __hot __attribute__((__hot__))
#define __cold __attribute__((__cold__))
#ifdef CONFIG_NOLIBC
void *__uring_memset(void *s, int c, size_t n);
void *__uring_malloc(size_t len);
void __uring_free(void *p);
static inline void *uring_malloc(size_t len)
{
#ifdef CONFIG_NOLIBC
return __uring_malloc(len);
#else
return malloc(len);
#define malloc(LEN) __uring_malloc(LEN)
#define free(PTR) __uring_free(PTR)
#define memset(PTR, C, LEN) __uring_memset(PTR, C, LEN)
#endif
}
static inline void uring_free(void *ptr)
{
#ifdef CONFIG_NOLIBC
__uring_free(ptr);
#else
free(ptr);
#endif
}
#endif /* #ifndef LIBURING_LIB_H */
src/liburing-ffi.map
+206
View File
@@ -0,0 +1,206 @@
LIBURING_2.4 {
global:
io_uring_get_probe;
io_uring_get_probe_ring;
io_uring_free_probe;
io_uring_get_sqe;
io_uring_peek_batch_cqe;
io_uring_queue_exit;
io_uring_queue_init;
io_uring_queue_init_params;
io_uring_queue_mmap;
io_uring_register_buffers;
io_uring_register_eventfd;
io_uring_register_eventfd_async;
io_uring_register_files;
io_uring_register_files_update;
io_uring_register_personality;
io_uring_register_probe;
io_uring_ring_dontfork;
io_uring_submit;
io_uring_submit_and_wait;
io_uring_unregister_buffers;
io_uring_unregister_eventfd;
io_uring_unregister_files;
io_uring_unregister_personality;
io_uring_wait_cqe_timeout;
io_uring_wait_cqes;
__io_uring_get_cqe;
__io_uring_sqring_wait;
io_uring_mlock_size_params;
io_uring_mlock_size;
io_uring_register_buffers_tags;
io_uring_register_buffers_update_tag;
io_uring_register_files_tags;
io_uring_register_files_update_tag;
io_uring_register_iowq_aff;
io_uring_unregister_iowq_aff;
io_uring_register_iowq_max_workers;
io_uring_submit_and_wait_timeout;
io_uring_register_ring_fd;
io_uring_unregister_ring_fd;
io_uring_register_files_sparse;
io_uring_register_buffers_sparse;
io_uring_register_buf_ring;
io_uring_unregister_buf_ring;
io_uring_close_ring_fd;
io_uring_setup_buf_ring;
io_uring_free_buf_ring;
io_uring_register_sync_cancel;
io_uring_register_file_alloc_range;
io_uring_enter;
io_uring_enter2;
io_uring_setup;
io_uring_register;
io_uring_get_events;
io_uring_submit_and_get_events;
io_uring_major_version;
io_uring_minor_version;
io_uring_check_version;
io_uring_peek_cqe;
io_uring_prep_timeout_update;
io_uring_buf_ring_init;
io_uring_prep_mkdirat;
io_uring_prep_recv_multishot;
io_uring_cq_advance;
io_uring_prep_multishot_accept;
io_uring_prep_fallocate;
io_uring_prep_link_timeout;
io_uring_prep_fsync;
io_uring_prep_openat_direct;
io_uring_prep_multishot_accept_direct;
io_uring_opcode_supported;
io_uring_prep_madvise;
io_uring_prep_send_set_addr;
io_uring_recvmsg_payload_length;
io_uring_prep_readv2;
io_uring_prep_msg_ring;
io_uring_prep_rename;
io_uring_prep_fadvise;
io_uring_prep_send_zc;
io_uring_buf_ring_advance;
io_uring_cqe_get_data;
io_uring_prep_symlinkat;
io_uring_prep_writev;
io_uring_cq_eventfd_toggle;
io_uring_prep_provide_buffers;
io_uring_cq_has_overflow;
io_uring_prep_cancel_fd;
io_uring_prep_socket;
io_uring_prep_close_direct;
io_uring_recvmsg_name;
io_uring_prep_timeout_remove;
io_uring_sqring_wait;
io_uring_cq_eventfd_enabled;
io_uring_prep_remove_buffers;
io_uring_prep_tee;
io_uring_prep_accept_direct;
io_uring_prep_nop;
io_uring_prep_getxattr;
io_uring_prep_link;
io_uring_prep_cancel;
io_uring_prep_readv;
io_uring_prep_connect;
io_uring_cq_ready;
io_uring_enable_rings;
io_uring_prep_shutdown;
io_uring_prep_openat;
io_uring_sq_space_left;
io_uring_recvmsg_payload;
io_uring_prep_send;
io_uring_buf_ring_add;
io_uring_prep_send_zc_fixed;
io_uring_prep_epoll_ctl;
io_uring_recvmsg_cmsg_firsthdr;
io_uring_prep_socket_direct;
io_uring_buf_ring_cq_advance;
__io_uring_buf_ring_cq_advance;
io_uring_prep_mkdir;
io_uring_wait_cqe_nr;
io_uring_prep_unlink;
io_uring_prep_writev2;
io_uring_prep_openat2_direct;
io_uring_sqe_set_flags;
io_uring_sqe_set_data;
io_uring_prep_accept;
io_uring_prep_poll_update;
io_uring_prep_splice;
io_uring_prep_poll_multishot;
io_uring_prep_symlink;
io_uring_sqe_set_data64;
io_uring_prep_cancel64;
io_uring_prep_fsetxattr;
io_uring_prep_recvmsg_multishot;
io_uring_cqe_seen;
io_uring_prep_sendmsg_zc;
io_uring_prep_read;
io_uring_prep_statx;
io_uring_prep_sendmsg;
io_uring_prep_unlinkat;
io_uring_prep_setxattr;
io_uring_cqe_get_data64;
io_uring_prep_renameat;
io_uring_prep_poll_remove;
io_uring_prep_close;
io_uring_sq_ready;
io_uring_prep_files_update;
io_uring_wait_cqe;
io_uring_prep_fgetxattr;
io_uring_prep_socket_direct_alloc;
io_uring_prep_sync_file_range;
io_uring_prep_read_fixed;
io_uring_prep_openat2;
io_uring_prep_recvmsg;
io_uring_recvmsg_cmsg_nexthdr;
io_uring_recvmsg_validate;
io_uring_prep_rw;
io_uring_prep_timeout;
io_uring_prep_linkat;
io_uring_prep_write_fixed;
io_uring_prep_poll_add;
io_uring_buf_ring_mask;
io_uring_register_restrictions;
io_uring_prep_write;
io_uring_prep_recv;
io_uring_prep_msg_ring_cqe_flags;
io_uring_prep_msg_ring_fd;
io_uring_prep_msg_ring_fd_alloc;
io_uring_prep_sendto;
io_uring_register_napi; /* Added in 2.6. */
io_uring_unregister_napi; /* Added in 2.6. */
local:
*;
};
LIBURING_2.5 {
global:
io_uring_queue_init_mem;
io_uring_prep_cmd_sock; /* Added in 2.5,
exported in 2.6. */
io_uring_prep_read_multishot; /* Added in 2.6. */
io_uring_prep_waitid; /* Added in 2.6. */
io_uring_prep_futex_wake; /* Added in 2.6. */
io_uring_prep_futex_wait; /* Added in 2.6. */
io_uring_prep_futex_waitv; /* Added in 2.6. */
} LIBURING_2.4;
LIBURING_2.6 {
global:
io_uring_prep_fixed_fd_install;
io_uring_buf_ring_available;
io_uring_prep_ftruncate;
io_uring_prep_send_bundle;
} LIBURING_2.5;
LIBURING_2.7 {
io_uring_prep_fadvise64;
io_uring_prep_madvise64;
io_uring_prep_bind;
io_uring_prep_listen;
} LIBURING_2.6;
src/liburing.map
+28
View File
@@ -67,3 +67,31 @@ LIBURING_2.3 {
io_uring_get_events;
io_uring_submit_and_get_events;
} LIBURING_2.2;
LIBURING_2.4 {
global:
io_uring_major_version;
io_uring_minor_version;
io_uring_check_version;
io_uring_close_ring_fd;
io_uring_enable_rings;
io_uring_register_restrictions;
io_uring_setup_buf_ring;
io_uring_free_buf_ring;
} LIBURING_2.3;
LIBURING_2.5 {
global:
io_uring_queue_init_mem;
} LIBURING_2.4;
LIBURING_2.6 {
global:
io_uring_buf_ring_head;
io_uring_register_napi;
io_uring_unregister_napi;
} LIBURING_2.5;
LIBURING_2.7 {
} LIBURING_2.6;
src/nolibc.c
+9 -2
View File
@@ -7,14 +7,21 @@
#include "lib.h"
#include "syscall.h"
void *memset(void *s, int c, size_t n)
void *__uring_memset(void *s, int c, size_t n)
{
size_t i;
unsigned char *p = s;
for (i = 0; i < n; i++)
for (i = 0; i < n; i++) {
p[i] = (unsigned char) c;
/*
* An empty inline ASM to avoid auto-vectorization
* because it's too bloated for liburing.
*/
__asm__ volatile ("");
}
return s;
}
src/queue.c
+10 -14
View File
@@ -81,7 +81,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
}
if (!cqe && !data->wait_nr && !data->submit) {
/*
* If we already looped once, we already entererd
* If we already looped once, we already entered
* the kernel. Since there's nothing to submit or
* wait for, don't keep retrying.
*/
@@ -201,7 +201,7 @@ again:
* Sync internal state with kernel ring state on the SQ side. Returns the
* number of pending items in the SQ ring, for the shared ring.
*/
unsigned __io_uring_flush_sq(struct io_uring *ring)
static unsigned __io_uring_flush_sq(struct io_uring *ring)
{
struct io_uring_sq *sq = &ring->sq;
unsigned tail = sq->sqe_tail;
@@ -212,22 +212,18 @@ unsigned __io_uring_flush_sq(struct io_uring *ring)
* Ensure kernel sees the SQE updates before the tail update.
*/
if (!(ring->flags & IORING_SETUP_SQPOLL))
IO_URING_WRITE_ONCE(*sq->ktail, tail);
*sq->ktail = tail;
else
io_uring_smp_store_release(sq->ktail, tail);
}
/*
* This _may_ look problematic, as we're not supposed to be reading
* SQ->head without acquire semantics. When we're in SQPOLL mode, the
* kernel submitter could be updating this right now. For non-SQPOLL,
* task itself does it, and there's no potential race. But even for
* SQPOLL, the load is going to be potentially out-of-date the very
* instant it's done, regardless or whether or not it's done
* atomically. Worst case, we're going to be over-estimating what
* we can submit. The point is, we need to be able to deal with this
* situation regardless of any perceived atomicity.
*/
return tail - *sq->khead;
* This load needs to be atomic, since sq->khead is written concurrently
* by the kernel, but it doesn't need to be load_acquire, since the
* kernel doesn't store to the submission queue; it advances khead just
* to indicate that it's finished reading the submission queue entries
* so they're available for us to write to.
*/
return tail - IO_URING_READ_ONCE(*sq->khead);
}
/*
src/register.c
+95 -96
View File
@@ -8,6 +8,21 @@
#include "liburing/compat.h"
#include "liburing/io_uring.h"
static inline int do_register(struct io_uring *ring, unsigned int opcode,
const void *arg, unsigned int nr_args)
{
int fd;
if (ring->int_flags & INT_FLAG_REG_REG_RING) {
opcode |= IORING_REGISTER_USE_REGISTERED_RING;
fd = ring->enter_ring_fd;
} else {
fd = ring->ring_fd;
}
return __sys_io_uring_register(fd, opcode, arg, nr_args);
}
int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
const struct iovec *iovecs,
const __u64 *tags,
@@ -20,8 +35,7 @@ int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
.nr = nr,
};
return __sys_io_uring_register(ring->ring_fd,IORING_REGISTER_BUFFERS_UPDATE, &up,
sizeof(up));
return do_register(ring, IORING_REGISTER_BUFFERS_UPDATE, &up, sizeof(up));
}
int io_uring_register_buffers_tags(struct io_uring *ring,
@@ -35,9 +49,7 @@ int io_uring_register_buffers_tags(struct io_uring *ring,
.tags = (unsigned long)tags,
};
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_BUFFERS2, &reg,
sizeof(reg));
return do_register(ring, IORING_REGISTER_BUFFERS2, &reg, sizeof(reg));
}
int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr)
@@ -47,27 +59,18 @@ int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr)
.nr = nr,
};
return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2,
&reg, sizeof(reg));
return do_register(ring, IORING_REGISTER_BUFFERS2, &reg, sizeof(reg));
}
int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
unsigned nr_iovecs)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS,
iovecs, nr_iovecs);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_REGISTER_BUFFERS, iovecs, nr_iovecs);
}
int io_uring_unregister_buffers(struct io_uring *ring)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS,
NULL, 0);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_UNREGISTER_BUFFERS, NULL, 0);
}
int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
@@ -81,9 +84,7 @@ int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
.nr = nr_files,
};
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILES_UPDATE2, &up,
sizeof(up));
return do_register(ring, IORING_REGISTER_FILES_UPDATE2, &up, sizeof(up));
}
/*
@@ -101,9 +102,7 @@ int io_uring_register_files_update(struct io_uring *ring, unsigned off,
.fds = (unsigned long) files,
};
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILES_UPDATE, &up,
nr_files);
return do_register(ring, IORING_REGISTER_FILES_UPDATE, &up, nr_files);
}
static int increase_rlimit_nofile(unsigned nr)
@@ -132,9 +131,7 @@ int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr)
int ret, did_increase = 0;
do {
ret = __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILES2, &reg,
sizeof(reg));
ret = do_register(ring, IORING_REGISTER_FILES2, &reg, sizeof(reg));
if (ret >= 0)
break;
if (ret == -EMFILE && !did_increase) {
@@ -159,9 +156,7 @@ int io_uring_register_files_tags(struct io_uring *ring, const int *files,
int ret, did_increase = 0;
do {
ret = __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILES2, &reg,
sizeof(reg));
ret = do_register(ring, IORING_REGISTER_FILES2, &reg, sizeof(reg));
if (ret >= 0)
break;
if (ret == -EMFILE && !did_increase) {
@@ -181,9 +176,7 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
int ret, did_increase = 0;
do {
ret = __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILES, files,
nr_files);
ret = do_register(ring, IORING_REGISTER_FILES, files, nr_files);
if (ret >= 0)
break;
if (ret == -EMFILE && !did_increase) {
@@ -199,79 +192,50 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
int io_uring_unregister_files(struct io_uring *ring)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES,
NULL, 0);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_UNREGISTER_FILES, NULL, 0);
}
int io_uring_register_eventfd(struct io_uring *ring, int event_fd)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD,
&event_fd, 1);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_REGISTER_EVENTFD, &event_fd, 1);
}
int io_uring_unregister_eventfd(struct io_uring *ring)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD,
NULL, 0);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_UNREGISTER_EVENTFD, NULL, 0);
}
int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_EVENTFD_ASYNC, &event_fd,
1);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_REGISTER_EVENTFD_ASYNC, &event_fd, 1);
}
int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p,
unsigned int nr_ops)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p,
nr_ops);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_REGISTER_PROBE, p, nr_ops);
}
int io_uring_register_personality(struct io_uring *ring)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_PERSONALITY, NULL, 0);
return do_register(ring, IORING_REGISTER_PERSONALITY, NULL, 0);
}
int io_uring_unregister_personality(struct io_uring *ring, int id)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_UNREGISTER_PERSONALITY, NULL, id);
return do_register(ring, IORING_UNREGISTER_PERSONALITY, NULL, id);
}
int io_uring_register_restrictions(struct io_uring *ring,
struct io_uring_restriction *res,
unsigned int nr_res)
{
int ret;
ret = __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_RESTRICTIONS, res,
nr_res);
return (ret < 0) ? ret : 0;
return do_register(ring, IORING_REGISTER_RESTRICTIONS, res, nr_res);
}
int io_uring_enable_rings(struct io_uring *ring)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_ENABLE_RINGS, NULL, 0);
return do_register(ring, IORING_REGISTER_ENABLE_RINGS, NULL, 0);
}
int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
@@ -280,21 +244,17 @@ int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
if (cpusz >= (1U << 31))
return -EINVAL;
return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF,
mask, (int) cpusz);
return do_register(ring, IORING_REGISTER_IOWQ_AFF, mask, (int) cpusz);
}
int io_uring_unregister_iowq_aff(struct io_uring *ring)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
return do_register(ring, IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
}
int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *val)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_IOWQ_MAX_WORKERS, val,
2);
return do_register(ring, IORING_REGISTER_IOWQ_MAX_WORKERS, val, 2);
}
int io_uring_register_ring_fd(struct io_uring *ring)
@@ -305,11 +265,16 @@ int io_uring_register_ring_fd(struct io_uring *ring)
};
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS,
&up, 1);
if (ring->int_flags & INT_FLAG_REG_RING)
return -EEXIST;
ret = do_register(ring, IORING_REGISTER_RING_FDS, &up, 1);
if (ret == 1) {
ring->enter_ring_fd = up.offset;
ring->int_flags |= INT_FLAG_REG_RING;
if (ring->features & IORING_FEAT_REG_REG_RING) {
ring->int_flags |= INT_FLAG_REG_REG_RING;
}
}
return ret;
}
@@ -322,48 +287,82 @@ int io_uring_unregister_ring_fd(struct io_uring *ring)
};
int ret;
ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_RING_FDS,
&up, 1);
if (!(ring->int_flags & INT_FLAG_REG_RING))
return -EINVAL;
ret = do_register(ring, IORING_UNREGISTER_RING_FDS, &up, 1);
if (ret == 1) {
ring->enter_ring_fd = ring->ring_fd;
ring->int_flags &= ~INT_FLAG_REG_RING;
ring->int_flags &= ~(INT_FLAG_REG_RING | INT_FLAG_REG_REG_RING);
}
return ret;
}
int io_uring_close_ring_fd(struct io_uring *ring)
{
if (!(ring->features & IORING_FEAT_REG_REG_RING))
return -EOPNOTSUPP;
if (!(ring->int_flags & INT_FLAG_REG_RING))
return -EINVAL;
if (ring->ring_fd == -1)
return -EBADF;
__sys_close(ring->ring_fd);
ring->ring_fd = -1;
return 1;
}
int io_uring_register_buf_ring(struct io_uring *ring,
struct io_uring_buf_reg *reg,
unsigned int __maybe_unused flags)
{
return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PBUF_RING,
reg, 1);
return do_register(ring, IORING_REGISTER_PBUF_RING, reg, 1);
}
int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid)
{
struct io_uring_buf_reg reg = { .bgid = bgid };
return __sys_io_uring_register(ring->ring_fd,
IORING_UNREGISTER_PBUF_RING, &reg, 1);
return do_register(ring, IORING_UNREGISTER_PBUF_RING, &reg, 1);
}
int io_uring_buf_ring_head(struct io_uring *ring, int buf_group, uint16_t *head)
{
struct io_uring_buf_status buf_status = {
.buf_group = buf_group,
};
int ret;
ret = do_register(ring, IORING_REGISTER_PBUF_STATUS, &buf_status, 1);
if (ret)
return ret;
*head = buf_status.head;
return 0;
}
int io_uring_register_sync_cancel(struct io_uring *ring,
struct io_uring_sync_cancel_reg *reg)
{
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_SYNC_CANCEL, reg, 1);
return do_register(ring, IORING_REGISTER_SYNC_CANCEL, reg, 1);
}
int io_uring_register_file_alloc_range(struct io_uring *ring,
unsigned off, unsigned len)
{
struct io_uring_file_index_range range;
struct io_uring_file_index_range range = {
.off = off,
.len = len
};
memset(&range, 0, sizeof(range));
range.off = off;
range.len = len;
return __sys_io_uring_register(ring->ring_fd,
IORING_REGISTER_FILE_ALLOC_RANGE, &range,
0);
return do_register(ring, IORING_REGISTER_FILE_ALLOC_RANGE, &range, 0);
}
int io_uring_register_napi(struct io_uring *ring, struct io_uring_napi *napi)
{
return do_register(ring, IORING_REGISTER_NAPI, napi, 1);
}
int io_uring_unregister_napi(struct io_uring *ring, struct io_uring_napi *napi)
{
return do_register(ring, IORING_UNREGISTER_NAPI, napi, 1);
}
src/setup.c
+412 -92
View File
@@ -5,16 +5,96 @@
#include "syscall.h"
#include "liburing.h"
#include "int_flags.h"
#include "setup.h"
#include "liburing/compat.h"
#include "liburing/io_uring.h"
#define KERN_MAX_ENTRIES 32768
#define KERN_MAX_CQ_ENTRIES (2 * KERN_MAX_ENTRIES)
static inline int __fls(int x)
{
if (!x)
return 0;
return 8 * sizeof(x) - __builtin_clz(x);
}
static unsigned roundup_pow2(unsigned depth)
{
return 1U << __fls(depth - 1);
}
static int get_sq_cq_entries(unsigned entries, struct io_uring_params *p,
unsigned *sq, unsigned *cq)
{
unsigned cq_entries;
if (!entries)
return -EINVAL;
if (entries > KERN_MAX_ENTRIES) {
if (!(p->flags & IORING_SETUP_CLAMP))
return -EINVAL;
entries = KERN_MAX_ENTRIES;
}
entries = roundup_pow2(entries);
if (p->flags & IORING_SETUP_CQSIZE) {
if (!p->cq_entries)
return -EINVAL;
cq_entries = p->cq_entries;
if (cq_entries > KERN_MAX_CQ_ENTRIES) {
if (!(p->flags & IORING_SETUP_CLAMP))
return -EINVAL;
cq_entries = KERN_MAX_CQ_ENTRIES;
}
cq_entries = roundup_pow2(cq_entries);
if (cq_entries < entries)
return -EINVAL;
} else {
cq_entries = 2 * entries;
}
*sq = entries;
*cq = cq_entries;
return 0;
}
static void io_uring_unmap_rings(struct io_uring_sq *sq, struct io_uring_cq *cq)
{
__sys_munmap(sq->ring_ptr, sq->ring_sz);
if (cq->ring_ptr && cq->ring_ptr != sq->ring_ptr)
if (sq->ring_sz)
__sys_munmap(sq->ring_ptr, sq->ring_sz);
if (cq->ring_ptr && cq->ring_sz && cq->ring_ptr != sq->ring_ptr)
__sys_munmap(cq->ring_ptr, cq->ring_sz);
}
static void io_uring_setup_ring_pointers(struct io_uring_params *p,
struct io_uring_sq *sq,
struct io_uring_cq *cq)
{
sq->khead = sq->ring_ptr + p->sq_off.head;
sq->ktail = sq->ring_ptr + p->sq_off.tail;
sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask;
sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
sq->kflags = sq->ring_ptr + p->sq_off.flags;
sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
sq->array = sq->ring_ptr + p->sq_off.array;
cq->khead = cq->ring_ptr + p->cq_off.head;
cq->ktail = cq->ring_ptr + p->cq_off.tail;
cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask;
cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries;
cq->koverflow = cq->ring_ptr + p->cq_off.overflow;
cq->cqes = cq->ring_ptr + p->cq_off.cqes;
if (p->cq_off.flags)
cq->kflags = cq->ring_ptr + p->cq_off.flags;
sq->ring_mask = *sq->kring_mask;
sq->ring_entries = *sq->kring_entries;
cq->ring_mask = *cq->kring_mask;
cq->ring_entries = *cq->kring_entries;
}
static int io_uring_mmap(int fd, struct io_uring_params *p,
struct io_uring_sq *sq, struct io_uring_cq *cq)
{
@@ -52,14 +132,6 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
}
}
sq->khead = sq->ring_ptr + p->sq_off.head;
sq->ktail = sq->ring_ptr + p->sq_off.tail;
sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask;
sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
sq->kflags = sq->ring_ptr + p->sq_off.flags;
sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
sq->array = sq->ring_ptr + p->sq_off.array;
size = sizeof(struct io_uring_sqe);
if (p->flags & IORING_SETUP_SQE128)
size += 64;
@@ -72,19 +144,7 @@ err:
return ret;
}
cq->khead = cq->ring_ptr + p->cq_off.head;
cq->ktail = cq->ring_ptr + p->cq_off.tail;
cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask;
cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries;
cq->koverflow = cq->ring_ptr + p->cq_off.overflow;
cq->cqes = cq->ring_ptr + p->cq_off.cqes;
if (p->cq_off.flags)
cq->kflags = cq->ring_ptr + p->cq_off.flags;
sq->ring_mask = *sq->kring_mask;
sq->ring_entries = *sq->kring_entries;
cq->ring_mask = *cq->kring_mask;
cq->ring_entries = *cq->kring_entries;
io_uring_setup_ring_pointers(p, sq, cq);
return 0;
}
@@ -97,16 +157,8 @@ err:
__cold int io_uring_queue_mmap(int fd, struct io_uring_params *p,
struct io_uring *ring)
{
int ret;
memset(ring, 0, sizeof(*ring));
ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
if (!ret) {
ring->flags = p->flags;
ring->ring_fd = ring->enter_ring_fd = fd;
ring->int_flags = 0;
}
return ret;
return io_uring_mmap(fd, p, &ring->sq, &ring->cq);
}
/*
@@ -144,33 +196,220 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring)
return 0;
}
__cold int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
struct io_uring_params *p)
/* FIXME */
static size_t huge_page_size = 2 * 1024 * 1024;
#define KRING_SIZE 64
/*
* Returns negative for error, or number of bytes used in the buffer on success
*/
static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
struct io_uring_sq *sq, struct io_uring_cq *cq,
void *buf, size_t buf_size)
{
int fd, ret;
unsigned long page_size = get_page_size();
unsigned sq_entries, cq_entries;
size_t ring_mem, sqes_mem, cqes_mem;
unsigned long mem_used = 0;
void *ptr;
int ret;
ret = get_sq_cq_entries(entries, p, &sq_entries, &cq_entries);
if (ret)
return ret;
ring_mem = KRING_SIZE;
sqes_mem = sq_entries * sizeof(struct io_uring_sqe);
sqes_mem = (sqes_mem + page_size - 1) & ~(page_size - 1);
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
sqes_mem += sq_entries * sizeof(unsigned);
cqes_mem = cq_entries * sizeof(struct io_uring_cqe);
if (p->flags & IORING_SETUP_CQE32)
cqes_mem *= 2;
ring_mem += sqes_mem + cqes_mem;
mem_used = ring_mem;
mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
/*
* A maxed-out number of CQ entries with IORING_SETUP_CQE32 fills a 2MB
* huge page by itself, so the SQ entries won't fit in the same huge
* page. For SQEs, that shouldn't be possible given KERN_MAX_ENTRIES,
* but check that too to future-proof (e.g. against different huge page
* sizes). Bail out early so we don't overrun.
*/
if (!buf && (sqes_mem > huge_page_size || ring_mem > huge_page_size))
return -ENOMEM;
if (buf) {
if (mem_used > buf_size)
return -ENOMEM;
ptr = buf;
} else {
int map_hugetlb = 0;
if (sqes_mem <= page_size)
buf_size = page_size;
else {
buf_size = huge_page_size;
map_hugetlb = MAP_HUGETLB;
}
ptr = __sys_mmap(NULL, buf_size, PROT_READ|PROT_WRITE,
MAP_SHARED|MAP_ANONYMOUS|map_hugetlb,
-1, 0);
if (IS_ERR(ptr))
return PTR_ERR(ptr);
}
sq->sqes = ptr;
if (mem_used <= buf_size) {
sq->ring_ptr = (void *) sq->sqes + sqes_mem;
/* clear ring sizes, we have just one mmap() to undo */
cq->ring_sz = 0;
sq->ring_sz = 0;
} else {
int map_hugetlb = 0;
if (ring_mem <= page_size)
buf_size = page_size;
else {
buf_size = huge_page_size;
map_hugetlb = MAP_HUGETLB;
}
ptr = __sys_mmap(NULL, buf_size, PROT_READ|PROT_WRITE,
MAP_SHARED|MAP_ANONYMOUS|map_hugetlb,
-1, 0);
if (IS_ERR(ptr)) {
__sys_munmap(sq->sqes, 1);
return PTR_ERR(ptr);
}
sq->ring_ptr = ptr;
sq->ring_sz = buf_size;
cq->ring_sz = 0;
}
cq->ring_ptr = (void *) sq->ring_ptr;
p->sq_off.user_addr = (unsigned long) sq->sqes;
p->cq_off.user_addr = (unsigned long) sq->ring_ptr;
return (int) mem_used;
}
int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
struct io_uring_params *p, void *buf,
size_t buf_size)
{
int fd, ret = 0;
unsigned *sq_array;
unsigned sq_entries, index;
fd = __sys_io_uring_setup(entries, p);
if (fd < 0)
return fd;
memset(ring, 0, sizeof(*ring));
ret = io_uring_queue_mmap(fd, p, ring);
if (ret) {
__sys_close(fd);
return ret;
/*
* The kernel does this check already, but checking it here allows us
* to avoid handling it below.
*/
if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY
&& !(p->flags & IORING_SETUP_NO_MMAP))
return -EINVAL;
if (p->flags & IORING_SETUP_NO_MMAP) {
ret = io_uring_alloc_huge(entries, p, &ring->sq, &ring->cq,
buf, buf_size);
if (ret < 0)
return ret;
if (buf)
ring->int_flags |= INT_FLAG_APP_MEM;
}
fd = __sys_io_uring_setup(entries, p);
if (fd < 0) {
if ((p->flags & IORING_SETUP_NO_MMAP) &&
!(ring->int_flags & INT_FLAG_APP_MEM)) {
__sys_munmap(ring->sq.sqes, 1);
io_uring_unmap_rings(&ring->sq, &ring->cq);
}
return fd;
}
if (!(p->flags & IORING_SETUP_NO_MMAP)) {
ret = io_uring_queue_mmap(fd, p, ring);
if (ret) {
__sys_close(fd);
return ret;
}
} else {
io_uring_setup_ring_pointers(p, &ring->sq, &ring->cq);
}
/*
* Directly map SQ slots to SQEs
*/
sq_array = ring->sq.array;
sq_entries = ring->sq.ring_entries;
for (index = 0; index < sq_entries; index++)
sq_array[index] = index;
if (!(p->flags & IORING_SETUP_NO_SQARRAY)) {
sq_array = ring->sq.array;
for (index = 0; index < sq_entries; index++)
sq_array[index] = index;
}
ring->features = p->features;
return 0;
ring->flags = p->flags;
ring->enter_ring_fd = fd;
if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY) {
ring->ring_fd = -1;
ring->int_flags |= INT_FLAG_REG_RING | INT_FLAG_REG_REG_RING;
} else {
ring->ring_fd = fd;
}
return ret;
}
static int io_uring_queue_init_try_nosqarr(unsigned entries, struct io_uring *ring,
struct io_uring_params *p, void *buf,
size_t buf_size)
{
unsigned flags = p->flags;
int ret;
p->flags |= IORING_SETUP_NO_SQARRAY;
ret = __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
/* don't fallback if explicitly asked for NOSQARRAY */
if (ret != -EINVAL || (flags & IORING_SETUP_NO_SQARRAY))
return ret;
p->flags = flags;
return __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
}
/*
* Like io_uring_queue_init_params(), except it allows the application to pass
* in a pre-allocated memory range that is used for the shared data between
* the kernel and the application. This includes the sqes array, and the two
* rings. The memory must be contiguous, the use case here is that the app
* allocates a huge page and passes it in.
*
* Returns the number of bytes used in the buffer, the app can then reuse
* the buffer with the returned offset to put more rings in the same huge
* page. Returns -ENOMEM if there's not enough room left in the buffer to
* host the ring.
*/
int io_uring_queue_init_mem(unsigned entries, struct io_uring *ring,
struct io_uring_params *p,
void *buf, size_t buf_size)
{
/* should already be set... */
p->flags |= IORING_SETUP_NO_MMAP;
return io_uring_queue_init_try_nosqarr(entries, ring, p, buf, buf_size);
}
int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
struct io_uring_params *p)
{
int ret;
ret = io_uring_queue_init_try_nosqarr(entries, ring, p, NULL, 0);
return ret >= 0 ? 0 : ret;
}
/*
@@ -194,18 +433,28 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
struct io_uring_cq *cq = &ring->cq;
size_t sqe_size;
sqe_size = sizeof(struct io_uring_sqe);
if (ring->flags & IORING_SETUP_SQE128)
sqe_size += 64;
__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
io_uring_unmap_rings(sq, cq);
if (!sq->ring_sz) {
sqe_size = sizeof(struct io_uring_sqe);
if (ring->flags & IORING_SETUP_SQE128)
sqe_size += 64;
__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
io_uring_unmap_rings(sq, cq);
} else {
if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
__sys_munmap(sq->sqes,
*sq->kring_entries * sizeof(struct io_uring_sqe));
io_uring_unmap_rings(sq, cq);
}
}
/*
* Not strictly required, but frees up the slot we used now rather
* than at process exit time.
*/
if (ring->int_flags & INT_FLAG_REG_RING)
io_uring_unregister_ring_fd(ring);
__sys_close(ring->ring_fd);
if (ring->ring_fd != -1)
__sys_close(ring->ring_fd);
}
__cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
@@ -215,7 +464,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
int r;
len = sizeof(*probe) + 256 * sizeof(struct io_uring_probe_op);
probe = uring_malloc(len);
probe = malloc(len);
if (!probe)
return NULL;
memset(probe, 0, len);
@@ -224,7 +473,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
if (r >= 0)
return probe;
uring_free(probe);
free(probe);
return NULL;
}
@@ -245,19 +494,7 @@ __cold struct io_uring_probe *io_uring_get_probe(void)
__cold void io_uring_free_probe(struct io_uring_probe *probe)
{
uring_free(probe);
}
static inline int __fls(unsigned long x)
{
if (!x)
return 0;
return 8 * sizeof(x) - __builtin_clzl(x);
}
static unsigned roundup_pow2(unsigned depth)
{
return 1U << __fls(depth - 1);
free(probe);
}
static size_t npages(size_t size, long page_size)
@@ -267,8 +504,6 @@ static size_t npages(size_t size, long page_size)
return __fls((int) size);
}
#define KRING_SIZE 320
static size_t rings_size(struct io_uring_params *p, unsigned entries,
unsigned cq_entries, long page_size)
{
@@ -290,9 +525,6 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
return pages * page_size;
}
#define KERN_MAX_ENTRIES 32768
#define KERN_MAX_CQ_ENTRIES (2 * KERN_MAX_ENTRIES)
/*
* Return the required ulimit -l memlock memory required for a given ring
* setup, in bytes. May return -errno on error. On newer (5.12+) kernels,
@@ -304,11 +536,14 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
__cold ssize_t io_uring_mlock_size_params(unsigned entries,
struct io_uring_params *p)
{
struct io_uring_params lp = { };
struct io_uring_params lp;
struct io_uring ring;
unsigned cq_entries;
unsigned cq_entries, sq;
long page_size;
ssize_t ret;
int cret;
memset(&lp, 0, sizeof(lp));
/*
* We only really use this inited ring to see if the kernel is newer
@@ -336,25 +571,12 @@ __cold ssize_t io_uring_mlock_size_params(unsigned entries,
entries = KERN_MAX_ENTRIES;
}
entries = roundup_pow2(entries);
if (p->flags & IORING_SETUP_CQSIZE) {
if (!p->cq_entries)
return -EINVAL;
cq_entries = p->cq_entries;
if (cq_entries > KERN_MAX_CQ_ENTRIES) {
if (!(p->flags & IORING_SETUP_CLAMP))
return -EINVAL;
cq_entries = KERN_MAX_CQ_ENTRIES;
}
cq_entries = roundup_pow2(cq_entries);
if (cq_entries < entries)
return -EINVAL;
} else {
cq_entries = 2 * entries;
}
cret = get_sq_cq_entries(entries, p, &sq, &cq_entries);
if (cret)
return cret;
page_size = get_page_size();
return rings_size(p, entries, cq_entries, page_size);
return rings_size(p, sq, cq_entries, page_size);
}
/*
@@ -363,7 +585,105 @@ __cold ssize_t io_uring_mlock_size_params(unsigned entries,
*/
__cold ssize_t io_uring_mlock_size(unsigned entries, unsigned flags)
{
struct io_uring_params p = { .flags = flags, };
struct io_uring_params p;
memset(&p, 0, sizeof(p));
p.flags = flags;
return io_uring_mlock_size_params(entries, &p);
}
#if defined(__hppa__)
static struct io_uring_buf_ring *br_setup(struct io_uring *ring,
unsigned int nentries, int bgid,
unsigned int flags, int *ret)
{
struct io_uring_buf_ring *br;
struct io_uring_buf_reg reg;
size_t ring_size;
off_t off;
int lret;
memset(&reg, 0, sizeof(reg));
reg.ring_entries = nentries;
reg.bgid = bgid;
reg.flags = IOU_PBUF_RING_MMAP;
*ret = 0;
lret = io_uring_register_buf_ring(ring, &reg, flags);
if (lret) {
*ret = lret;
return NULL;
}
off = IORING_OFF_PBUF_RING | (unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
ring_size = nentries * sizeof(struct io_uring_buf);
br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, ring->ring_fd, off);
if (IS_ERR(br)) {
*ret = PTR_ERR(br);
return NULL;
}
return br;
}
#else
static struct io_uring_buf_ring *br_setup(struct io_uring *ring,
unsigned int nentries, int bgid,
unsigned int flags, int *ret)
{
struct io_uring_buf_ring *br;
struct io_uring_buf_reg reg;
size_t ring_size;
int lret;
memset(&reg, 0, sizeof(reg));
ring_size = nentries * sizeof(struct io_uring_buf);
br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (IS_ERR(br)) {
*ret = PTR_ERR(br);
return NULL;
}
reg.ring_addr = (unsigned long) (uintptr_t) br;
reg.ring_entries = nentries;
reg.bgid = bgid;
*ret = 0;
lret = io_uring_register_buf_ring(ring, &reg, flags);
if (lret) {
__sys_munmap(br, ring_size);
*ret = lret;
br = NULL;
}
return br;
}
#endif
struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring,
unsigned int nentries,
int bgid, unsigned int flags,
int *ret)
{
struct io_uring_buf_ring *br;
br = br_setup(ring, nentries, bgid, flags, ret);
if (br)
io_uring_buf_ring_init(br);
return br;
}
int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
unsigned int nentries, int bgid)
{
int ret;
ret = io_uring_unregister_buf_ring(ring, bgid);
if (ret)
return ret;
__sys_munmap(br, nentries * sizeof(struct io_uring_buf));
return 0;
}
src/setup.h
+9
View File
@@ -0,0 +1,9 @@
/* SPDX-License-Identifier: MIT */
#ifndef LIBURING_SETUP_H
#define LIBURING_SETUP_H
int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
struct io_uring_params *p, void *buf,
size_t buf_size);
#endif
src/syscall.h
+2
View File
@@ -37,6 +37,8 @@ static inline bool IS_ERR(const void *ptr)
#include "arch/x86/syscall.h"
#elif defined(__aarch64__)
#include "arch/aarch64/syscall.h"
#elif defined(__riscv) && __riscv_xlen == 64
#include "arch/riscv64/syscall.h"
#else
/*
* We don't have native syscall wrappers
src/version.c
+21
View File
@@ -0,0 +1,21 @@
/* SPDX-License-Identifier: MIT */
#include "liburing.h"
#include "liburing/io_uring_version.h"
int io_uring_major_version(void)
{
return IO_URING_VERSION_MAJOR;
}
int io_uring_minor_version(void)
{
return IO_URING_VERSION_MINOR;
}
bool io_uring_check_version(int major, int minor)
{
return major > io_uring_major_version() ||
(major == io_uring_major_version() &&
minor > io_uring_minor_version());
}
test/232c93d07b74.c
+6 -6
View File
@@ -33,9 +33,9 @@ struct params {
__be16 bind_port;
};
pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
int rcv_ready = 0;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static int rcv_ready = 0;
static void set_rcv_ready(void)
{
@@ -64,8 +64,7 @@ static void *rcv(void *arg)
int res;
if (p->tcp) {
int val = 1;
int ret, val = 1;
s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP);
res = setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val));
@@ -77,7 +76,8 @@ static void *rcv(void *arg)
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr("127.0.0.1");
assert(t_bind_ephemeral_port(s0, &addr) == 0);
ret = t_bind_ephemeral_port(s0, &addr);
assert(!ret);
p->bind_port = addr.sin_port;
} else {
s0 = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
test/35fa71a030ca.c
+3 -3
View File
@@ -176,7 +176,7 @@ static void kill_and_wait(int pid, int* status)
}
#define SYZ_HAVE_SETUP_TEST 1
static void setup_test()
static void setup_test(void)
{
prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
setpgrp();
@@ -262,7 +262,7 @@ static void loop(void)
}
}
uint64_t r[1] = {0xffffffffffffffff};
static uint64_t r[1] = {0xffffffffffffffff};
void execute_call(int call)
{
@@ -320,7 +320,7 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
signal(SIGINT, sig_int);
mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
signal(SIGALRM, sig_int);
alarm(5);
test/500f9fbadef8.c
+2
View File
@@ -42,6 +42,8 @@ int main(int argc, char *argv[])
sprintf(buf, "./XXXXXX");
fd = mkostemp(buf, O_WRONLY | O_DIRECT | O_CREAT);
if (fd < 0) {
if (errno == EINVAL)
return T_EXIT_SKIP;
perror("mkostemp");
return T_EXIT_FAIL;
}
test/917257daa0fe.c
+1 -1
View File
@@ -19,7 +19,7 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
*(uint32_t*)0x20000000 = 0;
*(uint32_t*)0x20000004 = 0;
test/Makefile
+63 -13
View File
@@ -13,7 +13,8 @@ override CPPFLAGS += \
-D_GNU_SOURCE \
-D__SANE_USERSPACE_TYPES__ \
-I../src/include/ \
-include ../config-host.h
-include ../config-host.h \
-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
CFLAGS ?= -g -O3 -Wall -Wextra
XCFLAGS = -Wno-unused-parameter -Wno-sign-compare
@@ -45,15 +46,21 @@ test_srcs := \
a4c0b3decb33.c \
accept.c \
accept-link.c \
accept-non-empty.c \
accept-reuse.c \
accept-test.c \
across-fork.c \
b19062a56726.c \
b5837bd5311d.c \
bind-listen.c \
buf-ring.c \
buf-ring-nommap.c \
buf-ring-put.c \
ce593a6c480a.c \
close-opath.c \
connect.c \
connect-rep.c \
coredump.c \
cq-full.c \
cq-overflow.c \
cq-peek-batch.c \
@@ -63,19 +70,23 @@ test_srcs := \
d77a67ed5f27.c \
defer.c \
defer-taskrun.c \
defer-tw-timeout.c \
double-poll-crash.c \
drop-submit.c \
eeed8b54e0df.c \
empty-eownerdead.c \
eploop.c \
eventfd.c \
eventfd-disable.c \
eventfd-reg.c \
eventfd-ring.c \
evloop.c \
exec-target.c \
exit-no-cleanup.c \
fadvise.c \
fallocate.c \
fc2a85cb02ef.c \
fd-install.c \
fd-pass.c \
file-register.c \
files-exit-hang-poll.c \
@@ -83,14 +94,21 @@ test_srcs := \
file-update.c \
file-verify.c \
fixed-buf-iter.c \
fixed-buf-merge.c \
fixed-hugepage.c \
fixed-link.c \
fixed-reuse.c \
fpos.c \
fsnotify.c \
fsync.c \
futex.c \
hardlink.c \
ignore-single-mmap.c \
init-mem.c \
io-cancel.c \
iopoll.c \
iopoll-leak.c \
iopoll-overflow.c \
io_uring_enter.c \
io_uring_passthrough.c \
io_uring_register.c \
@@ -103,15 +121,21 @@ test_srcs := \
madvise.c \
mkdir.c \
msg-ring.c \
msg-ring-fd.c \
msg-ring-flags.c \
msg-ring-overflow.c \
multicqes_drain.c \
no-mmap-inval.c \
nolibc.c \
nop-all-sizes.c \
nop.c \
ooo-file-unreg.c \
openat2.c \
open-close.c \
open-direct-link.c \
open-direct-pick.c \
personality.c \
pipe-bug.c \
pipe-eof.c \
pipe-reuse.c \
poll.c \
@@ -120,42 +144,54 @@ test_srcs := \
poll-cancel-ton.c \
poll-link.c \
poll-many.c \
poll-mshot-update.c \
poll-mshot-overflow.c \
poll-mshot-update.c \
poll-race.c \
poll-race-mshot.c \
poll-ring.c \
poll-v-poll.c \
pollfree.c \
probe.c \
read-before-exit.c \
read-mshot.c \
read-mshot-empty.c \
read-write.c \
recv-msgall.c \
recv-msgall-stream.c \
recv-multishot.c \
reg-fd-only.c \
reg-hint.c \
reg-reg-ring.c \
regbuf-merge.c \
register-restrictions.c \
rename.c \
ringbuf-read.c \
ringbuf-status.c \
ring-leak2.c \
ring-leak.c \
rsrc_tags.c \
rw_merge_test.c \
self.c \
sendmsg_fs_cve.c \
recvsend_bundle.c \
send_recv.c \
send_recvmsg.c \
send-zerocopy.c \
shared-wq.c \
short-read.c \
shutdown.c \
sigfd-deadlock.c \
single-issuer.c \
skip-cqe.c \
socket.c \
socket-io-cmd.c \
socket-getsetsock-cmd.c \
socket-rw.c \
socket-rw-eagain.c \
socket-rw-offset.c \
splice.c \
sq-full.c \
sq-full-cpp.cc \
sqpoll-cancel-hang.c \
sqpoll-disable-exit.c \
sqpoll-exec.c \
sq-poll-dup.c \
sqpoll-exit-hang.c \
sq-poll-kthread.c \
@@ -166,20 +202,20 @@ test_srcs := \
submit-and-wait.c \
submit-link-fail.c \
submit-reuse.c \
sync-cancel.c \
symlink.c \
sync-cancel.c \
teardowns.c \
thread-exit.c \
timeout.c \
timeout-new.c \
timeout-overflow.c \
truncate.c \
tty-write-dpoll.c \
unlink.c \
version.c \
waitid.c \
wakeup-hang.c \
wq-aff.c \
xattr.c \
skip-cqe.c \
single-issuer.c \
send-zerocopy.c \
# EOL
all_targets :=
@@ -210,11 +246,22 @@ all: $(test_targets)
helpers.o: helpers.c
$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ -c $<
%.t: %.c $(helpers) helpers.h ../src/liburing.a
LIBURING := $(shell if [ -e ../src/liburing.a ]; then echo ../src/liburing.a; fi)
%.t: %.c $(helpers) helpers.h $(LIBURING)
$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
%.t: %.cc $(helpers) helpers.h ../src/liburing.a
$(QUIET_CXX)$(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
#
# Clang++ is not happy with -Wmissing-prototypes:
#
# cc1plus: warning: command-line option '-Wmissing-prototypes' \
# is valid for C/ObjC but not for C++
#
%.t: %.cc $(helpers) helpers.h $(LIBURING)
$(QUIET_CXX)$(CXX) \
$(patsubst -Wmissing-prototypes,,$(CPPFLAGS)) \
$(patsubst -Wmissing-prototypes,,$(CXXFLAGS)) \
-o $@ $< $(helpers) $(LDFLAGS)
install: $(test_targets) runtests.sh runtests-loop.sh
@@ -223,6 +270,9 @@ install: $(test_targets) runtests.sh runtests-loop.sh
$(INSTALL) -D -m 755 runtests.sh $(datadir)/liburing-test/
$(INSTALL) -D -m 755 runtests-loop.sh $(datadir)/liburing-test/
uninstall:
@rm -rf $(datadir)/liburing-test/
clean:
@rm -f $(all_targets) helpers.o output/*
@rm -rf output/
test/a0908ae19763.c
+2 -2
View File
@@ -14,13 +14,13 @@
#include "helpers.h"
#include "../src/syscall.h"
uint64_t r[1] = {0xffffffffffffffff};
static uint64_t r[1] = {0xffffffffffffffff};
int main(int argc, char *argv[])
{
if (argc > 1)
return T_EXIT_SKIP;
mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
intptr_t res = 0;
*(uint32_t*)0x20000080 = 0;
*(uint32_t*)0x20000084 = 0;
test/a4c0b3decb33.c
+3 -3
View File
@@ -95,7 +95,7 @@ static void kill_and_wait(int pid, int* status)
}
}
static void setup_test()
static void setup_test(void)
{
prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
setpgrp();
@@ -109,7 +109,7 @@ static void execute_one(void);
static void loop(void)
{
int iter;
for (iter = 0; iter < 5000; iter++) {
for (iter = 0; iter < 50; iter++) {
int pid = fork();
if (pid < 0)
exit(1);
@@ -175,7 +175,7 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
signal(SIGINT, sig_int);
mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
loop();
return T_EXIT_PASS;
}
test/accept-link.c
+4 -4
View File
@@ -16,8 +16,8 @@
#include "liburing.h"
#include "helpers.h"
pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static int recv_thread_ready = 0;
static int recv_thread_done = 0;
@@ -77,7 +77,7 @@ static void *send_thread(void *arg)
return NULL;
}
void *recv_thread(void *arg)
static void *recv_thread(void *arg)
{
struct data *data = arg;
struct io_uring ring;
@@ -195,7 +195,7 @@ static int test_accept_timeout(int do_connect, unsigned long timeout)
if (ret) {
fprintf(stderr, "queue_init: %d\n", ret);
return 1;
};
}
fast_poll = (p.features & IORING_FEAT_FAST_POLL) != 0;
io_uring_queue_exit(&ring);
test/accept-non-empty.c
+256
View File
@@ -0,0 +1,256 @@
/* SPDX-License-Identifier: MIT */
/*
* Check that kernels that support it will return IORING_CQE_F_SOCK_NONEMPTY
* on accepts requests where more connections are pending.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/un.h>
#include <netinet/tcp.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <pthread.h>
#include "liburing.h"
#include "helpers.h"
static int no_more_accept;
#define MAX_ACCEPTS 8
struct data {
pthread_t thread;
pthread_barrier_t barrier;
pthread_barrier_t conn_barrier;
int connects;
};
static int start_accept_listen(int port_off, int extra_flags)
{
struct sockaddr_in addr;
int32_t val = 1;
int fd, ret;
fd = socket(AF_INET, SOCK_STREAM | extra_flags, IPPROTO_TCP);
ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val));
assert(ret != -1);
ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
assert(ret != -1);
addr.sin_family = AF_INET;
addr.sin_port = htons(0x1235 + port_off);
addr.sin_addr.s_addr = inet_addr("127.0.0.1");
ret = bind(fd, (struct sockaddr *) &addr, sizeof(addr));
assert(ret != -1);
ret = listen(fd, 20000);
assert(ret != -1);
return fd;
}
static int test_maccept(struct data *d, int flags, int fixed)
{
struct io_uring_params p = { };
struct io_uring ring;
struct io_uring_cqe *cqe;
struct io_uring_sqe *sqe;
int err = 0, fd, ret, i, *fds;
p.flags = flags;
ret = io_uring_queue_init_params(8, &ring, &p);
if (ret == -EINVAL) {
return T_EXIT_SKIP;
} else if (ret < 0) {
fprintf(stderr, "ring setup failure: %d\n", ret);
return T_EXIT_FAIL;
}
if (!(p.features & IORING_FEAT_RECVSEND_BUNDLE)) {
no_more_accept = 1;
return 0;
}
fds = malloc(MAX_ACCEPTS * sizeof(int));
memset(fds, -1, MAX_ACCEPTS * sizeof(int));
if (fixed) {
io_uring_register_ring_fd(&ring);
ret = io_uring_register_files(&ring, fds, MAX_ACCEPTS);
if (ret) {
fprintf(stderr, "file reg %d\n", ret);
return -1;
}
}
fd = start_accept_listen(0, 0);
pthread_barrier_wait(&d->barrier);
if (d->connects > 1)
pthread_barrier_wait(&d->conn_barrier);
for (i = 0; i < d->connects; i++) {
sqe = io_uring_get_sqe(&ring);
if (fixed)
io_uring_prep_accept_direct(sqe, fd, NULL, NULL, 0, i);
else
io_uring_prep_accept(sqe, fd, NULL, NULL, 0);
ret = io_uring_submit_and_wait(&ring, 1);
assert(ret != -1);
ret = io_uring_wait_cqe(&ring, &cqe);
assert(!ret);
if (cqe->res < 0) {
fprintf(stderr, "res=%d\n", cqe->res);
break;
}
fds[i] = cqe->res;
if (d->connects == 1) {
if (cqe->flags & IORING_CQE_F_SOCK_NONEMPTY) {
fprintf(stderr, "Non-empty sock on single?\n");
err = 1;
break;
}
} else {
int last = i + 1 == d->connects;
if (last && cqe->flags & IORING_CQE_F_SOCK_NONEMPTY) {
fprintf(stderr, "Non-empty sock on last?\n");
err = 1;
break;
} else if (!last && !(cqe->flags & IORING_CQE_F_SOCK_NONEMPTY)) {
fprintf(stderr, "Empty on multi connect?\n");
err = 1;
break;
}
}
io_uring_cqe_seen(&ring, cqe);
}
close(fd);
if (!fixed) {
for (i = 0; i < MAX_ACCEPTS; i++)
if (fds[i] != -1)
close(fds[i]);
}
free(fds);
io_uring_queue_exit(&ring);
return err;
}
static void *connect_fn(void *data)
{
struct sockaddr_in addr = { };
struct data *d = data;
int i;
pthread_barrier_wait(&d->barrier);
addr.sin_family = AF_INET;
addr.sin_port = htons(0x1235);
addr.sin_addr.s_addr = inet_addr("127.0.0.1");
for (i = 0; i < d->connects; i++) {
int s;
s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (s < 0) {
perror("socket");
break;
}
if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
perror("connect");
break;
}
}
if (d->connects > 1)
pthread_barrier_wait(&d->conn_barrier);
return NULL;
}
static void setup_thread(struct data *d, int nconns)
{
d->connects = nconns;
pthread_barrier_init(&d->barrier, NULL, 2);
pthread_barrier_init(&d->conn_barrier, NULL, 2);
pthread_create(&d->thread, NULL, connect_fn, d);
}
static int test(int flags, int fixed)
{
struct data d;
void *tret;
int ret;
setup_thread(&d, 1);
ret = test_maccept(&d, flags, fixed);
if (ret) {
fprintf(stderr, "test conns=1 failed\n");
return ret;
}
if (no_more_accept)
return T_EXIT_SKIP;
pthread_join(d.thread, &tret);
setup_thread(&d, MAX_ACCEPTS);
ret = test_maccept(&d, flags, fixed);
if (ret) {
fprintf(stderr, "test conns=MAX failed\n");
return ret;
}
pthread_join(d.thread, &tret);
return 0;
}
int main(int argc, char *argv[])
{
int ret;
if (argc > 1)
return T_EXIT_SKIP;
ret = test(0, 0);
if (no_more_accept)
return T_EXIT_SKIP;
if (ret) {
fprintf(stderr, "test 0 0 failed\n");
return ret;
}
ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN, 0);
if (ret) {
fprintf(stderr, "test DEFER 0 failed\n");
return ret;
}
ret = test(0, 1);
if (ret) {
fprintf(stderr, "test 0 1 failed\n");
return ret;
}
ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN, 1);
if (ret) {
fprintf(stderr, "test DEFER 1 failed\n");
return ret;
}
return 0;
}
test/accept-reuse.c
+6 -8
View File
@@ -1,5 +1,4 @@
/* SPDX-License-Identifier: MIT */
#include <liburing.h>
#include <netdb.h>
#include <string.h>
#include <sys/socket.h>
@@ -11,17 +10,16 @@
#include "helpers.h"
#include "../src/syscall.h"
struct io_uring io_uring;
static struct io_uring io_uring;
int sys_io_uring_enter(const int fd,
const unsigned to_submit,
const unsigned min_complete,
const unsigned flags, sigset_t * const sig)
static int sys_io_uring_enter(const int fd, const unsigned to_submit,
const unsigned min_complete,
const unsigned flags, sigset_t * const sig)
{
return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig);
}
int submit_sqe(void)
static int submit_sqe(void)
{
struct io_uring_sq *sq = &io_uring.sq;
const unsigned tail = *sq->ktail;
@@ -47,7 +45,7 @@ int main(int argc, char **argv)
return T_EXIT_SKIP;
memset(&params, 0, sizeof(params));
ret = io_uring_queue_init_params(4, &io_uring, &params);
ret = t_io_uring_init_sqarray(4, &io_uring, &params);
if (ret) {
fprintf(stderr, "io_uring_init_failed: %d\n", ret);
return T_EXIT_FAIL;
test/accept.c
+56 -36
View File
@@ -195,7 +195,8 @@ static int start_accept_listen(struct sockaddr_in *addr, int port_off,
addr->sin_family = AF_INET;
addr->sin_addr.s_addr = inet_addr("127.0.0.1");
assert(!t_bind_ephemeral_port(fd, addr));
ret = t_bind_ephemeral_port(fd, addr);
assert(!ret);
ret = listen(fd, 128);
assert(ret != -1);
@@ -272,6 +273,7 @@ static int test_loop(struct io_uring *ring,
uint32_t multishot_mask = 0;
int nr_fds = multishot ? MAX_FDS : 1;
int multishot_idx = multishot ? INITIAL_USER_DATA : 0;
int err_ret = T_EXIT_FAIL;
if (args.overflow)
cause_overflow(ring);
@@ -298,6 +300,7 @@ static int test_loop(struct io_uring *ring,
no_accept_multi = 1;
else
no_accept = 1;
ret = T_EXIT_SKIP;
goto out;
} else if (s_fd[i] < 0) {
if (args.accept_should_error &&
@@ -308,6 +311,9 @@ static int test_loop(struct io_uring *ring,
multishot ? "Multishot" : "",
i, s_fd[i]);
goto err;
} else if (s_fd[i] == 195 && args.overflow) {
fprintf(stderr, "Broken overflow handling\n");
goto err;
}
if (multishot && fixed) {
@@ -360,10 +366,10 @@ static int test_loop(struct io_uring *ring,
out:
close_sock_fds(s_fd, c_fd, nr_fds, fixed);
return 0;
return T_EXIT_PASS;
err:
close_sock_fds(s_fd, c_fd, nr_fds, fixed);
return 1;
return err_ret;
}
static int test(struct io_uring *ring, struct accept_test_args args)
@@ -372,7 +378,7 @@ static int test(struct io_uring *ring, struct accept_test_args args)
int ret = 0;
int loop;
int32_t recv_s0 = start_accept_listen(&addr, 0,
args.nonblock ? O_NONBLOCK : 0);
args.nonblock ? SOCK_NONBLOCK : 0);
if (args.queue_accept_before_connect)
queue_accept_conn(ring, recv_s0, args);
for (loop = 0; loop < 1 + args.extra_loops; loop++) {
@@ -425,7 +431,7 @@ struct test_accept_many_args {
};
/*
* Test issue many accepts and see if we handle cancellation on exit
* Test issue many accepts and see if we handle cancelation on exit
*/
static int test_accept_many(struct test_accept_many_args args)
{
@@ -458,7 +464,7 @@ static int test_accept_many(struct test_accept_many_args args)
for (i = 0; i < nr_socks; i++)
fds[i] = start_accept_listen(NULL, i,
args.nonblock ? O_NONBLOCK : 0);
args.nonblock ? SOCK_NONBLOCK : 0);
for (i = 0; i < nr; i++) {
int sock_idx = args.single_sock ? 0 : i;
@@ -480,7 +486,7 @@ static int test_accept_many(struct test_accept_many_args args)
if (io_uring_peek_cqe(&m_io_uring, &cqe))
break;
if (cqe->res != -ECANCELED) {
fprintf(stderr, "Expected cqe to be cancelled %d\n", cqe->res);
fprintf(stderr, "Expected cqe to be canceled %d\n", cqe->res);
ret = 1;
goto out;
}
@@ -552,6 +558,9 @@ static int test_accept_cancel(unsigned usecs, unsigned int nr, bool multishot)
fprintf(stderr, "unexpected 0 user data\n");
goto err;
} else if (cqe->user_data <= nr) {
/* no multishot */
if (cqe->res == -EINVAL)
return T_EXIT_SKIP;
if (cqe->res != -EINTR && cqe->res != -ECANCELED) {
fprintf(stderr, "Cancelled accept got %d\n", cqe->res);
goto err;
@@ -611,7 +620,7 @@ static int test_multishot_accept(int count, bool before, bool overflow)
return ret;
}
static int test_accept_multishot_wrong_arg()
static int test_accept_multishot_wrong_arg(void)
{
struct io_uring m_io_uring;
struct io_uring_cqe *cqe;
@@ -676,7 +685,12 @@ static int test_accept_fixed(void)
ret = io_uring_queue_init(32, &m_io_uring, 0);
assert(ret >= 0);
ret = io_uring_register_files(&m_io_uring, &fd, 1);
assert(ret == 0);
if (ret) {
/* kernel doesn't support sparse registered files, skip */
if (ret == -EBADF || ret == -EINVAL)
return T_EXIT_SKIP;
return T_EXIT_FAIL;
}
ret = test(&m_io_uring, args);
io_uring_queue_exit(&m_io_uring);
return ret;
@@ -698,7 +712,12 @@ static int test_multishot_fixed_accept(void)
ret = io_uring_queue_init(MAX_FDS + 10, &m_io_uring, 0);
assert(ret >= 0);
ret = io_uring_register_files(&m_io_uring, fd, MAX_FDS);
assert(ret == 0);
if (ret) {
/* kernel doesn't support sparse registered files, skip */
if (ret == -EBADF || ret == -EINVAL)
return T_EXIT_SKIP;
return T_EXIT_FAIL;
}
ret = test(&m_io_uring, args);
io_uring_queue_exit(&m_io_uring);
return ret;
@@ -733,8 +752,9 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
ret = test_accept(1, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept failed\n");
return ret;
}
@@ -742,141 +762,141 @@ int main(int argc, char *argv[])
return T_EXIT_SKIP;
ret = test_accept(2, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept(2) failed\n");
return ret;
}
ret = test_accept(2, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept(2, true) failed\n");
return ret;
}
ret = test_accept_nonblock(false, 1);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_nonblock failed\n");
return ret;
}
ret = test_accept_nonblock(true, 1);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_nonblock(before, 1) failed\n");
return ret;
}
ret = test_accept_nonblock(true, 3);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_nonblock(before,3) failed\n");
return ret;
}
ret = test_accept_fixed();
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_fixed failed\n");
return ret;
}
ret = test_multishot_fixed_accept();
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_multishot_fixed_accept failed\n");
return ret;
}
ret = test_accept_multishot_wrong_arg();
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_multishot_wrong_arg failed\n");
return ret;
}
ret = test_accept_sqpoll();
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_sqpoll failed\n");
return ret;
}
ret = test_accept_cancel(0, 1, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel nodelay failed\n");
return ret;
}
ret = test_accept_cancel(10000, 1, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel delay failed\n");
return ret;
}
ret = test_accept_cancel(0, 4, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel nodelay failed\n");
return ret;
}
ret = test_accept_cancel(10000, 4, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel delay failed\n");
return ret;
}
ret = test_accept_cancel(0, 1, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel multishot nodelay failed\n");
return ret;
}
ret = test_accept_cancel(10000, 1, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel multishot delay failed\n");
return ret;
}
ret = test_accept_cancel(0, 4, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel multishot nodelay failed\n");
return ret;
}
ret = test_accept_cancel(10000, 4, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_cancel multishot delay failed\n");
return ret;
}
ret = test_multishot_accept(1, true, true);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_multishot_accept(1, false, true) failed\n");
return ret;
}
ret = test_multishot_accept(1, false, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_multishot_accept(1, false, false) failed\n");
return ret;
}
ret = test_multishot_accept(1, true, false);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_multishot_accept(1, true, false) failed\n");
return ret;
}
ret = test_accept_many((struct test_accept_many_args) {});
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_many failed\n");
return ret;
}
ret = test_accept_many((struct test_accept_many_args) {
.usecs = 100000 });
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_many(sleep) failed\n");
return ret;
}
ret = test_accept_many((struct test_accept_many_args) {
.nonblock = true });
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_many(nonblock) failed\n");
return ret;
}
@@ -885,13 +905,13 @@ int main(int argc, char *argv[])
.nonblock = true,
.single_sock = true,
.close_fds = true });
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_many(nonblock,close) failed\n");
return ret;
}
ret = test_accept_pending_on_exit();
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_accept_pending_on_exit failed\n");
return ret;
}
test/b19062a56726.c
+1 -1
View File
@@ -19,7 +19,7 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
*(uint32_t*)0x20000200 = 0;
*(uint32_t*)0x20000204 = 0;
test/bind-listen.c
+408
View File
@@ -0,0 +1,408 @@
/* SPDX-License-Identifier: MIT */
/*
* Configure and operate a TCP socket solely with io_uring.
*/
#include <stdio.h>
#include <string.h>
#include <liburing.h>
#include <err.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <sys/socket.h>
#include <unistd.h>
#include <stdlib.h>
#include <netinet/ip.h>
#include "liburing.h"
#include "helpers.h"
static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec)
{
ts->tv_sec = msec / 1000;
ts->tv_nsec = (msec % 1000) * 1000000;
}
static const char *magic = "Hello World!";
static int use_port = 8000;
enum {
SRV_INDEX = 0,
CLI_INDEX,
CONN_INDEX,
};
static int connect_client(struct io_uring *ring, unsigned short peer_port)
{
struct __kernel_timespec ts;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int head, ret, submitted = 0;
struct sockaddr_in peer_addr;
socklen_t addr_len = sizeof(peer_addr);
peer_addr.sin_family = AF_INET;
peer_addr.sin_port = peer_port;
peer_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
sqe = io_uring_get_sqe(ring);
io_uring_prep_socket_direct(sqe, AF_INET, SOCK_STREAM, 0,
CLI_INDEX, 0);
sqe->flags |= IOSQE_IO_LINK;
sqe = io_uring_get_sqe(ring);
io_uring_prep_connect(sqe, CLI_INDEX, (struct sockaddr*) &peer_addr, addr_len);
sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
sqe = io_uring_get_sqe(ring);
io_uring_prep_send(sqe, CLI_INDEX, magic, strlen(magic), 0);
sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
submitted = ret = io_uring_submit(ring);
if (ret < 0)
return T_SETUP_SKIP;
msec_to_ts(&ts, 300);
ret = io_uring_wait_cqes(ring, &cqe, submitted, &ts, NULL);
if (ret < 0)
return T_SETUP_SKIP;
io_uring_for_each_cqe(ring, head, cqe) {
ret = cqe->res;
if (ret < 0)
return T_SETUP_SKIP;
} io_uring_cq_advance(ring, submitted);
return T_SETUP_OK;
}
static int setup_srv(struct io_uring *ring, struct sockaddr_in *server_addr)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct __kernel_timespec ts;
int ret, val, submitted;
unsigned head;
memset(server_addr, 0, sizeof(struct sockaddr_in));
server_addr->sin_family = AF_INET;
server_addr->sin_port = htons(use_port++);
server_addr->sin_addr.s_addr = htons(INADDR_ANY);
sqe = io_uring_get_sqe(ring);
io_uring_prep_socket_direct(sqe, AF_INET, SOCK_STREAM, 0, SRV_INDEX, 0);
sqe->flags |= IOSQE_IO_LINK;
sqe = io_uring_get_sqe(ring);
val = 1;
io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_SETSOCKOPT, 0, SOL_SOCKET,
SO_REUSEADDR, &val, sizeof(val));
sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
sqe = io_uring_get_sqe(ring);
io_uring_prep_bind(sqe, SRV_INDEX, (struct sockaddr *) server_addr,
sizeof(struct sockaddr_in));
sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
sqe = io_uring_get_sqe(ring);
io_uring_prep_listen(sqe, SRV_INDEX, 1);
sqe->flags |= IOSQE_FIXED_FILE;
submitted = ret = io_uring_submit(ring);
if (ret < 0) {
fprintf(stderr, "submission failed. %d\n", ret);
return T_EXIT_FAIL;
}
msec_to_ts(&ts, 300);
ret = io_uring_wait_cqes(ring, &cqe, ret, &ts, NULL);
if (ret < 0) {
fprintf(stderr, "submission failed. %d\n", ret);
return T_EXIT_FAIL;
}
io_uring_for_each_cqe(ring, head, cqe) {
ret = cqe->res;
if (ret < 0) {
fprintf(stderr, "Server startup failed. step %d got %d \n", head, ret);
return T_EXIT_FAIL;
}
} io_uring_cq_advance(ring, submitted);
return T_SETUP_OK;
}
static int test_good_server(unsigned int ring_flags)
{
struct sockaddr_in server_addr;
struct __kernel_timespec ts;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
int ret;
int fds[3];
char buf[1024];
memset(fds, -1, sizeof(fds));
ret = t_create_ring(10, &ring, ring_flags | IORING_SETUP_SUBMIT_ALL);
if (ret < 0) {
fprintf(stderr, "queue_init: %s\n", strerror(-ret));
return T_SETUP_SKIP;
}
ret = io_uring_register_files(&ring, fds, 3);
if (ret) {
fprintf(stderr, "server file register %d\n", ret);
return T_SETUP_SKIP;
}
ret = setup_srv(&ring, &server_addr);
if (ret != T_SETUP_OK) {
fprintf(stderr, "srv startup failed.\n");
return T_EXIT_FAIL;
}
if (connect_client(&ring, server_addr.sin_port) != T_SETUP_OK) {
fprintf(stderr, "cli startup failed.\n");
return T_SETUP_SKIP;
}
/* Wait for a request */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_accept_direct(sqe, SRV_INDEX, NULL, NULL, 0, CONN_INDEX);
sqe->flags |= IOSQE_FIXED_FILE;
io_uring_submit(&ring);
io_uring_wait_cqe(&ring, &cqe);
if (cqe->res < 0) {
fprintf(stderr, "accept failed. %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(&ring, cqe);
sqe = io_uring_get_sqe(&ring);
io_uring_prep_recv(sqe, CONN_INDEX, buf, BUFSIZ, 0);
sqe->flags |= IOSQE_FIXED_FILE;
io_uring_submit(&ring);
io_uring_wait_cqe_timeout(&ring, &cqe, &ts);
if (cqe->res < 0) {
fprintf(stderr, "bad receive cqe. %d\n", cqe->res);
return T_EXIT_FAIL;
}
ret = cqe->res;
io_uring_cqe_seen(&ring, cqe);
io_uring_queue_exit(&ring);
if (ret != strlen(magic) || strncmp(buf, magic, ret)) {
fprintf(stderr, "didn't receive expected string. Got %d '%s'\n", ret, buf);
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
static int test_bad_bind(void)
{
struct sockaddr_in server_addr;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
int sock = -1, err;
int ret = T_EXIT_FAIL;
memset(&server_addr, 0, sizeof(struct sockaddr_in));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(9001);
server_addr.sin_addr.s_addr = htons(INADDR_ANY);
err = t_create_ring(1, &ring, 0);
if (err < 0) {
fprintf(stderr, "queue_init: %s\n", strerror(-ret));
return T_SETUP_SKIP;
}
sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock < 0) {
perror("socket");
goto fail;
}
/* Bind with size 0 */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_bind(sqe, sock, (struct sockaddr *) &server_addr, 0);
err = io_uring_submit(&ring);
if (err < 0)
goto fail;
err = io_uring_wait_cqe(&ring, &cqe);
if (err)
goto fail;
if (cqe->res != -EINVAL)
goto fail;
io_uring_cqe_seen(&ring, cqe);
/* Bind with bad fd */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_bind(sqe, 0, (struct sockaddr *) &server_addr, sizeof(struct sockaddr_in));
err = io_uring_submit(&ring);
if (err < 0)
goto fail;
err = io_uring_wait_cqe(&ring, &cqe);
if (err)
goto fail;
if (cqe->res != -ENOTSOCK)
goto fail;
io_uring_cqe_seen(&ring, cqe);
ret = T_EXIT_PASS;
/* bind with weird value */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_bind(sqe, sock, (struct sockaddr *) &server_addr, sizeof(struct sockaddr_in));
sqe->rw_flags = 1;
err = io_uring_submit(&ring);
if (err < 0)
goto fail;
err = io_uring_wait_cqe(&ring, &cqe);
if (err)
goto fail;
if (cqe->res != -EINVAL)
goto fail;
io_uring_cqe_seen(&ring, cqe);
ret = T_EXIT_PASS;
fail:
io_uring_queue_exit(&ring);
if (sock != -1)
close(sock);
return ret;
}
static int test_bad_listen(void)
{
struct sockaddr_in server_addr;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
int sock = -1, err;
int ret = T_EXIT_FAIL;
memset(&server_addr, 0, sizeof(struct sockaddr_in));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(8001);
server_addr.sin_addr.s_addr = htons(INADDR_ANY);
err = t_create_ring(1, &ring, 0);
if (err < 0) {
fprintf(stderr, "queue_init: %d\n", err);
return T_SETUP_SKIP;
}
sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock < 0) {
perror("socket");
goto fail;
}
err = t_bind_ephemeral_port(sock, &server_addr);
if (err) {
fprintf(stderr, "bind: %s\n", strerror(-err));
goto fail;
}
/* listen on bad sock */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_listen(sqe, 0, 1);
err = io_uring_submit(&ring);
if (err < 0)
goto fail;
err = io_uring_wait_cqe(&ring, &cqe);
if (err)
goto fail;
if (cqe->res != -ENOTSOCK)
goto fail;
io_uring_cqe_seen(&ring, cqe);
/* listen with weird parameters */
sqe = io_uring_get_sqe(&ring);
io_uring_prep_listen(sqe, sock, 1);
sqe->addr2 = 0xffffff;
err = io_uring_submit(&ring);
if (err < 0)
goto fail;
err = io_uring_wait_cqe(&ring, &cqe);
if (err)
goto fail;
if (cqe->res != -EINVAL)
goto fail;
io_uring_cqe_seen(&ring, cqe);
ret = T_EXIT_PASS;
fail:
io_uring_queue_exit(&ring);
if (sock != -1)
close(sock);
return ret;
}
int main(int argc, char *argv[])
{
struct io_uring_probe *probe;
int ret;
if (argc > 1)
return 0;
/*
* This test is not supported on older kernels. Check for
* OP_LISTEN, since that is the last feature required to support
* it.
*/
probe = io_uring_get_probe();
if (!probe)
return T_EXIT_SKIP;
if (!io_uring_opcode_supported(probe, IORING_OP_LISTEN))
return T_EXIT_SKIP;
ret = test_good_server(0);
if (ret) {
fprintf(stderr, "good 0 failed\n");
return T_EXIT_FAIL;
}
ret = test_good_server(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
if (ret) {
fprintf(stderr, "good defer failed\n");
return T_EXIT_FAIL;
}
ret = test_good_server(IORING_SETUP_SQPOLL);
if (ret) {
fprintf(stderr, "good sqpoll failed\n");
return T_EXIT_FAIL;
}
ret = test_bad_bind();
if (ret) {
fprintf(stderr, "bad bind failed\n");
return T_EXIT_FAIL;
}
ret = test_bad_listen();
if (ret) {
fprintf(stderr, "bad listen failed\n");
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
test/buf-ring-nommap.c
+123
View File
@@ -0,0 +1,123 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: test IOU_PBUF_RING_MMAP with a ring setup with a ring
* setup without mmap'ing sq/cq arrays
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#include "liburing.h"
#include "helpers.h"
static int bgid = 5;
static int bid = 89;
int main(int argc, char *argv[])
{
struct io_uring_buf_ring *br;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
size_t ring_size;
int ret, ring_mask, fds[2];
struct io_uring_buf_reg reg = {
.ring_entries = 1,
.bgid = bgid,
.flags = IOU_PBUF_RING_MMAP,
};
struct io_uring_params p = { };
void *ring_mem;
char buf[32];
off_t off;
if (argc > 1)
return T_EXIT_SKIP;
if (posix_memalign(&ring_mem, 16384, 16384))
return T_EXIT_FAIL;
memset(ring_mem, 0, 16384);
p.flags = IORING_SETUP_NO_MMAP;
ret = io_uring_queue_init_mem(1, &ring, &p, ring_mem, 16384);
if (ret < 0) {
if (ret == -EINVAL || ret == -ENOMEM)
return T_EXIT_SKIP;
fprintf(stderr, "queue init failed %d\n", ret);
return T_EXIT_FAIL;
}
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
ring_size = sizeof(struct io_uring_buf);
ring_mask = io_uring_buf_ring_mask(1);
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
if (ret == -EINVAL)
return T_EXIT_SKIP;
fprintf(stderr, "reg buf ring: %d\n", ret);
return T_EXIT_FAIL;
}
off = IORING_OFF_PBUF_RING |
(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
if (br == MAP_FAILED) {
if (errno == ENOMEM)
return T_EXIT_SKIP;
perror("mmap");
return T_EXIT_FAIL;
}
io_uring_buf_ring_add(br, buf, sizeof(buf), bid, ring_mask, 0);
io_uring_buf_ring_advance(br, 1);
sqe = io_uring_get_sqe(&ring);
io_uring_prep_read(sqe, fds[0], NULL, 0, 0);
sqe->flags |= IOSQE_BUFFER_SELECT;
sqe->buf_group = bgid;
io_uring_submit(&ring);
ret = write(fds[1], "Hello", 5);
if (ret < 0) {
perror("write");
return T_EXIT_FAIL;
} else if (ret != 5) {
fprintf(stderr, "short write %d\n", ret);
return T_EXIT_FAIL;
}
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret) {
fprintf(stderr, "wait %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res < 0) {
fprintf(stderr, "cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
if (!(cqe->flags & IORING_CQE_F_BUFFER)) {
fprintf(stderr, "buffer not selected in cqe\n");
return T_EXIT_FAIL;
}
if ((cqe->flags >> IORING_CQE_BUFFER_SHIFT) != bid) {
fprintf(stderr, "wrong buffer id returned\n");
return T_EXIT_FAIL;
}
io_uring_cqe_seen(&ring, cqe);
io_uring_queue_exit(&ring);
return T_EXIT_PASS;
}
test/buf-ring-put.c
+83
View File
@@ -0,0 +1,83 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: test persistence of mmap'ed provided ring buffers. Use a range
* of buffer group IDs that puts us into both the lower end array
* and higher end xarry.
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#include "liburing.h"
#include "helpers.h"
#define BGID_START 60
#define BGID_NR 10
#define ENTRIES 512
int main(int argc, char *argv[])
{
struct io_uring_buf_ring *br[BGID_NR];
struct io_uring ring;
size_t ring_size;
int ret, i, j;
if (argc > 1)
return T_EXIT_SKIP;
ret = io_uring_queue_init(1, &ring, 0);
if (ret) {
fprintf(stderr, "queue init failed %d\n", ret);
return T_EXIT_FAIL;
}
ring_size = ENTRIES * sizeof(struct io_uring_buf);
for (i = 0; i < BGID_NR; i++) {
int bgid = BGID_START + i;
struct io_uring_buf_reg reg = {
.ring_entries = ENTRIES,
.bgid = bgid,
.flags = IOU_PBUF_RING_MMAP,
};
off_t off;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
if (ret == -EINVAL)
return T_EXIT_SKIP;
fprintf(stderr, "reg buf ring: %d\n", ret);
return T_EXIT_FAIL;
}
off = IORING_OFF_PBUF_RING |
(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
br[i] = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
if (br[i] == MAP_FAILED) {
perror("mmap");
return T_EXIT_FAIL;
}
}
for (i = 0; i < BGID_NR; i++) {
ret = io_uring_unregister_buf_ring(&ring, BGID_START + i);
if (ret) {
fprintf(stderr, "reg buf ring: %d\n", ret);
return T_EXIT_FAIL;
}
}
for (j = 0; j < 1000; j++) {
for (i = 0; i < BGID_NR; i++)
memset(br[i], 0x5a, ring_size);
usleep(1000);
}
io_uring_queue_exit(&ring);
return T_EXIT_PASS;
}
test/buf-ring.c
+139 -86
View File
@@ -9,20 +9,22 @@
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <sys/mman.h>
#include "liburing.h"
#include "helpers.h"
static int no_buf_ring;
static int pagesize;
/* test trying to register classic group when ring group exists */
static int test_mixed_reg2(int bgid)
{
struct io_uring_buf_reg reg = { };
struct io_uring_buf_ring *br;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
void *ptr, *bufs;
void *bufs;
int ret;
ret = t_create_ring(1, &ring, 0);
@@ -31,15 +33,8 @@ static int test_mixed_reg2(int bgid)
else if (ret != T_SETUP_OK)
return 1;
if (posix_memalign(&ptr, 4096, 4096))
return 1;
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = 32;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
if (!br) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return 1;
}
@@ -60,6 +55,7 @@ static int test_mixed_reg2(int bgid)
}
io_uring_cqe_seen(&ring, cqe);
io_uring_free_buf_ring(&ring, br, 32, bgid);
io_uring_queue_exit(&ring);
return 0;
}
@@ -67,11 +63,11 @@ static int test_mixed_reg2(int bgid)
/* test trying to register ring group when classic group exists */
static int test_mixed_reg(int bgid)
{
struct io_uring_buf_reg reg = { };
struct io_uring_buf_ring *br;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct io_uring ring;
void *ptr, *bufs;
void *bufs;
int ret;
ret = t_create_ring(1, &ring, 0);
@@ -96,16 +92,9 @@ static int test_mixed_reg(int bgid)
}
io_uring_cqe_seen(&ring, cqe);
if (posix_memalign(&ptr, 4096, 4096))
return 1;
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = 32;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret != -EEXIST) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
if (br) {
fprintf(stderr, "Buffer ring setup succeeded unexpectedly %d\n", ret);
return 1;
}
@@ -116,8 +105,8 @@ static int test_mixed_reg(int bgid)
static int test_double_reg_unreg(int bgid)
{
struct io_uring_buf_reg reg = { };
struct io_uring_buf_ring *br;
struct io_uring ring;
void *ptr;
int ret;
ret = t_create_ring(1, &ring, 0);
@@ -126,21 +115,14 @@ static int test_double_reg_unreg(int bgid)
else if (ret != T_SETUP_OK)
return 1;
if (posix_memalign(&ptr, 4096, 4096))
return 1;
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = 32;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
if (!br) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return 1;
}
/* check that 2nd register with same bgid fails */
reg.ring_addr = (unsigned long) ptr;
reg.ring_addr = (unsigned long) br;
reg.ring_entries = 32;
reg.bgid = bgid;
@@ -150,7 +132,7 @@ static int test_double_reg_unreg(int bgid)
return 1;
}
ret = io_uring_unregister_buf_ring(&ring, bgid);
ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
if (ret) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return 1;
@@ -168,9 +150,8 @@ static int test_double_reg_unreg(int bgid)
static int test_reg_unreg(int bgid)
{
struct io_uring_buf_reg reg = { };
struct io_uring_buf_ring *br;
struct io_uring ring;
void *ptr;
int ret;
ret = t_create_ring(1, &ring, 0);
@@ -179,15 +160,8 @@ static int test_reg_unreg(int bgid)
else if (ret != T_SETUP_OK)
return 1;
if (posix_memalign(&ptr, 4096, 4096))
return 1;
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = 32;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
if (!br) {
if (ret == -EINVAL) {
no_buf_ring = 1;
return 0;
@@ -196,9 +170,9 @@ static int test_reg_unreg(int bgid)
return 1;
}
ret = io_uring_unregister_buf_ring(&ring, bgid);
ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
if (ret) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
fprintf(stderr, "Buffer ring unregister failed %d\n", ret);
return 1;
}
@@ -230,6 +204,54 @@ static int test_bad_reg(int bgid)
return !ret;
}
static int test_full_page_reg(int bgid)
{
#if defined(__hppa__)
return T_EXIT_SKIP;
#else
struct io_uring ring;
int ret;
void *ptr;
struct io_uring_buf_reg reg = { };
int entries = pagesize / sizeof(struct io_uring_buf);
ret = io_uring_queue_init(1, &ring, 0);
if (ret) {
fprintf(stderr, "queue init failed %d\n", ret);
return T_EXIT_FAIL;
}
ret = posix_memalign(&ptr, pagesize, pagesize * 2);
if (ret) {
fprintf(stderr, "posix_memalign failed %d\n", ret);
goto err;
}
ret = mprotect(ptr + pagesize, pagesize, PROT_NONE);
if (ret) {
fprintf(stderr, "mprotect failed %d\n", errno);
goto err1;
}
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = entries;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret)
fprintf(stderr, "register buf ring failed %d\n", ret);
if (mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_WRITE))
fprintf(stderr, "reverting mprotect failed %d\n", errno);
err1:
free(ptr);
err:
io_uring_queue_exit(&ring);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
#endif
}
static int test_one_read(int fd, int bgid, struct io_uring *ring)
{
int ret;
@@ -270,51 +292,63 @@ static int test_one_read(int fd, int bgid, struct io_uring *ring)
return cqe->flags >> 16;
}
static int test_running(int bgid, int entries, int loops)
static int test_running(int bgid, int entries, int loops, int use_mmap)
{
struct io_uring_buf_reg reg = { };
struct io_uring ring;
void *ptr;
char buffer[8];
int ret;
int ring_size = (entries * sizeof(struct io_uring_buf) + 4095) & (~4095);
int ring_mask = io_uring_buf_ring_mask(entries);
int loop, idx;
bool *buffers;
struct io_uring_buf_ring *br;
int read_fd;
int ret, loop, idx, read_fd;
struct io_uring ring;
char buffer[8];
bool *buffers;
ret = t_create_ring(1, &ring, 0);
if (ret == T_SETUP_SKIP)
return 0;
return T_EXIT_SKIP;
else if (ret != T_SETUP_OK)
return 1;
return T_EXIT_FAIL;
if (posix_memalign(&ptr, 4096, ring_size))
return 1;
if (!use_mmap) {
br = io_uring_setup_buf_ring(&ring, entries, bgid, 0, &ret);
if (!br) {
/* by now should have checked if this is supported or not */
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return T_EXIT_FAIL;
}
} else {
struct io_uring_buf_reg reg = {
.ring_entries = entries,
.bgid = bgid,
.flags = IOU_PBUF_RING_MMAP,
};
size_t ring_size;
off_t off;
br = (struct io_uring_buf_ring *)ptr;
io_uring_buf_ring_init(br);
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
if (ret == -EINVAL)
return T_EXIT_SKIP;
fprintf(stderr, "mmap ring register failed %d\n", ret);
return T_EXIT_FAIL;
}
off = IORING_OFF_PBUF_RING |
(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
ring_size = sizeof(struct io_uring_buf) * entries;
br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
if (br == MAP_FAILED) {
perror("mmap");
return T_EXIT_FAIL;
}
}
buffers = malloc(sizeof(bool) * entries);
if (!buffers)
return 1;
return T_EXIT_SKIP;
read_fd = open("/dev/zero", O_RDONLY);
if (read_fd < 0)
return 1;
reg.ring_addr = (unsigned long) ptr;
reg.ring_entries = entries;
reg.bgid = bgid;
ret = io_uring_register_buf_ring(&ring, &reg, 0);
if (ret) {
/* by now should have checked if this is supported or not */
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return 1;
}
return T_EXIT_SKIP;
for (loop = 0; loop < loops; loop++) {
memset(buffers, 0, sizeof(bool) * entries);
@@ -327,28 +361,28 @@ static int test_running(int bgid, int entries, int loops)
ret = test_one_read(read_fd, bgid, &ring);
if (ret < 0) {
fprintf(stderr, "bad run %d/%d = %d\n", loop, idx, ret);
return ret;
return T_EXIT_FAIL;
}
if (buffers[ret]) {
fprintf(stderr, "reused buffer %d/%d = %d!\n", loop, idx, ret);
return 1;
return T_EXIT_FAIL;
}
if (buffer[0] != 0) {
fprintf(stderr, "unexpected read %d %d/%d = %d!\n",
(int)buffer[0], loop, idx, ret);
return 1;
return T_EXIT_FAIL;
}
if (buffer[1] != 1) {
fprintf(stderr, "unexpected spilled read %d %d/%d = %d!\n",
(int)buffer[1], loop, idx, ret);
return 1;
return T_EXIT_FAIL;
}
buffers[ret] = true;
}
ret = test_one_read(read_fd, bgid, &ring);
if (ret != -ENOBUFS) {
fprintf(stderr, "expected enobufs run %d = %d\n", loop, ret);
return 1;
return T_EXIT_FAIL;
}
}
@@ -356,13 +390,13 @@ static int test_running(int bgid, int entries, int loops)
ret = io_uring_unregister_buf_ring(&ring, bgid);
if (ret) {
fprintf(stderr, "Buffer ring register failed %d\n", ret);
return 1;
return T_EXIT_FAIL;
}
close(read_fd);
io_uring_queue_exit(&ring);
free(buffers);
return 0;
return T_EXIT_PASS;
}
int main(int argc, char *argv[])
@@ -374,6 +408,8 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
pagesize = getpagesize();
for (i = 0; bgids[i] != -1; i++) {
ret = test_reg_unreg(bgids[i]);
if (ret) {
@@ -406,15 +442,32 @@ int main(int argc, char *argv[])
fprintf(stderr, "test_mixed_reg2 failed\n");
return T_EXIT_FAIL;
}
ret = test_full_page_reg(bgids[i]);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_full_page_reg failed\n");
return T_EXIT_FAIL;
}
}
for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
ret = test_running(2, entries[i], 3);
ret = test_running(2, entries[i], 3, 0);
if (ret) {
fprintf(stderr, "test_running(%d) failed\n", entries[i]);
return T_EXIT_FAIL;
}
}
for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
ret = test_running(2, entries[i], 3, 1);
if (ret == T_EXIT_SKIP) {
break;
} else if (ret != T_EXIT_PASS) {
fprintf(stderr, "test_running(%d) mmap failed\n", entries[i]);
return T_EXIT_FAIL;
}
}
return T_EXIT_PASS;
}
test/ce593a6c480a.c
+2 -2
View File
@@ -15,7 +15,7 @@
static int use_sqpoll = 0;
void notify_fd(int fd)
static void notify_fd(int fd)
{
char buf[8] = {0, 0, 0, 0, 0, 0, 1};
int ret;
@@ -25,7 +25,7 @@ void notify_fd(int fd)
perror("write");
}
void *delay_set_fd_from_thread(void *data)
static void *delay_set_fd_from_thread(void *data)
{
int fd = (intptr_t) data;
test/close-opath.c
+2 -1
View File
@@ -15,12 +15,13 @@
#include <errno.h>
#include <fcntl.h>
#include <liburing.h>
#include <sys/stat.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include "liburing.h"
typedef struct
{
const char *const flnames;
test/config
+4
View File
@@ -1,5 +1,9 @@
# Copy this to config.local, uncomment and define values
#
# NOTE: any files or devices added here will be used by tests that take
# a file or device arguments This includes tests that are destructive with
# respect to data contents. They may get erased or overwritten as part of tests.
#
# Define tests to exclude from running
# TEST_EXCLUDE=""
#
test/connect-rep.c
+204
View File
@@ -0,0 +1,204 @@
/* SPDX-License-Identifier: MIT */
/*
* Check that repeated IORING_OP_CONNECT to a socket without a listener keeps
* yielding -ECONNREFUSED rather than -ECONNABORTED. Based on a reproducer
* from:
*
* https://github.com/axboe/liburing/issues/828
*
* and adopted to our usual test cases. Other changes made like looping,
* using different ring types, adding a memset() for reuse, etc.
*
*/
#include <stdio.h>
#include <netinet/in.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include "liburing.h"
#include "helpers.h"
static unsigned long ud;
static int init_test_server(struct sockaddr_in *serv_addr)
{
socklen_t servaddr_len = sizeof(struct sockaddr_in);
int fd;
/* Init server socket. Bind but don't listen */
fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) {
perror("socket");
return -1;
}
serv_addr->sin_family = AF_INET;
serv_addr->sin_addr.s_addr = inet_addr("127.0.0.1");
if (bind(fd, (struct sockaddr *) serv_addr, servaddr_len) < 0) {
perror("bind");
return -1;
}
/*
* Get the addresses the socket is bound to because the port is chosen
* by the network stack.
*/
if (getsockname(fd, (struct sockaddr *)serv_addr, &servaddr_len) < 0) {
perror("getsockname");
return -1;
}
return fd;
}
static int init_test_client(void)
{
socklen_t addr_len = sizeof(struct sockaddr_in);
struct sockaddr_in client_addr = {};
int clientfd;
clientfd = socket(AF_INET, SOCK_STREAM, 0);
if (clientfd < 0) {
perror("socket");
return -1;
}
client_addr.sin_family = AF_INET;
client_addr.sin_addr.s_addr = inet_addr("127.0.0.1");
if (bind(clientfd, (struct sockaddr *)&client_addr, addr_len) < 0) {
perror("bind");
close(clientfd);
return -1;
}
/*
* Get the addresses the socket is bound to because the port is chosen
* by the network stack.
*/
if (getsockname(clientfd, (struct sockaddr *)&client_addr, &addr_len) < 0) {
perror("getsockname");
close(clientfd);
return -1;
}
return clientfd;
}
static int get_completion_and_print(struct io_uring *ring)
{
struct io_uring_cqe *cqe;
int ret, res;
ret = io_uring_wait_cqe(ring, &cqe);
if (ret < 0) {
fprintf(stderr, "wait_cqe=%d\n", ret);
return -1;
}
/* Mark this completion as seen */
res = cqe->res;
io_uring_cqe_seen(ring, cqe);
return res;
}
static int test_connect(struct io_uring *ring,
int clientfd, struct sockaddr_in *serv_addr)
{
struct sockaddr_in local_sa;
struct io_uring_sqe *sqe;
int ret;
sqe = io_uring_get_sqe(ring);
io_uring_prep_connect(sqe, clientfd, (const struct sockaddr *)serv_addr,
sizeof(struct sockaddr_in));
sqe->user_data = ++ud;
memcpy(&local_sa, serv_addr, sizeof(local_sa));
ret = io_uring_submit_and_wait(ring, 1);
if (ret != 1) {
fprintf(stderr, "submit=%d\n", ret);
return T_EXIT_FAIL;
}
/* check for reuse at the same time */
memset(&local_sa, 0xff, sizeof(local_sa));
ret = get_completion_and_print(ring);
if (ret != -ECONNREFUSED) {
fprintf(stderr, "Connect got %d\n", ret);
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
static int test(int flags)
{
struct io_uring_params params = { .flags = flags, };
struct sockaddr_in serv_addr = {};
struct io_uring ring;
int ret, clientfd, s_fd, i;
if (flags & IORING_SETUP_SQPOLL)
params.sq_thread_idle = 50;
ret = io_uring_queue_init_params(8, &ring, &params);
if (ret < 0) {
fprintf(stderr, "Queue init: %d\n", ret);
return T_EXIT_FAIL;
}
s_fd = init_test_server(&serv_addr);
if (s_fd < 0)
return T_EXIT_FAIL;
clientfd = init_test_client();
if (clientfd < 0) {
close(s_fd);
return T_EXIT_FAIL;
}
/* make sure SQPOLL thread is sleeping */
if (flags & IORING_SETUP_SQPOLL)
usleep(100000);
for (i = 0; i < 32; i++) {
ret = test_connect(&ring, clientfd, &serv_addr);
if (ret == T_EXIT_SKIP)
return T_EXIT_SKIP;
else if (ret == T_EXIT_PASS)
continue;
return T_EXIT_FAIL;
}
close(s_fd);
close(clientfd);
return T_EXIT_PASS;
}
int main(int argc, char *argv[])
{
int ret;
if (argc > 1)
return T_EXIT_SKIP;
ret = test(0);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test(0) failed\n");
return T_EXIT_FAIL;
}
ret = test(IORING_SETUP_SQPOLL);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test(SQPOLL) failed\n");
return T_EXIT_FAIL;
}
return 0;
}
test/connect.c
+54 -11
View File
@@ -15,6 +15,7 @@
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <sys/stat.h>
#include "liburing.h"
#include "helpers.h"
@@ -132,7 +133,7 @@ static int configure_connect(int fd, struct sockaddr_in* addr)
return ret;
}
static int connect_socket(struct io_uring *ring, int fd, int *code)
static int connect_socket(struct io_uring *ring, int fd, int *code, int async)
{
struct sockaddr_in addr;
int ret, res;
@@ -149,6 +150,8 @@ static int connect_socket(struct io_uring *ring, int fd, int *code)
}
io_uring_prep_connect(sqe, fd, (struct sockaddr*)&addr, sizeof(addr));
if (async)
sqe->flags |= IOSQE_ASYNC;
sqe->user_data = 1;
ret = submit_and_wait(ring, &res);
@@ -185,7 +188,7 @@ static int test_connect_with_no_peer(struct io_uring *ring)
if (connect_fd == -1)
return -1;
ret = connect_socket(ring, connect_fd, &code);
ret = connect_socket(ring, connect_fd, &code, 0);
if (ret == -1)
goto err;
@@ -208,7 +211,7 @@ err:
return -1;
}
static int test_connect(struct io_uring *ring)
static int test_connect(struct io_uring *ring, int async)
{
int accept_fd;
int connect_fd;
@@ -226,7 +229,7 @@ static int test_connect(struct io_uring *ring)
if (connect_fd == -1)
goto err1;
ret = connect_socket(ring, connect_fd, &code);
ret = connect_socket(ring, connect_fd, &code, async);
if (ret == -1)
goto err2;
@@ -256,6 +259,13 @@ static int test_connect_timeout(struct io_uring *ring)
struct sockaddr_in addr;
struct io_uring_sqe *sqe;
struct __kernel_timespec ts = {.tv_sec = 0, .tv_nsec = 100000};
struct stat sb;
/*
* Test reliably fails if syncookies isn't enabled
*/
if (stat("/proc/sys/net/ipv4/tcp_syncookies", &sb) < 0)
return T_EXIT_SKIP;
connect_fd[0] = create_socket();
if (connect_fd[0] == -1)
@@ -288,7 +298,7 @@ static int test_connect_timeout(struct io_uring *ring)
}
// We first connect with one client socket in order to fill the accept queue.
ret = connect_socket(ring, connect_fd[0], &code);
ret = connect_socket(ring, connect_fd[0], &code, 0);
if (ret == -1 || code != 0) {
fprintf(stderr, "unable to connect\n");
goto err;
@@ -355,15 +365,12 @@ err:
return -1;
}
int main(int argc, char *argv[])
static int test(int flags)
{
struct io_uring ring;
int ret;
if (argc > 1)
return T_EXIT_SKIP;
ret = io_uring_queue_init(8, &ring, 0);
ret = io_uring_queue_init(8, &ring, flags);
if (ret) {
fprintf(stderr, "io_uring_queue_setup() = %d\n", ret);
return T_EXIT_FAIL;
@@ -382,7 +389,13 @@ int main(int argc, char *argv[])
if (no_connect)
return T_EXIT_SKIP;
ret = test_connect(&ring);
ret = test_connect(&ring, 0);
if (ret == -1) {
fprintf(stderr, "test_connect(): failed\n");
return T_EXIT_FAIL;
}
ret = test_connect(&ring, 1);
if (ret == -1) {
fprintf(stderr, "test_connect(): failed\n");
return T_EXIT_FAIL;
@@ -397,3 +410,33 @@ int main(int argc, char *argv[])
io_uring_queue_exit(&ring);
return T_EXIT_PASS;
}
int main(int argc, char *argv[])
{
int ret;
if (argc > 1)
return T_EXIT_SKIP;
ret = test(0);
if (ret == -1) {
fprintf(stderr, "test 0 failed\n");
return T_EXIT_FAIL;
}
if (no_connect)
return T_EXIT_SKIP;
ret = test(IORING_SETUP_SQPOLL);
if (ret == -1) {
fprintf(stderr, "test SQPOLL failed\n");
return T_EXIT_FAIL;
}
ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
if (ret == -1) {
fprintf(stderr, "test DEFER failed\n");
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
test/coredump.c
+60
View File
@@ -0,0 +1,60 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: trigger segfault. A recent 6.4-rc kernel introduced a bug
* via vhost where segfaults for applications using io_uring
* would hang in D state forever upon trying to generate the
* core file. Perform a trivial test where a child process
* generates a NULL pointer dereference and ensure that we don't
* hang.
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/wait.h>
#include "liburing.h"
#include "helpers.h"
static void test(void)
{
struct io_uring_sqe *sqe;
struct io_uring ring;
int *ptr = NULL;
int fds[2];
char r1;
if (pipe(fds) < 0) {
perror("pipe");
exit(0);
}
io_uring_queue_init(8, &ring, 0);
sqe = io_uring_get_sqe(&ring);
io_uring_prep_read(sqe, fds[0], &r1, sizeof(r1), 0);
sqe->flags = IOSQE_ASYNC;
sqe->user_data = 1;
io_uring_submit(&ring);
*ptr = 0;
exit(0);
}
int main(int argc, char *argv[])
{
pid_t pid;
int wstat;
pid = fork();
if (pid < 0) {
perror("fork");
return T_EXIT_SKIP;
} else if (!pid) {
test();
}
wait(&wstat);
unlink("core");
return T_EXIT_PASS;
}
test/cq-overflow.c
+14 -8
View File
@@ -48,7 +48,8 @@ static struct iovec *vecs;
* bash -c "echo 1 > /proc/self/make-it-fail && exec ./cq-overflow.t"
*/
static int test_io(const char *file, unsigned long usecs, unsigned *drops, int fault)
static int test_io(const char *file, unsigned long usecs, unsigned *drops,
int fault)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
@@ -60,8 +61,10 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f
fd = open(file, O_RDONLY | O_DIRECT);
if (fd < 0) {
if (errno == EINVAL)
return T_EXIT_SKIP;
perror("file open");
return 1;
return T_EXIT_FAIL;
}
memset(&p, 0, sizeof(p));
@@ -69,7 +72,7 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f
if (ret) {
close(fd);
fprintf(stderr, "ring create failed: %d\n", ret);
return 1;
return T_EXIT_FAIL;
}
nodrop = 0;
if (p.features & IORING_FEAT_NODROP)
@@ -173,12 +176,12 @@ reap_it:
io_uring_queue_exit(&ring);
close(fd);
return 0;
return T_EXIT_PASS;
err:
if (fd != -1)
close(fd);
io_uring_queue_exit(&ring);
return 1;
return T_EXIT_SKIP;
}
static int reap_events(struct io_uring *ring, unsigned nr_events, int do_wait)
@@ -496,7 +499,10 @@ int main(int argc, char *argv[])
do {
drops = 0;
if (test_io(fname, usecs, &drops, 0)) {
ret = test_io(fname, usecs, &drops, 0);
if (ret == T_EXIT_SKIP)
break;
else if (ret != T_EXIT_PASS) {
fprintf(stderr, "test_io nofault failed\n");
goto err;
}
@@ -506,12 +512,12 @@ int main(int argc, char *argv[])
iters++;
} while (iters < 40);
if (test_io(fname, usecs, &drops, 0)) {
if (test_io(fname, usecs, &drops, 0) == T_EXIT_FAIL) {
fprintf(stderr, "test_io nofault failed\n");
goto err;
}
if (test_io(fname, usecs, &drops, 1)) {
if (test_io(fname, usecs, &drops, 1) == T_EXIT_FAIL) {
fprintf(stderr, "test_io fault failed\n");
goto err;
}
test/d4ae271dfaae.c
+1 -1
View File
@@ -29,7 +29,7 @@ int main(int argc, char *argv[])
memset(&p, 0, sizeof(p));
p.flags = IORING_SETUP_SQPOLL;
ret = t_create_ring_params(4, &ring, &p);
ret = t_create_ring_params(16, &ring, &p);
if (ret == T_SETUP_SKIP)
return T_EXIT_SKIP;
else if (ret < 0)
test/defer-taskrun.c
+64 -9
View File
@@ -4,7 +4,6 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <error.h>
#include <sys/eventfd.h>
#include <signal.h>
#include <poll.h>
@@ -57,9 +56,14 @@ static void eventfd_trigger(int fd)
assert(ret == sizeof(val));
}
#define CHECK(x) if (!(x)) { \
fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \
return -1; }
#define CHECK(x) \
do { \
if (!(x)) { \
fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \
return -1; \
} \
} while (0)
static int test_eventfd(void)
{
@@ -119,7 +123,7 @@ struct thread_data {
char buff[8];
};
void *thread(void *t)
static void *thread(void *t)
{
struct thread_data *td = t;
@@ -178,11 +182,11 @@ static int test_exec(const char *filename)
int wstatus;
CHECK(waitpid(fork_pid, &wstatus, 0) != (pid_t)-1);
if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) != T_EXIT_SKIP) {
if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) == T_EXIT_FAIL) {
fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus));
return -1;
}
return 0;
return T_EXIT_PASS;
}
ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER |
@@ -192,9 +196,15 @@ static int test_exec(const char *filename)
if (filename) {
fd = open(filename, O_RDONLY | O_DIRECT);
if (fd < 0 && errno == EINVAL)
return T_EXIT_SKIP;
} else {
t_create_file(EXEC_FILENAME, EXEC_FILESIZE);
fd = open(EXEC_FILENAME, O_RDONLY | O_DIRECT);
if (fd < 0 && errno == EINVAL) {
unlink(EXEC_FILENAME);
return T_EXIT_SKIP;
}
unlink(EXEC_FILENAME);
}
buff = (char*)malloc(EXEC_FILESIZE);
@@ -207,7 +217,7 @@ static int test_exec(const char *filename)
ret = execve("/proc/self/exe", new_argv, new_env);
/* if we get here it failed anyway */
fprintf(stderr, "execve failed %d\n", ret);
return -1;
return T_EXIT_FAIL;
}
static int test_flag(void)
@@ -283,6 +293,45 @@ static int test_ring_shutdown(void)
return 0;
}
static int test_drain(void)
{
struct io_uring ring;
int ret, i, fd[2];
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct iovec iovecs[128];
char buff[ARRAY_SIZE(iovecs)];
ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER |
IORING_SETUP_DEFER_TASKRUN |
IORING_SETUP_TASKRUN_FLAG);
CHECK(!ret);
for (i = 0; i < ARRAY_SIZE(iovecs); i++) {
iovecs[i].iov_base = &buff[i];
iovecs[i].iov_len = 1;
}
ret = t_create_socket_pair(fd, true);
CHECK(!ret);
sqe = io_uring_get_sqe(&ring);
io_uring_prep_writev(sqe, fd[1], &iovecs[0], ARRAY_SIZE(iovecs), 0);
sqe->flags |= IOSQE_IO_DRAIN;
io_uring_submit(&ring);
for (i = 0; i < ARRAY_SIZE(iovecs); i++)
iovecs[i].iov_base = NULL;
CHECK(io_uring_wait_cqe(&ring, &cqe) == 0);
CHECK(cqe->res == 128);
close(fd[0]);
close(fd[1]);
io_uring_queue_exit(&ring);
return 0;
}
int main(int argc, char *argv[])
{
int ret;
@@ -309,7 +358,7 @@ int main(int argc, char *argv[])
}
ret = test_exec(filename);
if (ret) {
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test_exec failed\n");
return T_EXIT_FAIL;
}
@@ -332,5 +381,11 @@ int main(int argc, char *argv[])
return T_EXIT_FAIL;
}
ret = test_drain();
if (ret) {
fprintf(stderr, "test_drain failed\n");
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
test/defer-tw-timeout.c
+173
View File
@@ -0,0 +1,173 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: test waiting for more events than what will be posted with
* a timeout with DEFER_TASKRUN. All kernels should time out,
* but a non-buggy kernel will end up with one CQE available
* for reaping. Buggy kernels will not have processed the
* task_work and will have 0 events.
*
*/
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include "liburing.h"
#include "helpers.h"
struct d {
int fd;
};
static void *thread_fn(void *data)
{
struct d *d = data;
int ret;
usleep(100000);
ret = write(d->fd, "Hello", 5);
if (ret < 0)
perror("write");
return NULL;
}
static int test_poll(struct io_uring *ring)
{
struct io_uring_cqe *cqe;
struct io_uring_sqe *sqe;
struct __kernel_timespec ts;
int ret, fds[2], i;
pthread_t thread;
char buf[32];
struct d d;
void *tret;
if (pipe(fds) < 0) {
perror("pipe");
return 1;
}
d.fd = fds[1];
sqe = io_uring_get_sqe(ring);
io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0);
pthread_create(&thread, NULL, thread_fn, &d);
ts.tv_sec = 1;
ts.tv_nsec = 0;
ret = io_uring_submit_and_wait_timeout(ring, &cqe, 2, &ts, NULL);
if (ret != 1) {
fprintf(stderr, "unexpected wait ret %d\n", ret);
return T_EXIT_FAIL;
}
for (i = 0; i < 2; i++) {
ret = io_uring_peek_cqe(ring, &cqe);
if (ret)
break;
io_uring_cqe_seen(ring, cqe);
}
if (i != 1) {
fprintf(stderr, "Got %d request, expected 1\n", i);
return T_EXIT_FAIL;
}
pthread_join(thread, &tret);
return T_EXIT_PASS;
}
static int test_file(struct io_uring *ring, char *__fname)
{
struct io_uring_cqe *cqe;
struct io_uring_sqe *sqe;
struct __kernel_timespec ts;
char filename[64], *fname;
int fd, ret, i;
void *buf;
if (!__fname) {
fname = filename;
sprintf(fname, ".defer-tw-timeout.%d", getpid());
t_create_file(fname, 128*1024);
} else {
fname = __fname;
}
fd = open(fname, O_RDONLY | O_DIRECT);
if (fd < 0) {
if (errno == EINVAL) {
if (!__fname)
unlink(fname);
return T_EXIT_SKIP;
}
perror("open");
if (!__fname)
unlink(fname);
return T_EXIT_FAIL;
}
if (!__fname)
unlink(fname);
if (posix_memalign(&buf, 4096, 4096)) {
close(fd);
return T_EXIT_FAIL;
}
sqe = io_uring_get_sqe(ring);
io_uring_prep_read(sqe, fd, buf, 4096, 0);
ts.tv_sec = 1;
ts.tv_nsec = 0;
ret = io_uring_submit_and_wait_timeout(ring, &cqe, 2, &ts, NULL);
if (ret != 1) {
fprintf(stderr, "unexpected wait ret %d\n", ret);
close(fd);
return T_EXIT_FAIL;
}
for (i = 0; i < 2; i++) {
ret = io_uring_peek_cqe(ring, &cqe);
if (ret)
break;
io_uring_cqe_seen(ring, cqe);
}
if (i != 1) {
fprintf(stderr, "Got %d request, expected 1\n", i);
close(fd);
return T_EXIT_FAIL;
}
close(fd);
return T_EXIT_PASS;
}
int main(int argc, char *argv[])
{
struct io_uring ring;
char *fname = NULL;
int ret;
ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN);
if (ret == -EINVAL)
return T_EXIT_SKIP;
if (argc > 1)
fname = argv[1];
ret = test_file(&ring, fname);
if (ret != T_EXIT_PASS)
return ret;
ret = test_poll(&ring);
if (ret != T_EXIT_PASS)
return ret;
return T_EXIT_PASS;
}
test/defer.c
+4 -4
View File
@@ -57,7 +57,7 @@ static int init_context(struct test_context *ctx, struct io_uring *ring, int nr,
case OP_REMOVE_BUFFERS:
io_uring_prep_remove_buffers(sqe, 10, 1);
break;
};
}
sqe->user_data = i;
ctx->sqes[i] = sqe;
}
@@ -88,7 +88,7 @@ static int wait_cqes(struct test_context *ctx)
return 0;
}
static int test_cancelled_userdata(struct io_uring *ring)
static int test_canceled_userdata(struct io_uring *ring)
{
struct test_context ctx;
int ret, i, nr = 100;
@@ -276,9 +276,9 @@ int main(int argc, char *argv[])
}
ret = test_cancelled_userdata(&poll_ring);
ret = test_canceled_userdata(&poll_ring);
if (ret) {
printf("test_cancelled_userdata failed\n");
printf("test_canceled_userdata failed\n");
return ret;
}
test/double-poll-crash.c
+3 -3
View File
@@ -109,7 +109,7 @@ static long syz_open_dev(volatile long a0, volatile long a1, volatile long a2)
}
}
uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff};
static uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff};
int main(int argc, char *argv[])
{
@@ -121,10 +121,10 @@ int main(int argc, char *argv[])
if (argc > 1)
return T_EXIT_SKIP;
mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul);
mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul);
if (mmap_ret == MAP_FAILED)
return T_EXIT_SKIP;
mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul);
mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul);
if (mmap_ret == MAP_FAILED)
return T_EXIT_SKIP;
intptr_t res = 0;
test/eeed8b54e0df.c
+8 -3
View File
@@ -102,13 +102,18 @@ int main(int argc, char *argv[])
goto err;
}
ret = T_EXIT_PASS;
if (cqe->res != -EAGAIN && cqe->res != 4096) {
printf("cqe error: %d\n", cqe->res);
goto err;
if (cqe->res == -EOPNOTSUPP) {
ret = T_EXIT_SKIP;
} else {
printf("cqe error: %d\n", cqe->res);
goto err;
}
}
close(fd);
return T_EXIT_PASS;
return ret;
err:
close(fd);
return T_EXIT_FAIL;
test/eploop.c
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: MIT */
/*
* Test that we don't recursively generate completion events if an io_uring
* fd is added to an epoll context, and the ring itself polls for events on
* the epollfd. Older kernels will stop on overflow, newer kernels will
* detect this earlier and abort correctly.
*/
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/epoll.h>
#include <sys/types.h>
#include <poll.h>
#include "liburing.h"
#include "helpers.h"
int main(int argc, char *argv[])
{
struct io_uring ring;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct epoll_event ev = { };
int epollfd, ret, i;
if (argc > 1)
return T_EXIT_SKIP;
ret = io_uring_queue_init(8, &ring, 0);
if (ret) {
fprintf(stderr, "Ring init failed: %d\n", ret);
return T_EXIT_FAIL;
}
epollfd = epoll_create1(0);
if (epollfd < 0) {
perror("epoll_create");
return T_EXIT_FAIL;
}
ev.events = EPOLLIN;
ev.data.fd = ring.ring_fd;
ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, ring.ring_fd, &ev);
if (ret < 0) {
perror("epoll_ctl");
return T_EXIT_FAIL;
}
sqe = io_uring_get_sqe(&ring);
io_uring_prep_poll_multishot(sqe, epollfd, POLLIN);
sqe->user_data = 1;
io_uring_submit(&ring);
sqe = io_uring_get_sqe(&ring);
sqe->user_data = 2;
io_uring_prep_nop(sqe);
io_uring_submit(&ring);
for (i = 0; i < 2; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret) {
fprintf(stderr, "wait_cqe ret = %d\n", ret);
break;
}
io_uring_cqe_seen(&ring, cqe);
}
ret = io_uring_peek_cqe(&ring, &cqe);
if (!ret) {
fprintf(stderr, "Generated too many events\n");
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
test/eventfd-reg.c
+1 -1
View File
@@ -43,7 +43,7 @@ int main(int argc, char *argv[])
return T_EXIT_FAIL;
}
/* Check that registrering again will get -EBUSY */
/* Check that registering again will get -EBUSY */
ret = io_uring_register_eventfd(&ring, evfd[1]);
if (ret != -EBUSY) {
fprintf(stderr, "unexpected 2nd register: %d\n", ret);
test/eventfd-ring.c
+1 -1
View File
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: run various nop tests
* Description: test use of eventfds with multiple rings
*
*/
#include <errno.h>
test/eventfd.c
+1 -1
View File
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: run various nop tests
* Description: run various eventfd tests
*
*/
#include <errno.h>
test/evloop.c
+73
View File
@@ -0,0 +1,73 @@
/* SPDX-License-Identifier: MIT */
/*
* Test that we don't recursively generate completion events if an io_uring
* has an eventfd registered that triggers on completions, and we add a poll
* request with multishot on the eventfd. Older kernels will stop on overflow,
* newer kernels will detect this earlier and abort correctly.
*/
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/eventfd.h>
#include <sys/types.h>
#include <poll.h>
#include <assert.h>
#include "liburing.h"
#include "helpers.h"
int main(int argc, char *argv[])
{
struct io_uring ring;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret, efd, i;
if (argc > 1)
return T_EXIT_SKIP;
ret = io_uring_queue_init(8, &ring, 0);
if (ret) {
fprintf(stderr, "Ring init failed: %d\n", ret);
return T_EXIT_FAIL;
}
efd = eventfd(0, 0);
if (efd < 0) {
perror("eventfd");
return T_EXIT_FAIL;
}
ret = io_uring_register_eventfd(&ring, efd);
if (ret) {
fprintf(stderr, "Ring eventfd register failed: %d\n", ret);
return T_EXIT_FAIL;
}
sqe = io_uring_get_sqe(&ring);
io_uring_prep_poll_multishot(sqe, efd, POLLIN);
sqe->user_data = 1;
io_uring_submit(&ring);
sqe = io_uring_get_sqe(&ring);
sqe->user_data = 2;
io_uring_prep_nop(sqe);
io_uring_submit(&ring);
for (i = 0; i < 2; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret) {
fprintf(stderr, "wait_cqe ret = %d\n", ret);
break;
}
io_uring_cqe_seen(&ring, cqe);
}
ret = io_uring_peek_cqe(&ring, &cqe);
if (!ret) {
fprintf(stderr, "Generated too many events\n");
return T_EXIT_FAIL;
}
return T_EXIT_PASS;
}
test/exit-no-cleanup.c
+1 -1
View File
@@ -26,7 +26,7 @@ static pthread_barrier_t init_barrier;
static int sleep_fd, notify_fd;
static sem_t sem;
void *thread_func(void *arg)
static void *thread_func(void *arg)
{
struct io_uring ring;
int res;
test/fadvise.c
+1 -1
View File
@@ -186,7 +186,7 @@ int main(int argc, char *argv[])
}
/* too hard to reliably test, just ignore */
if (0 && bad > good) {
if ((0) && bad > good) {
fprintf(stderr, "Suspicious timings\n");
goto err;
}
test/fallocate.c
+9
View File
@@ -12,6 +12,7 @@
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <signal.h>
#include "liburing.h"
#include "helpers.h"
@@ -216,14 +217,22 @@ err:
return 1;
}
static void sig_xfsz(int sig)
{
}
int main(int argc, char *argv[])
{
struct sigaction act = { };
struct io_uring ring;
int ret;
if (argc > 1)
return T_EXIT_SKIP;
act.sa_handler = sig_xfsz;
sigaction(SIGXFSZ, &act, NULL);
ret = io_uring_queue_init(8, &ring, 0);
if (ret) {
fprintf(stderr, "ring setup failed\n");
test/fc2a85cb02ef.c
+3 -3
View File
@@ -54,7 +54,7 @@ static int inject_fault(int nth)
return fd;
}
static int setup_fault()
static int setup_fault(void)
{
static struct {
const char* file;
@@ -79,13 +79,13 @@ static int setup_fault()
return 0;
}
uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff};
static uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff};
int main(int argc, char *argv[])
{
if (argc > 1)
return T_EXIT_SKIP;
mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, 0x32ul, -1, 0);
mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, MAP_ANON|MAP_PRIVATE, -1, 0);
if (setup_fault()) {
printf("Test needs failslab/fail_futex/fail_page_alloc enabled, skipped\n");
return T_EXIT_SKIP;
test/fd-install.c
+500
View File
@@ -0,0 +1,500 @@
/* SPDX-License-Identifier: MIT */
/*
* Description: test installing a direct descriptor into the regular
* file table
*
*/
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include "liburing.h"
#include "helpers.h"
static int no_fd_install;
/* test that O_CLOEXEC is accepted, and others are not */
static int test_flags(struct io_uring *ring, int async)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret, fds[2], fd;
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
ret = io_uring_register_files(ring, &fds[0], 1);
if (ret) {
fprintf(stderr, "failed register files %d\n", ret);
return T_EXIT_FAIL;
}
/* check that setting an invalid flag fails */
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, 1U << 17);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res != -EINVAL) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
/* check that IORING_FIXED_FD_NO_CLOEXEC is accepted */
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, IORING_FIXED_FD_NO_CLOEXEC);
if (async)
sqe->flags |= IOSQE_ASYNC;
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res < 0) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
fd = cqe->res;
io_uring_cqe_seen(ring, cqe);
close(fds[0]);
close(fds[1]);
close(fd);
io_uring_unregister_files(ring);
return T_EXIT_PASS;
}
static int test_linked(struct io_uring *ring)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret, fds[2], fd, i;
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
ret = io_uring_register_files(ring, &fds[0], 1);
if (ret) {
fprintf(stderr, "failed register files %d\n", ret);
return T_EXIT_FAIL;
}
sqe = io_uring_get_sqe(ring);
io_uring_prep_nop(sqe);
sqe->flags |= IOSQE_IO_LINK;
sqe->user_data = 1;
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, 0);
sqe->user_data = 2;
ret = io_uring_submit(ring);
if (ret != 2) {
fprintf(stderr, "submit: %d\n", ret);
return T_EXIT_FAIL;
}
fd = -1;
for (i = 0; i < 2; i++) {
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res < 0) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
if (cqe->user_data == 2)
fd = cqe->res;
io_uring_cqe_seen(ring, cqe);
}
close(fds[0]);
close(fds[1]);
if (fd != -1)
close(fd);
io_uring_unregister_files(ring);
return T_EXIT_PASS;
}
/* test not setting IOSQE_FIXED_FILE */
static int test_not_fixed(struct io_uring *ring)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret, fds[2];
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
ret = io_uring_register_files(ring, &fds[0], 1);
if (ret) {
fprintf(stderr, "failed register files %d\n", ret);
return T_EXIT_FAIL;
}
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, 0);
sqe->flags &= ~IOSQE_FIXED_FILE;
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res != -EBADF) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
close(fds[0]);
close(fds[1]);
io_uring_unregister_files(ring);
return T_EXIT_PASS;
}
/* test invalid direct descriptor indexes */
static int test_bad_fd(struct io_uring *ring, int some_fd)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret;
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, some_fd, 0);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res != -EBADF) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
return T_EXIT_PASS;
}
/* test basic functionality of shifting a direct descriptor to a normal file */
static int test_working(struct io_uring *ring)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret, fds[2];
char buf[32];
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
/* register read side */
ret = io_uring_register_files(ring, &fds[0], 1);
if (ret) {
fprintf(stderr, "failed register files %d\n", ret);
return T_EXIT_FAIL;
}
/* close normal descriptor */
close(fds[0]);
/* normal read should fail */
ret = read(fds[0], buf, 1);
if (ret != -1) {
fprintf(stderr, "unexpected read ret %d\n", ret);
return T_EXIT_FAIL;
}
if (errno != EBADF) {
fprintf(stderr, "unexpected read failure %d\n", errno);
return T_EXIT_FAIL;
}
/* verify we can read the data */
sqe = io_uring_get_sqe(ring);
io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0);
sqe->flags |= IOSQE_FIXED_FILE;
io_uring_submit(ring);
/* put some data in the pipe */
ret = write(fds[1], "Hello", 5);
if (ret < 0) {
perror("write");
return T_EXIT_FAIL;
} else if (ret != 5) {
fprintf(stderr, "short write %d\n", ret);
return T_EXIT_FAIL;
}
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res != 5) {
fprintf(stderr, "weird pipe read ret %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
/* fixed pipe read worked, now re-install as a regular fd */
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, 0);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res == -EINVAL) {
no_fd_install = 1;
return T_EXIT_SKIP;
}
if (cqe->res < 0) {
fprintf(stderr, "failed install fd: %d\n", cqe->res);
return T_EXIT_FAIL;
}
/* stash new pipe read side fd in old spot */
fds[0] = cqe->res;
io_uring_cqe_seen(ring, cqe);
ret = write(fds[1], "Hello", 5);
if (ret < 0) {
perror("write");
return T_EXIT_FAIL;
} else if (ret != 5) {
fprintf(stderr, "short write %d\n", ret);
return T_EXIT_FAIL;
}
/* normal pipe read should now work with new fd */
ret = read(fds[0], buf, sizeof(buf));
if (ret != 5) {
fprintf(stderr, "unexpected read ret %d\n", ret);
return T_EXIT_FAIL;
}
/* close fixed file */
sqe = io_uring_get_sqe(ring);
io_uring_prep_close_direct(sqe, 0);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res) {
fprintf(stderr, "close fixed fd %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
ret = write(fds[1], "Hello", 5);
if (ret < 0) {
perror("write");
return T_EXIT_FAIL;
} else if (ret != 5) {
fprintf(stderr, "short write %d\n", ret);
return T_EXIT_FAIL;
}
/* normal pipe read should still work with new fd */
ret = read(fds[0], buf, sizeof(buf));
if (ret != 5) {
fprintf(stderr, "unexpected read ret %d\n", ret);
return T_EXIT_FAIL;
}
/* fixed fd pipe read should now fail */
sqe = io_uring_get_sqe(ring);
io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0);
sqe->flags = IOSQE_FIXED_FILE;
io_uring_submit(ring);
/* put some data in the pipe */
ret = write(fds[1], "Hello", 5);
if (ret < 0) {
perror("write");
return T_EXIT_FAIL;
} else if (ret != 5) {
fprintf(stderr, "short write %d\n", ret);
return T_EXIT_FAIL;
}
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res != -EBADF) {
fprintf(stderr, "weird pipe read ret %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
close(fds[0]);
close(fds[1]);
io_uring_unregister_files(ring);
return T_EXIT_PASS;
}
static int test_creds(struct io_uring *ring, int async)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int cred_id, ret, fds[2];
if (pipe(fds) < 0) {
perror("pipe");
return T_EXIT_FAIL;
}
ret = io_uring_register_files(ring, &fds[0], 1);
if (ret) {
fprintf(stderr, "failed register files %d\n", ret);
return T_EXIT_FAIL;
}
cred_id = io_uring_register_personality(ring);
if (cred_id < 0) {
fprintf(stderr, "Failed registering creds: %d\n", cred_id);
return T_EXIT_FAIL;
}
/* check that asking for creds fails */
sqe = io_uring_get_sqe(ring);
io_uring_prep_fixed_fd_install(sqe, 0, 0);
if (async)
sqe->flags |= IOSQE_ASYNC;
sqe->personality = cred_id;
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret) {
fprintf(stderr, "wait cqe %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res > 0) {
fprintf(stderr, "install succeeded with creds\n");
return T_EXIT_FAIL;
}
if (cqe->res != -EPERM) {
fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
return T_EXIT_FAIL;
}
io_uring_cqe_seen(ring, cqe);
close(fds[0]);
close(fds[1]);
io_uring_unregister_files(ring);
io_uring_unregister_personality(ring, cred_id);
return T_EXIT_PASS;
}
int main(int argc, char *argv[])
{
struct io_uring ring;
int ret;
if (argc > 1)
return T_EXIT_SKIP;
ret = io_uring_queue_init(4, &ring, 0);
if (ret) {
fprintf(stderr, "ring setup failed: %d\n", ret);
return T_EXIT_FAIL;
}
ret = test_working(&ring);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_working failed\n");
return ret;
}
if (no_fd_install)
return T_EXIT_SKIP;
ret = test_bad_fd(&ring, 0);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_bad_fd 0 failed\n");
return ret;
}
ret = test_bad_fd(&ring, 500);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_bad_fd 500 failed\n");
return ret;
}
ret = test_not_fixed(&ring);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_not_fixed failed\n");
return ret;
}
ret = test_flags(&ring, 0);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_flags 0 failed\n");
return ret;
}
ret = test_flags(&ring, 1);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_flags 1 failed\n");
return ret;
}
ret = test_creds(&ring, 0);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_creds 0 failed\n");
return ret;
}
ret = test_creds(&ring, 1);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_creds 1 failed\n");
return ret;
}
ret = test_linked(&ring);
if (ret != T_EXIT_PASS) {
if (ret == T_EXIT_FAIL)
fprintf(stderr, "test_linked failed\n");
return ret;
}
return T_EXIT_PASS;
}
test/fd-pass.c
+67 -17
View File
@@ -13,8 +13,9 @@
#include "liburing.h"
#include "helpers.h"
#define FSIZE 128
#define PAT 0x9a
#define FSIZE 128
#define PAT 0x9a
#define USER_DATA 0x89
static int no_fd_pass;
@@ -49,7 +50,7 @@ static int verify_fixed_read(struct io_uring *ring, int fixed_fd, int fail)
return 0;
}
static int test(const char *filename)
static int test(const char *filename, int source_fd, int target_fd)
{
struct io_uring sring, dring;
struct io_uring_sqe *sqe;
@@ -79,10 +80,18 @@ static int test(const char *filename)
fprintf(stderr, "register files failed %d\n", ret);
return T_EXIT_FAIL;
}
if (target_fd == IORING_FILE_INDEX_ALLOC) {
/* we want to test installing into a non-zero slot */
ret = io_uring_register_file_alloc_range(&dring, 1, 1);
if (ret) {
fprintf(stderr, "io_uring_register_file_alloc_range %d\n", ret);
return T_EXIT_FAIL;
}
}
/* open direct descriptor */
sqe = io_uring_get_sqe(&sring);
io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, 0);
io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, source_fd);
io_uring_submit(&sring);
ret = io_uring_wait_cqe(&sring, &cqe);
if (ret) {
@@ -96,15 +105,19 @@ static int test(const char *filename)
io_uring_cqe_seen(&sring, cqe);
/* verify data is sane for source ring */
if (verify_fixed_read(&sring, 0, 0))
if (verify_fixed_read(&sring, source_fd, 0))
return T_EXIT_FAIL;
/* send direct descriptor to destination ring */
sqe = io_uring_get_sqe(&sring);
io_uring_prep_msg_ring(sqe, dring.ring_fd, 0, 0x89, 0);
sqe->addr = 1;
sqe->addr3 = 0;
sqe->file_index = 1;
if (target_fd == IORING_FILE_INDEX_ALLOC) {
io_uring_prep_msg_ring_fd_alloc(sqe, dring.ring_fd, source_fd,
USER_DATA, 0);
} else {
io_uring_prep_msg_ring_fd(sqe, dring.ring_fd, source_fd,
target_fd, USER_DATA, 0);
}
io_uring_submit(&sring);
ret = io_uring_wait_cqe(&sring, &cqe);
@@ -112,7 +125,7 @@ static int test(const char *filename)
fprintf(stderr, "wait cqe failed %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->res) {
if (cqe->res < 0) {
if (cqe->res == -EINVAL && !no_fd_pass) {
no_fd_pass = 1;
return T_EXIT_SKIP;
@@ -128,19 +141,30 @@ static int test(const char *filename)
fprintf(stderr, "wait cqe failed %d\n", ret);
return T_EXIT_FAIL;
}
if (cqe->user_data != 0x89) {
if (cqe->user_data != USER_DATA) {
fprintf(stderr, "bad user_data %ld\n", (long) cqe->res);
return T_EXIT_FAIL;
}
if (cqe->res < 0) {
fprintf(stderr, "bad result %i\n", cqe->res);
return T_EXIT_FAIL;
}
if (target_fd == IORING_FILE_INDEX_ALLOC) {
if (cqe->res != 1) {
fprintf(stderr, "invalid allocated index %i\n", cqe->res);
return T_EXIT_FAIL;
}
target_fd = cqe->res;
}
io_uring_cqe_seen(&dring, cqe);
/* now verify we can read the sane data from the destination ring */
if (verify_fixed_read(&dring, 0, 0))
if (verify_fixed_read(&dring, target_fd, 0))
return T_EXIT_FAIL;
/* close descriptor in source ring */
sqe = io_uring_get_sqe(&sring);
io_uring_prep_close_direct(sqe, 0);
io_uring_prep_close_direct(sqe, source_fd);
io_uring_submit(&sring);
ret = io_uring_wait_cqe(&sring, &cqe);
@@ -155,13 +179,15 @@ static int test(const char *filename)
io_uring_cqe_seen(&sring, cqe);
/* check that source ring fails after close */
if (verify_fixed_read(&sring, 0, 1))
if (verify_fixed_read(&sring, source_fd, 1))
return T_EXIT_FAIL;
/* check we can still read from destination ring */
if (verify_fixed_read(&dring, 0, 0))
if (verify_fixed_read(&dring, target_fd, 0))
return T_EXIT_FAIL;
io_uring_queue_exit(&sring);
io_uring_queue_exit(&dring);
return T_EXIT_PASS;
}
@@ -176,9 +202,33 @@ int main(int argc, char *argv[])
sprintf(fname, ".fd-pass.%d", getpid());
t_create_file_pattern(fname, FSIZE, PAT);
ret = test(fname);
ret = test(fname, 0, 1);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test failed\n");
fprintf(stderr, "test failed 0 1\n");
ret = T_EXIT_FAIL;
}
ret = test(fname, 0, 2);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test failed 0 2\n");
ret = T_EXIT_FAIL;
}
ret = test(fname, 1, 1);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test failed 1 1\n");
ret = T_EXIT_FAIL;
}
ret = test(fname, 1, 0);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test failed 1 0\n");
ret = T_EXIT_FAIL;
}
ret = test(fname, 1, IORING_FILE_INDEX_ALLOC);
if (ret == T_EXIT_FAIL) {
fprintf(stderr, "test failed 1 ALLOC\n");
ret = T_EXIT_FAIL;
}
test/file-register.c
+68 -3
View File
@@ -305,7 +305,7 @@ static int test_sparse(struct io_uring *ring)
files = open_files(100, 100, 0);
ret = io_uring_register_files(ring, files, 200);
if (ret) {
if (ret == -EBADF) {
if (ret == -EBADF || ret == -EINVAL) {
fprintf(stdout, "Sparse files not supported, skipping\n");
no_update = 1;
goto done;
@@ -352,10 +352,14 @@ err:
static int test_basic(struct io_uring *ring, int fail)
{
int *files;
int ret;
int ret, i;
int nr_files = fail ? 10 : 100;
files = open_files(nr_files, 0, 0);
files = open_files(nr_files, fail ? 90 : 0, 0);
if (fail) {
for (i = nr_files; i < nr_files + 90; i++)
files[i] = -2;
}
ret = io_uring_register_files(ring, files, 100);
if (ret) {
if (fail) {
@@ -935,6 +939,59 @@ static int test_zero_range_alloc(struct io_uring *ring, int fds[2])
return 0;
}
static int test_defer_taskrun(void)
{
struct io_uring_sqe *sqe;
struct io_uring ring;
int ret, fds[2];
char buff = 'x';
ret = io_uring_queue_init(8, &ring,
IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER);
if (ret) {
fprintf(stderr, "ring init\n");
return 1;
}
ret = pipe(fds);
if (ret) {
fprintf(stderr, "bad pipes\n");
return 1;
}
ret = io_uring_register_files(&ring, &fds[0], 2);
if (ret) {
fprintf(stderr, "bad register %d\n", ret);
return 1;
}
sqe = io_uring_get_sqe(&ring);
io_uring_prep_read(sqe, 0, &buff, 1, 0);
sqe->flags |= IOSQE_FIXED_FILE;
ret = io_uring_submit(&ring);
if (ret != 1) {
fprintf(stderr, "bad submit\n");
return 1;
}
ret = write(fds[1], &buff, 1);
if (ret != 1) {
fprintf(stderr, "bad pipe write\n");
return 1;
}
ret = io_uring_unregister_files(&ring);
if (ret) {
fprintf(stderr, "bad unregister %d\n", ret);
return 1;
}
close(fds[0]);
close(fds[1]);
io_uring_queue_exit(&ring);
return 0;
}
static int test_file_alloc_ranges(void)
{
struct io_uring ring;
@@ -1120,5 +1177,13 @@ int main(int argc, char *argv[])
return T_EXIT_FAIL;
}
if (t_probe_defer_taskrun()) {
ret = test_defer_taskrun();
if (ret) {
fprintf(stderr, "test_defer_taskrun failed\n");
return T_EXIT_FAIL;
}
}
return T_EXIT_PASS;
}
test/file-verify.c
+30 -9
View File
@@ -10,7 +10,6 @@
#include <string.h>
#include <fcntl.h>
#include <assert.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <linux/fs.h>
@@ -29,24 +28,42 @@
#define MAX_VECS 16
/*
* Can be anything, let's just do something for a bit of parallellism
* Can be anything, let's just do something for a bit of parallelism
*/
#define READ_BATCH 16
static void verify_buf_sync(void *buf, size_t size, bool registered)
{
#if defined(__hppa__)
if (registered) {
unsigned long off = (unsigned long) buf & 4095;
unsigned long p = (unsigned long) buf & ~4095;
int i;
size += off;
for (i = 0; i < size; i += 32)
asm volatile("fdc 0(%0)" : : "r" (p + i));
}
#endif
}
/*
* Each offset in the file has the offset / sizeof(int) stored for every
* sizeof(int) address.
*/
static int verify_buf(void *buf, size_t size, off_t off)
static int verify_buf(void *buf, size_t size, off_t off, bool registered)
{
int i, u_in_buf = size / sizeof(unsigned int);
unsigned int *ptr;
verify_buf_sync(buf, size, registered);
off /= sizeof(unsigned int);
ptr = buf;
for (i = 0; i < u_in_buf; i++) {
if (off != *ptr) {
fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off);
fprintf(stderr, "Found %u, wanted %llu\n", *ptr,
(unsigned long long) off);
return 1;
}
ptr++;
@@ -197,7 +214,7 @@ again:
goto err;
}
if (verify_buf(buf, CHUNK_SIZE / 2, 0))
if (verify_buf(buf, CHUNK_SIZE / 2, 0, false))
goto err;
/*
@@ -364,9 +381,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
v[i].iov_base = buf[i];
v[i].iov_len = CHUNK_SIZE;
}
ret = io_uring_register_buffers(ring, v, READ_BATCH);
ret = t_register_buffers(ring, v, READ_BATCH);
if (ret) {
fprintf(stderr, "Error buffer reg %d\n", ret);
if (ret == T_SETUP_SKIP) {
ret = 0;
goto free_bufs;
}
goto err;
}
}
@@ -445,12 +465,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
void *buf = vecs[index][j].iov_base;
size_t len = vecs[index][j].iov_len;
if (verify_buf(buf, len, voff))
if (verify_buf(buf, len, voff, registered))
goto err;
voff += len;
}
} else {
if (verify_buf(buf[index], CHUNK_SIZE, voff))
if (verify_buf(buf[index], CHUNK_SIZE, voff, registered))
goto err;
}
}
@@ -460,6 +480,7 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
done:
if (registered)
io_uring_unregister_buffers(ring);
free_bufs:
if (vectored) {
for (j = 0; j < READ_BATCH; j++)
for (i = 0; i < nr_vecs; i++)
test/files-exit-hang-timeout.c
+2 -2
View File
@@ -21,9 +21,9 @@
#define PORT 9100
struct io_uring ring;
static struct io_uring ring;
struct __kernel_timespec ts = {
static struct __kernel_timespec ts = {
.tv_sec = 300,
.tv_nsec = 0,
};
test/fixed-buf-merge.c
+101
View File
@@ -0,0 +1,101 @@
/* SPDX-License-Identifier: MIT */
/*
* Test fixed buffer merging/skipping
*
* Taken from: https://github.com/axboe/liburing/issues/994
*
*/
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include "liburing.h"
#include "helpers.h"
int main(int argc, char *argv[])
{
int ret, i, fd, initial_offset = 4096, num_requests = 3;
struct io_uring ring;
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
struct iovec iov;
char *buffer, *to_free;
unsigned head;
char filename[64];
ret = io_uring_queue_init(4, &ring, 0);
if (ret) {
fprintf(stderr, "queue_init: %d\n", ret);
return T_EXIT_FAIL;
}
sprintf(filename, ".fixed-buf-%d", getpid());
t_create_file(filename, 4 * 4096);
fd = open(filename, O_RDONLY | O_DIRECT, 0644);
if (fd < 0) {
if (errno == EINVAL) {
unlink(filename);
return T_EXIT_SKIP;
}
perror("open");
goto err_unlink;
}
to_free = buffer = aligned_alloc(4096, 128 * 4096);
if (!buffer) {
perror("aligned_alloc");
goto err_unlink;
}
/* Register buffer */
iov.iov_base = buffer;
iov.iov_len = 128 * 4096;
ret = io_uring_register_buffers(&ring, &iov, 1);
if (ret) {
fprintf(stderr, "buf register: %d\n", ret);
goto err_unlink;
}
/* Prepare read requests */
buffer += initial_offset;
for (i = 0; i < num_requests; i++) {
sqe = io_uring_get_sqe(&ring);
io_uring_prep_read_fixed(sqe, fd, buffer, 4096, 4096 * i, 0);
buffer += 4096;
}
/* Submit requests and reap completions */
ret = io_uring_submit_and_wait(&ring, num_requests);
if (ret != num_requests) {
fprintf(stderr, "Submit and wait: %d\n", ret);
goto err_unlink;
}
i = 0;
io_uring_for_each_cqe(&ring, head, cqe) {
if (cqe->res != 4096) {
fprintf(stderr, "cqe: %d\n", cqe->res);
goto err_unlink;
}
i++;
}
if (i != num_requests) {
fprintf(stderr, "Got %d completions\n", i);
goto err_unlink;
}
io_uring_cq_advance(&ring, i);
io_uring_queue_exit(&ring);
close(fd);
free(to_free);
unlink(filename);
return T_EXIT_PASS;
err_unlink:
unlink(filename);
return T_EXIT_FAIL;
}
test/fixed-hugepage.c
+411
View File
@@ -0,0 +1,411 @@
/* SPDX-License-Identifier: MIT */
/*
* Test fixed buffers consisting of hugepages.
*/
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <linux/mman.h>
#include "liburing.h"
#include "helpers.h"
/*
* Before testing
* echo (>=4) > /proc/sys/vm/nr_hugepages
* echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
* echo always > /sys/kernel/mm/transparent_hugepage/hugepages-16kB/enabled
*
* Not 100% guaranteed to get THP-backed memory, but in general it does.
*/
#define MTHP_16KB (16UL * 1024)
#define HUGEPAGE_SIZE (2UL * 1024 * 1024)
#define NR_BUFS 1
#define IN_FD "/dev/urandom"
#define OUT_FD "/dev/zero"
static int open_files(char *fname_in, int *fd_in, int *fd_out)
{
*fd_in = open(fname_in, O_RDONLY, 0644);
if (*fd_in < 0) {
printf("open %s failed\n", fname_in);
return -1;
}
*fd_out = open(OUT_FD, O_RDWR, 0644);
if (*fd_out < 0) {
printf("open %s failed\n", OUT_FD);
return -1;
}
return 0;
}
static void unmap(struct iovec *iov, int nr_bufs, size_t offset)
{
int i;
for (i = 0; i < nr_bufs; i++)
munmap(iov[i].iov_base - offset, iov[i].iov_len + offset);
}
static int mmap_hugebufs(struct iovec *iov, int nr_bufs, size_t buf_size, size_t offset)
{
int i;
for (i = 0; i < nr_bufs; i++) {
void *base = NULL;
base = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
if (base == MAP_FAILED) {
printf("Unable to map hugetlb page. Try increasing the "
"value in /proc/sys/vm/nr_hugepages\n");
unmap(iov, i, offset);
return -1;
}
memset(base, 0, buf_size);
iov[i].iov_base = base + offset;
iov[i].iov_len = buf_size - offset;
}
return 0;
}
/* map a hugepage and smaller page to a contiguous memory */
static int mmap_mixture(struct iovec *iov, int nr_bufs, size_t buf_size, bool huge_on_left)
{
int i;
void *small_base = NULL, *huge_base = NULL, *start = NULL,
*huge_start = NULL, *small_start = NULL;
size_t small_size = buf_size - HUGEPAGE_SIZE;
size_t seg_size = ((buf_size / HUGEPAGE_SIZE) + 1) * HUGEPAGE_SIZE;
start = mmap(NULL, seg_size * nr_bufs, PROT_NONE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
if (start == MAP_FAILED) {
printf("Unable to preserve the page mixture memory. "
"Try increasing the RLIMIT_MEMLOCK resource limit\n");
return -1;
}
for (i = 0; i < nr_bufs; i++) {
if (huge_on_left) {
huge_start = start;
small_start = start + HUGEPAGE_SIZE;
} else {
huge_start = start + HUGEPAGE_SIZE;
small_start = start + HUGEPAGE_SIZE - small_size;
}
huge_base = mmap(huge_start, HUGEPAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0);
if (huge_base == MAP_FAILED) {
printf("Unable to map hugetlb page in the page mixture. "
"Try increasing the value in /proc/sys/vm/nr_hugepages\n");
unmap(iov, nr_bufs, 0);
return -1;
}
small_base = mmap(small_start, small_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
if (small_base == MAP_FAILED) {
printf("Unable to map small page in the page mixture. "
"Try increasing the RLIMIT_MEMLOCK resource limit\n");
unmap(iov, nr_bufs, 0);
return -1;
}
if (huge_on_left) {
iov[i].iov_base = huge_base;
memset(huge_base, 0, buf_size);
}
else {
iov[i].iov_base = small_base;
memset(small_base, 0, buf_size);
}
iov[i].iov_len = buf_size;
start += seg_size;
}
return 0;
}
static void free_bufs(struct iovec *iov, int nr_bufs, size_t offset)
{
int i;
for (i = 0; i < nr_bufs; i++)
free(iov[i].iov_base - offset);
}
static int get_mthp_bufs(struct iovec *iov, int nr_bufs, size_t buf_size,
size_t alignment, size_t offset)
{
int i;
for (i = 0; i < nr_bufs; i++) {
void *base = NULL;
if (posix_memalign(&base, alignment, buf_size)) {
printf("Unable to allocate mthp pages. "
"Try increasing the RLIMIT_MEMLOCK resource limit\n");
free_bufs(iov, i, offset);
return -1;
}
memset(base, 0, buf_size);
iov[i].iov_base = base + offset;
iov[i].iov_len = buf_size - offset;
}
return 0;
}
static int do_read(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int i, ret;
for (i = 0; i < nr_bufs; i++) {
sqe = io_uring_get_sqe(ring);
if (!sqe) {
fprintf(stderr, "Could not get SQE.\n");
return -1;
}
io_uring_prep_read_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret < 0) {
fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
return -1;
}
if (cqe->res < 0) {
fprintf(stderr, "Error in async read operation: %s\n", strerror(-cqe->res));
return -1;
}
if (cqe->res != iov[i].iov_len) {
fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
return -1;
}
io_uring_cqe_seen(ring, cqe);
}
return 0;
}
static int do_write(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int i, ret;
for (i = 0; i < nr_bufs; i++) {
sqe = io_uring_get_sqe(ring);
if (!sqe) {
fprintf(stderr, "Could not get SQE.\n");
return -1;
}
io_uring_prep_write_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
io_uring_submit(ring);
ret = io_uring_wait_cqe(ring, &cqe);
if (ret < 0) {
fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
return -1;
}
if (cqe->res < 0) {
fprintf(stderr, "Error in async write operation: %s\n", strerror(-cqe->res));
return -1;
}
if (cqe->res != iov[i].iov_len) {
fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
return -1;
}
io_uring_cqe_seen(ring, cqe);
}
return 0;
}
static int register_submit(struct io_uring *ring, struct iovec *iov,
int nr_bufs, int fd_in, int fd_out)
{
int ret;
ret = io_uring_register_buffers(ring, iov, nr_bufs);
if (ret) {
fprintf(stderr, "Error registering buffers: %s\n", strerror(-ret));
return ret;
}
ret = do_read(ring, fd_in, iov, nr_bufs);
if (ret) {
fprintf(stderr, "Read test failed\n");
return ret;
}
ret = do_write(ring, fd_out, iov, nr_bufs);
if (ret) {
fprintf(stderr, "Write test failed\n");
return ret;
}
ret = io_uring_unregister_buffers(ring);
if (ret) {
fprintf(stderr, "Error unregistering buffers for one hugepage test: %s", strerror(-ret));
return ret;
}
return 0;
}
static int test_one_hugepage(struct io_uring *ring, int fd_in, int fd_out)
{
struct iovec iov[NR_BUFS];
size_t buf_size = HUGEPAGE_SIZE;
int ret;
if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
return T_EXIT_SKIP;
ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
unmap(iov, NR_BUFS, 0);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
}
static int test_multi_hugepages(struct io_uring *ring, int fd_in, int fd_out)
{
struct iovec iov[NR_BUFS];
size_t buf_size = 4 * HUGEPAGE_SIZE;
int ret;
if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
return T_EXIT_SKIP;
ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
unmap(iov, NR_BUFS, 0);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
}
static int test_unaligned_hugepage(struct io_uring *ring, int fd_in, int fd_out)
{
struct iovec iov[NR_BUFS];
size_t buf_size = 3 * HUGEPAGE_SIZE;
size_t offset = 0x1234;
int ret;
if (mmap_hugebufs(iov, NR_BUFS, buf_size, offset))
return T_EXIT_SKIP;
ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
unmap(iov, NR_BUFS, offset);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
}
static int test_multi_unaligned_mthps(struct io_uring *ring, int fd_in, int fd_out)
{
struct iovec iov[NR_BUFS];
int ret;
size_t buf_size = 3 * MTHP_16KB;
size_t offset = 0x1234;
if (get_mthp_bufs(iov, NR_BUFS, buf_size, MTHP_16KB, offset))
return T_EXIT_SKIP;
ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
free_bufs(iov, NR_BUFS, offset);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
}
/* Should not coalesce */
static int test_page_mixture(struct io_uring *ring, int fd_in, int fd_out, int huge_on_left)
{
struct iovec iov[NR_BUFS];
size_t buf_size = HUGEPAGE_SIZE + MTHP_16KB;
int ret;
if (mmap_mixture(iov, NR_BUFS, buf_size, huge_on_left))
return T_EXIT_SKIP;
ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
unmap(iov, NR_BUFS, 0);
return ret ? T_EXIT_FAIL : T_EXIT_PASS;
}
int main(int argc, char *argv[])
{
struct io_uring ring;
int ret, fd_in, fd_out;
char *fname_in;
if (argc > 1)
fname_in = argv[1];
else
fname_in = IN_FD;
if (open_files(fname_in, &fd_in, &fd_out))
return T_EXIT_SKIP;
ret = t_create_ring(8, &ring, 0);
if (ret == T_SETUP_SKIP)
return T_EXIT_SKIP;
else if (ret < 0)
return T_EXIT_FAIL;
ret = test_one_hugepage(&ring, fd_in, fd_out);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test one hugepage failed.\n");
return ret;
}
ret = test_multi_hugepages(&ring, fd_in, fd_out);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test multi hugepages failed.\n");
return ret;
}
ret = test_unaligned_hugepage(&ring, fd_in, fd_out);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test unaligned hugepage failed.\n");
return ret;
}
ret = test_multi_unaligned_mthps(&ring, fd_in, fd_out);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test unaligned multi-size'd THPs failed.\n");
return ret;
}
ret = test_page_mixture(&ring, fd_in, fd_out, true);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test huge small page mixture (start with huge) failed.\n");
return ret;
}
ret = test_page_mixture(&ring, fd_in, fd_out, false);
if (ret != T_EXIT_PASS) {
if (ret != T_EXIT_SKIP)
fprintf(stderr, "Test huge small page mixture (start with small) failed.\n");
return ret;
}
io_uring_queue_exit(&ring);
return T_EXIT_PASS;
}

Some files were not shown because too many files have changed in this diff Show More