!23 liburing升级2.7

Merge pull request !23 from f房芷仪/1014

.github/workflows/build.yml

@@ -26,7 +26,8 @@ jobs:
             cxx_pkg: clang
             cc: clang
             cxx: clang++
-            extra_flags: -Wshorten-64-to-32
+            liburing_extra_flags: -Wshorten-64-to-32
+            extra_flags: -Wmissing-prototypes -Wstrict-prototypes -Wunreachable-code-loop-increment -Wunreachable-code -Wmissing-variable-declarations -Wextra-semi-stmt
 
           # x86 (32-bit) gcc
           - arch: i686
@@ -49,6 +50,13 @@ jobs:
             cc: arm-linux-gnueabi-gcc
             cxx: arm-linux-gnueabi-g++
 
+          # riscv64
+          - arch: riscv64
+            cc_pkg: gcc-riscv64-linux-gnu
+            cxx_pkg: g++-riscv64-linux-gnu
+            cc: riscv64-linux-gnu-gcc
+            cxx: riscv64-linux-gnu-g++
+
           # powerpc64
           - arch: powerpc64
             cc_pkg: gcc-powerpc64-linux-gnu
@@ -84,23 +92,31 @@ jobs:
             cc: mips-linux-gnu-gcc
             cxx: mips-linux-gnu-g++
 
+          # hppa
+          - arch: hppa
+            cc_pkg: gcc-hppa-linux-gnu
+            cxx_pkg: g++-hppa-linux-gnu
+            cc: hppa-linux-gnu-gcc
+            cxx: hppa-linux-gnu-g++
+
     env:
-      FLAGS: -g -O3 -Wall -Wextra -Werror
+      FLAGS: -g -O3 -Wall -Wextra -Werror -Wno-sign-compare ${{matrix.extra_flags}}
 
       # Flags for building sources in src/ dir only.
-      LIBURING_CFLAGS: ${{matrix.extra_flags}}
+      LIBURING_CFLAGS: ${{matrix.liburing_extra_flags}}
 
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Install Compilers
       run: |
         if [[ "${{matrix.cc_pkg}}" == "clang" ]]; then \
             wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh; \
-            sudo bash /tmp/llvm.sh 16; \
+            sudo apt-get purge --auto-remove llvm python3-lldb-14 llvm-14 -y; \
-            sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 400; \
+            sudo bash /tmp/llvm.sh 17; \
-            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 400; \
+            sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 400; \
+            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 400; \
         else \
             sudo apt-get update -y; \
             sudo apt-get install -y ${{matrix.cc_pkg}} ${{matrix.cxx_pkg}}; \
@@ -116,16 +132,6 @@ jobs:
         ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}};
         make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
 
-    - name: Build nolibc
-      run: |
-        if [[ "${{matrix.arch}}" == "x86_64" || "${{matrix.arch}}" == "i686" || "${{matrix.arch}}" == "aarch64" ]]; then \
-            make clean; \
-            ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}} --nolibc; \
-            make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS"; \
-        else \
-            echo "Skipping nolibc build, this arch doesn't support building liburing without libc"; \
-        fi;
-
     - name: Test install command
       run: |
         sudo make install;

.github/workflows/shellcheck.yml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Display shellcheck version
       run: shellcheck --version

.gitignore

@@ -9,15 +9,23 @@
 
 /src/liburing.a
 /src/liburing.so*
+/src/liburing-ffi.a
+/src/liburing-ffi.so*
 /src/include/liburing/compat.h
+/src/include/liburing/io_uring_version.h
 
+/examples/io_uring-close-test
 /examples/io_uring-cp
 /examples/io_uring-test
 /examples/io_uring-udp
 /examples/link-cp
+/examples/napi-busy-poll-client
+/examples/napi-busy-poll-server
 /examples/ucontext-cp
 /examples/poll-bench
+/examples/proxy
 /examples/send-zerocopy
+/examples/rsrc-update-bench
 
 /test/*.t
 /test/*.dmesg
@@ -28,5 +36,6 @@ config-host.mak
 config.log
 
 liburing.pc
+liburing-ffi.pc
 
 cscope.out

CHANGELOG

@@ -1,3 +1,68 @@
+liburing-2.7 release
+
+- Man page updates
+- Sync with kernel 6.10
+	- send/recv bundle support
+	- accept nowait and CQE_F_MORE
+- Add and update test cases
+- Fix io_uring_queue_init_mem() returning a value that was too small,
+  potentially causing memory corruption in userspace by overwriting
+  64 bytes beyond the returned value. Also add test case for that.
+- Add 64-bit length variants of io_uring_prep_{m,f}advise()
+- Add BIND/LISTEN support and helpers / man pages
+- Add io_uring_enable_rings.3 man page
+- Fix bug in io_uring_prep_read_multishot()
+- Fixup bundle test cases
+- Add fixed-hugepage test case
+- Fix io_uring_prep_fixed_fd_install.3 man page
+- Note 'len' == 0 requirement in io_uring_prep_send.3 man page
+- Fix some test cases for skipping on older kernels
+
+liburing-2.6 release
+
+- Add getsockopt and setsockopt socket commands
+- Add test cases to test/hardlink
+- Man page fixes
+- Add futex support, and test cases
+- Add waitid support, and test cases
+- Add read multishot, and test cases
+- Add support for IORING_SETUP_NO_SQARRAY
+- Use IORING_SETUP_NO_SQARRAY as the default
+- Add support for IORING_OP_FIXED_FD_INSTALL
+- Add io_uring_prep_fixed_fd_install() helper
+- Support for napi busy polling
+- Improve/add test cases
+- Man page fixes
+- Add sample 'proxy' example
+
+liburing-2.5 release
+
+- Add support for io_uring_prep_cmd_sock()
+- Add support for application allocated ring memory, for placing rings
+  in huge mem. Available through io_uring_queue_init_mem().
+- Add support for registered ring fds
+- Various documentation updates
+- Various fixes
+
+liburing-2.4 release
+
+- Add io_uring_{major,minor,check}_version() functions.
+- Add IO_URING_{MAJOR,MINOR,CHECK}_VERSION() macros.
+- FFI support (for non-C/C++ languages integration).
+- Add io_uring_prep_msg_ring_cqe_flags() function.
+- Deprecate --nolibc configure option.
+- CONFIG_NOLIBC is always enabled on x86-64, x86, and aarch64.
+- Add support for IORING_REGISTER_USE_REGISTERED_RING and use if available.
+- Add io_uring_close_ring_fd() function.
+- Add io_uring_prep_msg_ring_fd_alloc function.
+- Add io_uring_free_buf_ring() and io_uring_setup_buf_ring() functions.
+- Ensure that io_uring_prep_accept_direct(), io_uring_prep_openat_direct(),
+  io_uring_prep_openat2_direct(), io_uring_prep_msg_ring_fd(), and
+  io_uring_prep_socket_direct() factor in being called with
+  IORING_FILE_INDEX_ALLOC for allocating a direct descriptor.
+- Add io_uring_prep_sendto() function.
+- Add io_uring_prep_cmd_sock() function.
+
 liburing-2.3 release
 
 - Support non-libc build for aarch64.

Makefile

@@ -11,11 +11,11 @@ all:
 	@$(MAKE) -C test
 	@$(MAKE) -C examples
 
-.PHONY: all install default clean test
+library:
-.PHONY: FORCE cscope
+	@$(MAKE) -C src
 
-partcheck: all
+.PHONY: all install default clean test library
-	@echo "make partcheck => TODO add tests with out kernel support"
+.PHONY: FORCE cscope
 
 runtests: all
 	@$(MAKE) -C test runtests
@@ -25,7 +25,7 @@ runtests-parallel: all
 	@$(MAKE) -C test runtests-parallel
 
 config-host.mak: configure
-	@if [ ! -e "$@" ]; then					\
+	+@if [ ! -e "$@" ]; then				\
 	  echo "Running configure ...";				\
 	  ./configure;						\
 	else							\
@@ -45,13 +45,14 @@ endif
 	    -e "s%@VERSION@%$(VERSION)%g" \
 	    $< >$@
 
-install: $(NAME).pc
+install: $(NAME).pc $(NAME)-ffi.pc
 	@$(MAKE) -C src install prefix=$(DESTDIR)$(prefix) \
 		includedir=$(DESTDIR)$(includedir) \
 		libdir=$(DESTDIR)$(libdir) \
 		libdevdir=$(DESTDIR)$(libdevdir) \
 		relativelibdir=$(relativelibdir)
 	$(INSTALL) -D -m 644 $(NAME).pc $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME).pc
+	$(INSTALL) -D -m 644 $(NAME)-ffi.pc $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME)-ffi.pc
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man2
 	$(INSTALL) -m 644 man/*.2 $(DESTDIR)$(mandir)/man2
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man3
@@ -59,11 +60,22 @@ install: $(NAME).pc
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man7
 	$(INSTALL) -m 644 man/*.7 $(DESTDIR)$(mandir)/man7
 
+uninstall:
+	@$(MAKE) -C src uninstall prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
+	@rm -f $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME).pc
+	@rm -f $(DESTDIR)$(libdevdir)/pkgconfig/$(NAME)-ffi.pc
+	@rm -rf $(DESTDIR)$(mandir)/man2/io_uring*.2
+	@rm -rf $(DESTDIR)$(mandir)/man3/io_uring*.3
+	@rm -rf $(DESTDIR)$(mandir)/man7/io_uring*.7
+
 install-tests:
 	@$(MAKE) -C test install prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
 
+uninstall-tests:
+	@$(MAKE) -C test uninstall prefix=$(DESTDIR)$(prefix) datadir=$(DESTDIR)$(datadir)
+
 clean:
-	@rm -f config-host.mak config-host.h cscope.out $(NAME).pc test/*.dmesg
+	@rm -f config-host.mak config-host.h cscope.out $(NAME).pc $(NAME)-ffi.pc test/*.dmesg
 	@$(MAKE) -C src clean
 	@$(MAKE) -C test clean
 	@$(MAKE) -C examples clean

Makefile.common

@@ -3,4 +3,5 @@ NAME=liburing
 SPECFILE=$(TOP)/$(NAME).spec
 VERSION=$(shell awk '/Version:/ { print $$2 }' $(SPECFILE))
 VERSION_MAJOR=$(shell echo $(VERSION) | cut -d. -f1)
+VERSION_MINOR=$(shell echo $(VERSION) | cut -d. -f2)
 TAG = $(NAME)-$(VERSION)

README

@@ -47,6 +47,54 @@ the kernel io_uring support. Please note that this suite isn't expected to
 pass on older kernels, and may even crash or hang older kernels!
 
 
+Building liburing
+-----------------
+
+    #
+    # Prepare build config (optional).
+    #
+    #  --cc  specifies the C   compiler.
+    #  --cxx specifies the C++ compiler.
+    #
+    ./configure --cc=gcc --cxx=g++;
+
+    #
+    # Build liburing.
+    #
+    make -j$(nproc);
+
+    #
+    # Install liburing (headers, shared/static libs, and manpage).
+    #
+    sudo make install;
+
+See './configure --help' for more information about build config options.
+
+
+FFI support
+-----------
+
+By default, the build results in 4 lib files:
+
+    2 shared libs:
+
+        liburing.so
+        liburing-ffi.so
+
+    2 static libs:
+
+        liburing.a
+        liburing-ffi.a
+
+Languages and applications that can't use 'static inline' functions in
+liburing.h should use the FFI variants.
+
+liburing's main public interface lives in liburing.h as 'static inline'
+functions. Users wishing to consume liburing purely as a binary dependency
+should link against liburing-ffi. It contains definitions for every 'static
+inline' function.
+
+
 License
 -------
 

README.OpenSource

@@ -3,9 +3,9 @@
         "Name": "liburing",
         "License": "MIT License",
         "License File": "LICENSE",
-        "Version Number": "2.3",
+        "Version Number": "2.7",
         "Owner": "mailto:maojingjing1@huawei.com",
-        "Upstream URL": "https://github.com/axboe/liburing.git",
+        "Upstream URL": "https://github.com/axboe/liburing/releases/tag/liburing-2.7",
         "Description": "liburing provides helpers to setup and reardown io_uring instances, and also a simplified interface for applications that don't need (or want) to deal with the full kernel side implementation."
     }
 ]

bundle.json

@@ -1,7 +1,7 @@
 {
     "name": "@ohos/liburing",
     "description": "liburing provides helpers to setup and teardown io_uring instances, and also a simplified interface for applications that don't need (or want) to deal with the full kernel side implementation.",
-    "version": "2.3",
+    "version": "2.7",
     "license": "MIT License",
     "publishAs": "code-segment",
     "segment": {

configure

@@ -10,7 +10,7 @@ for opt do
   case "$opt" in
   --help|-h) show_help=yes
   ;;
-  --prefix=*) prefix="$optarg"
+  --prefix=*) prefix="$(realpath -s $optarg)"
   ;;
   --includedir=*) includedir="$optarg"
   ;;
@@ -26,7 +26,7 @@ for opt do
   ;;
   --cxx=*) cxx="$optarg"
   ;;
-  --nolibc) liburing_nolibc="yes"
+  --use-libc) use_libc=yes
   ;;
   *)
     echo "ERROR: unknown option $opt"
@@ -75,7 +75,7 @@ Options: [defaults in brackets after descriptions]
   --datadir=PATH           install shared data in PATH [$datadir]
   --cc=CMD                 use CMD as the C compiler
   --cxx=CMD                use CMD as the C++ compiler
-  --nolibc                 build liburing without libc
+  --use-libc               use libc for liburing (useful for hardening)
 EOF
 exit 0
 fi
@@ -115,7 +115,7 @@ print_config() {
 }
 
 # Default CFLAGS
-CFLAGS="-D_GNU_SOURCE -include config-host.h"
+CFLAGS="-D_GNU_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -include config-host.h"
 BUILD_CFLAGS=""
 
 # Print configure header at the top of $config_host_h
@@ -202,6 +202,15 @@ print_and_output_mak "relativelibdir" "$relativelibdir"
 print_and_output_mak "mandir" "$mandir"
 print_and_output_mak "datadir" "$datadir"
 
+####################################################
+# Check for correct compiler runtime library to link with
+libgcc_link_flag="-lgcc"
+if $cc -print-libgcc-file-name >/dev/null 2>&1; then
+  libgcc_link_flag="$($cc $CFLAGS $LDFLAGS -print-libgcc-file-name)"
+fi
+print_and_output_mak "libgcc_link_flag" "$libgcc_link_flag"
+####################################################
+
 ##########################################
 # check for compiler -Wstringop-overflow
 stringop_overflow="no"
@@ -384,14 +393,87 @@ if compile_prog "" "" "nvme uring cmd"; then
 fi
 print_config "NVMe uring command support" "$nvme_uring_cmd"
 
+##########################################
+# Check futexv support
+futexv="no"
+cat > $TMPC << EOF
+#include <linux/futex.h>
+#include <unistd.h>
+#include <string.h>
+int main(void)
+{
+  struct futex_waitv fw;
+
+  memset(&fw, FUTEX_32, sizeof(fw));
+
+  return sizeof(struct futex_waitv);
+}
+EOF
+if compile_prog "" "" "futexv"; then
+  futexv="yes"
+fi
+print_config "futex waitv support" "$futexv"
+
+##########################################
+# Check idtype_t support
+has_idtype_t="no"
+cat > $TMPC << EOF
+#include <sys/wait.h>
+int main(void)
+{
+  idtype_t v;
+  return 0;
+}
+EOF
+if compile_prog "" "" "idtype_t"; then
+  has_idtype_t="yes"
+fi
+print_config "has_idtype_t" "$has_idtype_t"
+
 #############################################################################
+liburing_nolibc="no"
+if test "$use_libc" != "yes"; then
+
+  #
+  # Currently, CONFIG_NOLIBC only supports x86-64, x86 (32-bit), aarch64 and riscv64.
+  #
+  cat > $TMPC << EOF
+int main(void){
+#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || (defined(__riscv) && __riscv_xlen == 64)
+  return 0;
+#else
+#error libc is needed
+#endif
+}
+EOF
+
+  if compile_prog "" "" "nolibc"; then
+    liburing_nolibc="yes"
+  fi
+fi
+
+print_config "nolibc" "$liburing_nolibc";
+#############################################################################
+
+####################################################
+# Most Android devices don't have sys/fanotify.h
+has_fanotify="no"
+cat > $TMPC << EOF
+#include <sys/fanotify.h>
+int main(void)
+{
+  return 0;
+}
+EOF
+if compile_prog "" "" "fanotify"; then
+  has_fanotify="yes"
+fi
+print_config "has_fanotify" "$has_fanotify"
+####################################################
+
 if test "$liburing_nolibc" = "yes"; then
   output_sym "CONFIG_NOLIBC"
-else
-  liburing_nolibc="no"
 fi
-print_config "liburing_nolibc" "$liburing_nolibc"
-
 if test "$__kernel_rwf_t" = "yes"; then
   output_sym "CONFIG_HAVE_KERNEL_RWF_T"
 fi
@@ -422,12 +504,36 @@ fi
 if test "$nvme_uring_cmd" = "yes"; then
   output_sym "CONFIG_HAVE_NVME_URING"
 fi
+if test "$has_fanotify" = "yes"; then
+  output_sym "CONFIG_HAVE_FANOTIFY"
+fi
+if test "$futexv" = "yes"; then
+  output_sym "CONFIG_HAVE_FUTEXV"
+fi
 
 echo "CC=$cc" >> $config_host_mak
 print_config "CC" "$cc"
 echo "CXX=$cxx" >> $config_host_mak
 print_config "CXX" "$cxx"
 
+# generate io_uring_version.h
+# Reset MAKEFLAGS
+MAKEFLAGS=
+MAKE_PRINT_VARS="include Makefile.common\nprint-%%: ; @echo \$(\$*)\n"
+VERSION_MAJOR=$(printf "$MAKE_PRINT_VARS" | make -s --no-print-directory -f - print-VERSION_MAJOR)
+VERSION_MINOR=$(printf "$MAKE_PRINT_VARS" | make -s --no-print-directory -f - print-VERSION_MINOR)
+io_uring_version_h="src/include/liburing/io_uring_version.h"
+cat > $io_uring_version_h << EOF
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_VERSION_H
+#define LIBURING_VERSION_H
+
+#define IO_URING_VERSION_MAJOR $VERSION_MAJOR
+#define IO_URING_VERSION_MINOR $VERSION_MINOR
+
+#endif
+EOF
+
 # generate compat.h
 compat_h="src/include/liburing/compat.h"
 cat > $compat_h << EOF
@@ -452,10 +558,15 @@ struct __kernel_timespec {
 	long long	tv_nsec;
 };
 
+/* <linux/time_types.h> is not available, so it can't be included */
+#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
+
 EOF
 else
 cat >> $compat_h << EOF
 #include <linux/time_types.h>
+/* <linux/time_types.h> is included above and not needed again */
+#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
 
 EOF
 fi
@@ -481,7 +592,33 @@ cat >> $compat_h << EOF
 
 EOF
 fi
+if test "$futexv" != "yes"; then
+cat >> $compat_h << EOF
+#include <inttypes.h>
 
+#define FUTEX_32	2
+#define FUTEX_WAITV_MAX	128
+
+struct futex_waitv {
+	uint64_t	val;
+	uint64_t	uaddr;
+	uint32_t	flags;
+	uint32_t	__reserved;
+};
+
+EOF
+fi
+
+if test "$has_idtype_t" != "yes"; then
+cat >> $compat_h << EOF
+typedef enum
+{
+  P_ALL,		/* Wait for any child.  */
+  P_PID,		/* Wait for specified process.  */
+  P_PGID		/* Wait for members of process group.  */
+} idtype_t;
+EOF
+fi
 cat >> $compat_h << EOF
 #endif
 EOF

debian/changelog

@@ -1,3 +1,14 @@
+liburing (2.2-1) stable; urgency=low
+
+  * Update to 2.2
+  * Bump up so version to 2
+  * Drop liburing1-udeb
+  * Package using dh instead of using dh_* helpers manually
+  * Add linux header dependency to liburing-dev
+  * Bump up debhelper-compact level to 13
+
+ -- Kefu Chai <tchaikov@gmail.com>  Sun, 16 Oct 2022 16:30:48 +0800
+
 liburing (0.7-1) stable; urgency=low
 
   * Update to 0.7

debian/compat

@@ -1 +0,0 @@
-9

debian/control

@@ -2,13 +2,14 @@ Source: liburing
 Section: libs
 Priority: optional
 Maintainer: Liu Changcheng <changcheng.liu@intel.com>
-Build-Depends: debhelper (>=9)
+Build-Depends:
+  debhelper-compat (= 13)
 Standards-Version: 4.1.4
 Homepage: https://git.kernel.dk/cgit/liburing/tree/README
 Vcs-Git: https://git.kernel.dk/liburing
 Vcs-Browser: https://git.kernel.dk/cgit/liburing/
 
-Package: liburing1
+Package: liburing2
 Architecture: linux-any
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
@@ -21,24 +22,14 @@ Description: userspace library for using io_uring
  .
  This package contains the shared library.
 
-Package: liburing1-udeb
-Package-Type: udeb
-Section: debian-installer
-Architecture: linux-any
-Depends: ${misc:Depends}, ${shlibs:Depends},
-Description: userspace library for using io_uring
- io_uring is kernel feature to improve development
- The newese Linux IO interface, io_uring could improve
- system performance a lot. liburing is the userpace
- library to use io_uring feature.
- .
- This package contains the udeb shared library.
-
 Package: liburing-dev
 Section: libdevel
 Architecture: linux-any
 Multi-Arch: same
-Depends: ${misc:Depends}, liburing1 (= ${binary:Version}),
+Depends:
+  ${misc:Depends},
+  liburing2 (= ${binary:Version}),
+  linux-libc-dev (>= 5.1)
 Description: userspace library for using io_uring
  io_uring is kernel feature to improve development
  The newese Linux IO interface, io_uring could improve

debian/liburing-dev.manpages

@@ -1,6 +1,5 @@
-man/io_uring_setup.2
+usr/share/man/man2/io_uring_*.2
-man/io_uring_enter.2
+usr/share/man/man3/io_uring_*.3
-man/io_uring_register.2
+usr/share/man/man7/io_uring.7
-man/io_uring_queue_exit.3
+usr/share/man/man3/IO_URING_*.3
-man/io_uring_queue_init.3
+usr/share/man/man3/__io_uring_*.3
-man/io_uring_get_sqe.3

debian/liburing1-udeb.install

@@ -1 +0,0 @@
-lib/*/lib*.so.*

debian/liburing1.install

@@ -1 +0,0 @@
-lib/*/lib*.so.*

debian/liburing1.symbols

@@ -1,32 +0,0 @@
-liburing.so.1 liburing1 #MINVER#
- (symver)LIBURING_0.1 0.1-1
- io_uring_get_sqe@LIBURING_0.1 0.1-1
- io_uring_queue_exit@LIBURING_0.1 0.1-1
- io_uring_queue_init@LIBURING_0.1 0.1-1
- io_uring_queue_mmap@LIBURING_0.1 0.1-1
- io_uring_register_buffers@LIBURING_0.1 0.1-1
- io_uring_register_eventfd@LIBURING_0.1 0.1-1
- io_uring_register_eventfd_async@LIBURING_0.6 0.6-1
- io_uring_register_files@LIBURING_0.1 0.1-1
- io_uring_submit@LIBURING_0.1 0.1-1
- io_uring_submit_and_wait@LIBURING_0.1 0.1-1
- io_uring_unregister_buffers@LIBURING_0.1 0.1-1
- io_uring_unregister_files@LIBURING_0.1 0.1-1
- (symver)LIBURING_0.2 0.2-1
- __io_uring_get_cqe@LIBURING_0.2 0.2-1
- io_uring_queue_init_params@LIBURING_0.2 0.2-1
- io_uring_register_files_update@LIBURING_0.2 0.2-1
- io_uring_peek_batch_cqe@LIBURING_0.2 0.2-1
- io_uring_wait_cqe_timeout@LIBURING_0.2 0.2-1
- io_uring_wait_cqes@LIBURING_0.2 0.2-1
- (symver)LIBURING_0.3 0.3-1
- (symver)LIBURING_0.4 0.4-1
- (symver)LIBURING_0.5 0.5-1
- (symver)LIBURING_0.6 0.6-1
- (symver)LIBURING_0.7 0.7-1
- io_uring_get_probe@LIBURING_0.4 0.4-1
- io_uring_get_probe_ring@LIBURING_0.4 0.4-1
- io_uring_register_personality@LIBURING_0.4 0.4-1
- io_uring_register_probe@LIBURING_0.4 0.4-1
- io_uring_ring_dontfork@LIBURING_0.4 0.4-1
- io_uring_unregister_personality@LIBURING_0.4 0.4-1

debian/liburing2.install

@@ -0,0 +1 @@
+usr/lib/*/lib*.so.*

debian/liburing2.symbols

@@ -0,0 +1,56 @@
+liburing.so.2 liburing2 #MINVER#                                                                                                                                                                                                                                    [47/1887]
+ LIBURING_2.0@LIBURING_2.0 0.7-1
+ LIBURING_2.1@LIBURING_2.1 0.7-1
+ LIBURING_2.2@LIBURING_2.2 0.7-1
+ LIBURING_2.3@LIBURING_2.3 0.7-1
+ __io_uring_get_cqe@LIBURING_2.0 0.7-1
+ __io_uring_sqring_wait@LIBURING_2.0 0.7-1
+ io_uring_enter2@LIBURING_2.3 0.7-1
+ io_uring_enter@LIBURING_2.3 0.7-1
+ io_uring_free_probe@LIBURING_2.0 0.7-1
+ io_uring_get_events@LIBURING_2.3 0.7-1
+ io_uring_get_probe@LIBURING_2.0 0.7-1
+ io_uring_get_probe_ring@LIBURING_2.0 0.7-1
+ io_uring_get_sqe@LIBURING_2.0 0.7-1
+ io_uring_mlock_size@LIBURING_2.1 0.7-1
+ io_uring_mlock_size_params@LIBURING_2.1 0.7-1
+ io_uring_peek_batch_cqe@LIBURING_2.0 0.7-1
+ io_uring_queue_exit@LIBURING_2.0 0.7-1
+ io_uring_queue_init@LIBURING_2.0 0.7-1
+ io_uring_queue_init_params@LIBURING_2.0 0.7-1
+ io_uring_queue_mmap@LIBURING_2.0 0.7-1
+ io_uring_register@LIBURING_2.3 0.7-1
+ io_uring_register_buf_ring@LIBURING_2.2 0.7-1
+ io_uring_register_buffers@LIBURING_2.0 0.7-1
+ io_uring_register_buffers_sparse@LIBURING_2.2 0.7-1
+ io_uring_register_buffers_tags@LIBURING_2.1 0.7-1
+ io_uring_register_buffers_update_tag@LIBURING_2.1 0.7-1
+ io_uring_register_eventfd@LIBURING_2.0 0.7-1
+ io_uring_register_eventfd_async@LIBURING_2.0 0.7-1
+ io_uring_register_file_alloc_range@LIBURING_2.3 0.7-1
+ io_uring_register_files@LIBURING_2.0 0.7-1
+ io_uring_register_files_sparse@LIBURING_2.2 0.7-1
+ io_uring_register_files_tags@LIBURING_2.1 0.7-1
+ io_uring_register_files_update@LIBURING_2.0 0.7-1
+ io_uring_register_files_update_tag@LIBURING_2.1 0.7-1
+ io_uring_register_iowq_aff@LIBURING_2.1 0.7-1
+ io_uring_register_iowq_max_workers@LIBURING_2.1 0.7-1
+ io_uring_register_personality@LIBURING_2.0 0.7-1
+ io_uring_register_probe@LIBURING_2.0 0.7-1
+ io_uring_register_ring_fd@LIBURING_2.2 0.7-1
+ io_uring_register_sync_cancel@LIBURING_2.3 0.7-1
+ io_uring_ring_dontfork@LIBURING_2.0 0.7-1
+ io_uring_setup@LIBURING_2.3 0.7-1
+ io_uring_submit@LIBURING_2.0 0.7-1
+ io_uring_submit_and_get_events@LIBURING_2.3 0.7-1
+ io_uring_submit_and_wait@LIBURING_2.0 0.7-1
+ io_uring_submit_and_wait_timeout@LIBURING_2.2 0.7-1
+ io_uring_unregister_buf_ring@LIBURING_2.2 0.7-1
+ io_uring_unregister_buffers@LIBURING_2.0 0.7-1
+ io_uring_unregister_eventfd@LIBURING_2.0 0.7-1
+ io_uring_unregister_files@LIBURING_2.0 0.7-1
+ io_uring_unregister_iowq_aff@LIBURING_2.1 0.7-1
+ io_uring_unregister_personality@LIBURING_2.0 0.7-1
+ io_uring_unregister_ring_fd@LIBURING_2.2 0.7-1
+ io_uring_wait_cqe_timeout@LIBURING_2.0 0.7-1
+ io_uring_wait_cqes@LIBURING_2.0 0.7-1

debian/rules

@@ -5,77 +5,25 @@
 
 DEB_BUILD_MAINT_OPTIONS = hardening=+bindnow
 DEB_CFLAGS_MAINT_PREPEND = -Wall
+DEB_BUILD_OPTIONS += nocheck
 
 include /usr/share/dpkg/default.mk
 include /usr/share/dpkg/buildtools.mk
 
-export CC
+%:
+	dh $@ --parallel
 
-lib := liburing1
+override_dh_auto_configure:
-libdbg := $(lib)-dbg
+	./configure \
-libudeb := $(lib)-udeb
+	  --prefix=/usr \
-libdev := liburing-dev
+	  --includedir=/usr/include \
-
+	  --datadir=/usr/share \
-build-indep:
+	  --mandir=/usr/share/man \
-
+	  --libdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
-build-arch:
+	  --libdevdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
-	dh_testdir
+	  --cc=$(CC)
-
-	$(MAKE) CPPFLAGS="$(CPPFLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
-
-build: build-indep build-arch
-
-clean:
-	dh_testdir
-	dh_testroot
-
-	$(MAKE) clean
-
-	dh_clean
-
-check-arch: build-arch
-	dh_testdir
 
+override_dh_auto_test:
 ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-	$(MAKE) CPPFLAGS="$(CPPFLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
+	$(MAKE) runtests
-	        partcheck
 endif
-
-install-arch: check-arch
-	dh_testdir
-	dh_testroot
-	dh_clean
-	dh_installdirs
-
-	$(MAKE) install \
-	  DESTDIR=$(CURDIR)/debian/tmp \
-	  libdir=/lib/$(DEB_HOST_MULTIARCH) \
-	  libdevdir=/usr/lib/$(DEB_HOST_MULTIARCH) \
-	  relativelibdir=/lib/$(DEB_HOST_MULTIARCH)/
-
-binary: binary-indep binary-arch
-
-binary-indep:
-	# Nothing to do.
-
-binary-arch: install-arch
-	dh_testdir
-	dh_testroot
-	dh_install -a
-	dh_installdocs -a
-	dh_installexamples -a
-	dh_installman -a
-	dh_lintian -a
-	dh_link -a
-	dh_strip -a --ddeb-migration='$(libdbg) (<< 0.3)'
-	dh_compress -a
-	dh_fixperms -a
-	dh_makeshlibs -a --add-udeb '$(libudeb)'
-	dh_shlibdeps -a
-	dh_installdeb -a
-	dh_gencontrol -a
-	dh_md5sums -a
-	dh_builddeb -a
-
-.PHONY: clean build-indep build-arch build
-.PHONY: install-arch binary-indep binary-arch binary

examples/Makefile

@@ -10,13 +10,21 @@ ifneq ($(MAKECMDGOALS),clean)
 include ../config-host.mak
 endif
 
+LDFLAGS ?=
+override LDFLAGS += -L../src/ -luring -lpthread
+
 example_srcs := \
+	io_uring-close-test.c \
 	io_uring-cp.c \
 	io_uring-test.c \
 	io_uring-udp.c \
 	link-cp.c \
+	napi-busy-poll-client.c \
+	napi-busy-poll-server.c \
 	poll-bench.c \
-	send-zerocopy.c
+	send-zerocopy.c \
+	rsrc-update-bench.c \
+	proxy.c
 
 all_targets :=
 
@@ -24,16 +32,20 @@ all_targets :=
 ifdef CONFIG_HAVE_UCONTEXT
 	example_srcs += ucontext-cp.c
 endif
-all_targets += ucontext-cp
+all_targets += ucontext-cp helpers.o
 
 example_targets := $(patsubst %.c,%,$(patsubst %.cc,%,$(example_srcs)))
 all_targets += $(example_targets)
 
+helpers = helpers.o
 
 all: $(example_targets)
 
-%: %.c ../src/liburing.a
+helpers.o: helpers.c
-	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS)
+	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ -c $<
+
+%: %.c $(helpers) ../src/liburing.a
+	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
 
 clean:
 	@rm -f $(all_targets)

examples/helpers.c

@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: MIT */
+#include <fcntl.h>
+#include <stdint.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "helpers.h"
+
+int setup_listening_socket(int port, int ipv6)
+{
+	struct sockaddr_in srv_addr = { };
+	struct sockaddr_in6 srv_addr6 = { };
+	int fd, enable, ret, domain;
+
+	if (ipv6)
+		domain = AF_INET6;
+	else
+		domain = AF_INET;
+
+	fd = socket(domain, SOCK_STREAM, 0);
+	if (fd == -1) {
+		perror("socket()");
+		return -1;
+	}
+
+	enable = 1;
+	ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
+	if (ret < 0) {
+		perror("setsockopt(SO_REUSEADDR)");
+		return -1;
+	}
+
+	if (ipv6) {
+		srv_addr6.sin6_family = AF_INET6;
+		srv_addr6.sin6_port = htons(port);
+		srv_addr6.sin6_addr = in6addr_any;
+		ret = bind(fd, (const struct sockaddr *)&srv_addr6, sizeof(srv_addr6));
+	} else {
+		srv_addr.sin_family = AF_INET;
+		srv_addr.sin_port = htons(port);
+		srv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
+		ret = bind(fd, (const struct sockaddr *)&srv_addr, sizeof(srv_addr));
+	}
+
+	if (ret < 0) {
+		perror("bind()");
+		return -1;
+	}
+
+	if (listen(fd, 1024) < 0) {
+		perror("listen()");
+		return -1;
+	}
+
+	return fd;
+}

examples/helpers.h

@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_EX_HELPERS_H
+#define LIBURING_EX_HELPERS_H
+
+int setup_listening_socket(int port, int ipv6);
+
+#endif

examples/io_uring-close-test.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Simple app that demonstrates how to setup an io_uring interface, and use it
+ * via a registered ring fd, without leaving the original fd open.
+ *
+ * gcc -Wall -O2 -D_GNU_SOURCE -o io_uring-close-test io_uring-close-test.c -luring
+ */
+#include <stdio.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include "liburing.h"
+
+#define QD	4
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	int i, fd, ret, pending, done;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct iovec *iovecs;
+	struct stat sb;
+	ssize_t fsize;
+	off_t offset;
+	void *buf;
+
+	if (argc < 2) {
+		printf("%s: file\n", argv[0]);
+		return 1;
+	}
+
+	ret = io_uring_queue_init(QD, &ring, 0);
+	if (ret < 0) {
+		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	ret = io_uring_register_ring_fd(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "register_ring_fd: %s\n", strerror(-ret));
+		return 1;
+	}
+	ret = io_uring_close_ring_fd(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "close_ring_fd: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	fd = open(argv[1], O_RDONLY);
+	if (fd < 0) {
+		perror("open");
+		return 1;
+	}
+
+	if (fstat(fd, &sb) < 0) {
+		perror("fstat");
+		return 1;
+	}
+
+	fsize = 0;
+	iovecs = calloc(QD, sizeof(struct iovec));
+	for (i = 0; i < QD; i++) {
+		if (posix_memalign(&buf, 4096, 4096))
+			return 1;
+		iovecs[i].iov_base = buf;
+		iovecs[i].iov_len = 4096;
+		fsize += 4096;
+	}
+
+	offset = 0;
+	i = 0;
+	do {
+		sqe = io_uring_get_sqe(&ring);
+		if (!sqe)
+			break;
+		io_uring_prep_readv(sqe, fd, &iovecs[i], 1, offset);
+		offset += iovecs[i].iov_len;
+		i++;
+		if (offset > sb.st_size)
+			break;
+	} while (1);
+
+	ret = io_uring_submit(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_submit: %s\n", strerror(-ret));
+		return 1;
+	} else if (ret != i) {
+		fprintf(stderr, "io_uring_submit submitted less %d\n", ret);
+		return 1;
+	}
+
+	done = 0;
+	pending = ret;
+	fsize = 0;
+	for (i = 0; i < pending; i++) {
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		if (ret < 0) {
+			fprintf(stderr, "io_uring_wait_cqe: %s\n", strerror(-ret));
+			return 1;
+		}
+
+		done++;
+		ret = 0;
+		if (cqe->res != 4096 && cqe->res + fsize != sb.st_size) {
+			fprintf(stderr, "ret=%d, wanted 4096\n", cqe->res);
+			ret = 1;
+		}
+		fsize += cqe->res;
+		io_uring_cqe_seen(&ring, cqe);
+		if (ret)
+			break;
+	}
+
+	printf("Submitted=%d, completed=%d, bytes=%lu\n", pending, done,
+						(unsigned long) fsize);
+	close(fd);
+	io_uring_queue_exit(&ring);
+	return 0;
+}

examples/io_uring-test.c

@@ -69,7 +69,7 @@ int main(int argc, char *argv[])
 		io_uring_prep_readv(sqe, fd, &iovecs[i], 1, offset);
 		offset += iovecs[i].iov_len;
 		i++;
-		if (offset > sb.st_size)
+		if (offset >= sb.st_size)
 			break;
 	} while (1);
 

examples/io_uring-udp.c

@@ -271,14 +271,22 @@ static int process_cqe_recv(struct ctx *ctx, struct io_uring_cqe *cqe,
 	}
 
 	if (ctx->verbose) {
+		struct sockaddr_in *addr = io_uring_recvmsg_name(o);
+		struct sockaddr_in6 *addr6 = (void *)addr;
 		char buff[INET6_ADDRSTRLEN + 1];
 		const char *name;
-		struct sockaddr_in *addr = io_uring_recvmsg_name(o);
+		void *paddr;
 
-		name = inet_ntop(ctx->af, addr, buff, sizeof(buff));
+		if (ctx->af == AF_INET6)
+			paddr = &addr6->sin6_addr;
+		else
+			paddr = &addr->sin_addr;
+
+		name = inet_ntop(ctx->af, paddr, buff, sizeof(buff));
 		if (!name)
 			name = "<INVALID>";
-		fprintf(stderr, "received %u bytes %d from %s:%d\n",
+
+		fprintf(stderr, "received %u bytes %d from [%s]:%d\n",
 			io_uring_recvmsg_payload_length(o, cqe->res, &ctx->msg),
 			o->namelen, name, (int)ntohs(addr->sin_port));
 	}

examples/napi-busy-poll-client.c

@@ -0,0 +1,509 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Simple ping/pong client which can use the io_uring NAPI support.
+ *
+ * Needs to be run as root because it sets SCHED_FIFO scheduling class,
+ * but will work without that.
+ *
+ * Example:
+ *
+ * sudo examples/napi-busy-poll-client -a 192.168.2.2 -n100000 -p4444 \
+ *	-b -t10 -u
+ *
+ * send and receive 100k packets, using NAPI.
+ */
+#include <ctype.h>
+#include <errno.h>
+#include <float.h>
+#include <getopt.h>
+#include <liburing.h>
+#include <math.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <time.h>
+#include <unistd.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <netinet/in.h>
+
+#define MAXBUFLEN 100
+#define PORTNOLEN 10
+#define ADDRLEN   80
+#define RINGSIZE  1024
+
+#define printable(ch) (isprint((unsigned char)ch) ? ch : '#')
+
+enum {
+	IOURING_RECV,
+	IOURING_SEND,
+	IOURING_RECVMSG,
+	IOURING_SENDMSG
+};
+
+struct ctx
+{
+	struct io_uring ring;
+	union {
+		struct sockaddr_in6 saddr6;
+		struct sockaddr_in saddr;
+	};
+
+	int sockfd;
+	int buffer_len;
+	int num_pings;
+	bool napi_check;
+
+	union {
+		char buffer[MAXBUFLEN];
+		struct timespec ts;
+	};
+
+	int rtt_index;
+	double *rtt;
+};
+
+struct options
+{
+	int  num_pings;
+	__u32  timeout;
+
+	bool sq_poll;
+	bool defer_tw;
+	bool busy_loop;
+	bool prefer_busy_poll;
+	bool ipv6;
+
+	char port[PORTNOLEN];
+	char addr[ADDRLEN];
+};
+
+static struct option longopts[] =
+{
+	{"address"  , 1, NULL, 'a'},
+	{"busy"     , 0, NULL, 'b'},
+	{"help"     , 0, NULL, 'h'},
+	{"num_pings", 1, NULL, 'n'},
+	{"port"     , 1, NULL, 'p'},
+	{"prefer"   , 1, NULL, 'u'},
+	{"sqpoll"   , 0, NULL, 's'},
+	{"timeout"  , 1, NULL, 't'},
+	{NULL       , 0, NULL,  0 }
+};
+
+static void printUsage(const char *name)
+{
+	fprintf(stderr,
+	"Usage: %s [-l|--listen] [-a|--address ip_address] [-p|--port port-no] [-s|--sqpoll]"
+	" [-b|--busy] [-n|--num pings] [-t|--timeout busy-poll-timeout] [-u||--prefer] [-6] [-h|--help]\n"
+	"--address\n"
+	"-a        : remote or local ipv6 address\n"
+	"--busy\n"
+	"-b        : busy poll io_uring instead of blocking.\n"
+	"--num_pings\n"
+	"-n        : number of pings\n"
+	"--port\n"
+	"-p        : port\n"
+	"--sqpoll\n"
+	"-s        : Configure io_uring to use SQPOLL thread\n"
+	"--timeout\n"
+	"-t        : Configure NAPI busy poll timeout"
+	"--prefer\n"
+	"-u        : prefer NAPI busy poll\n"
+	"-6        : use IPV6\n"
+	"--help\n"
+	"-h        : Display this usage message\n\n",
+	name);
+}
+
+static void printError(const char *msg, int opt)
+{
+	if (msg && opt)
+		fprintf(stderr, "%s (-%c)\n", msg, printable(opt));
+}
+
+static void setProcessScheduler(void)
+{
+	struct sched_param param;
+
+	param.sched_priority = sched_get_priority_max(SCHED_FIFO);
+	if (sched_setscheduler(0, SCHED_FIFO, &param) < 0)
+		fprintf(stderr, "sched_setscheduler() failed: (%d) %s\n",
+			errno, strerror(errno));
+}
+
+static double diffTimespec(const struct timespec *time1, const struct timespec *time0)
+{
+	return (time1->tv_sec - time0->tv_sec)
+		+ (time1->tv_nsec - time0->tv_nsec) / 1000000000.0;
+}
+
+static uint64_t encodeUserData(char type, int fd)
+{
+	return (uint32_t)fd | ((uint64_t)type << 56);
+}
+
+static void decodeUserData(uint64_t data, char *type, int *fd)
+{
+	*type = data >> 56;
+	*fd   = data & 0xffffffffU;
+}
+
+static const char *opTypeToStr(char type)
+{
+	const char *res;
+
+	switch (type) {
+	case IOURING_RECV:
+		res = "IOURING_RECV";
+		break;
+	case IOURING_SEND:
+		res = "IOURING_SEND";
+		break;
+	case IOURING_RECVMSG:
+		res = "IOURING_RECVMSG";
+		break;
+	case IOURING_SENDMSG:
+		res = "IOURING_SENDMSG";
+		break;
+	default:
+		res = "Unknown";
+	}
+
+	return res;
+}
+
+static void reportNapi(struct ctx *ctx)
+{
+	unsigned int napi_id = 0;
+	socklen_t len = sizeof(napi_id);
+
+	getsockopt(ctx->sockfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);
+	if (napi_id)
+		printf(" napi id: %d\n", napi_id);
+	else
+		printf(" unassigned napi id\n");
+
+	ctx->napi_check = true;
+}
+
+static void sendPing(struct ctx *ctx)
+{
+	struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
+
+	clock_gettime(CLOCK_REALTIME, (struct timespec *)ctx->buffer);
+	io_uring_prep_send(sqe, ctx->sockfd, ctx->buffer, sizeof(struct timespec), 0);
+	sqe->user_data = encodeUserData(IOURING_SEND, ctx->sockfd);
+}
+
+static void receivePing(struct ctx *ctx)
+{
+	struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
+
+	io_uring_prep_recv(sqe, ctx->sockfd, ctx->buffer, MAXBUFLEN, 0);
+	sqe->user_data = encodeUserData(IOURING_RECV, ctx->sockfd);
+}
+
+static void recordRTT(struct ctx *ctx)
+{
+	struct timespec startTs = ctx->ts;
+
+	// Send next ping.
+	sendPing(ctx);
+
+	// Store round-trip time.
+	ctx->rtt[ctx->rtt_index] = diffTimespec(&ctx->ts, &startTs);
+	ctx->rtt_index++;
+}
+
+static void printStats(struct ctx *ctx)
+{
+	double minRTT    = DBL_MAX;
+	double maxRTT    = 0.0;
+	double avgRTT    = 0.0;
+	double stddevRTT = 0.0;
+
+	// Calculate min, max, avg.
+	for (int i = 0; i < ctx->rtt_index; i++) {
+		if (ctx->rtt[i] < minRTT)
+			minRTT = ctx->rtt[i];
+		if (ctx->rtt[i] > maxRTT)
+			maxRTT = ctx->rtt[i];
+
+		avgRTT += ctx->rtt[i];
+	}
+	avgRTT /= ctx->rtt_index;
+
+	// Calculate stddev.
+	for (int i = 0; i < ctx->rtt_index; i++)
+		stddevRTT += fabs(ctx->rtt[i] - avgRTT);
+	stddevRTT /= ctx->rtt_index;
+
+	fprintf(stdout, " rtt(us) min/avg/max/mdev = %.3f/%.3f/%.3f/%.3f\n",
+		minRTT * 1000000, avgRTT * 1000000, maxRTT * 1000000, stddevRTT * 1000000);
+}
+
+static int completion(struct ctx *ctx, struct io_uring_cqe *cqe)
+{
+	char type;
+	int  fd;
+	int  res = cqe->res;
+
+	decodeUserData(cqe->user_data, &type, &fd);
+	if (res < 0) {
+		fprintf(stderr, "unexpected %s failure: (%d) %s\n",
+			opTypeToStr(type), -res, strerror(-res));
+		return -1;
+	}
+
+	switch (type) {
+	case IOURING_SEND:
+		receivePing(ctx);
+		break;
+	case IOURING_RECV:
+		if (res != sizeof(struct timespec)) {
+			fprintf(stderr, "unexpected ping reply len: %d\n", res);
+			abort();
+		}
+
+		if (!ctx->napi_check) {
+			reportNapi(ctx);
+			sendPing(ctx);
+		} else {
+			recordRTT(ctx);
+		}
+
+		--ctx->num_pings;
+		break;
+
+	default:
+		fprintf(stderr, "unexpected %s completion\n",
+			opTypeToStr(type));
+		return -1;
+		break;
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct ctx       ctx;
+	struct options   opt;
+	struct __kernel_timespec *tsPtr;
+	struct __kernel_timespec ts;
+	struct io_uring_params params;
+	struct io_uring_napi napi;
+	int flag, ret, af;
+
+	memset(&opt, 0, sizeof(struct options));
+
+	// Process flags.
+	while ((flag = getopt_long(argc, argv, ":hs:bua:n:p:t:6d:", longopts, NULL)) != -1) {
+		switch (flag) {
+		case 'a':
+			strcpy(opt.addr, optarg);
+			break;
+		case 'b':
+			opt.busy_loop = true;
+			break;
+		case 'h':
+			printUsage(argv[0]);
+			exit(0);
+			break;
+		case 'n':
+			opt.num_pings = atoi(optarg) + 1;
+			break;
+		case 'p':
+			strcpy(opt.port, optarg);
+			break;
+		case 's':
+			opt.sq_poll = !!atoi(optarg);
+			break;
+		case 't':
+			opt.timeout = atoi(optarg);
+			break;
+		case 'u':
+			opt.prefer_busy_poll = true;
+			break;
+		case '6':
+			opt.ipv6 = true;
+			break;
+		case 'd':
+			opt.defer_tw = !!atoi(optarg);
+			break;
+		case ':':
+			printError("Missing argument", optopt);
+			printUsage(argv[0]);
+			exit(-1);
+			break;
+		case '?':
+			printError("Unrecognized option", optopt);
+			printUsage(argv[0]);
+			exit(-1);
+			break;
+
+		default:
+			fprintf(stderr, "Fatal: Unexpected case in CmdLineProcessor switch()\n");
+			exit(-1);
+			break;
+		}
+	}
+
+	if (strlen(opt.addr) == 0) {
+		fprintf(stderr, "address option is mandatory\n");
+		printUsage(argv[0]);
+		exit(1);
+	}
+
+	if (opt.ipv6) {
+		af = AF_INET6;
+		ctx.saddr6.sin6_port   = htons(atoi(opt.port));
+		ctx.saddr6.sin6_family = AF_INET6;
+	} else {
+		af = AF_INET;
+		ctx.saddr.sin_port   = htons(atoi(opt.port));
+		ctx.saddr.sin_family = AF_INET;
+	}
+
+	if (opt.ipv6)
+		ret = inet_pton(af, opt.addr, &ctx.saddr6.sin6_addr);
+	else
+		ret = inet_pton(af, opt.addr, &ctx.saddr.sin_addr);
+	if (ret <= 0) {
+		fprintf(stderr, "inet_pton error for %s\n", optarg);
+		printUsage(argv[0]);
+		exit(1);
+	}
+
+	// Connect to server.
+	fprintf(stdout, "Connecting to %s... (port=%s) to send %d pings\n", opt.addr, opt.port, opt.num_pings - 1);
+
+	if ((ctx.sockfd = socket(af, SOCK_DGRAM, 0)) < 0) {
+		fprintf(stderr, "socket() failed: (%d) %s\n", errno, strerror(errno));
+		exit(1);
+	}
+
+	if (opt.ipv6)
+		ret = connect(ctx.sockfd, (struct sockaddr *)&ctx.saddr6, sizeof(struct sockaddr_in6));
+	else
+		ret = connect(ctx.sockfd, (struct sockaddr *)&ctx.saddr, sizeof(struct sockaddr_in));
+	if (ret < 0) {
+		fprintf(stderr, "connect() failed: (%d) %s\n", errno, strerror(errno));
+		exit(1);
+	}
+
+	// Setup ring.
+	memset(&params, 0, sizeof(params));
+	memset(&ts, 0, sizeof(ts));
+	memset(&napi, 0, sizeof(napi));
+
+	params.flags = IORING_SETUP_SINGLE_ISSUER;
+	if (opt.defer_tw) {
+		params.flags |= IORING_SETUP_DEFER_TASKRUN;
+	} else if (opt.sq_poll) {
+		params.flags = IORING_SETUP_SQPOLL;
+		params.sq_thread_idle = 50;
+	} else {
+		params.flags |= IORING_SETUP_COOP_TASKRUN;
+	}
+
+	ret = io_uring_queue_init_params(RINGSIZE, &ctx.ring, &params);
+	if (ret) {
+		fprintf(stderr, "io_uring_queue_init_params() failed: (%d) %s\n",
+			ret, strerror(-ret));
+		exit(1);
+	}
+
+	if (opt.timeout || opt.prefer_busy_poll) {
+		napi.prefer_busy_poll = opt.prefer_busy_poll;
+		napi.busy_poll_to = opt.timeout;
+
+		ret = io_uring_register_napi(&ctx.ring, &napi);
+		if (ret) {
+			fprintf(stderr, "io_uring_register_napi: %d\n", ret);
+			exit(1);
+		}
+	}
+
+	if (opt.busy_loop)
+		tsPtr = &ts;
+	else
+		tsPtr = NULL;
+
+	// Use realtime scheduler.
+	setProcessScheduler();
+
+	// Copy payload.
+	clock_gettime(CLOCK_REALTIME, &ctx.ts);
+
+	// Setup context.
+	ctx.napi_check = false;
+	ctx.buffer_len = sizeof(struct timespec);
+	ctx.num_pings  = opt.num_pings;
+
+	ctx.rtt_index = 0;
+	ctx.rtt = (double *)malloc(sizeof(double) * opt.num_pings);
+	if (!ctx.rtt) {
+		fprintf(stderr, "Cannot allocate results array\n");
+		exit(1);
+	}
+
+	// Send initial message to get napi id.
+	sendPing(&ctx);
+
+	while (ctx.num_pings != 0) {
+		int res;
+		unsigned num_completed = 0;
+		unsigned head;
+		struct io_uring_cqe *cqe;
+
+		do {
+			res = io_uring_submit_and_wait_timeout(&ctx.ring, &cqe, 1, tsPtr, NULL);
+			if (res >= 0)
+				break;
+			else if (res == -ETIME)
+				continue;
+			fprintf(stderr, "submit_and_wait: %d\n", res);
+			exit(1);
+		} while (1);
+
+		io_uring_for_each_cqe(&ctx.ring, head, cqe) {
+			++num_completed;
+			if (completion(&ctx, cqe))
+				goto out;
+		}
+
+		if (num_completed)
+			io_uring_cq_advance(&ctx.ring, num_completed);
+	}
+
+	printStats(&ctx);
+
+out:
+	// Clean up.
+	if (opt.timeout || opt.prefer_busy_poll) {
+		ret = io_uring_unregister_napi(&ctx.ring, &napi);
+		if (ret)
+			fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
+		if (opt.timeout          != napi.busy_poll_to ||
+		    opt.prefer_busy_poll != napi.prefer_busy_poll) {
+			fprintf(stderr, "Expected busy poll to = %d, got %d\n",
+				opt.timeout, napi.busy_poll_to);
+			fprintf(stderr, "Expected prefer busy poll = %d, got %d\n",
+				opt.prefer_busy_poll, napi.prefer_busy_poll);
+		}
+	} else {
+		ret = io_uring_unregister_napi(&ctx.ring, NULL);
+		if (ret)
+			fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
+	}
+
+	io_uring_queue_exit(&ctx.ring);
+	free(ctx.rtt);
+	close(ctx.sockfd);
+	return 0;
+}

examples/napi-busy-poll-server.c

@@ -0,0 +1,450 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Simple ping/pong backend which can use the io_uring NAPI support.
+ *
+ * Needs to be run as root because it sets SCHED_FIFO scheduling class,
+ * but will work without that.
+ *
+ * Example:
+ *
+ * sudo examples/napi-busy-poll-server -l -a 192.168.2.2 -n100000 \
+ *	-p4444 -t10 -b -u
+ *
+ * will respond to 100k packages, using NAPI.
+ */
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <liburing.h>
+#include <math.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <time.h>
+#include <unistd.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <netinet/in.h>
+
+#define MAXBUFLEN 100
+#define PORTNOLEN 10
+#define ADDRLEN   80
+#define RINGSIZE  1024
+
+#define printable(ch) (isprint((unsigned char)ch) ? ch : '#')
+
+enum {
+	IOURING_RECV,
+	IOURING_SEND,
+	IOURING_RECVMSG,
+	IOURING_SENDMSG
+};
+
+struct ctx
+{
+	struct io_uring     ring;
+	union {
+		struct sockaddr_in6 saddr6;
+		struct sockaddr_in saddr;
+	};
+	struct iovec        iov;
+	struct msghdr       msg;
+
+	int sockfd;
+	int buffer_len;
+	int num_pings;
+	bool napi_check;
+
+	union {
+		char buffer[MAXBUFLEN];
+		struct timespec ts;
+	};
+};
+
+struct options
+{
+	int  num_pings;
+	__u32 timeout;
+
+	bool listen;
+	bool defer_tw;
+	bool sq_poll;
+	bool busy_loop;
+	bool prefer_busy_poll;
+	bool ipv6;
+
+	char port[PORTNOLEN];
+	char addr[ADDRLEN];
+};
+
+static struct options opt;
+
+static struct option longopts[] =
+{
+	{"address"  , 1, NULL, 'a'},
+	{"busy"     , 0, NULL, 'b'},
+	{"help"     , 0, NULL, 'h'},
+	{"listen"   , 0, NULL, 'l'},
+	{"num_pings", 1, NULL, 'n'},
+	{"port"     , 1, NULL, 'p'},
+	{"prefer"   , 1, NULL, 'u'},
+	{"sqpoll"   , 0, NULL, 's'},
+	{"timeout"  , 1, NULL, 't'},
+	{NULL       , 0, NULL,  0 }
+};
+
+static void printUsage(const char *name)
+{
+	fprintf(stderr,
+	"Usage: %s [-l|--listen] [-a|--address ip_address] [-p|--port port-no] [-s|--sqpoll]"
+	" [-b|--busy] [-n|--num pings] [-t|--timeout busy-poll-timeout] [-u|--prefer] [-6] [-h|--help]\n"
+	" --listen\n"
+	"-l        : Server mode\n"
+	"--address\n"
+	"-a        : remote or local ipv6 address\n"
+	"--busy\n"
+	"-b        : busy poll io_uring instead of blocking.\n"
+	"--num_pings\n"
+	"-n        : number of pings\n"
+	"--port\n"
+	"-p        : port\n"
+	"--sqpoll\n"
+	"-s        : Configure io_uring to use SQPOLL thread\n"
+	"--timeout\n"
+	"-t        : Configure NAPI busy poll timeout"
+	"--prefer\n"
+	"-u        : prefer NAPI busy poll\n"
+	"-6        : use IPV6\n"
+	"--help\n"
+	"-h        : Display this usage message\n\n",
+	name);
+}
+
+static void printError(const char *msg, int opt)
+{
+	if (msg && opt)
+		fprintf(stderr, "%s (-%c)\n", msg, printable(opt));
+}
+
+static void setProcessScheduler(void)
+{
+	struct sched_param param;
+
+	param.sched_priority = sched_get_priority_max(SCHED_FIFO);
+	if (sched_setscheduler(0, SCHED_FIFO, &param) < 0)
+		fprintf(stderr, "sched_setscheduler() failed: (%d) %s\n",
+			errno, strerror(errno));
+}
+
+static uint64_t encodeUserData(char type, int fd)
+{
+	return (uint32_t)fd | ((__u64)type << 56);
+}
+
+static void decodeUserData(uint64_t data, char *type, int *fd)
+{
+	*type = data >> 56;
+	*fd   = data & 0xffffffffU;
+}
+
+static const char *opTypeToStr(char type)
+{
+	const char *res;
+
+	switch (type) {
+	case IOURING_RECV:
+		res = "IOURING_RECV";
+		break;
+	case IOURING_SEND:
+		res = "IOURING_SEND";
+		break;
+	case IOURING_RECVMSG:
+		res = "IOURING_RECVMSG";
+		break;
+	case IOURING_SENDMSG:
+		res = "IOURING_SENDMSG";
+		break;
+	default:
+		res = "Unknown";
+	}
+
+	return res;
+}
+
+static void reportNapi(struct ctx *ctx)
+{
+	unsigned int napi_id = 0;
+	socklen_t len = sizeof(napi_id);
+
+	getsockopt(ctx->sockfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);
+	if (napi_id)
+		printf(" napi id: %d\n", napi_id);
+	else
+		printf(" unassigned napi id\n");
+
+	ctx->napi_check = true;
+}
+
+static void sendPing(struct ctx *ctx)
+{
+	struct io_uring_sqe *sqe = io_uring_get_sqe(&ctx->ring);
+
+	io_uring_prep_sendmsg(sqe, ctx->sockfd, &ctx->msg, 0);
+	sqe->user_data = encodeUserData(IOURING_SENDMSG, ctx->sockfd);
+}
+
+static void receivePing(struct ctx *ctx)
+{
+	struct io_uring_sqe *sqe;
+
+	bzero(&ctx->msg, sizeof(struct msghdr));
+	if (opt.ipv6) {
+		ctx->msg.msg_name    = &ctx->saddr6;
+		ctx->msg.msg_namelen = sizeof(struct sockaddr_in6);
+	} else {
+		ctx->msg.msg_name    = &ctx->saddr;
+		ctx->msg.msg_namelen = sizeof(struct sockaddr_in);
+	}
+	ctx->iov.iov_base    = ctx->buffer;
+	ctx->iov.iov_len     = MAXBUFLEN;
+	ctx->msg.msg_iov     = &ctx->iov;
+	ctx->msg.msg_iovlen  = 1;
+
+	sqe = io_uring_get_sqe(&ctx->ring);
+	io_uring_prep_recvmsg(sqe, ctx->sockfd, &ctx->msg, 0);
+	sqe->user_data = encodeUserData(IOURING_RECVMSG, ctx->sockfd);
+}
+
+static void completion(struct ctx *ctx, struct io_uring_cqe *cqe)
+{
+	char type;
+	int  fd;
+	int  res = cqe->res;
+
+	decodeUserData(cqe->user_data, &type, &fd);
+	if (res < 0) {
+		fprintf(stderr, "unexpected %s failure: (%d) %s\n",
+			opTypeToStr(type), -res, strerror(-res));
+		abort();
+	}
+
+	switch (type) {
+	case IOURING_SENDMSG:
+		receivePing(ctx);
+		--ctx->num_pings;
+		break;
+	case IOURING_RECVMSG:
+		ctx->iov.iov_len = res;
+		sendPing(ctx);
+		if (!ctx->napi_check)
+			reportNapi(ctx);
+		break;
+	default:
+		fprintf(stderr, "unexpected %s completion\n",
+			opTypeToStr(type));
+		abort();
+		break;
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	int flag;
+	struct ctx       ctx;
+	struct __kernel_timespec *tsPtr;
+	struct __kernel_timespec ts;
+	struct io_uring_params params;
+	struct io_uring_napi napi;
+	int ret, af;
+
+	memset(&opt, 0, sizeof(struct options));
+
+	// Process flags.
+	while ((flag = getopt_long(argc, argv, ":lhs:bua:n:p:t:6d:", longopts, NULL)) != -1) {
+		switch (flag) {
+		case 'a':
+			strcpy(opt.addr, optarg);
+			break;
+		case 'b':
+			opt.busy_loop = true;
+			break;
+		case 'h':
+			printUsage(argv[0]);
+			exit(0);
+			break;
+		case 'l':
+			opt.listen = true;
+			break;
+		case 'n':
+			opt.num_pings = atoi(optarg) + 1;
+			break;
+		case 'p':
+			strcpy(opt.port, optarg);
+			break;
+		case 's':
+			opt.sq_poll = !!atoi(optarg);
+			break;
+		case 't':
+			opt.timeout = atoi(optarg);
+			break;
+		case 'u':
+			opt.prefer_busy_poll = true;
+			break;
+		case '6':
+			opt.ipv6 = true;
+			break;
+		case 'd':
+			opt.defer_tw = !!atoi(optarg);
+			break;
+		case ':':
+			printError("Missing argument", optopt);
+			printUsage(argv[0]);
+			exit(-1);
+			break;
+		case '?':
+			printError("Unrecognized option", optopt);
+			printUsage(argv[0]);
+			exit(-1);
+			break;
+
+		default:
+			fprintf(stderr, "Fatal: Unexpected case in CmdLineProcessor switch()\n");
+			exit(-1);
+			break;
+		}
+	}
+
+	if (strlen(opt.addr) == 0) {
+		fprintf(stderr, "address option is mandatory\n");
+		printUsage(argv[0]);
+		exit(1);
+	}
+
+	if (opt.ipv6) {
+		af = AF_INET6;
+		ctx.saddr6.sin6_port   = htons(atoi(opt.port));
+		ctx.saddr6.sin6_family = AF_INET6;
+	} else {
+		af = AF_INET;
+		ctx.saddr.sin_port   = htons(atoi(opt.port));
+		ctx.saddr.sin_family = AF_INET;
+	}
+
+	if (opt.ipv6)
+		ret = inet_pton(AF_INET6, opt.addr, &ctx.saddr6.sin6_addr);
+	else
+		ret = inet_pton(AF_INET, opt.addr, &ctx.saddr.sin_addr);
+	if (ret <= 0) {
+		fprintf(stderr, "inet_pton error for %s\n", optarg);
+		printUsage(argv[0]);
+		exit(1);
+	}
+
+	// Connect to server.
+	fprintf(stdout, "Listening %s : %s...\n", opt.addr, opt.port);
+
+	if ((ctx.sockfd = socket(af, SOCK_DGRAM, 0)) < 0) {
+		fprintf(stderr, "socket() failed: (%d) %s\n", errno, strerror(errno));
+		exit(1);
+	}
+
+	if (opt.ipv6)
+		ret = bind(ctx.sockfd, (struct sockaddr *)&ctx.saddr6, sizeof(struct sockaddr_in6));
+	else
+		ret = bind(ctx.sockfd, (struct sockaddr *)&ctx.saddr, sizeof(struct sockaddr_in));
+	if (ret < 0) {
+		fprintf(stderr, "bind() failed: (%d) %s\n", errno, strerror(errno));
+		exit(1);
+	}
+
+	// Setup ring.
+	memset(&params, 0, sizeof(params));
+	memset(&ts, 0, sizeof(ts));
+	memset(&napi, 0, sizeof(napi));
+
+	params.flags = IORING_SETUP_SINGLE_ISSUER;
+	if (opt.defer_tw) {
+		params.flags |= IORING_SETUP_DEFER_TASKRUN;
+	} else if (opt.sq_poll) {
+		params.flags = IORING_SETUP_SQPOLL;
+		params.sq_thread_idle = 50;
+	} else {
+		params.flags |= IORING_SETUP_COOP_TASKRUN;
+	}
+
+	ret = io_uring_queue_init_params(RINGSIZE, &ctx.ring, &params);
+	if (ret) {
+		fprintf(stderr, "io_uring_queue_init_params() failed: (%d) %s\n",
+			ret, strerror(-ret));
+		exit(1);
+	}
+
+	if (opt.timeout || opt.prefer_busy_poll) {
+		napi.prefer_busy_poll = opt.prefer_busy_poll;
+		napi.busy_poll_to = opt.timeout;
+
+		ret = io_uring_register_napi(&ctx.ring, &napi);
+		if (ret) {
+			fprintf(stderr, "io_uring_register_napi: %d\n", ret);
+			exit(1);
+		}
+	}
+
+	if (opt.busy_loop)
+		tsPtr = &ts;
+	else
+		tsPtr = NULL;
+
+	// Use realtime scheduler.
+	setProcessScheduler();
+
+	// Copy payload.
+	clock_gettime(CLOCK_REALTIME, &ctx.ts);
+
+	// Setup context.
+	ctx.napi_check = false;
+	ctx.buffer_len = sizeof(struct timespec);
+	ctx.num_pings  = opt.num_pings;
+
+	// Receive initial message to get napi id.
+	receivePing(&ctx);
+
+	while (ctx.num_pings != 0) {
+		int res;
+		unsigned int num_completed = 0;
+		unsigned int head;
+		struct io_uring_cqe *cqe;
+
+		do {
+			res = io_uring_submit_and_wait_timeout(&ctx.ring, &cqe, 1, tsPtr, NULL);
+			if (res >= 0)
+				break;
+			else if (res == -ETIME)
+				continue;
+			fprintf(stderr, "submit_and_wait: %d\n", res);
+			exit(1);
+		} while (1);
+
+		io_uring_for_each_cqe(&ctx.ring, head, cqe) {
+			++num_completed;
+			completion(&ctx, cqe);
+		}
+
+		if (num_completed)
+			io_uring_cq_advance(&ctx.ring, num_completed);
+	}
+
+	// Clean up.
+	if (opt.timeout || opt.prefer_busy_poll) {
+		ret = io_uring_unregister_napi(&ctx.ring, &napi);
+		if (ret)
+			fprintf(stderr, "io_uring_unregister_napi: %d\n", ret);
+	}
+
+	io_uring_queue_exit(&ctx.ring);
+	close(ctx.sockfd);
+	return 0;
+}

examples/proxy.h

@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_PROXY_H
+#define LIBURING_PROXY_H
+
+#include <sys/time.h>
+
+/*
+ * Generic opcode agnostic encoding to sqe/cqe->user_data
+ */
+struct userdata {
+	union {
+		struct {
+			uint16_t op_tid; /* 4 bits op, 12 bits tid */
+			uint16_t bid;
+			uint16_t fd;
+		};
+		uint64_t val;
+	};
+};
+
+#define OP_SHIFT	(12)
+#define TID_MASK	((1U << 12) - 1)
+
+/*
+ * Packs the information that we will need at completion time into the
+ * sqe->user_data field, which is passed back in the completion in
+ * cqe->user_data. Some apps would need more space than this, and in fact
+ * I'd love to pack the requested IO size in here, and it's not uncommon to
+ * see apps use this field as just a cookie to either index a data structure
+ * at completion time, or even just put the pointer to the associated
+ * structure into this field.
+ */
+static inline void __encode_userdata(struct io_uring_sqe *sqe, int tid, int op,
+				     int bid, int fd)
+{
+	struct userdata ud = {
+		.op_tid = (op << OP_SHIFT) | tid,
+		.bid = bid,
+		.fd = fd
+	};
+
+	io_uring_sqe_set_data64(sqe, ud.val);
+}
+
+static inline uint64_t __raw_encode(int tid, int op, int bid, int fd)
+{
+	struct userdata ud = {
+		.op_tid = (op << OP_SHIFT) | tid,
+		.bid = bid,
+		.fd = fd
+	};
+
+	return ud.val;
+}
+
+static inline int cqe_to_op(struct io_uring_cqe *cqe)
+{
+	struct userdata ud = { .val = cqe->user_data };
+
+	return ud.op_tid >> OP_SHIFT;
+}
+
+static inline int cqe_to_bid(struct io_uring_cqe *cqe)
+{
+	struct userdata ud = { .val = cqe->user_data };
+
+	return ud.bid;
+}
+
+static inline int cqe_to_fd(struct io_uring_cqe *cqe)
+{
+	struct userdata ud = { .val = cqe->user_data };
+
+	return ud.fd;
+}
+
+static unsigned long long mtime_since(const struct timeval *s,
+				      const struct timeval *e)
+{
+	long long sec, usec;
+
+	sec = e->tv_sec - s->tv_sec;
+	usec = (e->tv_usec - s->tv_usec);
+	if (sec > 0 && usec < 0) {
+		sec--;
+		usec += 1000000;
+	}
+
+	sec *= 1000;
+	usec /= 1000;
+	return sec + usec;
+}
+
+static unsigned long long mtime_since_now(struct timeval *tv)
+{
+	struct timeval end;
+
+	gettimeofday(&end, NULL);
+	return mtime_since(tv, &end);
+}
+
+#endif

examples/rsrc-update-bench.c

@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: MIT */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <poll.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+
+static unsigned long runtime_ms = 10000;
+
+static unsigned long gettimeofday_ms(void)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+}
+
+int main(void)
+{
+	unsigned long tstop;
+	unsigned long nr_reqs = 0;
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	struct io_uring ring;
+	int pipe1[2];
+	int ret, i, qd = 32;
+	int table_size = 128;
+
+	if (pipe(pipe1) != 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	ret = io_uring_queue_init(1024, &ring, IORING_SETUP_SINGLE_ISSUER |
+					       IORING_SETUP_DEFER_TASKRUN);
+	if (ret) {
+		fprintf(stderr, "io_uring_queue_init failed: %d\n", ret);
+		return 1;
+	}
+	ret = io_uring_register_ring_fd(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_ring_fd failed\n");
+		return 1;
+	}
+	ret = io_uring_register_files_sparse(&ring, table_size);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_files_sparse failed\n");
+		return 1;
+	}
+
+	for (i = 0; i < table_size; i++) {
+		ret = io_uring_register_files_update(&ring, i, pipe1, 1);
+		if (ret < 0) {
+			fprintf(stderr, "io_uring_register_files_update failed\n");
+			return 1;
+		}
+	}
+
+	srand(time(NULL));
+
+	tstop = gettimeofday_ms() + runtime_ms;
+	do {
+		int off = rand();
+
+		for (i = 0; i < qd; i++) {
+			sqe = io_uring_get_sqe(&ring);
+			int roff = (off + i) % table_size;
+			io_uring_prep_files_update(sqe, pipe1, 1, roff);
+		}
+
+		ret = io_uring_submit(&ring);
+		if (ret != qd) {
+			fprintf(stderr, "child: sqe submit failed: %d\n", ret);
+			return 1;
+		}
+
+		for (i = 0; i < qd; i++) {
+			ret = io_uring_wait_cqe(&ring, &cqe);
+			if (ret < 0) {
+				fprintf(stderr, "child: wait completion %d\n", ret);
+				break;
+			}
+			io_uring_cqe_seen(&ring, cqe);
+			nr_reqs++;
+		}
+	} while (gettimeofday_ms() < tstop);
+
+	fprintf(stderr, "max updates/s: %lu\n", nr_reqs * 1000UL / runtime_ms);
+
+	io_uring_queue_exit(&ring);
+	close(pipe1[0]);
+	close(pipe1[1]);
+	return 0;
+}

examples/send-zerocopy.c

@@ -5,15 +5,17 @@
 #include <stdint.h>
 #include <assert.h>
 #include <errno.h>
-#include <error.h>
 #include <limits.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <stdbool.h>
+#include <stdarg.h>
 #include <string.h>
+#include <pthread.h>
 
+#include <poll.h>
+#include <sched.h>
 #include <arpa/inet.h>
-#include <linux/errqueue.h>
 #include <linux/if_packet.h>
 #include <linux/ipv6.h>
 #include <linux/socket.h>
@@ -35,27 +37,107 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
+#include <signal.h>
 
 #include "liburing.h"
 
 #define ZC_TAG 0xfffffffULL
 #define MAX_SUBMIT_NR 512
+#define MAX_THREADS 100
+
+struct thread_data {
+	pthread_t thread;
+	void *ret;
+	int idx;
+	unsigned long long packets;
+	unsigned long long bytes;
+	unsigned long long dt_ms;
+	struct sockaddr_storage dst_addr;
+	int fd;
+};
 
 static bool cfg_reg_ringfd = true;
 static bool cfg_fixed_files = 1;
 static bool cfg_zc = 1;
 static int  cfg_nr_reqs = 8;
 static bool cfg_fixed_buf = 1;
+static bool cfg_hugetlb = 0;
+static bool cfg_defer_taskrun = 0;
+static int  cfg_cpu = -1;
+static bool cfg_rx = 0;
+static unsigned  cfg_nr_threads = 1;
 
 static int  cfg_family		= PF_UNSPEC;
+static int  cfg_type		= 0;
 static int  cfg_payload_len;
 static int  cfg_port		= 8000;
 static int  cfg_runtime_ms	= 4200;
+static bool cfg_rx_poll		= false;
 
 static socklen_t cfg_alen;
-static struct sockaddr_storage cfg_dst_addr;
+static char *str_addr = NULL;
 
-static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
+static char payload_buf[IP_MAXPACKET] __attribute__((aligned(4096)));
+static char *payload;
+static struct thread_data threads[MAX_THREADS];
+static pthread_barrier_t barrier;
+
+static bool should_stop = false;
+
+static void sigint_handler(__attribute__((__unused__)) int sig)
+{
+	/* kill if should_stop can't unblock threads fast enough */
+	if (should_stop)
+		_exit(-1);
+	should_stop = true;
+}
+
+/*
+ * Implementation of error(3), prints an error message and exits.
+ */
+static void t_error(int status, int errnum, const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);
+
+	vfprintf(stderr, format, args);
+	if (errnum)
+		fprintf(stderr, ": %s", strerror(errnum));
+
+	fprintf(stderr, "\n");
+	va_end(args);
+	exit(status);
+}
+
+static void set_cpu_affinity(void)
+{
+	cpu_set_t mask;
+
+	if (cfg_cpu == -1)
+		return;
+
+	CPU_ZERO(&mask);
+	CPU_SET(cfg_cpu, &mask);
+	if (sched_setaffinity(0, sizeof(mask), &mask))
+		t_error(1, errno, "unable to pin cpu\n");
+}
+
+static void set_iowq_affinity(struct io_uring *ring)
+{
+	cpu_set_t mask;
+	int ret;
+
+	if (cfg_cpu == -1)
+		return;
+
+	CPU_ZERO(&mask);
+	CPU_SET(cfg_cpu, &mask);
+	ret = io_uring_register_iowq_aff(ring, 1, &mask);
+	if (ret)
+		t_error(1, ret, "unabled to set io-wq affinity\n");
+}
 
 static unsigned long gettimeofday_ms(void)
 {
@@ -68,7 +150,7 @@ static unsigned long gettimeofday_ms(void)
 static void do_setsockopt(int fd, int level, int optname, int val)
 {
 	if (setsockopt(fd, level, optname, &val, sizeof(val)))
-		error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
+		t_error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
 }
 
 static void setup_sockaddr(int domain, const char *str_addr,
@@ -76,42 +158,156 @@ static void setup_sockaddr(int domain, const char *str_addr,
 {
 	struct sockaddr_in6 *addr6 = (void *) sockaddr;
 	struct sockaddr_in *addr4 = (void *) sockaddr;
+	int port = cfg_port;
 
 	switch (domain) {
 	case PF_INET:
 		memset(addr4, 0, sizeof(*addr4));
 		addr4->sin_family = AF_INET;
-		addr4->sin_port = htons(cfg_port);
+		addr4->sin_port = htons(port);
 		if (str_addr &&
 		    inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
-			error(1, 0, "ipv4 parse error: %s", str_addr);
+			t_error(1, 0, "ipv4 parse error: %s", str_addr);
 		break;
 	case PF_INET6:
 		memset(addr6, 0, sizeof(*addr6));
 		addr6->sin6_family = AF_INET6;
-		addr6->sin6_port = htons(cfg_port);
+		addr6->sin6_port = htons(port);
 		if (str_addr &&
 		    inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
-			error(1, 0, "ipv6 parse error: %s", str_addr);
+			t_error(1, 0, "ipv6 parse error: %s", str_addr);
 		break;
 	default:
-		error(1, 0, "illegal domain");
+		t_error(1, 0, "illegal domain");
 	}
 }
 
-static int do_setup_tx(int domain, int type, int protocol)
+static int do_poll(int fd, int events)
 {
-	int fd;
+	struct pollfd pfd;
+	int ret;
+
+	pfd.events = events;
+	pfd.revents = 0;
+	pfd.fd = fd;
+
+	ret = poll(&pfd, 1, -1);
+	if (ret == -1)
+		t_error(1, errno, "poll");
+
+	return ret && (pfd.revents & events);
+}
+
+/* Flush all outstanding bytes for the tcp receive queue */
+static int do_flush_tcp(struct thread_data *td, int fd)
+{
+	int ret;
+
+	/* MSG_TRUNC flushes up to len bytes */
+	ret = recv(fd, NULL, 1 << 21, MSG_TRUNC | MSG_DONTWAIT);
+	if (ret == -1 && errno == EAGAIN)
+		return 0;
+	if (ret == -1)
+		t_error(1, errno, "flush");
+	if (!ret)
+		return 1;
+
+	td->packets++;
+	td->bytes += ret;
+	return 0;
+}
+
+/* Flush all outstanding datagrams. Verify first few bytes of each. */
+static int do_flush_datagram(struct thread_data *td, int fd)
+{
+	long ret, off = 0;
+	char buf[64];
+
+	/* MSG_TRUNC will return full datagram length */
+	ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT | MSG_TRUNC);
+	if (ret == -1 && errno == EAGAIN)
+		return 0;
+
+	if (ret == -1)
+		t_error(1, errno, "recv");
+	if (ret != cfg_payload_len)
+		t_error(1, 0, "recv: ret=%u != %u", ret, cfg_payload_len);
+	if ((unsigned long) ret > sizeof(buf) - off)
+		ret = sizeof(buf) - off;
+	if (memcmp(buf + off, payload, ret))
+		t_error(1, 0, "recv: data mismatch");
+
+	td->packets++;
+	td->bytes += cfg_payload_len;
+	return 0;
+}
+
+static void do_setup_rx(int domain, int type, int protocol)
+{
+	struct sockaddr_storage addr = {};
+	struct thread_data *td;
+	int listen_fd, fd;
+	unsigned int i;
 
 	fd = socket(domain, type, protocol);
 	if (fd == -1)
-		error(1, errno, "socket t");
+		t_error(1, errno, "socket r");
 
-	do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);
+	do_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, 1);
 
-	if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))
+	setup_sockaddr(cfg_family, str_addr, &addr);
-		error(1, errno, "connect");
+
-	return fd;
+	if (bind(fd, (void *)&addr, cfg_alen))
+		t_error(1, errno, "bind");
+
+	if (type != SOCK_STREAM) {
+		if (cfg_nr_threads != 1)
+			t_error(1, 0, "udp rx cant multithread");
+		threads[0].fd = fd;
+		return;
+	}
+
+	listen_fd = fd;
+	if (listen(listen_fd, cfg_nr_threads))
+		t_error(1, errno, "listen");
+
+	for (i = 0; i < cfg_nr_threads; i++) {
+		td = &threads[i];
+
+		fd = accept(listen_fd, NULL, NULL);
+		if (fd == -1)
+			t_error(1, errno, "accept");
+		td->fd = fd;
+	}
+
+	if (close(listen_fd))
+		t_error(1, errno, "close listen sock");
+}
+
+static void *do_rx(void *arg)
+{
+	struct thread_data *td = arg;
+	const int cfg_receiver_wait_ms = 400;
+	uint64_t tstop;
+	int ret, fd = td->fd;
+
+	tstop = gettimeofday_ms() + cfg_runtime_ms + cfg_receiver_wait_ms;
+	do {
+		if (cfg_type == SOCK_STREAM)
+			ret = do_flush_tcp(td, fd);
+		else
+			ret = do_flush_datagram(td, fd);
+
+		if (ret)
+			break;
+
+		do_poll(fd, POLLIN);
+	} while (gettimeofday_ms() < tstop);
+
+	if (close(fd))
+		t_error(1, errno, "close");
+	pthread_exit(&td->ret);
+	return NULL;
 }
 
 static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
@@ -125,35 +321,47 @@ static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
 
 	ret = io_uring_wait_cqe(ring, &cqe);
 	if (ret)
-		error(1, ret, "wait cqe");
+		t_error(1, ret, "wait cqe");
 	return cqe;
 }
 
-static void do_tx(int domain, int type, int protocol)
+static void do_tx(struct thread_data *td, int domain, int type, int protocol)
 {
-	unsigned long packets = 0;
+	const int notif_slack = 128;
-	unsigned long bytes = 0;
 	struct io_uring ring;
 	struct iovec iov;
-	uint64_t tstop;
+	uint64_t tstart;
 	int i, fd, ret;
 	int compl_cqes = 0;
+	int ring_flags = IORING_SETUP_COOP_TASKRUN | IORING_SETUP_SINGLE_ISSUER;
+	unsigned loop = 0;
 
-	fd = do_setup_tx(domain, type, protocol);
+	if (cfg_defer_taskrun)
+		ring_flags |= IORING_SETUP_DEFER_TASKRUN;
 
-	ret = io_uring_queue_init(512, &ring, IORING_SETUP_COOP_TASKRUN);
+	fd = socket(domain, type, protocol);
+	if (fd == -1)
+		t_error(1, errno, "socket t");
+
+	if (connect(fd, (void *)&td->dst_addr, cfg_alen))
+		t_error(1, errno, "connect, idx %i", td->idx);
+
+	ret = io_uring_queue_init(512, &ring, ring_flags);
 	if (ret)
-		error(1, ret, "io_uring: queue init");
+		t_error(1, ret, "io_uring: queue init");
+
+	set_cpu_affinity();
+	set_iowq_affinity(&ring);
 
 	if (cfg_fixed_files) {
 		ret = io_uring_register_files(&ring, &fd, 1);
 		if (ret < 0)
-			error(1, ret, "io_uring: files registration");
+			t_error(1, ret, "io_uring: files registration");
 	}
 	if (cfg_reg_ringfd) {
 		ret = io_uring_register_ring_fd(&ring);
 		if (ret < 0)
-			error(1, ret, "io_uring: io_uring_register_ring_fd");
+			t_error(1, ret, "io_uring: io_uring_register_ring_fd");
 	}
 
 	iov.iov_base = payload;
@@ -161,9 +369,22 @@ static void do_tx(int domain, int type, int protocol)
 
 	ret = io_uring_register_buffers(&ring, &iov, 1);
 	if (ret)
-		error(1, ret, "io_uring: buffer registration");
+		t_error(1, ret, "io_uring: buffer registration");
 
-	tstop = gettimeofday_ms() + cfg_runtime_ms;
+	if (cfg_rx_poll) {
+		struct io_uring_sqe *sqe;
+
+		sqe = io_uring_get_sqe(&ring);
+		io_uring_prep_poll_add(sqe, fd, POLLIN);
+
+		ret = io_uring_submit(&ring);
+		if (ret != 1)
+			t_error(1, ret, "submit poll");
+	}
+
+	pthread_barrier_wait(&barrier);
+
+	tstart = gettimeofday_ms();
 	do {
 		struct io_uring_sqe *sqe;
 		struct io_uring_cqe *cqe;
@@ -191,16 +412,20 @@ static void do_tx(int domain, int type, int protocol)
 			}
 		}
 
-		ret = io_uring_submit(&ring);
+		if (cfg_defer_taskrun && compl_cqes >= notif_slack)
+			ret = io_uring_submit_and_get_events(&ring);
+		else
+			ret = io_uring_submit(&ring);
+
 		if (ret != cfg_nr_reqs)
-			error(1, ret, "submit");
+			t_error(1, ret, "submit");
 
 		for (i = 0; i < cfg_nr_reqs; i++) {
 			cqe = wait_cqe_fast(&ring);
 
 			if (cqe->flags & IORING_CQE_F_NOTIF) {
 				if (cqe->flags & IORING_CQE_F_MORE)
-					error(1, -EINVAL, "F_MORE notif");
+					t_error(1, -EINVAL, "F_MORE notif");
 				compl_cqes--;
 				i--;
 				io_uring_cqe_seen(&ring, cqe);
@@ -210,28 +435,27 @@ static void do_tx(int domain, int type, int protocol)
 				compl_cqes++;
 
 			if (cqe->res >= 0) {
-				packets++;
+				td->packets++;
-				bytes += cqe->res;
+				td->bytes += cqe->res;
 			} else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE ||
 				   cqe->res == -ECONNRESET) {
-				fprintf(stderr, "Connection failure");
+				fprintf(stderr, "Connection failure\n");
 				goto out_fail;
 			} else if (cqe->res != -EAGAIN) {
-				error(1, cqe->res, "send failed");
+				t_error(1, cqe->res, "send failed");
 			}
 			io_uring_cqe_seen(&ring, cqe);
 		}
-	} while (gettimeofday_ms() < tstop);
+		if (should_stop)
+			break;
+	} while ((++loop % 16 != 0) || gettimeofday_ms() < tstart + cfg_runtime_ms);
+
+	td->dt_ms = gettimeofday_ms() - tstart;
 
 out_fail:
 	shutdown(fd, SHUT_RDWR);
 	if (close(fd))
-		error(1, errno, "close");
+		t_error(1, errno, "close");
-
-	fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
-			packets, bytes >> 20,
-			packets / (cfg_runtime_ms / 1000),
-			(bytes >> 20) / (cfg_runtime_ms / 1000));
 
 	while (compl_cqes) {
 		struct io_uring_cqe *cqe = wait_cqe_fast(&ring);
@@ -242,47 +466,67 @@ out_fail:
 	io_uring_queue_exit(&ring);
 }
 
-static void do_test(int domain, int type, int protocol)
+static void *do_test(void *arg)
 {
-	int i;
+	struct thread_data *td = arg;
+	int protocol = 0;
 
-	for (i = 0; i < IP_MAXPACKET; i++)
+	setup_sockaddr(cfg_family, str_addr, &td->dst_addr);
-		payload[i] = 'a' + (i % 26);
 
-	do_tx(domain, type, protocol);
+	do_tx(td, cfg_family, cfg_type, protocol);
+	pthread_exit(&td->ret);
+	return NULL;
 }
 
 static void usage(const char *filepath)
 {
-	error(1, 0, "Usage: %s [-n<N>] [-z<val>] [-s<payload size>] "
+	printf("Usage:\t%s <protocol> <ip-version> -D<addr> [options]\n", filepath);
-		    "(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
+	printf("\t%s <protocol> <ip-version> -R [options]\n\n", filepath);
+
+	printf("  -4\t\tUse IPv4\n");
+	printf("  -6\t\tUse IPv4\n");
+	printf("  -D <address>\tDestination address\n");
+	printf("  -p <port>\tServer port to listen on/connect to\n");
+	printf("  -s <size>\tBytes per request\n");
+	printf("  -s <size>\tBytes per request\n");
+	printf("  -n <nr>\tNumber of parallel requests\n");
+	printf("  -z <mode>\tZerocopy mode, 0 to disable, enabled otherwise\n");
+	printf("  -b <mode>\tUse registered buffers\n");
+	printf("  -l <mode>\tUse huge pages\n");
+	printf("  -d\t\tUse defer taskrun\n");
+	printf("  -C <cpu>\tPin to the specified CPU\n");
+	printf("  -T <nr>\tNumber of threads to use for sending\n");
+	printf("  -R\t\tPlay the server role\n");
+	printf("  -t <seconds>\tTime in seconds\n");
 }
 
 static void parse_opts(int argc, char **argv)
 {
-	const int max_payload_len = sizeof(payload) -
+	const int max_payload_len = IP_MAXPACKET -
 				    sizeof(struct ipv6hdr) -
 				    sizeof(struct tcphdr) -
 				    40 /* max tcp options */;
 	int c;
 	char *daddr = NULL;
 
-	if (argc <= 1)
+	if (argc <= 1) {
 		usage(argv[0]);
+		exit(0);
+	}
 
 	cfg_payload_len = max_payload_len;
 
-	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:k")) != -1) {
+	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:l:dC:T:Ry")) != -1) {
 		switch (c) {
 		case '4':
 			if (cfg_family != PF_UNSPEC)
-				error(1, 0, "Pass one of -4 or -6");
+				t_error(1, 0, "Pass one of -4 or -6");
 			cfg_family = PF_INET;
 			cfg_alen = sizeof(struct sockaddr_in);
 			break;
 		case '6':
 			if (cfg_family != PF_UNSPEC)
-				error(1, 0, "Pass one of -4 or -6");
+				t_error(1, 0, "Pass one of -4 or -6");
 			cfg_family = PF_INET6;
 			cfg_alen = sizeof(struct sockaddr_in6);
 			break;
@@ -307,15 +551,35 @@ static void parse_opts(int argc, char **argv)
 		case 'b':
 			cfg_fixed_buf = strtoul(optarg, NULL, 0);
 			break;
+		case 'l':
+			cfg_hugetlb = strtoul(optarg, NULL, 0);
+			break;
+		case 'd':
+			cfg_defer_taskrun = 1;
+			break;
+		case 'C':
+			cfg_cpu = strtol(optarg, NULL, 0);
+			break;
+		case 'T':
+			cfg_nr_threads = strtol(optarg, NULL, 0);
+			if (cfg_nr_threads > MAX_THREADS)
+				t_error(1, 0, "too many threads\n");
+			break;
+		case 'R':
+			cfg_rx = 1;
+			break;
+		case 'y':
+			cfg_rx_poll = 1;
+			break;
 		}
 	}
 
 	if (cfg_nr_reqs > MAX_SUBMIT_NR)
-		error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
+		t_error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
 	if (cfg_payload_len > max_payload_len)
-		error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
+		t_error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
 
-	setup_sockaddr(cfg_family, daddr, &cfg_dst_addr);
+	str_addr = daddr;
 
 	if (optind != argc - 1)
 		usage(argv[0]);
@@ -323,17 +587,72 @@ static void parse_opts(int argc, char **argv)
 
 int main(int argc, char **argv)
 {
+	unsigned long long tsum = 0;
+	unsigned long long packets = 0, bytes = 0;
+	struct thread_data *td;
 	const char *cfg_test;
+	unsigned int i;
+	void *res;
 
 	parse_opts(argc, argv);
+	set_cpu_affinity();
+
+	payload = payload_buf;
+	if (cfg_hugetlb) {
+		payload = mmap(NULL, 2*1024*1024, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS,
+				-1, 0);
+		if (payload == MAP_FAILED) {
+			fprintf(stderr, "hugetlb alloc failed\n");
+			return 1;
+		}
+	}
 
 	cfg_test = argv[argc - 1];
 	if (!strcmp(cfg_test, "tcp"))
-		do_test(cfg_family, SOCK_STREAM, 0);
+		cfg_type = SOCK_STREAM;
 	else if (!strcmp(cfg_test, "udp"))
-		do_test(cfg_family, SOCK_DGRAM, 0);
+		cfg_type = SOCK_DGRAM;
 	else
-		error(1, 0, "unknown cfg_test %s", cfg_test);
+		t_error(1, 0, "unknown cfg_test %s", cfg_test);
 
+	pthread_barrier_init(&barrier, NULL, cfg_nr_threads);
+
+	for (i = 0; i < IP_MAXPACKET; i++)
+		payload[i] = 'a' + (i % 26);
+
+	for (i = 0; i < cfg_nr_threads; i++) {
+		td = &threads[i];
+		td->idx = i;
+	}
+
+	if (cfg_rx)
+		do_setup_rx(cfg_family, cfg_type, 0);
+
+	if (!cfg_rx)
+		signal(SIGINT, sigint_handler);
+
+	for (i = 0; i < cfg_nr_threads; i++)
+		pthread_create(&threads[i].thread, NULL,
+				!cfg_rx ? do_test : do_rx, &threads[i]);
+
+	for (i = 0; i < cfg_nr_threads; i++) {
+		td = &threads[i];
+		pthread_join(td->thread, &res);
+		packets += td->packets;
+		bytes += td->bytes;
+		tsum += td->dt_ms;
+	}
+	tsum = tsum / cfg_nr_threads;
+
+	if (!tsum) {
+		printf("The run is too short, can't gather stats\n");
+	} else {
+		printf("packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n",
+			packets, bytes >> 20,
+			packets * 1000 / tsum,
+			(bytes >> 20) * 1000 / tsum);
+	}
+	pthread_barrier_destroy(&barrier);
 	return 0;
 }

examples/ucontext-cp.c

@@ -68,23 +68,8 @@ DEFINE_AWAIT_OP(readv)
 DEFINE_AWAIT_OP(writev)
 #undef DEFINE_AWAIT_OP
 
-int await_poll(async_context *pctx, int fd, short poll_mask) {
+static int await_delay(async_context *pctx, time_t seconds)
-	struct io_uring_sqe *sqe = io_uring_get_sqe(pctx->ring);
+{
-	struct io_uring_cqe *cqe;
-	if (!sqe)
-		return -1;
-
-	io_uring_prep_poll_add(sqe, fd, poll_mask);
-	io_uring_sqe_set_data(sqe, pctx);
-	swapcontext(&pctx->ctx_fnew, &pctx->ctx_main);
-	io_uring_peek_cqe(pctx->ring, &cqe);
-	assert(cqe);
-	io_uring_cqe_seen(pctx->ring, cqe);
-
-	return cqe->res;
-}
-
-int await_delay(async_context *pctx, time_t seconds) {
 	struct io_uring_sqe *sqe = io_uring_get_sqe(pctx->ring);
 	struct io_uring_cqe *cqe;
 	struct __kernel_timespec ts = {

liburing-ffi.pc.in

@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=${prefix}
+libdir=@libdir@
+includedir=@includedir@
+
+Name: @NAME@
+Version: @VERSION@
+Description: io_uring FFI library
+URL: https://git.kernel.dk/cgit/liburing/
+
+Libs: -L${libdir} -luring-ffi
+Cflags: -I${includedir}

liburing.pc.in

@@ -6,7 +6,7 @@ includedir=@includedir@
 Name: @NAME@
 Version: @VERSION@
 Description: io_uring library
-URL: http://git.kernel.dk/cgit/liburing/
+URL: https://git.kernel.dk/cgit/liburing/
 
 Libs: -L${libdir} -luring
-Cflags: -I${includedir}
+Cflags: -I${includedir} -D_GNU_SOURCE

liburing.spec

@@ -1,5 +1,5 @@
 Name: liburing
-Version: 2.3
+Version: 2.7
 Release: 1%{?dist}
 Summary: Linux-native io_uring I/O access library
 License: (GPLv2 with exceptions and LGPLv2+) or MIT
@@ -27,7 +27,7 @@ for the Linux-native io_uring.
 
 %build
 %set_build_flags
-./configure --prefix=%{_prefix} --libdir=/%{_libdir} --libdevdir=/%{_libdir} --mandir=%{_mandir} --includedir=%{_includedir}
+./configure --prefix=%{_prefix} --libdir=%{_libdir} --libdevdir=%{_libdir} --mandir=%{_mandir} --includedir=%{_includedir}
 
 %make_build
 

make-debs.sh

@@ -13,16 +13,18 @@
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 set -xe
 
 # Create dir for build
 base=${1:-/tmp/release}
-codename=$(lsb_release -sc)
+distro=unstable
 releasedir=$base/$(lsb_release -si)/liburing
 rm -rf $releasedir
 mkdir -p $releasedir
+HEAD=$(which head)
+DCH=$(which dch)
 
 src_dir=$(readlink -e `basename $0`)
 liburing_dir=$(dirname $src_dir)
@@ -38,12 +40,12 @@ cd ${releasedir}/${outfile}
 git clean -dxf
 
 # Change changelog if it's needed
-cur_ver=`head -l debian/changelog | sed -n -e 's/.* (\(.*\)) .*/\1/p'`
+cur_ver=`$HEAD < debian/changelog | sed -n -e 's/.* (\(.*\)) .*/\1/p'`
 if [ "$cur_ver" != "$version-1" ]; then
-	dch -D $codename --force-distribution -b -v "$version-1" "new version"
+	$DCH -D $distro --force-distribution -b -v "$version-1" "new version"
 fi
 
-# Create tar archieve
+# Create tar archive
 cd ../
 tar cvzf ${outfile}.tar.gz ${outfile}
 ln -s ${outfile}.tar.gz ${orgfile}.orig.tar.gz

src/Makefile

@@ -8,23 +8,29 @@ libdevdir ?= $(prefix)/lib
 LIBURING_CFLAGS ?=
 CPPFLAGS ?=
 override CPPFLAGS += -D_GNU_SOURCE \
-	-Iinclude/ -include ../config-host.h
+	-Iinclude/ -include ../config-host.h \
-CFLAGS ?= -g -O3 -Wall -Wextra -fno-stack-protector
+	-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
-override CFLAGS += -Wno-unused-parameter -Wno-sign-compare \
+CFLAGS ?= -O3 -Wall -Wextra -fno-stack-protector
+override CFLAGS += -Wno-unused-parameter \
 	-DLIBURING_INTERNAL \
 	$(LIBURING_CFLAGS)
 SO_CFLAGS=-fPIC $(CFLAGS)
 L_CFLAGS=$(CFLAGS)
-LINK_FLAGS=
+LINK_FLAGS=-Wl,-z,defs
 LINK_FLAGS+=$(LDFLAGS)
 ENABLE_SHARED ?= 1
 
 soname=liburing.so.$(VERSION_MAJOR)
 libname=liburing.so.$(VERSION)
+ffi_soname=liburing-ffi.so.$(VERSION_MAJOR)
+ffi_libname=liburing-ffi.so.$(VERSION)
+
 all_targets += liburing.a
+all_targets += liburing-ffi.a
 
 ifeq ($(ENABLE_SHARED),1)
 all_targets += $(libname)
+all_targets += $(ffi_libname)
 endif
 
 include ../Makefile.quiet
@@ -35,18 +41,20 @@ endif
 
 all: $(all_targets)
 
-liburing_srcs := setup.c queue.c register.c syscall.c
+liburing_srcs := setup.c queue.c register.c syscall.c version.c
 
 ifeq ($(CONFIG_NOLIBC),y)
 	liburing_srcs += nolibc.c
-	override CFLAGS += -nostdlib -nodefaultlibs -ffreestanding
+	override CFLAGS += -nostdlib -nodefaultlibs -ffreestanding -fno-builtin -fno-stack-protector
-	override CPPFLAGS += -nostdlib -nodefaultlibs -ffreestanding
+	override CPPFLAGS += -nostdlib -nodefaultlibs -ffreestanding -fno-builtin -fno-stack-protector
-	override LINK_FLAGS += -nostdlib -nodefaultlibs
+	override LINK_FLAGS += -nostdlib -nodefaultlibs $(libgcc_link_flag)
 endif
 
 override CPPFLAGS += -MT "$@" -MMD -MP -MF "$@.d"
 liburing_objs := $(patsubst %.c,%.ol,$(liburing_srcs))
 liburing_sobjs := $(patsubst %.c,%.os,$(liburing_srcs))
+liburing_ffi_objs := ffi.ol
+liburing_ffi_sobjs := ffi.os
 
 %.os: %.c
 	$(QUIET_CC)$(CC) $(CPPFLAGS) $(SO_CFLAGS) -c -o $@ $<
@@ -65,25 +73,56 @@ liburing.a: $(liburing_objs)
 	$(QUIET_AR)$(AR) r liburing.a $^
 	$(QUIET_RANLIB)$(RANLIB) liburing.a
 
+liburing-ffi.a: $(liburing_objs) $(liburing_ffi_objs)
+	@rm -f liburing-ffi.a
+	$(QUIET_AR)$(AR) r liburing-ffi.a $^
+	$(QUIET_RANLIB)$(RANLIB) liburing-ffi.a
+
 $(libname): $(liburing_sobjs) liburing.map
 	$(QUIET_CC)$(CC) $(SO_CFLAGS) -shared -Wl,--version-script=liburing.map -Wl,-soname=$(soname) -o $@ $(liburing_sobjs) $(LINK_FLAGS)
 
+$(ffi_libname): $(liburing_ffi_objs) $(liburing_ffi_sobjs) $(liburing_sobjs) liburing-ffi.map
+	$(QUIET_CC)$(CC) $(SO_CFLAGS) -shared -Wl,--version-script=liburing-ffi.map -Wl,-soname=$(ffi_soname) -o $@ $(liburing_sobjs) $(liburing_ffi_sobjs) $(LINK_FLAGS)
+
 install: $(all_targets)
 	install -D -m 644 include/liburing/io_uring.h $(includedir)/liburing/io_uring.h
 	install -D -m 644 include/liburing.h $(includedir)/liburing.h
 	install -D -m 644 include/liburing/compat.h $(includedir)/liburing/compat.h
 	install -D -m 644 include/liburing/barrier.h $(includedir)/liburing/barrier.h
+	install -D -m 644 include/liburing/io_uring_version.h $(includedir)/liburing/io_uring_version.h
 	install -D -m 644 liburing.a $(libdevdir)/liburing.a
+	install -D -m 644 liburing-ffi.a $(libdevdir)/liburing-ffi.a
 ifeq ($(ENABLE_SHARED),1)
 	install -D -m 755 $(libname) $(libdir)/$(libname)
+	install -D -m 755 $(ffi_libname) $(libdir)/$(ffi_libname)
 	ln -sf $(libname) $(libdir)/$(soname)
 	ln -sf $(relativelibdir)$(libname) $(libdevdir)/liburing.so
+	ln -sf $(ffi_libname) $(libdir)/$(ffi_soname)
+	ln -sf $(relativelibdir)$(ffi_libname) $(libdevdir)/liburing-ffi.so
+endif
+
+uninstall:
+	@rm -f $(includedir)/liburing/io_uring.h
+	@rm -f $(includedir)/liburing.h
+	@rm -f $(includedir)/liburing/compat.h
+	@rm -f $(includedir)/liburing/barrier.h
+	@rm -f $(includedir)/liburing/io_uring_version.h
+	@rm -f $(libdevdir)/liburing.a
+	@rm -f $(libdevdir)/liburing-ffi.a
+ifeq ($(ENABLE_SHARED),1)
+	@rm -f $(libdir)/$(libname)
+	@rm -f $(libdir)/$(ffi_libname)
+	@rm -f $(libdir)/$(soname)
+	@rm -f $(libdevdir)/liburing.so
+	@rm -f $(libdir)/$(ffi_soname)
+	@rm -f $(libdevdir)/liburing-ffi.so
 endif
 
 clean:
-	@rm -f $(all_targets) $(liburing_objs) $(liburing_sobjs) $(soname).new
+	@rm -f $(all_targets) $(liburing_objs) $(liburing_sobjs) $(liburing_ffi_objs) $(liburing_ffi_sobjs) $(soname).new
 	@rm -f *.so* *.a *.o *.d
 	@rm -f include/liburing/compat.h
+	@rm -f include/liburing/io_uring_version.h
 
 	@# When cleaning, we don't include ../config-host.mak,
 	@# so the nolibc objects are always skipped, clean them up!

src/arch/aarch64/lib.h

@@ -4,7 +4,6 @@
 #define LIBURING_ARCH_AARCH64_LIB_H
 
 #include <elf.h>
-#include <sys/auxv.h>
 #include "../../syscall.h"
 
 static inline long __get_page_size(void)
@@ -21,7 +20,7 @@ static inline long __get_page_size(void)
 		ssize_t x;
 
 		x = __sys_read(fd, buf, sizeof(buf));
-		if (x < sizeof(buf))
+		if (x < (long) sizeof(buf))
 			break;
 
 		if (buf[0] == AT_PAGESZ) {

src/arch/riscv64/lib.h

@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: MIT */
+
+#ifndef LIBURING_ARCH_RISCV64_LIB_H
+#define LIBURING_ARCH_RISCV64_LIB_H
+
+#include <elf.h>
+#include <sys/auxv.h>
+#include "../../syscall.h"
+
+static inline long __get_page_size(void)
+{
+	Elf64_Off buf[2];
+	long ret = 4096;
+	int fd;
+
+	fd = __sys_open("/proc/self/auxv", O_RDONLY, 0);
+	if (fd < 0)
+		return ret;
+
+	while (1) {
+		ssize_t x;
+
+		x = __sys_read(fd, buf, sizeof(buf));
+		if (x < (long) sizeof(buf))
+			break;
+
+		if (buf[0] == AT_PAGESZ) {
+			ret = buf[1];
+			break;
+		}
+	}
+
+	__sys_close(fd);
+	return ret;
+}
+
+static inline long get_page_size(void)
+{
+	static long cache_val;
+
+	if (cache_val)
+		return cache_val;
+
+	cache_val = __get_page_size();
+	return cache_val;
+}
+
+#endif /* #ifndef LIBURING_ARCH_RISCV64_LIB_H */

src/arch/riscv64/syscall.h

@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: MIT */
+
+#ifndef LIBURING_ARCH_RISCV64_SYSCALL_H
+#define LIBURING_ARCH_RISCV64_SYSCALL_H
+
+#if defined(__riscv) && __riscv_xlen == 64
+
+#define __do_syscallM(...) ({						\
+	__asm__ volatile (						\
+		"ecall"							\
+		: "=r"(a0)						\
+		: __VA_ARGS__						\
+		: "memory", "a1");					\
+	(long) a0;							\
+})
+
+#define __do_syscallN(...) ({						\
+	__asm__ volatile (						\
+		"ecall"							\
+		: "=r"(a0)						\
+		: __VA_ARGS__						\
+		: "memory");					\
+	(long) a0;							\
+})
+
+#define __do_syscall0(__n) ({						\
+	register long a7 __asm__("a7") = __n;				\
+	register long a0 __asm__("a0");					\
+									\
+	__do_syscallM("r" (a7));					\
+})
+
+#define __do_syscall1(__n, __a) ({					\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+									\
+	__do_syscallM("r" (a7), "0" (a0));				\
+})
+
+#define __do_syscall2(__n, __a, __b) ({					\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1));			\
+})
+
+#define __do_syscall3(__n, __a, __b, __c) ({				\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2));		\
+})
+
+#define __do_syscall4(__n, __a, __b, __c, __d) ({			\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3));\
+})
+
+#define __do_syscall5(__n, __a, __b, __c, __d, __e) ({			\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+	register __typeof__(__e) a4 __asm__("a4") = __e;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3),	\
+			"r"(a4));					\
+})
+
+#define __do_syscall6(__n, __a, __b, __c, __d, __e, __f) ({		\
+	register long a7 __asm__("a7") = __n;				\
+	register __typeof__(__a) a0 __asm__("a0") = __a;		\
+	register __typeof__(__b) a1 __asm__("a1") = __b;		\
+	register __typeof__(__c) a2 __asm__("a2") = __c;		\
+	register __typeof__(__d) a3 __asm__("a3") = __d;		\
+	register __typeof__(__e) a4 __asm__("a4") = __e;		\
+	register __typeof__(__f) a5 __asm__("a5") = __f;		\
+									\
+	__do_syscallN("r" (a7), "0" (a0), "r" (a1), "r" (a2), "r" (a3),	\
+			"r" (a4), "r"(a5));				\
+})
+
+#include "../syscall-defs.h"
+
+#else /* #if defined(__riscv) && __riscv_xlen == 64 */
+
+#include "../generic/syscall.h"
+
+#endif /*  #if defined(__riscv) && __riscv_xlen == 64 */
+
+#endif /* #ifndef LIBURING_ARCH_RISCV64_SYSCALL_H */

src/ffi.c

@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+#define IOURINGINLINE
+
+#ifdef __clang__
+// clang doesn't seem to particularly like that we're including a header that
+// deliberately contains function definitions so we explicitly silence it
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wmissing-prototypes"
+#endif
+
+#include "liburing.h"
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif

src/include/liburing/io_uring.h

@@ -12,12 +12,11 @@
 #include <linux/types.h>
 /*
  * this file is shared with liburing and that has to autodetect
- * if linux/time_types.h is available
+ * if linux/time_types.h is available or not, it can
+ * define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
+ * if linux/time_types.h is not available
  */
-#ifdef __KERNEL__
+#ifndef UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
-#define HAVE_LINUX_TIME_TYPES_H 1
-#endif
-#ifdef HAVE_LINUX_TIME_TYPES_H
 #include <linux/time_types.h>
 #endif
 
@@ -44,6 +43,10 @@ struct io_uring_sqe {
 	union {
 		__u64	addr;	/* pointer to buffer or iovecs */
 		__u64	splice_off_in;
+		struct {
+			__u32	level;
+			__u32	optname;
+		};
 	};
 	__u32	len;		/* buffer size or number of iovecs */
 	union {
@@ -66,6 +69,10 @@ struct io_uring_sqe {
 		__u32		xattr_flags;
 		__u32		msg_ring_flags;
 		__u32		uring_cmd_flags;
+		__u32		waitid_flags;
+		__u32		futex_flags;
+		__u32		install_fd_flags;
+		__u32		nop_flags;
 	};
 	__u64	user_data;	/* data to be passed back at completion time */
 	/* pack this to avoid bogus arm OABI complaints */
@@ -80,6 +87,7 @@ struct io_uring_sqe {
 	union {
 		__s32	splice_fd_in;
 		__u32	file_index;
+		__u32	optlen;
 		struct {
 			__u16	addr_len;
 			__u16	__pad3[1];
@@ -90,6 +98,7 @@ struct io_uring_sqe {
 			__u64	addr3;
 			__u64	__pad2[1];
 		};
+		__u64	optval;
 		/*
 		 * If the ring is initialized with IORING_SETUP_SQE128, then
 		 * this field is used for 80 bytes of arbitrary command data
@@ -174,6 +183,23 @@ enum {
  */
 #define IORING_SETUP_DEFER_TASKRUN	(1U << 13)
 
+/*
+ * Application provides ring memory
+ */
+#define IORING_SETUP_NO_MMAP		(1U << 14)
+
+/*
+ * Register the ring fd in itself for use with
+ * IORING_REGISTER_USE_REGISTERED_RING; return a registered fd index rather
+ * than an fd.
+ */
+#define IORING_SETUP_REGISTERED_FD_ONLY	(1U << 15)
+
+/*
+ * Removes indirection through the SQ index array.
+ */
+#define IORING_SETUP_NO_SQARRAY		(1U << 16)
+
 enum io_uring_op {
 	IORING_OP_NOP,
 	IORING_OP_READV,
@@ -224,6 +250,15 @@ enum io_uring_op {
 	IORING_OP_URING_CMD,
 	IORING_OP_SEND_ZC,
 	IORING_OP_SENDMSG_ZC,
+	IORING_OP_READ_MULTISHOT,
+	IORING_OP_WAITID,
+	IORING_OP_FUTEX_WAIT,
+	IORING_OP_FUTEX_WAKE,
+	IORING_OP_FUTEX_WAITV,
+	IORING_OP_FIXED_FD_INSTALL,
+	IORING_OP_FTRUNCATE,
+	IORING_OP_BIND,
+	IORING_OP_LISTEN,
 
 	/* this goes last, obviously */
 	IORING_OP_LAST,
@@ -231,7 +266,7 @@ enum io_uring_op {
 
 /*
  * sqe->uring_cmd_flags
- * IORING_URING_CMD_FIXED	use registered buffer; pass thig flag
+ * IORING_URING_CMD_FIXED	use registered buffer; pass this flag
  *				along with setting sqe->buf_index.
  */
 #define IORING_URING_CMD_FIXED	(1U << 0)
@@ -251,6 +286,7 @@ enum io_uring_op {
 #define IORING_TIMEOUT_REALTIME		(1U << 3)
 #define IORING_LINK_TIMEOUT_UPDATE	(1U << 4)
 #define IORING_TIMEOUT_ETIME_SUCCESS	(1U << 5)
+#define IORING_TIMEOUT_MULTISHOT	(1U << 6)
 #define IORING_TIMEOUT_CLOCK_MASK	(IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
 #define IORING_TIMEOUT_UPDATE_MASK	(IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
 /*
@@ -281,7 +317,7 @@ enum io_uring_op {
  * ASYNC_CANCEL flags.
  *
  * IORING_ASYNC_CANCEL_ALL	Cancel all requests that match the given key
- * IORING_ASYNC_CANCEL_FD	Key off 'fd' for cancelation rather than the
+ * IORING_ASYNC_CANCEL_FD	  Key off 'fd' for cancelation rather than the
  *				request 'user_data'
  * IORING_ASYNC_CANCEL_ANY	Match any request
  * IORING_ASYNC_CANCEL_FD_FIXED	'fd' passed in is a fixed descriptor
@@ -305,15 +341,44 @@ enum io_uring_op {
  *
  * IORING_RECVSEND_FIXED_BUF	Use registered buffers, the index is stored in
  *				the buf_index field.
+ *
+ * IORING_SEND_ZC_REPORT_USAGE
+ *				If set, SEND[MSG]_ZC should report
+ *				the zerocopy usage in cqe.res
+ *				for the IORING_CQE_F_NOTIF cqe.
+ *				0 is reported if zerocopy was actually possible.
+ *				IORING_NOTIF_USAGE_ZC_COPIED if data was copied
+ *				(at least partially).
+ *
+ * IORING_RECVSEND_BUNDLE	Used with IOSQE_BUFFER_SELECT. If set, send wil
+ *				grab as many buffers from the buffer group ID
+ *				given and send them all. The completion result
+ *				will be the number of buffers send, with the
+ *				starting buffer ID in cqe->flags as per usual
+ *				for provided buffer usage. The buffers will be
+ *				contigious from the starting buffer ID.
  */
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)
 #define IORING_RECV_MULTISHOT		(1U << 1)
 #define IORING_RECVSEND_FIXED_BUF	(1U << 2)
+#define IORING_SEND_ZC_REPORT_USAGE	(1U << 3)
+#define IORING_RECVSEND_BUNDLE		(1U << 4)
+
+/*
+ * cqe.res for IORING_CQE_F_NOTIF if
+ * IORING_SEND_ZC_REPORT_USAGE was requested
+ *
+ * It should be treated as a flag, all other
+ * bits of cqe.res should be treated as reserved!
+ */
+#define IORING_NOTIF_USAGE_ZC_COPIED    (1U << 31)
 
 /*
  * accept flags stored in sqe->ioprio
  */
 #define IORING_ACCEPT_MULTISHOT	(1U << 0)
+#define IORING_ACCEPT_DONTWAIT	(1U << 1)
+#define IORING_ACCEPT_POLL_FIRST	(1U << 2)
 
 /*
  * IORING_OP_MSG_RING command types, stored in sqe->addr
@@ -330,12 +395,28 @@ enum {
  *				applicable for IORING_MSG_DATA, obviously.
  */
 #define IORING_MSG_RING_CQE_SKIP	(1U << 0)
+/* Pass through the flags from sqe->file_index to cqe->flags */
+#define IORING_MSG_RING_FLAGS_PASS	(1U << 1)
+
+/*
+ * IORING_OP_FIXED_FD_INSTALL flags (sqe->install_fd_flags)
+ *
+ * IORING_FIXED_FD_NO_CLOEXEC	Don't mark the fd as O_CLOEXEC
+ */
+#define IORING_FIXED_FD_NO_CLOEXEC	(1U << 0)
+
+/*
+ * IORING_OP_NOP flags (sqe->nop_flags)
+ *
+ * IORING_NOP_INJECT_RESULT	Inject result from sqe->result
+ */
+#define IORING_NOP_INJECT_RESULT	(1U << 0)
 
 /*
  * IO completion data structure (Completion Queue Entry)
  */
 struct io_uring_cqe {
-	__u64	user_data;	/* sqe->data submission passed back */
+	__u64	user_data;	/* sqe->user_data value passed back */
 	__s32	res;		/* result code for this event */
 	__u32	flags;
 
@@ -370,6 +451,9 @@ enum {
 #define IORING_OFF_SQ_RING		0ULL
 #define IORING_OFF_CQ_RING		0x8000000ULL
 #define IORING_OFF_SQES			0x10000000ULL
+#define IORING_OFF_PBUF_RING		0x80000000ULL
+#define IORING_OFF_PBUF_SHIFT		16
+#define IORING_OFF_MMAP_MASK		0xf8000000ULL
 
 /*
  * Filled with the offset for mmap(2)
@@ -383,7 +467,7 @@ struct io_sqring_offsets {
 	__u32 dropped;
 	__u32 array;
 	__u32 resv1;
-	__u64 resv2;
+	__u64 user_addr;
 };
 
 /*
@@ -402,7 +486,7 @@ struct io_cqring_offsets {
 	__u32 cqes;
 	__u32 flags;
 	__u32 resv1;
-	__u64 resv2;
+	__u64 user_addr;
 };
 
 /*
@@ -453,6 +537,8 @@ struct io_uring_params {
 #define IORING_FEAT_RSRC_TAGS		(1U << 10)
 #define IORING_FEAT_CQE_SKIP		(1U << 11)
 #define IORING_FEAT_LINKED_FILE		(1U << 12)
+#define IORING_FEAT_REG_REG_RING	(1U << 13)
+#define IORING_FEAT_RECVSEND_BUNDLE	(1U << 14)
 
 /*
  * io_uring_register(2) opcodes and arguments
@@ -499,8 +585,18 @@ enum {
 	/* register a range of fixed file slots for automatic slot allocation */
 	IORING_REGISTER_FILE_ALLOC_RANGE	= 25,
 
+	/* return status information for a buffer group */
+	IORING_REGISTER_PBUF_STATUS		= 26,
+
+	/* set/clear busy poll settings */
+	IORING_REGISTER_NAPI			= 27,
+	IORING_UNREGISTER_NAPI			= 28,
+
 	/* this goes last */
-	IORING_REGISTER_LAST
+	IORING_REGISTER_LAST,
+
+	/* flag added to the opcode to use a registered ring fd */
+	IORING_REGISTER_USE_REGISTERED_RING	= 1U << 31
 };
 
 /* io-wq worker categories */
@@ -545,19 +641,6 @@ struct io_uring_rsrc_update2 {
 	__u32 resv2;
 };
 
-struct io_uring_notification_slot {
-	__u64 tag;
-	__u64 resv[3];
-};
-
-struct io_uring_notification_register {
-	__u32 nr_slots;
-	__u32 resv;
-	__u64 resv2;
-	__u64 data;
-	__u64 resv3;
-};
-
 /* Skip updating fd indexes set to this value in the fd table */
 #define IORING_REGISTER_FILES_SKIP	(-2)
 
@@ -612,15 +695,44 @@ struct io_uring_buf_ring {
 	};
 };
 
+/*
+ * Flags for IORING_REGISTER_PBUF_RING.
+ *
+ * IOU_PBUF_RING_MMAP:	If set, kernel will allocate the memory for the ring.
+ *			The application must not set a ring_addr in struct
+ *			io_uring_buf_reg, instead it must subsequently call
+ *			mmap(2) with the offset set as:
+ *			IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT)
+ *			to get a virtual mapping for the ring.
+ */
+enum {
+	IOU_PBUF_RING_MMAP	= 1,
+};
+
 /* argument for IORING_(UN)REGISTER_PBUF_RING */
 struct io_uring_buf_reg {
 	__u64	ring_addr;
 	__u32	ring_entries;
 	__u16	bgid;
-	__u16	pad;
+	__u16	flags;
 	__u64	resv[3];
 };
 
+/* argument for IORING_REGISTER_PBUF_STATUS */
+struct io_uring_buf_status {
+	__u32	buf_group;	/* input */
+	__u32	head;		/* output */
+	__u32	resv[8];
+};
+
+/* argument for IORING_(UN)REGISTER_NAPI */
+struct io_uring_napi {
+	__u32   busy_poll_to;
+	__u8    prefer_busy_poll;
+	__u8    pad[3];
+	__u64   resv;
+};
+
 /*
  * io_uring_restriction->opcode values
  */
@@ -675,6 +787,16 @@ struct io_uring_recvmsg_out {
 	__u32 flags;
 };
 
+/*
+ * Argument for IORING_OP_URING_CMD when file is a socket
+ */
+enum {
+	SOCKET_URING_OP_SIOCINQ		= 0,
+	SOCKET_URING_OP_SIOCOUTQ,
+	SOCKET_URING_OP_GETSOCKOPT,
+	SOCKET_URING_OP_SETSOCKOPT,
+};
+
 #ifdef __cplusplus
 }
 #endif

src/int_flags.h

@@ -4,6 +4,8 @@
 
 enum {
 	INT_FLAG_REG_RING	= 1,
+	INT_FLAG_REG_REG_RING	= 2,
+	INT_FLAG_APP_MEM	= 4,
 };
 
 #endif

src/lib.h

@@ -10,6 +10,8 @@
 #include "arch/x86/lib.h"
 #elif defined(__aarch64__)
 #include "arch/aarch64/lib.h"
+#elif defined(__riscv) && __riscv_xlen == 64
+#include "arch/riscv64/lib.h"
 #else
 /*
  * We don't have nolibc support for this arch. Must use libc!
@@ -37,25 +39,14 @@
 #define __hot			__attribute__((__hot__))
 #define __cold			__attribute__((__cold__))
 
+#ifdef CONFIG_NOLIBC
+void *__uring_memset(void *s, int c, size_t n);
 void *__uring_malloc(size_t len);
 void __uring_free(void *p);
 
-static inline void *uring_malloc(size_t len)
+#define malloc(LEN)		__uring_malloc(LEN)
-{
+#define free(PTR)		__uring_free(PTR)
-#ifdef CONFIG_NOLIBC
+#define memset(PTR, C, LEN)	__uring_memset(PTR, C, LEN)
-	return __uring_malloc(len);
-#else
-	return malloc(len);
 #endif
-}
-
-static inline void uring_free(void *ptr)
-{
-#ifdef CONFIG_NOLIBC
-	__uring_free(ptr);
-#else
-	free(ptr);
-#endif
-}
 
 #endif /* #ifndef LIBURING_LIB_H */

src/liburing-ffi.map

@@ -0,0 +1,206 @@
+LIBURING_2.4 {
+	global:
+		io_uring_get_probe;
+		io_uring_get_probe_ring;
+		io_uring_free_probe;
+		io_uring_get_sqe;
+		io_uring_peek_batch_cqe;
+		io_uring_queue_exit;
+		io_uring_queue_init;
+		io_uring_queue_init_params;
+		io_uring_queue_mmap;
+		io_uring_register_buffers;
+		io_uring_register_eventfd;
+		io_uring_register_eventfd_async;
+		io_uring_register_files;
+		io_uring_register_files_update;
+		io_uring_register_personality;
+		io_uring_register_probe;
+		io_uring_ring_dontfork;
+		io_uring_submit;
+		io_uring_submit_and_wait;
+		io_uring_unregister_buffers;
+		io_uring_unregister_eventfd;
+		io_uring_unregister_files;
+		io_uring_unregister_personality;
+		io_uring_wait_cqe_timeout;
+		io_uring_wait_cqes;
+
+		__io_uring_get_cqe;
+		__io_uring_sqring_wait;
+
+		io_uring_mlock_size_params;
+		io_uring_mlock_size;
+		io_uring_register_buffers_tags;
+		io_uring_register_buffers_update_tag;
+		io_uring_register_files_tags;
+		io_uring_register_files_update_tag;
+		io_uring_register_iowq_aff;
+		io_uring_unregister_iowq_aff;
+		io_uring_register_iowq_max_workers;
+
+		io_uring_submit_and_wait_timeout;
+		io_uring_register_ring_fd;
+		io_uring_unregister_ring_fd;
+		io_uring_register_files_sparse;
+		io_uring_register_buffers_sparse;
+		io_uring_register_buf_ring;
+		io_uring_unregister_buf_ring;
+		io_uring_close_ring_fd;
+		io_uring_setup_buf_ring;
+		io_uring_free_buf_ring;
+
+		io_uring_register_sync_cancel;
+		io_uring_register_file_alloc_range;
+		io_uring_enter;
+		io_uring_enter2;
+		io_uring_setup;
+		io_uring_register;
+		io_uring_get_events;
+		io_uring_submit_and_get_events;
+
+		io_uring_major_version;
+		io_uring_minor_version;
+		io_uring_check_version;
+
+		io_uring_peek_cqe;
+		io_uring_prep_timeout_update;
+		io_uring_buf_ring_init;
+		io_uring_prep_mkdirat;
+		io_uring_prep_recv_multishot;
+		io_uring_cq_advance;
+		io_uring_prep_multishot_accept;
+		io_uring_prep_fallocate;
+		io_uring_prep_link_timeout;
+		io_uring_prep_fsync;
+		io_uring_prep_openat_direct;
+		io_uring_prep_multishot_accept_direct;
+		io_uring_opcode_supported;
+		io_uring_prep_madvise;
+		io_uring_prep_send_set_addr;
+		io_uring_recvmsg_payload_length;
+		io_uring_prep_readv2;
+		io_uring_prep_msg_ring;
+		io_uring_prep_rename;
+		io_uring_prep_fadvise;
+		io_uring_prep_send_zc;
+		io_uring_buf_ring_advance;
+		io_uring_cqe_get_data;
+		io_uring_prep_symlinkat;
+		io_uring_prep_writev;
+		io_uring_cq_eventfd_toggle;
+		io_uring_prep_provide_buffers;
+		io_uring_cq_has_overflow;
+		io_uring_prep_cancel_fd;
+		io_uring_prep_socket;
+		io_uring_prep_close_direct;
+		io_uring_recvmsg_name;
+		io_uring_prep_timeout_remove;
+		io_uring_sqring_wait;
+		io_uring_cq_eventfd_enabled;
+		io_uring_prep_remove_buffers;
+		io_uring_prep_tee;
+		io_uring_prep_accept_direct;
+		io_uring_prep_nop;
+		io_uring_prep_getxattr;
+		io_uring_prep_link;
+		io_uring_prep_cancel;
+		io_uring_prep_readv;
+		io_uring_prep_connect;
+		io_uring_cq_ready;
+		io_uring_enable_rings;
+		io_uring_prep_shutdown;
+		io_uring_prep_openat;
+		io_uring_sq_space_left;
+		io_uring_recvmsg_payload;
+		io_uring_prep_send;
+		io_uring_buf_ring_add;
+		io_uring_prep_send_zc_fixed;
+		io_uring_prep_epoll_ctl;
+		io_uring_recvmsg_cmsg_firsthdr;
+		io_uring_prep_socket_direct;
+		io_uring_buf_ring_cq_advance;
+		__io_uring_buf_ring_cq_advance;
+		io_uring_prep_mkdir;
+		io_uring_wait_cqe_nr;
+		io_uring_prep_unlink;
+		io_uring_prep_writev2;
+		io_uring_prep_openat2_direct;
+		io_uring_sqe_set_flags;
+		io_uring_sqe_set_data;
+		io_uring_prep_accept;
+		io_uring_prep_poll_update;
+		io_uring_prep_splice;
+		io_uring_prep_poll_multishot;
+		io_uring_prep_symlink;
+		io_uring_sqe_set_data64;
+		io_uring_prep_cancel64;
+		io_uring_prep_fsetxattr;
+		io_uring_prep_recvmsg_multishot;
+		io_uring_cqe_seen;
+		io_uring_prep_sendmsg_zc;
+		io_uring_prep_read;
+		io_uring_prep_statx;
+		io_uring_prep_sendmsg;
+		io_uring_prep_unlinkat;
+		io_uring_prep_setxattr;
+		io_uring_cqe_get_data64;
+		io_uring_prep_renameat;
+		io_uring_prep_poll_remove;
+		io_uring_prep_close;
+		io_uring_sq_ready;
+		io_uring_prep_files_update;
+		io_uring_wait_cqe;
+		io_uring_prep_fgetxattr;
+		io_uring_prep_socket_direct_alloc;
+		io_uring_prep_sync_file_range;
+		io_uring_prep_read_fixed;
+		io_uring_prep_openat2;
+		io_uring_prep_recvmsg;
+		io_uring_recvmsg_cmsg_nexthdr;
+		io_uring_recvmsg_validate;
+		io_uring_prep_rw;
+		io_uring_prep_timeout;
+		io_uring_prep_linkat;
+		io_uring_prep_write_fixed;
+		io_uring_prep_poll_add;
+		io_uring_buf_ring_mask;
+		io_uring_register_restrictions;
+		io_uring_prep_write;
+		io_uring_prep_recv;
+		io_uring_prep_msg_ring_cqe_flags;
+		io_uring_prep_msg_ring_fd;
+		io_uring_prep_msg_ring_fd_alloc;
+		io_uring_prep_sendto;
+		io_uring_register_napi;		/* Added in 2.6. */
+		io_uring_unregister_napi;	/* Added in 2.6. */
+	local:
+		*;
+};
+
+LIBURING_2.5 {
+	global:
+		io_uring_queue_init_mem;
+		io_uring_prep_cmd_sock;		/* Added in 2.5,
+						   exported in 2.6. */
+		io_uring_prep_read_multishot;	/* Added in 2.6. */
+		io_uring_prep_waitid;		/* Added in 2.6. */
+		io_uring_prep_futex_wake;	/* Added in 2.6. */
+		io_uring_prep_futex_wait;	/* Added in 2.6. */
+		io_uring_prep_futex_waitv;	/* Added in 2.6. */
+} LIBURING_2.4;
+
+LIBURING_2.6 {
+	global:
+		io_uring_prep_fixed_fd_install;
+		io_uring_buf_ring_available;
+		io_uring_prep_ftruncate;
+		io_uring_prep_send_bundle;
+} LIBURING_2.5;
+
+LIBURING_2.7 {
+		io_uring_prep_fadvise64;
+		io_uring_prep_madvise64;
+		io_uring_prep_bind;
+		io_uring_prep_listen;
+} LIBURING_2.6;

src/liburing.map

@@ -67,3 +67,31 @@ LIBURING_2.3 {
 		io_uring_get_events;
 		io_uring_submit_and_get_events;
 } LIBURING_2.2;
+
+LIBURING_2.4 {
+	global:
+		io_uring_major_version;
+		io_uring_minor_version;
+		io_uring_check_version;
+
+		io_uring_close_ring_fd;
+		io_uring_enable_rings;
+		io_uring_register_restrictions;
+		io_uring_setup_buf_ring;
+		io_uring_free_buf_ring;
+} LIBURING_2.3;
+
+LIBURING_2.5 {
+	global:
+		io_uring_queue_init_mem;
+} LIBURING_2.4;
+
+LIBURING_2.6 {
+	global:
+		io_uring_buf_ring_head;
+		io_uring_register_napi;
+		io_uring_unregister_napi;
+} LIBURING_2.5;
+
+LIBURING_2.7 {
+} LIBURING_2.6;

src/nolibc.c

@@ -7,14 +7,21 @@
 #include "lib.h"
 #include "syscall.h"
 
-void *memset(void *s, int c, size_t n)
+void *__uring_memset(void *s, int c, size_t n)
 {
 	size_t i;
 	unsigned char *p = s;
 
-	for (i = 0; i < n; i++)
+	for (i = 0; i < n; i++) {
 		p[i] = (unsigned char) c;
 
+		/*
+		 * An empty inline ASM to avoid auto-vectorization
+		 * because it's too bloated for liburing.
+		 */
+		__asm__ volatile ("");
+	}
+
 	return s;
 }
 

src/queue.c

@@ -81,7 +81,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 		}
 		if (!cqe && !data->wait_nr && !data->submit) {
 			/*
-			 * If we already looped once, we already entererd
+			 * If we already looped once, we already entered
 			 * the kernel. Since there's nothing to submit or
 			 * wait for, don't keep retrying.
 			 */
@@ -201,7 +201,7 @@ again:
  * Sync internal state with kernel ring state on the SQ side. Returns the
  * number of pending items in the SQ ring, for the shared ring.
  */
-unsigned __io_uring_flush_sq(struct io_uring *ring)
+static unsigned __io_uring_flush_sq(struct io_uring *ring)
 {
 	struct io_uring_sq *sq = &ring->sq;
 	unsigned tail = sq->sqe_tail;
@@ -212,22 +212,18 @@ unsigned __io_uring_flush_sq(struct io_uring *ring)
 		 * Ensure kernel sees the SQE updates before the tail update.
 		 */
 		if (!(ring->flags & IORING_SETUP_SQPOLL))
-			IO_URING_WRITE_ONCE(*sq->ktail, tail);
+			*sq->ktail = tail;
 		else
 			io_uring_smp_store_release(sq->ktail, tail);
 	}
 	/*
-	 * This _may_ look problematic, as we're not supposed to be reading
+	* This load needs to be atomic, since sq->khead is written concurrently
-	 * SQ->head without acquire semantics. When we're in SQPOLL mode, the
+	* by the kernel, but it doesn't need to be load_acquire, since the
-	 * kernel submitter could be updating this right now. For non-SQPOLL,
+	* kernel doesn't store to the submission queue; it advances khead just
-	 * task itself does it, and there's no potential race. But even for
+	* to indicate that it's finished reading the submission queue entries
-	 * SQPOLL, the load is going to be potentially out-of-date the very
+	* so they're available for us to write to.
-	 * instant it's done, regardless or whether or not it's done
+	*/
-	 * atomically. Worst case, we're going to be over-estimating what
+	return tail - IO_URING_READ_ONCE(*sq->khead);
-	 * we can submit. The point is, we need to be able to deal with this
-	 * situation regardless of any perceived atomicity.
-	 */
-	return tail - *sq->khead;
 }
 
 /*

src/register.c

@@ -8,6 +8,21 @@
 #include "liburing/compat.h"
 #include "liburing/io_uring.h"
 
+static inline int do_register(struct io_uring *ring, unsigned int opcode,
+			      const void *arg, unsigned int nr_args)
+{
+	int fd;
+
+	if (ring->int_flags & INT_FLAG_REG_REG_RING) {
+		opcode |= IORING_REGISTER_USE_REGISTERED_RING;
+		fd = ring->enter_ring_fd;
+	} else {
+		fd = ring->ring_fd;
+	}
+
+	return __sys_io_uring_register(fd, opcode, arg, nr_args);
+}
+
 int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
 					 const struct iovec *iovecs,
 					 const __u64 *tags,
@@ -20,8 +35,7 @@ int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
 		.nr = nr,
 	};
 
-	return __sys_io_uring_register(ring->ring_fd,IORING_REGISTER_BUFFERS_UPDATE, &up,
+	return do_register(ring, IORING_REGISTER_BUFFERS_UPDATE, &up, sizeof(up));
-					 sizeof(up));
 }
 
 int io_uring_register_buffers_tags(struct io_uring *ring,
@@ -35,9 +49,7 @@ int io_uring_register_buffers_tags(struct io_uring *ring,
 		.tags = (unsigned long)tags,
 	};
 
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_BUFFERS2, &reg, sizeof(reg));
-				       IORING_REGISTER_BUFFERS2, &reg,
-				       sizeof(reg));
 }
 
 int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr)
@@ -47,27 +59,18 @@ int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr)
 		.nr = nr,
 	};
 
-	return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2,
+	return do_register(ring, IORING_REGISTER_BUFFERS2, &reg, sizeof(reg));
-				       &reg, sizeof(reg));
 }
 
 int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
 			      unsigned nr_iovecs)
 {
-	int ret;
+	return do_register(ring, IORING_REGISTER_BUFFERS, iovecs, nr_iovecs);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS,
-				      iovecs, nr_iovecs);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_unregister_buffers(struct io_uring *ring)
 {
-	int ret;
+	return do_register(ring, IORING_UNREGISTER_BUFFERS, NULL, 0);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS,
-				      NULL, 0);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
@@ -81,9 +84,7 @@ int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
 		.nr = nr_files,
 	};
 
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_FILES_UPDATE2, &up, sizeof(up));
-				       IORING_REGISTER_FILES_UPDATE2, &up,
-				       sizeof(up));
 }
 
 /*
@@ -101,9 +102,7 @@ int io_uring_register_files_update(struct io_uring *ring, unsigned off,
 		.fds	= (unsigned long) files,
 	};
 
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_FILES_UPDATE, &up, nr_files);
-				       IORING_REGISTER_FILES_UPDATE, &up,
-				       nr_files);
 }
 
 static int increase_rlimit_nofile(unsigned nr)
@@ -132,9 +131,7 @@ int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr)
 	int ret, did_increase = 0;
 
 	do {
-		ret = __sys_io_uring_register(ring->ring_fd,
+		ret = do_register(ring, IORING_REGISTER_FILES2, &reg, sizeof(reg));
-					      IORING_REGISTER_FILES2, &reg,
-					      sizeof(reg));
 		if (ret >= 0)
 			break;
 		if (ret == -EMFILE && !did_increase) {
@@ -159,9 +156,7 @@ int io_uring_register_files_tags(struct io_uring *ring, const int *files,
 	int ret, did_increase = 0;
 
 	do {
-		ret = __sys_io_uring_register(ring->ring_fd,
+		ret = do_register(ring, IORING_REGISTER_FILES2, &reg, sizeof(reg));
-					      IORING_REGISTER_FILES2, &reg,
-					      sizeof(reg));
 		if (ret >= 0)
 			break;
 		if (ret == -EMFILE && !did_increase) {
@@ -181,9 +176,7 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
 	int ret, did_increase = 0;
 
 	do {
-		ret = __sys_io_uring_register(ring->ring_fd,
+		ret = do_register(ring, IORING_REGISTER_FILES, files, nr_files);
-					      IORING_REGISTER_FILES, files,
-					      nr_files);
 		if (ret >= 0)
 			break;
 		if (ret == -EMFILE && !did_increase) {
@@ -199,79 +192,50 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
 
 int io_uring_unregister_files(struct io_uring *ring)
 {
-	int ret;
+	return do_register(ring, IORING_UNREGISTER_FILES, NULL, 0);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES,
-				      NULL, 0);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_register_eventfd(struct io_uring *ring, int event_fd)
 {
-	int ret;
+	return do_register(ring, IORING_REGISTER_EVENTFD, &event_fd, 1);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD,
-				      &event_fd, 1);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_unregister_eventfd(struct io_uring *ring)
 {
-	int ret;
+	return do_register(ring, IORING_UNREGISTER_EVENTFD, NULL, 0);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD,
-				      NULL, 0);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd)
 {
-	int ret;
+	return do_register(ring, IORING_REGISTER_EVENTFD_ASYNC, &event_fd, 1);
-
-	ret = __sys_io_uring_register(ring->ring_fd,
-				      IORING_REGISTER_EVENTFD_ASYNC, &event_fd,
-				      1);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p,
 			    unsigned int nr_ops)
 {
-	int ret;
+	return do_register(ring, IORING_REGISTER_PROBE, p, nr_ops);
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p,
-				      nr_ops);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_register_personality(struct io_uring *ring)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_PERSONALITY, NULL, 0);
-				       IORING_REGISTER_PERSONALITY, NULL, 0);
 }
 
 int io_uring_unregister_personality(struct io_uring *ring, int id)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_UNREGISTER_PERSONALITY, NULL, id);
-				       IORING_UNREGISTER_PERSONALITY, NULL, id);
 }
 
 int io_uring_register_restrictions(struct io_uring *ring,
 				   struct io_uring_restriction *res,
 				   unsigned int nr_res)
 {
-	int ret;
+	return do_register(ring, IORING_REGISTER_RESTRICTIONS, res, nr_res);
-
-	ret = __sys_io_uring_register(ring->ring_fd,
-				      IORING_REGISTER_RESTRICTIONS, res,
-				      nr_res);
-	return (ret < 0) ? ret : 0;
 }
 
 int io_uring_enable_rings(struct io_uring *ring)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_ENABLE_RINGS, NULL, 0);
-				       IORING_REGISTER_ENABLE_RINGS, NULL, 0);
 }
 
 int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
@@ -280,21 +244,17 @@ int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
 	if (cpusz >= (1U << 31))
 		return -EINVAL;
 
-	return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF,
+	return do_register(ring, IORING_REGISTER_IOWQ_AFF, mask, (int) cpusz);
-					mask, (int) cpusz);
 }
 
 int io_uring_unregister_iowq_aff(struct io_uring *ring)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
-				       IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
 }
 
 int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *val)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_IOWQ_MAX_WORKERS, val, 2);
-				       IORING_REGISTER_IOWQ_MAX_WORKERS, val,
-				       2);
 }
 
 int io_uring_register_ring_fd(struct io_uring *ring)
@@ -305,11 +265,16 @@ int io_uring_register_ring_fd(struct io_uring *ring)
 	};
 	int ret;
 
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS,
+	if (ring->int_flags & INT_FLAG_REG_RING)
-				      &up, 1);
+		return -EEXIST;
+
+	ret = do_register(ring, IORING_REGISTER_RING_FDS, &up, 1);
 	if (ret == 1) {
 		ring->enter_ring_fd = up.offset;
 		ring->int_flags |= INT_FLAG_REG_RING;
+		if (ring->features & IORING_FEAT_REG_REG_RING) {
+			ring->int_flags |= INT_FLAG_REG_REG_RING;
+		}
 	}
 	return ret;
 }
@@ -322,48 +287,82 @@ int io_uring_unregister_ring_fd(struct io_uring *ring)
 	};
 	int ret;
 
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_RING_FDS,
+	if (!(ring->int_flags & INT_FLAG_REG_RING))
-				      &up, 1);
+		return -EINVAL;
+
+	ret = do_register(ring, IORING_UNREGISTER_RING_FDS, &up, 1);
 	if (ret == 1) {
 		ring->enter_ring_fd = ring->ring_fd;
-		ring->int_flags &= ~INT_FLAG_REG_RING;
+		ring->int_flags &= ~(INT_FLAG_REG_RING | INT_FLAG_REG_REG_RING);
 	}
 	return ret;
 }
 
+int io_uring_close_ring_fd(struct io_uring *ring)
+{
+	if (!(ring->features & IORING_FEAT_REG_REG_RING))
+		return -EOPNOTSUPP;
+	if (!(ring->int_flags & INT_FLAG_REG_RING))
+		return -EINVAL;
+	if (ring->ring_fd == -1)
+		return -EBADF;
+
+	__sys_close(ring->ring_fd);
+	ring->ring_fd = -1;
+	return 1;
+}
+
 int io_uring_register_buf_ring(struct io_uring *ring,
 			       struct io_uring_buf_reg *reg,
 			       unsigned int __maybe_unused flags)
 {
-	return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PBUF_RING,
+	return do_register(ring, IORING_REGISTER_PBUF_RING, reg, 1);
-				       reg, 1);
 }
 
 int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid)
 {
 	struct io_uring_buf_reg reg = { .bgid = bgid };
 
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_UNREGISTER_PBUF_RING, &reg, 1);
-				       IORING_UNREGISTER_PBUF_RING, &reg, 1);
+}
+
+int io_uring_buf_ring_head(struct io_uring *ring, int buf_group, uint16_t *head)
+{
+	struct io_uring_buf_status buf_status = {
+		.buf_group	= buf_group,
+	};
+	int ret;
+
+	ret = do_register(ring, IORING_REGISTER_PBUF_STATUS, &buf_status, 1);
+	if (ret)
+		return ret;
+	*head = buf_status.head;
+	return 0;
 }
 
 int io_uring_register_sync_cancel(struct io_uring *ring,
 				  struct io_uring_sync_cancel_reg *reg)
 {
-	return __sys_io_uring_register(ring->ring_fd,
+	return do_register(ring, IORING_REGISTER_SYNC_CANCEL, reg, 1);
-				       IORING_REGISTER_SYNC_CANCEL, reg, 1);
 }
 
 int io_uring_register_file_alloc_range(struct io_uring *ring,
 					unsigned off, unsigned len)
 {
-	struct io_uring_file_index_range range;
+	struct io_uring_file_index_range range = {
+		.off = off,
+		.len = len
+	};
 
-	memset(&range, 0, sizeof(range));
+	return do_register(ring, IORING_REGISTER_FILE_ALLOC_RANGE, &range, 0);
-	range.off = off;
+}
-	range.len = len;
+
-
+int io_uring_register_napi(struct io_uring *ring, struct io_uring_napi *napi)
-	return __sys_io_uring_register(ring->ring_fd,
+{
-				       IORING_REGISTER_FILE_ALLOC_RANGE, &range,
+	return do_register(ring, IORING_REGISTER_NAPI, napi, 1);
-				       0);
+}
+
+int io_uring_unregister_napi(struct io_uring *ring, struct io_uring_napi *napi)
+{
+	return do_register(ring, IORING_UNREGISTER_NAPI, napi, 1);
 }

src/setup.c

@@ -5,16 +5,96 @@
 #include "syscall.h"
 #include "liburing.h"
 #include "int_flags.h"
+#include "setup.h"
 #include "liburing/compat.h"
 #include "liburing/io_uring.h"
 
+#define KERN_MAX_ENTRIES	32768
+#define KERN_MAX_CQ_ENTRIES	(2 * KERN_MAX_ENTRIES)
+
+static inline int __fls(int x)
+{
+	if (!x)
+		return 0;
+	return 8 * sizeof(x) - __builtin_clz(x);
+}
+
+static unsigned roundup_pow2(unsigned depth)
+{
+	return 1U << __fls(depth - 1);
+}
+
+static int get_sq_cq_entries(unsigned entries, struct io_uring_params *p,
+			     unsigned *sq, unsigned *cq)
+{
+	unsigned cq_entries;
+
+	if (!entries)
+		return -EINVAL;
+	if (entries > KERN_MAX_ENTRIES) {
+		if (!(p->flags & IORING_SETUP_CLAMP))
+			return -EINVAL;
+		entries = KERN_MAX_ENTRIES;
+	}
+
+	entries = roundup_pow2(entries);
+	if (p->flags & IORING_SETUP_CQSIZE) {
+		if (!p->cq_entries)
+			return -EINVAL;
+		cq_entries = p->cq_entries;
+		if (cq_entries > KERN_MAX_CQ_ENTRIES) {
+			if (!(p->flags & IORING_SETUP_CLAMP))
+				return -EINVAL;
+			cq_entries = KERN_MAX_CQ_ENTRIES;
+		}
+		cq_entries = roundup_pow2(cq_entries);
+		if (cq_entries < entries)
+			return -EINVAL;
+	} else {
+		cq_entries = 2 * entries;
+	}
+
+	*sq = entries;
+	*cq = cq_entries;
+	return 0;
+}
+
 static void io_uring_unmap_rings(struct io_uring_sq *sq, struct io_uring_cq *cq)
 {
-	__sys_munmap(sq->ring_ptr, sq->ring_sz);
+	if (sq->ring_sz)
-	if (cq->ring_ptr && cq->ring_ptr != sq->ring_ptr)
+		__sys_munmap(sq->ring_ptr, sq->ring_sz);
+	if (cq->ring_ptr && cq->ring_sz && cq->ring_ptr != sq->ring_ptr)
 		__sys_munmap(cq->ring_ptr, cq->ring_sz);
 }
 
+static void io_uring_setup_ring_pointers(struct io_uring_params *p,
+					 struct io_uring_sq *sq,
+					 struct io_uring_cq *cq)
+{
+	sq->khead = sq->ring_ptr + p->sq_off.head;
+	sq->ktail = sq->ring_ptr + p->sq_off.tail;
+	sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask;
+	sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
+	sq->kflags = sq->ring_ptr + p->sq_off.flags;
+	sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		sq->array = sq->ring_ptr + p->sq_off.array;
+
+	cq->khead = cq->ring_ptr + p->cq_off.head;
+	cq->ktail = cq->ring_ptr + p->cq_off.tail;
+	cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask;
+	cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries;
+	cq->koverflow = cq->ring_ptr + p->cq_off.overflow;
+	cq->cqes = cq->ring_ptr + p->cq_off.cqes;
+	if (p->cq_off.flags)
+		cq->kflags = cq->ring_ptr + p->cq_off.flags;
+
+	sq->ring_mask = *sq->kring_mask;
+	sq->ring_entries = *sq->kring_entries;
+	cq->ring_mask = *cq->kring_mask;
+	cq->ring_entries = *cq->kring_entries;
+}
+
 static int io_uring_mmap(int fd, struct io_uring_params *p,
 			 struct io_uring_sq *sq, struct io_uring_cq *cq)
 {
@@ -52,14 +132,6 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
 		}
 	}
 
-	sq->khead = sq->ring_ptr + p->sq_off.head;
-	sq->ktail = sq->ring_ptr + p->sq_off.tail;
-	sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask;
-	sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
-	sq->kflags = sq->ring_ptr + p->sq_off.flags;
-	sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
-	sq->array = sq->ring_ptr + p->sq_off.array;
-
 	size = sizeof(struct io_uring_sqe);
 	if (p->flags & IORING_SETUP_SQE128)
 		size += 64;
@@ -72,19 +144,7 @@ err:
 		return ret;
 	}
 
-	cq->khead = cq->ring_ptr + p->cq_off.head;
+	io_uring_setup_ring_pointers(p, sq, cq);
-	cq->ktail = cq->ring_ptr + p->cq_off.tail;
-	cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask;
-	cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries;
-	cq->koverflow = cq->ring_ptr + p->cq_off.overflow;
-	cq->cqes = cq->ring_ptr + p->cq_off.cqes;
-	if (p->cq_off.flags)
-		cq->kflags = cq->ring_ptr + p->cq_off.flags;
-
-	sq->ring_mask = *sq->kring_mask;
-	sq->ring_entries = *sq->kring_entries;
-	cq->ring_mask = *cq->kring_mask;
-	cq->ring_entries = *cq->kring_entries;
 	return 0;
 }
 
@@ -97,16 +157,8 @@ err:
 __cold int io_uring_queue_mmap(int fd, struct io_uring_params *p,
 			       struct io_uring *ring)
 {
-	int ret;
-
 	memset(ring, 0, sizeof(*ring));
-	ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
+	return io_uring_mmap(fd, p, &ring->sq, &ring->cq);
-	if (!ret) {
-		ring->flags = p->flags;
-		ring->ring_fd = ring->enter_ring_fd = fd;
-		ring->int_flags = 0;
-	}
-	return ret;
 }
 
 /*
@@ -144,33 +196,220 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring)
 	return 0;
 }
 
-__cold int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+/* FIXME */
-				      struct io_uring_params *p)
+static size_t huge_page_size = 2 * 1024 * 1024;
+
+#define KRING_SIZE	64
+
+/*
+ * Returns negative for error, or number of bytes used in the buffer on success
+ */
+static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
+			       struct io_uring_sq *sq, struct io_uring_cq *cq,
+			       void *buf, size_t buf_size)
 {
-	int fd, ret;
+	unsigned long page_size = get_page_size();
+	unsigned sq_entries, cq_entries;
+	size_t ring_mem, sqes_mem, cqes_mem;
+	unsigned long mem_used = 0;
+	void *ptr;
+	int ret;
+
+	ret = get_sq_cq_entries(entries, p, &sq_entries, &cq_entries);
+	if (ret)
+		return ret;
+
+	ring_mem = KRING_SIZE;
+
+	sqes_mem = sq_entries * sizeof(struct io_uring_sqe);
+	sqes_mem = (sqes_mem + page_size - 1) & ~(page_size - 1);
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		sqes_mem += sq_entries * sizeof(unsigned);
+
+	cqes_mem = cq_entries * sizeof(struct io_uring_cqe);
+	if (p->flags & IORING_SETUP_CQE32)
+		cqes_mem *= 2;
+	ring_mem += sqes_mem + cqes_mem;
+	mem_used = ring_mem;
+	mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
+
+	/*
+	 * A maxed-out number of CQ entries with IORING_SETUP_CQE32 fills a 2MB
+	 * huge page by itself, so the SQ entries won't fit in the same huge
+	 * page. For SQEs, that shouldn't be possible given KERN_MAX_ENTRIES,
+	 * but check that too to future-proof (e.g. against different huge page
+	 * sizes). Bail out early so we don't overrun.
+	 */
+	if (!buf && (sqes_mem > huge_page_size || ring_mem > huge_page_size))
+		return -ENOMEM;
+
+	if (buf) {
+		if (mem_used > buf_size)
+			return -ENOMEM;
+		ptr = buf;
+	} else {
+		int map_hugetlb = 0;
+		if (sqes_mem <= page_size)
+			buf_size = page_size;
+		else {
+			buf_size = huge_page_size;
+			map_hugetlb = MAP_HUGETLB;
+		}
+		ptr = __sys_mmap(NULL, buf_size, PROT_READ|PROT_WRITE,
+					MAP_SHARED|MAP_ANONYMOUS|map_hugetlb,
+					-1, 0);
+		if (IS_ERR(ptr))
+			return PTR_ERR(ptr);
+	}
+
+	sq->sqes = ptr;
+	if (mem_used <= buf_size) {
+		sq->ring_ptr = (void *) sq->sqes + sqes_mem;
+		/* clear ring sizes, we have just one mmap() to undo */
+		cq->ring_sz = 0;
+		sq->ring_sz = 0;
+	} else {
+		int map_hugetlb = 0;
+		if (ring_mem <= page_size)
+			buf_size = page_size;
+		else {
+			buf_size = huge_page_size;
+			map_hugetlb = MAP_HUGETLB;
+		}
+		ptr = __sys_mmap(NULL, buf_size, PROT_READ|PROT_WRITE,
+					MAP_SHARED|MAP_ANONYMOUS|map_hugetlb,
+					-1, 0);
+		if (IS_ERR(ptr)) {
+			__sys_munmap(sq->sqes, 1);
+			return PTR_ERR(ptr);
+		}
+		sq->ring_ptr = ptr;
+		sq->ring_sz = buf_size;
+		cq->ring_sz = 0;
+	}
+
+	cq->ring_ptr = (void *) sq->ring_ptr;
+	p->sq_off.user_addr = (unsigned long) sq->sqes;
+	p->cq_off.user_addr = (unsigned long) sq->ring_ptr;
+	return (int) mem_used;
+}
+
+int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+				 struct io_uring_params *p, void *buf,
+				 size_t buf_size)
+{
+	int fd, ret = 0;
 	unsigned *sq_array;
 	unsigned sq_entries, index;
 
-	fd = __sys_io_uring_setup(entries, p);
+	memset(ring, 0, sizeof(*ring));
-	if (fd < 0)
-		return fd;
 
-	ret = io_uring_queue_mmap(fd, p, ring);
+	/*
-	if (ret) {
+	 * The kernel does this check already, but checking it here allows us
-		__sys_close(fd);
+	 * to avoid handling it below.
-		return ret;
+	 */
+	if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY
+	    && !(p->flags & IORING_SETUP_NO_MMAP))
+		return -EINVAL;
+
+	if (p->flags & IORING_SETUP_NO_MMAP) {
+		ret = io_uring_alloc_huge(entries, p, &ring->sq, &ring->cq,
+						buf, buf_size);
+		if (ret < 0)
+			return ret;
+		if (buf)
+			ring->int_flags |= INT_FLAG_APP_MEM;
+	}
+
+	fd = __sys_io_uring_setup(entries, p);
+	if (fd < 0) {
+		if ((p->flags & IORING_SETUP_NO_MMAP) &&
+		    !(ring->int_flags & INT_FLAG_APP_MEM)) {
+			__sys_munmap(ring->sq.sqes, 1);
+			io_uring_unmap_rings(&ring->sq, &ring->cq);
+		}
+		return fd;
+	}
+
+	if (!(p->flags & IORING_SETUP_NO_MMAP)) {
+		ret = io_uring_queue_mmap(fd, p, ring);
+		if (ret) {
+			__sys_close(fd);
+			return ret;
+		}
+	} else {
+		io_uring_setup_ring_pointers(p, &ring->sq, &ring->cq);
 	}
 
 	/*
 	 * Directly map SQ slots to SQEs
 	 */
-	sq_array = ring->sq.array;
 	sq_entries = ring->sq.ring_entries;
-	for (index = 0; index < sq_entries; index++)
-		sq_array[index] = index;
 
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY)) {
+		sq_array = ring->sq.array;
+		for (index = 0; index < sq_entries; index++)
+			sq_array[index] = index;
+	}
 	ring->features = p->features;
-	return 0;
+	ring->flags = p->flags;
+	ring->enter_ring_fd = fd;
+	if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY) {
+		ring->ring_fd = -1;
+		ring->int_flags |= INT_FLAG_REG_RING | INT_FLAG_REG_REG_RING;
+	} else {
+		ring->ring_fd = fd;
+	}
+
+	return ret;
+}
+
+static int io_uring_queue_init_try_nosqarr(unsigned entries, struct io_uring *ring,
+					   struct io_uring_params *p, void *buf,
+					   size_t buf_size)
+{
+	unsigned flags = p->flags;
+	int ret;
+
+	p->flags |= IORING_SETUP_NO_SQARRAY;
+	ret = __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
+
+	/* don't fallback if explicitly asked for NOSQARRAY */
+	if (ret != -EINVAL || (flags & IORING_SETUP_NO_SQARRAY))
+		return ret;
+
+	p->flags = flags;
+	return __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
+}
+
+/*
+ * Like io_uring_queue_init_params(), except it allows the application to pass
+ * in a pre-allocated memory range that is used for the shared data between
+ * the kernel and the application. This includes the sqes array, and the two
+ * rings. The memory must be contiguous, the use case here is that the app
+ * allocates a huge page and passes it in.
+ *
+ * Returns the number of bytes used in the buffer, the app can then reuse
+ * the buffer with the returned offset to put more rings in the same huge
+ * page. Returns -ENOMEM if there's not enough room left in the buffer to
+ * host the ring.
+ */
+int io_uring_queue_init_mem(unsigned entries, struct io_uring *ring,
+			    struct io_uring_params *p,
+			    void *buf, size_t buf_size)
+{
+	/* should already be set... */
+	p->flags |= IORING_SETUP_NO_MMAP;
+	return io_uring_queue_init_try_nosqarr(entries, ring, p, buf, buf_size);
+}
+
+int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+			       struct io_uring_params *p)
+{
+	int ret;
+
+	ret = io_uring_queue_init_try_nosqarr(entries, ring, p, NULL, 0);
+	return ret >= 0 ? 0 : ret;
 }
 
 /*
@@ -194,18 +433,28 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
 	struct io_uring_cq *cq = &ring->cq;
 	size_t sqe_size;
 
-	sqe_size = sizeof(struct io_uring_sqe);
+	if (!sq->ring_sz) {
-	if (ring->flags & IORING_SETUP_SQE128)
+		sqe_size = sizeof(struct io_uring_sqe);
-		sqe_size += 64;
+		if (ring->flags & IORING_SETUP_SQE128)
-	__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
+			sqe_size += 64;
-	io_uring_unmap_rings(sq, cq);
+		__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
+		io_uring_unmap_rings(sq, cq);
+	} else {
+		if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
+			__sys_munmap(sq->sqes,
+				*sq->kring_entries * sizeof(struct io_uring_sqe));
+			io_uring_unmap_rings(sq, cq);
+		}
+	}
+
 	/*
 	 * Not strictly required, but frees up the slot we used now rather
 	 * than at process exit time.
 	 */
 	if (ring->int_flags & INT_FLAG_REG_RING)
 		io_uring_unregister_ring_fd(ring);
-	__sys_close(ring->ring_fd);
+	if (ring->ring_fd != -1)
+		__sys_close(ring->ring_fd);
 }
 
 __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
@@ -215,7 +464,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
 	int r;
 
 	len = sizeof(*probe) + 256 * sizeof(struct io_uring_probe_op);
-	probe = uring_malloc(len);
+	probe = malloc(len);
 	if (!probe)
 		return NULL;
 	memset(probe, 0, len);
@@ -224,7 +473,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
 	if (r >= 0)
 		return probe;
 
-	uring_free(probe);
+	free(probe);
 	return NULL;
 }
 
@@ -245,19 +494,7 @@ __cold struct io_uring_probe *io_uring_get_probe(void)
 
 __cold void io_uring_free_probe(struct io_uring_probe *probe)
 {
-	uring_free(probe);
+	free(probe);
-}
-
-static inline int __fls(unsigned long x)
-{
-	if (!x)
-		return 0;
-	return 8 * sizeof(x) - __builtin_clzl(x);
-}
-
-static unsigned roundup_pow2(unsigned depth)
-{
-	return 1U << __fls(depth - 1);
 }
 
 static size_t npages(size_t size, long page_size)
@@ -267,8 +504,6 @@ static size_t npages(size_t size, long page_size)
 	return __fls((int) size);
 }
 
-#define KRING_SIZE	320
-
 static size_t rings_size(struct io_uring_params *p, unsigned entries,
 			 unsigned cq_entries, long page_size)
 {
@@ -290,9 +525,6 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
 	return pages * page_size;
 }
 
-#define KERN_MAX_ENTRIES	32768
-#define KERN_MAX_CQ_ENTRIES	(2 * KERN_MAX_ENTRIES)
-
 /*
  * Return the required ulimit -l memlock memory required for a given ring
  * setup, in bytes. May return -errno on error. On newer (5.12+) kernels,
@@ -304,11 +536,14 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
 __cold ssize_t io_uring_mlock_size_params(unsigned entries,
 					  struct io_uring_params *p)
 {
-	struct io_uring_params lp = { };
+	struct io_uring_params lp;
 	struct io_uring ring;
-	unsigned cq_entries;
+	unsigned cq_entries, sq;
 	long page_size;
 	ssize_t ret;
+	int cret;
+
+	memset(&lp, 0, sizeof(lp));
 
 	/*
 	 * We only really use this inited ring to see if the kernel is newer
@@ -336,25 +571,12 @@ __cold ssize_t io_uring_mlock_size_params(unsigned entries,
 		entries = KERN_MAX_ENTRIES;
 	}
 
-	entries = roundup_pow2(entries);
+	cret = get_sq_cq_entries(entries, p, &sq, &cq_entries);
-	if (p->flags & IORING_SETUP_CQSIZE) {
+	if (cret)
-		if (!p->cq_entries)
+		return cret;
-			return -EINVAL;
-		cq_entries = p->cq_entries;
-		if (cq_entries > KERN_MAX_CQ_ENTRIES) {
-			if (!(p->flags & IORING_SETUP_CLAMP))
-				return -EINVAL;
-			cq_entries = KERN_MAX_CQ_ENTRIES;
-		}
-		cq_entries = roundup_pow2(cq_entries);
-		if (cq_entries < entries)
-			return -EINVAL;
-	} else {
-		cq_entries = 2 * entries;
-	}
 
 	page_size = get_page_size();
-	return rings_size(p, entries, cq_entries, page_size);
+	return rings_size(p, sq, cq_entries, page_size);
 }
 
 /*
@@ -363,7 +585,105 @@ __cold ssize_t io_uring_mlock_size_params(unsigned entries,
  */
 __cold ssize_t io_uring_mlock_size(unsigned entries, unsigned flags)
 {
-	struct io_uring_params p = { .flags = flags, };
+	struct io_uring_params p;
 
+	memset(&p, 0, sizeof(p));
+	p.flags = flags;
 	return io_uring_mlock_size_params(entries, &p);
 }
+
+#if defined(__hppa__)
+static struct io_uring_buf_ring *br_setup(struct io_uring *ring,
+					  unsigned int nentries, int bgid,
+					  unsigned int flags, int *ret)
+{
+	struct io_uring_buf_ring *br;
+	struct io_uring_buf_reg reg;
+	size_t ring_size;
+	off_t off;
+	int lret;
+
+	memset(&reg, 0, sizeof(reg));
+	reg.ring_entries = nentries;
+	reg.bgid = bgid;
+	reg.flags = IOU_PBUF_RING_MMAP;
+
+	*ret = 0;
+	lret = io_uring_register_buf_ring(ring, &reg, flags);
+	if (lret) {
+		*ret = lret;
+		return NULL;
+	}
+
+	off = IORING_OFF_PBUF_RING | (unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
+	ring_size = nentries * sizeof(struct io_uring_buf);
+	br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
+			MAP_SHARED | MAP_POPULATE, ring->ring_fd, off);
+	if (IS_ERR(br)) {
+		*ret = PTR_ERR(br);
+		return NULL;
+	}
+
+	return br;
+}
+#else
+static struct io_uring_buf_ring *br_setup(struct io_uring *ring,
+					  unsigned int nentries, int bgid,
+					  unsigned int flags, int *ret)
+{
+	struct io_uring_buf_ring *br;
+	struct io_uring_buf_reg reg;
+	size_t ring_size;
+	int lret;
+
+	memset(&reg, 0, sizeof(reg));
+	ring_size = nentries * sizeof(struct io_uring_buf);
+	br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
+			MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+	if (IS_ERR(br)) {
+		*ret = PTR_ERR(br);
+		return NULL;
+	}
+
+	reg.ring_addr = (unsigned long) (uintptr_t) br;
+	reg.ring_entries = nentries;
+	reg.bgid = bgid;
+
+	*ret = 0;
+	lret = io_uring_register_buf_ring(ring, &reg, flags);
+	if (lret) {
+		__sys_munmap(br, ring_size);
+		*ret = lret;
+		br = NULL;
+	}
+
+	return br;
+}
+#endif
+
+struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring,
+						  unsigned int nentries,
+						  int bgid, unsigned int flags,
+						  int *ret)
+{
+	struct io_uring_buf_ring *br;
+
+	br = br_setup(ring, nentries, bgid, flags, ret);
+	if (br)
+		io_uring_buf_ring_init(br);
+
+	return br;
+}
+
+int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
+			   unsigned int nentries, int bgid)
+{
+	int ret;
+
+	ret = io_uring_unregister_buf_ring(ring, bgid);
+	if (ret)
+		return ret;
+
+	__sys_munmap(br, nentries * sizeof(struct io_uring_buf));
+	return 0;
+}

src/setup.h

@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_SETUP_H
+#define LIBURING_SETUP_H
+
+int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+				 struct io_uring_params *p, void *buf,
+				 size_t buf_size);
+
+#endif

src/syscall.h

@@ -37,6 +37,8 @@ static inline bool IS_ERR(const void *ptr)
 #include "arch/x86/syscall.h"
 #elif defined(__aarch64__)
 #include "arch/aarch64/syscall.h"
+#elif defined(__riscv) && __riscv_xlen == 64
+#include "arch/riscv64/syscall.h"
 #else
 /*
  * We don't have native syscall wrappers

src/version.c

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "liburing.h"
+#include "liburing/io_uring_version.h"
+
+int io_uring_major_version(void)
+{
+	return IO_URING_VERSION_MAJOR;
+}
+
+int io_uring_minor_version(void)
+{
+	return IO_URING_VERSION_MINOR;
+}
+
+bool io_uring_check_version(int major, int minor)
+{
+	return major > io_uring_major_version() ||
+		(major == io_uring_major_version() &&
+		 minor > io_uring_minor_version());
+}

test/232c93d07b74.c

@@ -33,9 +33,9 @@ struct params {
 	__be16 bind_port;
 };
 
-pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
-int rcv_ready = 0;
+static int rcv_ready = 0;
 
 static void set_rcv_ready(void)
 {
@@ -64,8 +64,7 @@ static void *rcv(void *arg)
 	int res;
 
 	if (p->tcp) {
-		int val = 1;
+		int ret, val = 1;
-                
 
 		s0 = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP);
 		res = setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val));
@@ -77,7 +76,8 @@ static void *rcv(void *arg)
 
 		addr.sin_family = AF_INET;
 		addr.sin_addr.s_addr = inet_addr("127.0.0.1");
-		assert(t_bind_ephemeral_port(s0, &addr) == 0);
+		ret = t_bind_ephemeral_port(s0, &addr);
+		assert(!ret);
 		p->bind_port = addr.sin_port;
 	} else {
 		s0 = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);

test/35fa71a030ca.c

@@ -176,7 +176,7 @@ static void kill_and_wait(int pid, int* status)
 }
 
 #define SYZ_HAVE_SETUP_TEST 1
-static void setup_test()
+static void setup_test(void)
 {
   prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
   setpgrp();
@@ -262,7 +262,7 @@ static void loop(void)
   }
 }
 
-uint64_t r[1] = {0xffffffffffffffff};
+static uint64_t r[1] = {0xffffffffffffffff};
 
 void execute_call(int call)
 {
@@ -320,7 +320,7 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return T_EXIT_SKIP;
 	signal(SIGINT, sig_int);
-	mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
+	mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
 	signal(SIGALRM, sig_int);
 	alarm(5);
 

test/500f9fbadef8.c

@@ -42,6 +42,8 @@ int main(int argc, char *argv[])
 	sprintf(buf, "./XXXXXX");
 	fd = mkostemp(buf, O_WRONLY | O_DIRECT | O_CREAT);
 	if (fd < 0) {
+		if (errno == EINVAL)
+			return T_EXIT_SKIP;
 		perror("mkostemp");
 		return T_EXIT_FAIL;
 	}

test/917257daa0fe.c

@@ -19,7 +19,7 @@ int main(int argc, char *argv[])
   if (argc > 1)
     return T_EXIT_SKIP;
 
-  mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
+  mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
 
   *(uint32_t*)0x20000000 = 0;
   *(uint32_t*)0x20000004 = 0;

test/Makefile

@@ -13,7 +13,8 @@ override CPPFLAGS += \
 	-D_GNU_SOURCE \
 	-D__SANE_USERSPACE_TYPES__ \
 	-I../src/include/ \
-	-include ../config-host.h
+	-include ../config-host.h \
+	-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
 
 CFLAGS ?= -g -O3 -Wall -Wextra
 XCFLAGS = -Wno-unused-parameter -Wno-sign-compare
@@ -45,15 +46,21 @@ test_srcs := \
 	a4c0b3decb33.c \
 	accept.c \
 	accept-link.c \
+	accept-non-empty.c \
 	accept-reuse.c \
 	accept-test.c \
 	across-fork.c \
 	b19062a56726.c \
 	b5837bd5311d.c \
+	bind-listen.c \
 	buf-ring.c \
+	buf-ring-nommap.c \
+	buf-ring-put.c \
 	ce593a6c480a.c \
 	close-opath.c \
 	connect.c \
+	connect-rep.c \
+	coredump.c \
 	cq-full.c \
 	cq-overflow.c \
 	cq-peek-batch.c \
@@ -63,19 +70,23 @@ test_srcs := \
 	d77a67ed5f27.c \
 	defer.c \
 	defer-taskrun.c \
+	defer-tw-timeout.c \
 	double-poll-crash.c \
 	drop-submit.c \
 	eeed8b54e0df.c \
 	empty-eownerdead.c \
+	eploop.c \
 	eventfd.c \
 	eventfd-disable.c \
 	eventfd-reg.c \
 	eventfd-ring.c \
+	evloop.c \
 	exec-target.c \
 	exit-no-cleanup.c \
 	fadvise.c \
 	fallocate.c \
 	fc2a85cb02ef.c \
+	fd-install.c \
 	fd-pass.c \
 	file-register.c \
 	files-exit-hang-poll.c \
@@ -83,14 +94,21 @@ test_srcs := \
 	file-update.c \
 	file-verify.c \
 	fixed-buf-iter.c \
+	fixed-buf-merge.c \
+	fixed-hugepage.c \
 	fixed-link.c \
 	fixed-reuse.c \
 	fpos.c \
+	fsnotify.c \
 	fsync.c \
+	futex.c \
 	hardlink.c \
+	ignore-single-mmap.c \
+	init-mem.c \
 	io-cancel.c \
 	iopoll.c \
 	iopoll-leak.c \
+	iopoll-overflow.c \
 	io_uring_enter.c \
 	io_uring_passthrough.c \
 	io_uring_register.c \
@@ -103,15 +121,21 @@ test_srcs := \
 	madvise.c \
 	mkdir.c \
 	msg-ring.c \
+	msg-ring-fd.c \
+	msg-ring-flags.c \
+	msg-ring-overflow.c \
 	multicqes_drain.c \
+	no-mmap-inval.c \
 	nolibc.c \
 	nop-all-sizes.c \
 	nop.c \
+	ooo-file-unreg.c \
 	openat2.c \
 	open-close.c \
 	open-direct-link.c \
 	open-direct-pick.c \
 	personality.c \
+	pipe-bug.c \
 	pipe-eof.c \
 	pipe-reuse.c \
 	poll.c \
@@ -120,42 +144,54 @@ test_srcs := \
 	poll-cancel-ton.c \
 	poll-link.c \
 	poll-many.c \
-	poll-mshot-update.c \
 	poll-mshot-overflow.c \
+	poll-mshot-update.c \
+	poll-race.c \
+	poll-race-mshot.c \
 	poll-ring.c \
 	poll-v-poll.c \
-	pollfree.c \
 	probe.c \
 	read-before-exit.c \
+	read-mshot.c \
+	read-mshot-empty.c \
 	read-write.c \
 	recv-msgall.c \
 	recv-msgall-stream.c \
 	recv-multishot.c \
+	reg-fd-only.c \
+	reg-hint.c \
+	reg-reg-ring.c \
+	regbuf-merge.c \
 	register-restrictions.c \
 	rename.c \
 	ringbuf-read.c \
+	ringbuf-status.c \
 	ring-leak2.c \
 	ring-leak.c \
 	rsrc_tags.c \
 	rw_merge_test.c \
 	self.c \
-	sendmsg_fs_cve.c \
+	recvsend_bundle.c \
 	send_recv.c \
 	send_recvmsg.c \
+	send-zerocopy.c \
 	shared-wq.c \
 	short-read.c \
 	shutdown.c \
 	sigfd-deadlock.c \
+	single-issuer.c \
 	skip-cqe.c \
 	socket.c \
+	socket-io-cmd.c \
+	socket-getsetsock-cmd.c \
 	socket-rw.c \
 	socket-rw-eagain.c \
 	socket-rw-offset.c \
 	splice.c \
 	sq-full.c \
 	sq-full-cpp.cc \
-	sqpoll-cancel-hang.c \
 	sqpoll-disable-exit.c \
+	sqpoll-exec.c \
 	sq-poll-dup.c \
 	sqpoll-exit-hang.c \
 	sq-poll-kthread.c \
@@ -166,20 +202,20 @@ test_srcs := \
 	submit-and-wait.c \
 	submit-link-fail.c \
 	submit-reuse.c \
-	sync-cancel.c \
 	symlink.c \
+	sync-cancel.c \
 	teardowns.c \
 	thread-exit.c \
 	timeout.c \
 	timeout-new.c \
-	timeout-overflow.c \
+	truncate.c \
 	tty-write-dpoll.c \
 	unlink.c \
+	version.c \
+	waitid.c \
 	wakeup-hang.c \
+	wq-aff.c \
 	xattr.c \
-	skip-cqe.c \
-	single-issuer.c \
-	send-zerocopy.c \
 	# EOL
 
 all_targets :=
@@ -210,11 +246,22 @@ all: $(test_targets)
 helpers.o: helpers.c
 	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ -c $<
 
-%.t: %.c $(helpers) helpers.h ../src/liburing.a
+LIBURING := $(shell if [ -e ../src/liburing.a ]; then echo ../src/liburing.a; fi)
+
+%.t: %.c $(helpers) helpers.h $(LIBURING)
 	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
 
-%.t: %.cc $(helpers) helpers.h ../src/liburing.a
+#
-	$(QUIET_CXX)$(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
+# Clang++ is not happy with -Wmissing-prototypes:
+#
+#   cc1plus: warning: command-line option '-Wmissing-prototypes' \
+#   is valid for C/ObjC but not for C++
+#
+%.t: %.cc $(helpers) helpers.h $(LIBURING)
+	$(QUIET_CXX)$(CXX) \
+	$(patsubst -Wmissing-prototypes,,$(CPPFLAGS)) \
+	$(patsubst -Wmissing-prototypes,,$(CXXFLAGS)) \
+	-o $@ $< $(helpers) $(LDFLAGS)
 
 
 install: $(test_targets) runtests.sh runtests-loop.sh
@@ -223,6 +270,9 @@ install: $(test_targets) runtests.sh runtests-loop.sh
 	$(INSTALL) -D -m 755 runtests.sh  $(datadir)/liburing-test/
 	$(INSTALL) -D -m 755 runtests-loop.sh  $(datadir)/liburing-test/
 
+uninstall:
+	@rm -rf $(datadir)/liburing-test/
+
 clean:
 	@rm -f $(all_targets) helpers.o output/*
 	@rm -rf output/

test/a0908ae19763.c

@@ -14,13 +14,13 @@
 #include "helpers.h"
 #include "../src/syscall.h"
 
-uint64_t r[1] = {0xffffffffffffffff};
+static uint64_t r[1] = {0xffffffffffffffff};
 
 int main(int argc, char *argv[])
 {
   if (argc > 1)
     return T_EXIT_SKIP;
-  mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
+  mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
   intptr_t res = 0;
   *(uint32_t*)0x20000080 = 0;
   *(uint32_t*)0x20000084 = 0;

test/a4c0b3decb33.c

@@ -95,7 +95,7 @@ static void kill_and_wait(int pid, int* status)
 	}
 }
 
-static void setup_test()
+static void setup_test(void)
 {
 	prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
 	setpgrp();
@@ -109,7 +109,7 @@ static void execute_one(void);
 static void loop(void)
 {
 	int iter;
-	for (iter = 0; iter < 5000; iter++) {
+	for (iter = 0; iter < 50; iter++) {
 		int pid = fork();
 		if (pid < 0)
 			exit(1);
@@ -175,7 +175,7 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return T_EXIT_SKIP;
 	signal(SIGINT, sig_int);
-	mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
+	mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
 	loop();
 	return T_EXIT_PASS;
 }

test/accept-link.c

@@ -16,8 +16,8 @@
 #include "liburing.h"
 #include "helpers.h"
 
-pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
 
 static int recv_thread_ready = 0;
 static int recv_thread_done = 0;
@@ -77,7 +77,7 @@ static void *send_thread(void *arg)
 	return NULL;
 }
 
-void *recv_thread(void *arg)
+static void *recv_thread(void *arg)
 {
 	struct data *data = arg;
 	struct io_uring ring;
@@ -195,7 +195,7 @@ static int test_accept_timeout(int do_connect, unsigned long timeout)
 	if (ret) {
 		fprintf(stderr, "queue_init: %d\n", ret);
 		return 1;
-	};
+	}
 
 	fast_poll = (p.features & IORING_FEAT_FAST_POLL) != 0;
 	io_uring_queue_exit(&ring);

test/accept-non-empty.c

@@ -0,0 +1,256 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Check that kernels that support it will return IORING_CQE_F_SOCK_NONEMPTY
+ * on accepts requests where more connections are pending.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/un.h>
+#include <netinet/tcp.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <pthread.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static int no_more_accept;
+
+#define MAX_ACCEPTS	8
+
+struct data {
+	pthread_t thread;
+	pthread_barrier_t barrier;
+	pthread_barrier_t conn_barrier;
+	int connects;
+};
+
+static int start_accept_listen(int port_off, int extra_flags)
+{
+	struct sockaddr_in addr;
+	int32_t val = 1;
+	int fd, ret;
+
+	fd = socket(AF_INET, SOCK_STREAM | extra_flags, IPPROTO_TCP);
+
+	ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val));
+	assert(ret != -1);
+	ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
+	assert(ret != -1);
+
+	addr.sin_family = AF_INET;
+	addr.sin_port = htons(0x1235 + port_off);
+	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+
+	ret = bind(fd, (struct sockaddr *) &addr, sizeof(addr));
+	assert(ret != -1);
+	ret = listen(fd, 20000);
+	assert(ret != -1);
+
+	return fd;
+}
+
+static int test_maccept(struct data *d, int flags, int fixed)
+{
+	struct io_uring_params p = { };
+	struct io_uring ring;
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int err = 0, fd, ret, i, *fds;
+
+	p.flags = flags;
+	ret = io_uring_queue_init_params(8, &ring, &p);
+	if (ret == -EINVAL) {
+		return T_EXIT_SKIP;
+	} else if (ret < 0) {
+		fprintf(stderr, "ring setup failure: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	if (!(p.features & IORING_FEAT_RECVSEND_BUNDLE)) {
+		no_more_accept = 1;
+		return 0;
+	}
+
+	fds = malloc(MAX_ACCEPTS * sizeof(int));
+	memset(fds, -1, MAX_ACCEPTS * sizeof(int));
+
+	if (fixed) {
+		io_uring_register_ring_fd(&ring);
+
+		ret = io_uring_register_files(&ring, fds, MAX_ACCEPTS);
+		if (ret) {
+			fprintf(stderr, "file reg %d\n", ret);
+			return -1;
+		}
+	}
+
+	fd = start_accept_listen(0, 0);
+
+	pthread_barrier_wait(&d->barrier);
+
+	if (d->connects > 1)
+		pthread_barrier_wait(&d->conn_barrier);
+
+	for (i = 0; i < d->connects; i++) {
+		sqe = io_uring_get_sqe(&ring);
+		if (fixed)
+			io_uring_prep_accept_direct(sqe, fd, NULL, NULL, 0, i);
+		else
+			io_uring_prep_accept(sqe, fd, NULL, NULL, 0);
+
+		ret = io_uring_submit_and_wait(&ring, 1);
+		assert(ret != -1);
+
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		assert(!ret);
+		if (cqe->res < 0) {
+			fprintf(stderr, "res=%d\n", cqe->res);
+			break;
+		}
+		fds[i] = cqe->res;
+		if (d->connects == 1) {
+			if (cqe->flags & IORING_CQE_F_SOCK_NONEMPTY) {
+				fprintf(stderr, "Non-empty sock on single?\n");
+				err = 1;
+				break;
+			}
+		} else {
+			int last = i + 1 == d->connects;
+
+			if (last && cqe->flags & IORING_CQE_F_SOCK_NONEMPTY) {
+				fprintf(stderr, "Non-empty sock on last?\n");
+				err = 1;
+				break;
+			} else if (!last && !(cqe->flags & IORING_CQE_F_SOCK_NONEMPTY)) {
+				fprintf(stderr, "Empty on multi connect?\n");
+				err = 1;
+				break;
+			}
+		}
+		io_uring_cqe_seen(&ring, cqe);
+	}
+
+	close(fd);
+	if (!fixed) {
+		for (i = 0; i < MAX_ACCEPTS; i++)
+			if (fds[i] != -1)
+				close(fds[i]);
+	}
+	free(fds);
+	io_uring_queue_exit(&ring);
+	return err;
+}
+
+static void *connect_fn(void *data)
+{
+	struct sockaddr_in addr = { };
+	struct data *d = data;
+	int i;
+
+	pthread_barrier_wait(&d->barrier);
+
+	addr.sin_family = AF_INET;
+	addr.sin_port = htons(0x1235);
+	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+
+	for (i = 0; i < d->connects; i++) {
+		int s;
+
+		s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+		if (s < 0) {
+			perror("socket");
+			break;
+		}
+		if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
+			perror("connect");
+			break;
+		}
+	}
+
+	if (d->connects > 1)
+		pthread_barrier_wait(&d->conn_barrier);
+
+	return NULL;
+}
+
+static void setup_thread(struct data *d, int nconns)
+{
+	d->connects = nconns;
+	pthread_barrier_init(&d->barrier, NULL, 2);
+	pthread_barrier_init(&d->conn_barrier, NULL, 2);
+	pthread_create(&d->thread, NULL, connect_fn, d);
+}
+
+static int test(int flags, int fixed)
+{
+	struct data d;
+	void *tret;
+	int ret;
+
+	setup_thread(&d, 1);
+	ret = test_maccept(&d, flags, fixed);
+	if (ret) {
+		fprintf(stderr, "test conns=1 failed\n");
+		return ret;
+	}
+	if (no_more_accept)
+		return T_EXIT_SKIP;
+
+	pthread_join(d.thread, &tret);
+
+	setup_thread(&d, MAX_ACCEPTS);
+	ret = test_maccept(&d, flags, fixed);
+	if (ret) {
+		fprintf(stderr, "test conns=MAX failed\n");
+		return ret;
+	}
+
+	pthread_join(d.thread, &tret);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = test(0, 0);
+	if (no_more_accept)
+		return T_EXIT_SKIP;
+	if (ret) {
+		fprintf(stderr, "test 0 0 failed\n");
+		return ret;
+	}
+
+	ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN, 0);
+	if (ret) {
+		fprintf(stderr, "test DEFER 0 failed\n");
+		return ret;
+	}
+
+	ret = test(0, 1);
+	if (ret) {
+		fprintf(stderr, "test 0 1 failed\n");
+		return ret;
+	}
+
+	ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN, 1);
+	if (ret) {
+		fprintf(stderr, "test DEFER 1 failed\n");
+		return ret;
+	}
+
+	return 0;
+}

test/accept-reuse.c

@@ -1,5 +1,4 @@
 /* SPDX-License-Identifier: MIT */
-#include <liburing.h>
 #include <netdb.h>
 #include <string.h>
 #include <sys/socket.h>
@@ -11,17 +10,16 @@
 #include "helpers.h"
 #include "../src/syscall.h"
 
-struct io_uring io_uring;
+static struct io_uring io_uring;
 
-int sys_io_uring_enter(const int fd,
+static int sys_io_uring_enter(const int fd, const unsigned to_submit,
-		       const unsigned to_submit,
+			      const unsigned min_complete,
-		       const unsigned min_complete,
+			      const unsigned flags, sigset_t * const sig)
-		       const unsigned flags, sigset_t * const sig)
 {
 	return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig);
 }
 
-int submit_sqe(void)
+static int submit_sqe(void)
 {
 	struct io_uring_sq *sq = &io_uring.sq;
 	const unsigned tail = *sq->ktail;
@@ -47,7 +45,7 @@ int main(int argc, char **argv)
 		return T_EXIT_SKIP;
 
 	memset(&params, 0, sizeof(params));
-	ret = io_uring_queue_init_params(4, &io_uring, &params);
+	ret = t_io_uring_init_sqarray(4, &io_uring, &params);
 	if (ret) {
 		fprintf(stderr, "io_uring_init_failed: %d\n", ret);
 		return T_EXIT_FAIL;

test/accept.c

@@ -195,7 +195,8 @@ static int start_accept_listen(struct sockaddr_in *addr, int port_off,
 
 	addr->sin_family = AF_INET;
 	addr->sin_addr.s_addr = inet_addr("127.0.0.1");
-	assert(!t_bind_ephemeral_port(fd, addr));
+	ret = t_bind_ephemeral_port(fd, addr);
+	assert(!ret);
 	ret = listen(fd, 128);
 	assert(ret != -1);
 
@@ -272,6 +273,7 @@ static int test_loop(struct io_uring *ring,
 	uint32_t multishot_mask = 0;
 	int nr_fds = multishot ? MAX_FDS : 1;
 	int multishot_idx = multishot ? INITIAL_USER_DATA : 0;
+	int err_ret = T_EXIT_FAIL;
 
 	if (args.overflow)
 		cause_overflow(ring);
@@ -298,6 +300,7 @@ static int test_loop(struct io_uring *ring,
 				no_accept_multi = 1;
 			else
 				no_accept = 1;
+			ret = T_EXIT_SKIP;
 			goto out;
 		} else if (s_fd[i] < 0) {
 			if (args.accept_should_error &&
@@ -308,6 +311,9 @@ static int test_loop(struct io_uring *ring,
 				multishot ? "Multishot" : "",
 				i, s_fd[i]);
 			goto err;
+		} else if (s_fd[i] == 195 && args.overflow) {
+			fprintf(stderr, "Broken overflow handling\n");
+			goto err;
 		}
 
 		if (multishot && fixed) {
@@ -360,10 +366,10 @@ static int test_loop(struct io_uring *ring,
 
 out:
 	close_sock_fds(s_fd, c_fd, nr_fds, fixed);
-	return 0;
+	return T_EXIT_PASS;
 err:
 	close_sock_fds(s_fd, c_fd, nr_fds, fixed);
-	return 1;
+	return err_ret;
 }
 
 static int test(struct io_uring *ring, struct accept_test_args args)
@@ -372,7 +378,7 @@ static int test(struct io_uring *ring, struct accept_test_args args)
 	int ret = 0;
 	int loop;
 	int32_t recv_s0 = start_accept_listen(&addr, 0,
-					      args.nonblock ? O_NONBLOCK : 0);
+					      args.nonblock ? SOCK_NONBLOCK : 0);
 	if (args.queue_accept_before_connect)
 		queue_accept_conn(ring, recv_s0, args);
 	for (loop = 0; loop < 1 + args.extra_loops; loop++) {
@@ -425,7 +431,7 @@ struct test_accept_many_args {
 };
 
 /*
- * Test issue many accepts and see if we handle cancellation on exit
+ * Test issue many accepts and see if we handle cancelation on exit
  */
 static int test_accept_many(struct test_accept_many_args args)
 {
@@ -458,7 +464,7 @@ static int test_accept_many(struct test_accept_many_args args)
 
 	for (i = 0; i < nr_socks; i++)
 		fds[i] = start_accept_listen(NULL, i,
-					     args.nonblock ? O_NONBLOCK : 0);
+					     args.nonblock ? SOCK_NONBLOCK : 0);
 
 	for (i = 0; i < nr; i++) {
 		int sock_idx = args.single_sock ? 0 : i;
@@ -480,7 +486,7 @@ static int test_accept_many(struct test_accept_many_args args)
 		if (io_uring_peek_cqe(&m_io_uring, &cqe))
 			break;
 		if (cqe->res != -ECANCELED) {
-			fprintf(stderr, "Expected cqe to be cancelled %d\n", cqe->res);
+			fprintf(stderr, "Expected cqe to be canceled %d\n", cqe->res);
 			ret = 1;
 			goto out;
 		}
@@ -552,6 +558,9 @@ static int test_accept_cancel(unsigned usecs, unsigned int nr, bool multishot)
 			fprintf(stderr, "unexpected 0 user data\n");
 			goto err;
 		} else if (cqe->user_data <= nr) {
+			/* no multishot */
+			if (cqe->res == -EINVAL)
+				return T_EXIT_SKIP;
 			if (cqe->res != -EINTR && cqe->res != -ECANCELED) {
 				fprintf(stderr, "Cancelled accept got %d\n", cqe->res);
 				goto err;
@@ -611,7 +620,7 @@ static int test_multishot_accept(int count, bool before, bool overflow)
 	return ret;
 }
 
-static int test_accept_multishot_wrong_arg()
+static int test_accept_multishot_wrong_arg(void)
 {
 	struct io_uring m_io_uring;
 	struct io_uring_cqe *cqe;
@@ -676,7 +685,12 @@ static int test_accept_fixed(void)
 	ret = io_uring_queue_init(32, &m_io_uring, 0);
 	assert(ret >= 0);
 	ret = io_uring_register_files(&m_io_uring, &fd, 1);
-	assert(ret == 0);
+	if (ret) {
+		/* kernel doesn't support sparse registered files, skip */
+		if (ret == -EBADF || ret == -EINVAL)
+			return T_EXIT_SKIP;
+		return T_EXIT_FAIL;
+	}
 	ret = test(&m_io_uring, args);
 	io_uring_queue_exit(&m_io_uring);
 	return ret;
@@ -698,7 +712,12 @@ static int test_multishot_fixed_accept(void)
 	ret = io_uring_queue_init(MAX_FDS + 10, &m_io_uring, 0);
 	assert(ret >= 0);
 	ret = io_uring_register_files(&m_io_uring, fd, MAX_FDS);
-	assert(ret == 0);
+	if (ret) {
+		/* kernel doesn't support sparse registered files, skip */
+		if (ret == -EBADF || ret == -EINVAL)
+			return T_EXIT_SKIP;
+		return T_EXIT_FAIL;
+	}
 	ret = test(&m_io_uring, args);
 	io_uring_queue_exit(&m_io_uring);
 	return ret;
@@ -733,8 +752,9 @@ int main(int argc, char *argv[])
 
 	if (argc > 1)
 		return T_EXIT_SKIP;
+
 	ret = test_accept(1, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept failed\n");
 		return ret;
 	}
@@ -742,141 +762,141 @@ int main(int argc, char *argv[])
 		return T_EXIT_SKIP;
 
 	ret = test_accept(2, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept(2) failed\n");
 		return ret;
 	}
 
 	ret = test_accept(2, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept(2, true) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_nonblock(false, 1);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_nonblock failed\n");
 		return ret;
 	}
 
 	ret = test_accept_nonblock(true, 1);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_nonblock(before, 1) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_nonblock(true, 3);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_nonblock(before,3) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_fixed();
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_fixed failed\n");
 		return ret;
 	}
 
 	ret = test_multishot_fixed_accept();
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_multishot_fixed_accept failed\n");
 		return ret;
 	}
 
 	ret = test_accept_multishot_wrong_arg();
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_multishot_wrong_arg failed\n");
 		return ret;
 	}
 
 	ret = test_accept_sqpoll();
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_sqpoll failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(0, 1, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel nodelay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(10000, 1, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel delay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(0, 4, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel nodelay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(10000, 4, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel delay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(0, 1, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel multishot nodelay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(10000, 1, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel multishot delay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(0, 4, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel multishot nodelay failed\n");
 		return ret;
 	}
 
 	ret = test_accept_cancel(10000, 4, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_cancel multishot delay failed\n");
 		return ret;
 	}
 
 	ret = test_multishot_accept(1, true, true);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_multishot_accept(1, false, true) failed\n");
 		return ret;
 	}
 
 	ret = test_multishot_accept(1, false, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_multishot_accept(1, false, false) failed\n");
 		return ret;
 	}
 
 	ret = test_multishot_accept(1, true, false);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_multishot_accept(1, true, false) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_many((struct test_accept_many_args) {});
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_many failed\n");
 		return ret;
 	}
 
 	ret = test_accept_many((struct test_accept_many_args) {
 				.usecs = 100000 });
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_many(sleep) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_many((struct test_accept_many_args) {
 				.nonblock = true });
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_many(nonblock) failed\n");
 		return ret;
 	}
@@ -885,13 +905,13 @@ int main(int argc, char *argv[])
 				.nonblock = true,
 				.single_sock = true,
 				.close_fds = true });
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_many(nonblock,close) failed\n");
 		return ret;
 	}
 
 	ret = test_accept_pending_on_exit();
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_accept_pending_on_exit failed\n");
 		return ret;
 	}

test/b19062a56726.c

@@ -19,7 +19,7 @@ int main(int argc, char *argv[])
   if (argc > 1)
     return T_EXIT_SKIP;
 
-  mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0);
+  mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0);
 
   *(uint32_t*)0x20000200 = 0;
   *(uint32_t*)0x20000204 = 0;

test/bind-listen.c

@@ -0,0 +1,408 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Configure and operate a TCP socket solely with io_uring.
+ */
+#include <stdio.h>
+#include <string.h>
+#include <liburing.h>
+#include <err.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <netinet/ip.h>
+#include "liburing.h"
+#include "helpers.h"
+
+static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec)
+{
+        ts->tv_sec = msec / 1000;
+        ts->tv_nsec = (msec % 1000) * 1000000;
+}
+
+static const char *magic = "Hello World!";
+static int use_port = 8000;
+
+enum {
+	SRV_INDEX = 0,
+	CLI_INDEX,
+	CONN_INDEX,
+};
+
+static int connect_client(struct io_uring *ring, unsigned short peer_port)
+{
+	struct __kernel_timespec ts;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int head, ret, submitted = 0;
+	struct sockaddr_in peer_addr;
+ 	socklen_t addr_len = sizeof(peer_addr);
+
+	peer_addr.sin_family = AF_INET;
+	peer_addr.sin_port = peer_port;
+	peer_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_socket_direct(sqe, AF_INET, SOCK_STREAM, 0,
+				    CLI_INDEX, 0);
+	sqe->flags |= IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_connect(sqe, CLI_INDEX, (struct sockaddr*) &peer_addr, addr_len);
+	sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_send(sqe, CLI_INDEX, magic, strlen(magic), 0);
+	sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
+
+	submitted = ret = io_uring_submit(ring);
+	if (ret < 0)
+		return T_SETUP_SKIP;
+
+	msec_to_ts(&ts, 300);
+	ret = io_uring_wait_cqes(ring, &cqe, submitted, &ts, NULL);
+	if (ret < 0)
+		return T_SETUP_SKIP;
+
+	io_uring_for_each_cqe(ring, head, cqe) {
+		ret = cqe->res;
+		if (ret < 0)
+			return T_SETUP_SKIP;
+	} io_uring_cq_advance(ring, submitted);
+
+	return T_SETUP_OK;
+}
+
+static int setup_srv(struct io_uring *ring, struct sockaddr_in *server_addr)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct __kernel_timespec ts;
+	int ret, val, submitted;
+	unsigned head;
+
+	memset(server_addr, 0, sizeof(struct sockaddr_in));
+	server_addr->sin_family = AF_INET;
+	server_addr->sin_port = htons(use_port++);
+	server_addr->sin_addr.s_addr = htons(INADDR_ANY);
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_socket_direct(sqe, AF_INET, SOCK_STREAM, 0, SRV_INDEX, 0);
+	sqe->flags |= IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	val = 1;
+	io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_SETSOCKOPT, 0, SOL_SOCKET,
+			       SO_REUSEADDR, &val, sizeof(val));
+	sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_bind(sqe, SRV_INDEX, (struct sockaddr *) server_addr,
+			   sizeof(struct sockaddr_in));
+	sqe->flags |= IOSQE_FIXED_FILE | IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_listen(sqe, SRV_INDEX, 1);
+	sqe->flags |= IOSQE_FIXED_FILE;
+
+	submitted = ret = io_uring_submit(ring);
+	if (ret < 0) {
+		fprintf(stderr, "submission failed. %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	msec_to_ts(&ts, 300);
+	ret = io_uring_wait_cqes(ring, &cqe, ret, &ts, NULL);
+	if (ret < 0) {
+		fprintf(stderr, "submission failed. %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_for_each_cqe(ring, head, cqe) {
+		ret = cqe->res;
+		if (ret < 0) {
+			fprintf(stderr, "Server startup failed. step %d got %d \n", head, ret);
+			return T_EXIT_FAIL;
+		}
+	} io_uring_cq_advance(ring, submitted);
+
+	return T_SETUP_OK;
+}
+
+static int test_good_server(unsigned int ring_flags)
+{
+	struct sockaddr_in server_addr;
+	struct __kernel_timespec ts;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int ret;
+	int fds[3];
+	char buf[1024];
+
+	memset(fds, -1, sizeof(fds));
+
+	ret = t_create_ring(10, &ring, ring_flags | IORING_SETUP_SUBMIT_ALL);
+	if (ret < 0) {
+		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+		return T_SETUP_SKIP;
+	}
+
+	ret = io_uring_register_files(&ring, fds, 3);
+	if (ret) {
+		fprintf(stderr, "server file register %d\n", ret);
+		return T_SETUP_SKIP;
+	}
+
+	ret = setup_srv(&ring, &server_addr);
+	if (ret != T_SETUP_OK) {
+		fprintf(stderr, "srv startup failed.\n");
+		return T_EXIT_FAIL;
+	}
+
+	if (connect_client(&ring, server_addr.sin_port) != T_SETUP_OK) {
+		fprintf(stderr, "cli startup failed.\n");
+		return T_SETUP_SKIP;
+	}
+
+	/* Wait for a request */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_accept_direct(sqe, SRV_INDEX, NULL, NULL, 0, CONN_INDEX);
+	sqe->flags |= IOSQE_FIXED_FILE;
+
+	io_uring_submit(&ring);
+	io_uring_wait_cqe(&ring, &cqe);
+	if (cqe->res < 0) {
+		fprintf(stderr, "accept failed. %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(&ring, cqe);
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_recv(sqe, CONN_INDEX, buf, BUFSIZ, 0);
+	sqe->flags |= IOSQE_FIXED_FILE;
+
+	io_uring_submit(&ring);
+	io_uring_wait_cqe_timeout(&ring, &cqe, &ts);
+
+	if (cqe->res < 0) {
+		fprintf(stderr, "bad receive cqe. %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	ret = cqe->res;
+	io_uring_cqe_seen(&ring, cqe);
+
+	io_uring_queue_exit(&ring);
+
+	if (ret != strlen(magic) || strncmp(buf, magic, ret)) {
+		fprintf(stderr, "didn't receive expected string. Got %d '%s'\n", ret, buf);
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}
+
+static int test_bad_bind(void)
+{
+	struct sockaddr_in server_addr;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int sock = -1, err;
+	int ret = T_EXIT_FAIL;
+
+	memset(&server_addr, 0, sizeof(struct sockaddr_in));
+	server_addr.sin_family = AF_INET;
+	server_addr.sin_port = htons(9001);
+	server_addr.sin_addr.s_addr = htons(INADDR_ANY);
+
+	err = t_create_ring(1, &ring, 0);
+	if (err < 0) {
+		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+		return T_SETUP_SKIP;
+	}
+
+	sock = socket(AF_INET, SOCK_STREAM, 0);
+	if (sock < 0) {
+		perror("socket");
+		goto fail;
+	}
+
+	/* Bind with size 0 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_bind(sqe, sock, (struct sockaddr *) &server_addr, 0);
+	err = io_uring_submit(&ring);
+	if (err < 0)
+		goto fail;
+
+	err = io_uring_wait_cqe(&ring, &cqe);
+	if (err)
+		goto fail;
+
+	if (cqe->res != -EINVAL)
+		goto fail;
+	io_uring_cqe_seen(&ring, cqe);
+
+	/* Bind with bad fd */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_bind(sqe, 0, (struct sockaddr *) &server_addr,  sizeof(struct sockaddr_in));
+	err = io_uring_submit(&ring);
+	if (err < 0)
+		goto fail;
+
+	err = io_uring_wait_cqe(&ring, &cqe);
+	if (err)
+		goto fail;
+	if (cqe->res != -ENOTSOCK)
+		goto fail;
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = T_EXIT_PASS;
+
+	/* bind with weird value */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_bind(sqe, sock, (struct sockaddr *) &server_addr,  sizeof(struct sockaddr_in));
+	sqe->rw_flags = 1;
+	err = io_uring_submit(&ring);
+	if (err < 0)
+		goto fail;
+
+	err = io_uring_wait_cqe(&ring, &cqe);
+	if (err)
+		goto fail;
+	if (cqe->res != -EINVAL)
+		goto fail;
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = T_EXIT_PASS;
+
+fail:
+	io_uring_queue_exit(&ring);
+	if (sock != -1)
+		close(sock);
+	return ret;
+}
+
+static int test_bad_listen(void)
+{
+	struct sockaddr_in server_addr;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int sock = -1, err;
+	int ret = T_EXIT_FAIL;
+
+	memset(&server_addr, 0, sizeof(struct sockaddr_in));
+	server_addr.sin_family = AF_INET;
+	server_addr.sin_port = htons(8001);
+	server_addr.sin_addr.s_addr = htons(INADDR_ANY);
+
+	err = t_create_ring(1, &ring, 0);
+	if (err < 0) {
+		fprintf(stderr, "queue_init: %d\n", err);
+		return T_SETUP_SKIP;
+	}
+
+	sock = socket(AF_INET, SOCK_STREAM, 0);
+	if (sock < 0) {
+		perror("socket");
+		goto fail;
+	}
+
+	err = t_bind_ephemeral_port(sock, &server_addr);
+	if (err) {
+		fprintf(stderr, "bind: %s\n", strerror(-err));
+		goto fail;
+	}
+
+	/* listen on bad sock */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_listen(sqe, 0, 1);
+	err = io_uring_submit(&ring);
+	if (err < 0)
+		goto fail;
+
+	err = io_uring_wait_cqe(&ring, &cqe);
+	if (err)
+		goto fail;
+
+	if (cqe->res != -ENOTSOCK)
+		goto fail;
+	io_uring_cqe_seen(&ring, cqe);
+
+	/* listen with weird parameters */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_listen(sqe, sock, 1);
+	sqe->addr2 = 0xffffff;
+	err = io_uring_submit(&ring);
+	if (err < 0)
+		goto fail;
+
+	err = io_uring_wait_cqe(&ring, &cqe);
+	if (err)
+		goto fail;
+
+	if (cqe->res != -EINVAL)
+		goto fail;
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = T_EXIT_PASS;
+fail:
+	io_uring_queue_exit(&ring);
+	if (sock != -1)
+		close(sock);
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_probe *probe;
+	int ret;
+
+	if (argc > 1)
+		return 0;
+
+	/*
+	 * This test is not supported on older kernels. Check for
+	 * OP_LISTEN, since that is the last feature required to support
+	 * it.
+	 */
+	probe = io_uring_get_probe();
+	if (!probe)
+		return T_EXIT_SKIP;
+	if (!io_uring_opcode_supported(probe, IORING_OP_LISTEN))
+		return T_EXIT_SKIP;
+
+	ret = test_good_server(0);
+	if (ret) {
+		fprintf(stderr, "good 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_good_server(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
+	if (ret) {
+		fprintf(stderr, "good defer failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_good_server(IORING_SETUP_SQPOLL);
+	if (ret) {
+		fprintf(stderr, "good sqpoll failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_bad_bind();
+	if (ret) {
+		fprintf(stderr, "bad bind failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_bad_listen();
+	if (ret) {
+		fprintf(stderr, "bad listen failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}

test/buf-ring-nommap.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test IOU_PBUF_RING_MMAP with a ring setup with a ring
+ *		setup without mmap'ing sq/cq arrays
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/mman.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static int bgid = 5;
+static int bid = 89;
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_buf_ring *br;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	size_t ring_size;
+	int ret, ring_mask, fds[2];
+	struct io_uring_buf_reg reg = {
+		.ring_entries = 1,
+		.bgid = bgid,
+		.flags = IOU_PBUF_RING_MMAP,
+	};
+	struct io_uring_params p = { };
+	void *ring_mem;
+	char buf[32];
+	off_t off;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	if (posix_memalign(&ring_mem, 16384, 16384))
+		return T_EXIT_FAIL;
+
+	memset(ring_mem, 0, 16384);
+
+	p.flags = IORING_SETUP_NO_MMAP;
+	ret = io_uring_queue_init_mem(1, &ring, &p, ring_mem, 16384);
+	if (ret < 0) {
+		if (ret == -EINVAL || ret == -ENOMEM)
+			return T_EXIT_SKIP;
+		fprintf(stderr, "queue init failed %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ring_size = sizeof(struct io_uring_buf);
+	ring_mask = io_uring_buf_ring_mask(1);
+
+	ret = io_uring_register_buf_ring(&ring, &reg, 0);
+	if (ret) {
+		if (ret == -EINVAL)
+			return T_EXIT_SKIP;
+		fprintf(stderr, "reg buf ring: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	off = IORING_OFF_PBUF_RING |
+			(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
+	br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
+	if (br == MAP_FAILED) {
+		if (errno == ENOMEM)
+			return T_EXIT_SKIP;
+		perror("mmap");
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_buf_ring_add(br, buf, sizeof(buf), bid, ring_mask, 0);
+	io_uring_buf_ring_advance(br, 1);
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], NULL, 0, 0);
+	sqe->flags |= IOSQE_BUFFER_SELECT;
+	sqe->buf_group = bgid;
+
+	io_uring_submit(&ring);
+
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return T_EXIT_FAIL;
+	} else if (ret != 5) {
+		fprintf(stderr, "short write %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res < 0) {
+		fprintf(stderr, "cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	if (!(cqe->flags & IORING_CQE_F_BUFFER)) {
+		fprintf(stderr, "buffer not selected in cqe\n");
+		return T_EXIT_FAIL;
+	}
+	if ((cqe->flags >> IORING_CQE_BUFFER_SHIFT) != bid) {
+		fprintf(stderr, "wrong buffer id returned\n");
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_cqe_seen(&ring, cqe);
+
+	io_uring_queue_exit(&ring);
+	return T_EXIT_PASS;
+}

test/buf-ring-put.c

@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test persistence of mmap'ed provided ring buffers. Use a range
+ *		of buffer group IDs that puts us into both the lower end array
+ *		and higher end xarry.
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/mman.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#define BGID_START	60
+#define BGID_NR		10
+#define ENTRIES		512
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_buf_ring *br[BGID_NR];
+	struct io_uring ring;
+	size_t ring_size;
+	int ret, i, j;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = io_uring_queue_init(1, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue init failed %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ring_size = ENTRIES * sizeof(struct io_uring_buf);
+
+	for (i = 0; i < BGID_NR; i++) {
+		int bgid = BGID_START + i;
+		struct io_uring_buf_reg reg = {
+			.ring_entries = ENTRIES,
+			.bgid = bgid,
+			.flags = IOU_PBUF_RING_MMAP,
+		};
+		off_t off;
+
+		ret = io_uring_register_buf_ring(&ring, &reg, 0);
+		if (ret) {
+			if (ret == -EINVAL)
+				return T_EXIT_SKIP;
+			fprintf(stderr, "reg buf ring: %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+
+		off = IORING_OFF_PBUF_RING |
+			(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
+		br[i] = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
+		if (br[i] == MAP_FAILED) {
+			perror("mmap");
+			return T_EXIT_FAIL;
+		}
+	}
+
+	for (i = 0; i < BGID_NR; i++) {
+		ret = io_uring_unregister_buf_ring(&ring, BGID_START + i);
+		if (ret) {
+			fprintf(stderr, "reg buf ring: %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+	}
+
+	for (j = 0; j < 1000; j++) {
+		for (i = 0; i < BGID_NR; i++)
+			memset(br[i], 0x5a, ring_size);
+		usleep(1000);
+	}
+
+	io_uring_queue_exit(&ring);
+	return T_EXIT_PASS;
+}

test/buf-ring.c

@@ -9,20 +9,22 @@
 #include <stdlib.h>
 #include <string.h>
 #include <fcntl.h>
+#include <sys/mman.h>
 
 #include "liburing.h"
 #include "helpers.h"
 
 static int no_buf_ring;
+static int pagesize;
 
 /* test trying to register classic group when ring group exists */
 static int test_mixed_reg2(int bgid)
 {
-	struct io_uring_buf_reg reg = { };
+	struct io_uring_buf_ring *br;
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	struct io_uring ring;
-	void *ptr, *bufs;
+	void *bufs;
 	int ret;
 
 	ret = t_create_ring(1, &ring, 0);
@@ -31,15 +33,8 @@ static int test_mixed_reg2(int bgid)
 	else if (ret != T_SETUP_OK)
 		return 1;
 
-	if (posix_memalign(&ptr, 4096, 4096))
+	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
-		return 1;
+	if (!br) {
-
-	reg.ring_addr = (unsigned long) ptr;
-	reg.ring_entries = 32;
-	reg.bgid = bgid;
-
-	ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	if (ret) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
 	}
@@ -60,6 +55,7 @@ static int test_mixed_reg2(int bgid)
 	}
 	io_uring_cqe_seen(&ring, cqe);
 
+	io_uring_free_buf_ring(&ring, br, 32, bgid);
 	io_uring_queue_exit(&ring);
 	return 0;
 }
@@ -67,11 +63,11 @@ static int test_mixed_reg2(int bgid)
 /* test trying to register ring group when  classic group exists */
 static int test_mixed_reg(int bgid)
 {
-	struct io_uring_buf_reg reg = { };
+	struct io_uring_buf_ring *br;
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	struct io_uring ring;
-	void *ptr, *bufs;
+	void *bufs;
 	int ret;
 
 	ret = t_create_ring(1, &ring, 0);
@@ -96,16 +92,9 @@ static int test_mixed_reg(int bgid)
 	}
 	io_uring_cqe_seen(&ring, cqe);
 
-	if (posix_memalign(&ptr, 4096, 4096))
+	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
-		return 1;
+	if (br) {
-
+		fprintf(stderr, "Buffer ring setup succeeded unexpectedly %d\n", ret);
-	reg.ring_addr = (unsigned long) ptr;
-	reg.ring_entries = 32;
-	reg.bgid = bgid;
-
-	ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	if (ret != -EEXIST) {
-		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
 	}
 
@@ -116,8 +105,8 @@ static int test_mixed_reg(int bgid)
 static int test_double_reg_unreg(int bgid)
 {
 	struct io_uring_buf_reg reg = { };
+	struct io_uring_buf_ring *br;
 	struct io_uring ring;
-	void *ptr;
 	int ret;
 
 	ret = t_create_ring(1, &ring, 0);
@@ -126,21 +115,14 @@ static int test_double_reg_unreg(int bgid)
 	else if (ret != T_SETUP_OK)
 		return 1;
 
-	if (posix_memalign(&ptr, 4096, 4096))
+	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
-		return 1;
+	if (!br) {
-
-	reg.ring_addr = (unsigned long) ptr;
-	reg.ring_entries = 32;
-	reg.bgid = bgid;
-
-	ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	if (ret) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
 	}
 
 	/* check that 2nd register with same bgid fails */
-	reg.ring_addr = (unsigned long) ptr;
+	reg.ring_addr = (unsigned long) br;
 	reg.ring_entries = 32;
 	reg.bgid = bgid;
 
@@ -150,7 +132,7 @@ static int test_double_reg_unreg(int bgid)
 		return 1;
 	}
 
-	ret = io_uring_unregister_buf_ring(&ring, bgid);
+	ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
 	if (ret) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
@@ -168,9 +150,8 @@ static int test_double_reg_unreg(int bgid)
 
 static int test_reg_unreg(int bgid)
 {
-	struct io_uring_buf_reg reg = { };
+	struct io_uring_buf_ring *br;
 	struct io_uring ring;
-	void *ptr;
 	int ret;
 
 	ret = t_create_ring(1, &ring, 0);
@@ -179,15 +160,8 @@ static int test_reg_unreg(int bgid)
 	else if (ret != T_SETUP_OK)
 		return 1;
 
-	if (posix_memalign(&ptr, 4096, 4096))
+	br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret);
-		return 1;
+	if (!br) {
-
-	reg.ring_addr = (unsigned long) ptr;
-	reg.ring_entries = 32;
-	reg.bgid = bgid;
-
-	ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	if (ret) {
 		if (ret == -EINVAL) {
 			no_buf_ring = 1;
 			return 0;
@@ -196,9 +170,9 @@ static int test_reg_unreg(int bgid)
 		return 1;
 	}
 
-	ret = io_uring_unregister_buf_ring(&ring, bgid);
+	ret = io_uring_free_buf_ring(&ring, br, 32, bgid);
 	if (ret) {
-		fprintf(stderr, "Buffer ring register failed %d\n", ret);
+		fprintf(stderr, "Buffer ring unregister failed %d\n", ret);
 		return 1;
 	}
 
@@ -230,6 +204,54 @@ static int test_bad_reg(int bgid)
 	return !ret;
 }
 
+static int test_full_page_reg(int bgid)
+{
+#if defined(__hppa__)
+	return T_EXIT_SKIP;
+#else
+	struct io_uring ring;
+	int ret;
+	void *ptr;
+	struct io_uring_buf_reg reg = { };
+	int entries = pagesize / sizeof(struct io_uring_buf);
+
+	ret = io_uring_queue_init(1, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue init failed %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = posix_memalign(&ptr, pagesize, pagesize * 2);
+	if (ret) {
+		fprintf(stderr, "posix_memalign failed %d\n", ret);
+		goto err;
+	}
+
+	ret = mprotect(ptr + pagesize, pagesize, PROT_NONE);
+	if (ret) {
+		fprintf(stderr, "mprotect failed %d\n", errno);
+		goto err1;
+	}
+
+	reg.ring_addr = (unsigned long) ptr;
+	reg.ring_entries = entries;
+	reg.bgid = bgid;
+
+	ret = io_uring_register_buf_ring(&ring, &reg, 0);
+	if (ret)
+		fprintf(stderr, "register buf ring failed %d\n", ret);
+
+	if (mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_WRITE))
+		fprintf(stderr, "reverting mprotect failed %d\n", errno);
+
+err1:
+	free(ptr);
+err:
+	io_uring_queue_exit(&ring);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+#endif
+}
+
 static int test_one_read(int fd, int bgid, struct io_uring *ring)
 {
 	int ret;
@@ -270,51 +292,63 @@ static int test_one_read(int fd, int bgid, struct io_uring *ring)
 	return cqe->flags >> 16;
 }
 
-static int test_running(int bgid, int entries, int loops)
+static int test_running(int bgid, int entries, int loops, int use_mmap)
 {
-	struct io_uring_buf_reg reg = { };
-	struct io_uring ring;
-	void *ptr;
-	char buffer[8];
-	int ret;
-	int ring_size = (entries * sizeof(struct io_uring_buf) + 4095) & (~4095);
 	int ring_mask = io_uring_buf_ring_mask(entries);
-
-	int loop, idx;
-	bool *buffers;
 	struct io_uring_buf_ring *br;
-	int read_fd;
+	int ret, loop, idx, read_fd;
+	struct io_uring ring;
+	char buffer[8];
+	bool *buffers;
 
 	ret = t_create_ring(1, &ring, 0);
 	if (ret == T_SETUP_SKIP)
-		return 0;
+		return T_EXIT_SKIP;
 	else if (ret != T_SETUP_OK)
-		return 1;
+		return T_EXIT_FAIL;
 
-	if (posix_memalign(&ptr, 4096, ring_size))
+	if (!use_mmap) {
-		return 1;
+		br = io_uring_setup_buf_ring(&ring, entries, bgid, 0, &ret);
+		if (!br) {
+			/* by now should have checked if this is supported or not */
+			fprintf(stderr, "Buffer ring register failed %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+	} else {
+		struct io_uring_buf_reg reg = {
+			.ring_entries = entries,
+			.bgid = bgid,
+			.flags = IOU_PBUF_RING_MMAP,
+		};
+		size_t ring_size;
+		off_t off;
 
-	br = (struct io_uring_buf_ring *)ptr;
+		ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	io_uring_buf_ring_init(br);
+		if (ret) {
+			if (ret == -EINVAL)
+				return T_EXIT_SKIP;
+			fprintf(stderr, "mmap ring register failed %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+
+		off = IORING_OFF_PBUF_RING |
+			(unsigned long long) bgid << IORING_OFF_PBUF_SHIFT;
+		ring_size = sizeof(struct io_uring_buf) * entries;
+		br = mmap(NULL, ring_size, PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE, ring.ring_fd, off);
+		if (br == MAP_FAILED) {
+			perror("mmap");
+			return T_EXIT_FAIL;
+		}
+	}
 
 	buffers = malloc(sizeof(bool) * entries);
 	if (!buffers)
-		return 1;
+		return T_EXIT_SKIP;
 
 	read_fd = open("/dev/zero", O_RDONLY);
 	if (read_fd < 0)
-		return 1;
+		return T_EXIT_SKIP;
-
-	reg.ring_addr = (unsigned long) ptr;
-	reg.ring_entries = entries;
-	reg.bgid = bgid;
-
-	ret = io_uring_register_buf_ring(&ring, &reg, 0);
-	if (ret) {
-		/* by now should have checked if this is supported or not */
-		fprintf(stderr, "Buffer ring register failed %d\n", ret);
-		return 1;
-	}
 
 	for (loop = 0; loop < loops; loop++) {
 		memset(buffers, 0, sizeof(bool) * entries);
@@ -327,28 +361,28 @@ static int test_running(int bgid, int entries, int loops)
 			ret = test_one_read(read_fd, bgid, &ring);
 			if (ret < 0) {
 				fprintf(stderr, "bad run %d/%d = %d\n", loop, idx, ret);
-				return ret;
+				return T_EXIT_FAIL;
 			}
 			if (buffers[ret]) {
 				fprintf(stderr, "reused buffer %d/%d = %d!\n", loop, idx, ret);
-				return 1;
+				return T_EXIT_FAIL;
 			}
 			if (buffer[0] != 0) {
 				fprintf(stderr, "unexpected read %d %d/%d = %d!\n",
 						(int)buffer[0], loop, idx, ret);
-				return 1;
+				return T_EXIT_FAIL;
 			}
 			if (buffer[1] != 1) {
 				fprintf(stderr, "unexpected spilled read %d %d/%d = %d!\n",
 						(int)buffer[1], loop, idx, ret);
-				return 1;
+				return T_EXIT_FAIL;
 			}
 			buffers[ret] = true;
 		}
 		ret = test_one_read(read_fd, bgid, &ring);
 		if (ret != -ENOBUFS) {
 			fprintf(stderr, "expected enobufs run %d = %d\n", loop, ret);
-			return 1;
+			return T_EXIT_FAIL;
 		}
 
 	}
@@ -356,13 +390,13 @@ static int test_running(int bgid, int entries, int loops)
 	ret = io_uring_unregister_buf_ring(&ring, bgid);
 	if (ret) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
-		return 1;
+		return T_EXIT_FAIL;
 	}
 
 	close(read_fd);
 	io_uring_queue_exit(&ring);
 	free(buffers);
-	return 0;
+	return T_EXIT_PASS;
 }
 
 int main(int argc, char *argv[])
@@ -374,6 +408,8 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
+	pagesize = getpagesize();
+
 	for (i = 0; bgids[i] != -1; i++) {
 		ret = test_reg_unreg(bgids[i]);
 		if (ret) {
@@ -406,15 +442,32 @@ int main(int argc, char *argv[])
 			fprintf(stderr, "test_mixed_reg2 failed\n");
 			return T_EXIT_FAIL;
 		}
+
+		ret = test_full_page_reg(bgids[i]);
+		if (ret == T_EXIT_FAIL) {
+			fprintf(stderr, "test_full_page_reg failed\n");
+			return T_EXIT_FAIL;
+		}
 	}
 
 	for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
-		ret = test_running(2, entries[i], 3);
+		ret = test_running(2, entries[i], 3, 0);
 		if (ret) {
 			fprintf(stderr, "test_running(%d) failed\n", entries[i]);
 			return T_EXIT_FAIL;
 		}
 	}
 
+	for (i = 0; !no_buf_ring && entries[i] != -1; i++) {
+		ret = test_running(2, entries[i], 3, 1);
+		if (ret == T_EXIT_SKIP) {
+			break;
+		} else if (ret != T_EXIT_PASS) {
+			fprintf(stderr, "test_running(%d) mmap failed\n", entries[i]);
+			return T_EXIT_FAIL;
+		}
+	}
+
+
 	return T_EXIT_PASS;
 }

test/ce593a6c480a.c

@@ -15,7 +15,7 @@
 
 static int use_sqpoll = 0;
 
-void notify_fd(int fd)
+static void notify_fd(int fd)
 {
 	char buf[8] = {0, 0, 0, 0, 0, 0, 1};
 	int ret;
@@ -25,7 +25,7 @@ void notify_fd(int fd)
 		perror("write");
 }
 
-void *delay_set_fd_from_thread(void *data)
+static void *delay_set_fd_from_thread(void *data)
 {
 	int fd = (intptr_t) data;
 

test/close-opath.c

@@ -15,12 +15,13 @@
 
 #include <errno.h>
 #include <fcntl.h>
-#include <liburing.h>
 #include <sys/stat.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
+#include "liburing.h"
+
 typedef struct
 {
 	const char *const flnames;

test/config

@@ -1,5 +1,9 @@
 # Copy this to config.local, uncomment and define values
 #
+# NOTE: any files or devices added here will be used by tests that take
+# a file or device arguments This includes tests that are destructive with
+# respect to data contents. They may get erased or overwritten as part of tests.
+#
 # Define tests to exclude from running
 # TEST_EXCLUDE=""
 #

test/connect-rep.c

@@ -0,0 +1,204 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Check that repeated IORING_OP_CONNECT to a socket without a listener keeps
+ * yielding -ECONNREFUSED rather than -ECONNABORTED. Based on a reproducer
+ * from:
+ *
+ * https://github.com/axboe/liburing/issues/828
+ *
+ * and adopted to our usual test cases. Other changes made like looping,
+ * using different ring types, adding a memset() for reuse, etc.
+ *
+ */
+#include <stdio.h>
+#include <netinet/in.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <arpa/inet.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static unsigned long ud;
+
+static int init_test_server(struct sockaddr_in *serv_addr)
+{
+	socklen_t servaddr_len = sizeof(struct sockaddr_in);
+	int fd;
+
+	/* Init server socket. Bind but don't listen */
+	fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) {
+		perror("socket");
+		return -1;
+	}
+
+	serv_addr->sin_family = AF_INET;
+	serv_addr->sin_addr.s_addr = inet_addr("127.0.0.1");
+
+	if (bind(fd, (struct sockaddr *) serv_addr, servaddr_len) < 0) {
+		perror("bind");
+		return -1;
+	}
+
+	/*
+	 * Get the addresses the socket is bound to because the port is chosen
+	 * by the network stack.
+	 */
+	if (getsockname(fd, (struct sockaddr *)serv_addr, &servaddr_len) < 0) {
+		perror("getsockname");
+		return -1;
+	}
+
+	return fd;
+}
+
+static int init_test_client(void)
+{
+	socklen_t addr_len = sizeof(struct sockaddr_in);
+	struct sockaddr_in client_addr = {};
+	int clientfd;
+
+	clientfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (clientfd < 0) {
+		perror("socket");
+		return -1;
+	}
+
+	client_addr.sin_family = AF_INET;
+	client_addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+
+	if (bind(clientfd, (struct sockaddr *)&client_addr, addr_len) < 0) {
+		perror("bind");
+		close(clientfd);
+		return -1;
+	}
+
+	/*
+	 * Get the addresses the socket is bound to because the port is chosen
+	 * by the network stack.
+	 */
+	 if (getsockname(clientfd, (struct sockaddr *)&client_addr, &addr_len) < 0) {
+		 perror("getsockname");
+		 close(clientfd);
+		 return -1;
+	 }
+
+	 return clientfd;
+}
+
+static int get_completion_and_print(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	int ret, res;
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait_cqe=%d\n", ret);
+		return -1;
+	}
+
+	/* Mark this completion as seen */
+	res = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+	return res;
+}
+
+static int test_connect(struct io_uring *ring,
+			int clientfd, struct sockaddr_in *serv_addr)
+{
+	struct sockaddr_in local_sa;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_connect(sqe, clientfd, (const struct sockaddr *)serv_addr,
+				sizeof(struct sockaddr_in));
+	sqe->user_data = ++ud;
+
+	memcpy(&local_sa, serv_addr, sizeof(local_sa));
+
+	ret = io_uring_submit_and_wait(ring, 1);
+	if (ret != 1) {
+		fprintf(stderr, "submit=%d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* check for reuse at the same time */
+	memset(&local_sa, 0xff, sizeof(local_sa));
+
+	ret = get_completion_and_print(ring);
+	if (ret != -ECONNREFUSED) {
+		fprintf(stderr, "Connect got %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	return T_EXIT_PASS;
+}
+
+static int test(int flags)
+{
+	struct io_uring_params params = { .flags = flags, };
+	struct sockaddr_in serv_addr = {};
+	struct io_uring ring;
+	int ret, clientfd, s_fd, i;
+
+	if (flags & IORING_SETUP_SQPOLL)
+		params.sq_thread_idle = 50;
+
+	ret = io_uring_queue_init_params(8, &ring, &params);
+	if (ret < 0) {
+		fprintf(stderr, "Queue init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	s_fd = init_test_server(&serv_addr);
+	if (s_fd < 0)
+		return T_EXIT_FAIL;
+
+	clientfd = init_test_client();
+	if (clientfd < 0) {
+		close(s_fd);
+		return T_EXIT_FAIL;
+	}
+
+	/* make sure SQPOLL thread is sleeping */
+	if (flags & IORING_SETUP_SQPOLL)
+		usleep(100000);
+
+	for (i = 0; i < 32; i++) {
+		ret = test_connect(&ring, clientfd, &serv_addr);
+		if (ret == T_EXIT_SKIP)
+			return T_EXIT_SKIP;
+		else if (ret == T_EXIT_PASS)
+			continue;
+
+		return T_EXIT_FAIL;
+	}
+
+	close(s_fd);
+	close(clientfd);
+	return T_EXIT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = test(0);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test(0) failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(IORING_SETUP_SQPOLL);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test(SQPOLL) failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	return 0;
+}

test/connect.c

@@ -15,6 +15,7 @@
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
+#include <sys/stat.h>
 
 #include "liburing.h"
 #include "helpers.h"
@@ -132,7 +133,7 @@ static int configure_connect(int fd, struct sockaddr_in* addr)
 	return ret;
 }
 
-static int connect_socket(struct io_uring *ring, int fd, int *code)
+static int connect_socket(struct io_uring *ring, int fd, int *code, int async)
 {
 	struct sockaddr_in addr;
 	int ret, res;
@@ -149,6 +150,8 @@ static int connect_socket(struct io_uring *ring, int fd, int *code)
 	}
 
 	io_uring_prep_connect(sqe, fd, (struct sockaddr*)&addr, sizeof(addr));
+	if (async)
+		sqe->flags |= IOSQE_ASYNC;
 	sqe->user_data = 1;
 
 	ret = submit_and_wait(ring, &res);
@@ -185,7 +188,7 @@ static int test_connect_with_no_peer(struct io_uring *ring)
 	if (connect_fd == -1)
 		return -1;
 
-	ret = connect_socket(ring, connect_fd, &code);
+	ret = connect_socket(ring, connect_fd, &code, 0);
 	if (ret == -1)
 		goto err;
 
@@ -208,7 +211,7 @@ err:
 	return -1;
 }
 
-static int test_connect(struct io_uring *ring)
+static int test_connect(struct io_uring *ring, int async)
 {
 	int accept_fd;
 	int connect_fd;
@@ -226,7 +229,7 @@ static int test_connect(struct io_uring *ring)
 	if (connect_fd == -1)
 		goto err1;
 
-	ret = connect_socket(ring, connect_fd, &code);
+	ret = connect_socket(ring, connect_fd, &code, async);
 	if (ret == -1)
 		goto err2;
 
@@ -256,6 +259,13 @@ static int test_connect_timeout(struct io_uring *ring)
 	struct sockaddr_in addr;
 	struct io_uring_sqe *sqe;
 	struct __kernel_timespec ts = {.tv_sec = 0, .tv_nsec = 100000};
+	struct stat sb;
+
+	/*
+	 * Test reliably fails if syncookies isn't enabled
+	 */
+	if (stat("/proc/sys/net/ipv4/tcp_syncookies", &sb) < 0)
+		return T_EXIT_SKIP;
 
 	connect_fd[0] = create_socket();
 	if (connect_fd[0] == -1)
@@ -288,7 +298,7 @@ static int test_connect_timeout(struct io_uring *ring)
 	}
 
 	// We first connect with one client socket in order to fill the accept queue.
-	ret = connect_socket(ring, connect_fd[0], &code);
+	ret = connect_socket(ring, connect_fd[0], &code, 0);
 	if (ret == -1 || code != 0) {
 		fprintf(stderr, "unable to connect\n");
 		goto err;
@@ -355,15 +365,12 @@ err:
 	return -1;
 }
 
-int main(int argc, char *argv[])
+static int test(int flags)
 {
 	struct io_uring ring;
 	int ret;
 
-	if (argc > 1)
+	ret = io_uring_queue_init(8, &ring, flags);
-		return T_EXIT_SKIP;
-
-	ret = io_uring_queue_init(8, &ring, 0);
 	if (ret) {
 		fprintf(stderr, "io_uring_queue_setup() = %d\n", ret);
 		return T_EXIT_FAIL;
@@ -382,7 +389,13 @@ int main(int argc, char *argv[])
 	if (no_connect)
 		return T_EXIT_SKIP;
 
-	ret = test_connect(&ring);
+	ret = test_connect(&ring, 0);
+	if (ret == -1) {
+		fprintf(stderr, "test_connect(): failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_connect(&ring, 1);
 	if (ret == -1) {
 		fprintf(stderr, "test_connect(): failed\n");
 		return T_EXIT_FAIL;
@@ -397,3 +410,33 @@ int main(int argc, char *argv[])
 	io_uring_queue_exit(&ring);
 	return T_EXIT_PASS;
 }
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = test(0);
+	if (ret == -1) {
+		fprintf(stderr, "test 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+	if (no_connect)
+		return T_EXIT_SKIP;
+
+	ret = test(IORING_SETUP_SQPOLL);
+	if (ret == -1) {
+		fprintf(stderr, "test SQPOLL failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
+	if (ret == -1) {
+		fprintf(stderr, "test DEFER failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}

test/coredump.c

@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: trigger segfault. A recent 6.4-rc kernel introduced a bug
+ *		via vhost where segfaults for applications using io_uring
+ *		would hang in D state forever upon trying to generate the
+ *		core file. Perform a trivial test where a child process
+ *		generates a NULL pointer dereference and ensure that we don't
+ *		hang.
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static void test(void)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring ring;
+	int *ptr = NULL;
+	int fds[2];
+	char r1;
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		exit(0);
+	}
+
+	io_uring_queue_init(8, &ring, 0);
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], &r1, sizeof(r1), 0);
+	sqe->flags = IOSQE_ASYNC;
+	sqe->user_data = 1;
+
+	io_uring_submit(&ring);
+	*ptr = 0;
+	exit(0);
+}
+
+int main(int argc, char *argv[])
+{
+	pid_t pid;
+	int wstat;
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fork");
+		return T_EXIT_SKIP;
+	} else if (!pid) {
+		test();
+	}
+
+	wait(&wstat);
+	unlink("core");
+	return T_EXIT_PASS;
+}

test/cq-overflow.c

@@ -48,7 +48,8 @@ static struct iovec *vecs;
  * bash -c "echo 1 > /proc/self/make-it-fail && exec ./cq-overflow.t"
  */
 
-static int test_io(const char *file, unsigned long usecs, unsigned *drops, int fault)
+static int test_io(const char *file, unsigned long usecs, unsigned *drops,
+		   int fault)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
@@ -60,8 +61,10 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f
 
 	fd = open(file, O_RDONLY | O_DIRECT);
 	if (fd < 0) {
+		if (errno == EINVAL)
+			return T_EXIT_SKIP;
 		perror("file open");
-		return 1;
+		return T_EXIT_FAIL;
 	}
 
 	memset(&p, 0, sizeof(p));
@@ -69,7 +72,7 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f
 	if (ret) {
 		close(fd);
 		fprintf(stderr, "ring create failed: %d\n", ret);
-		return 1;
+		return T_EXIT_FAIL;
 	}
 	nodrop = 0;
 	if (p.features & IORING_FEAT_NODROP)
@@ -173,12 +176,12 @@ reap_it:
 
 	io_uring_queue_exit(&ring);
 	close(fd);
-	return 0;
+	return T_EXIT_PASS;
 err:
 	if (fd != -1)
 		close(fd);
 	io_uring_queue_exit(&ring);
-	return 1;
+	return T_EXIT_SKIP;
 }
 
 static int reap_events(struct io_uring *ring, unsigned nr_events, int do_wait)
@@ -496,7 +499,10 @@ int main(int argc, char *argv[])
 	do {
 		drops = 0;
 
-		if (test_io(fname, usecs, &drops, 0)) {
+		ret = test_io(fname, usecs, &drops, 0);
+		if (ret == T_EXIT_SKIP)
+			break;
+		else if (ret != T_EXIT_PASS) {
 			fprintf(stderr, "test_io nofault failed\n");
 			goto err;
 		}
@@ -506,12 +512,12 @@ int main(int argc, char *argv[])
 		iters++;
 	} while (iters < 40);
 
-	if (test_io(fname, usecs, &drops, 0)) {
+	if (test_io(fname, usecs, &drops, 0) == T_EXIT_FAIL) {
 		fprintf(stderr, "test_io nofault failed\n");
 		goto err;
 	}
 
-	if (test_io(fname, usecs, &drops, 1)) {
+	if (test_io(fname, usecs, &drops, 1) == T_EXIT_FAIL) {
 		fprintf(stderr, "test_io fault failed\n");
 		goto err;
 	}

test/d4ae271dfaae.c

@@ -29,7 +29,7 @@ int main(int argc, char *argv[])
 
 	memset(&p, 0, sizeof(p));
 	p.flags = IORING_SETUP_SQPOLL;
-	ret = t_create_ring_params(4, &ring, &p);
+	ret = t_create_ring_params(16, &ring, &p);
 	if (ret == T_SETUP_SKIP)
 		return T_EXIT_SKIP;
 	else if (ret < 0)

test/defer-taskrun.c

@@ -4,7 +4,6 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <string.h>
-#include <error.h>
 #include <sys/eventfd.h>
 #include <signal.h>
 #include <poll.h>
@@ -57,9 +56,14 @@ static void eventfd_trigger(int fd)
 	assert(ret == sizeof(val));
 }
 
-#define CHECK(x) if (!(x)) { \
+#define CHECK(x)								\
-		fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \
+do {										\
-		return -1; }
+	if (!(x)) {								\
+		fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x);	\
+		return -1;							\
+	}									\
+} while (0)
+
 
 static int test_eventfd(void)
 {
@@ -119,7 +123,7 @@ struct thread_data {
 	char buff[8];
 };
 
-void *thread(void *t)
+static void *thread(void *t)
 {
 	struct thread_data *td = t;
 
@@ -178,11 +182,11 @@ static int test_exec(const char *filename)
 		int wstatus;
 
 		CHECK(waitpid(fork_pid, &wstatus, 0) != (pid_t)-1);
-		if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) != T_EXIT_SKIP) {
+		if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) == T_EXIT_FAIL) {
 			fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus));
 			return -1;
 		}
-		return 0;
+		return T_EXIT_PASS;
 	}
 
 	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER |
@@ -192,9 +196,15 @@ static int test_exec(const char *filename)
 
 	if (filename) {
 		fd = open(filename, O_RDONLY | O_DIRECT);
+		if (fd < 0 && errno == EINVAL)
+			return T_EXIT_SKIP;
 	} else {
 		t_create_file(EXEC_FILENAME, EXEC_FILESIZE);
 		fd = open(EXEC_FILENAME, O_RDONLY | O_DIRECT);
+		if (fd < 0 && errno == EINVAL) {
+			unlink(EXEC_FILENAME);
+			return T_EXIT_SKIP;
+		}
 		unlink(EXEC_FILENAME);
 	}
 	buff = (char*)malloc(EXEC_FILESIZE);
@@ -207,7 +217,7 @@ static int test_exec(const char *filename)
 	ret = execve("/proc/self/exe", new_argv, new_env);
 	/* if we get here it failed anyway */
 	fprintf(stderr, "execve failed %d\n", ret);
-	return -1;
+	return T_EXIT_FAIL;
 }
 
 static int test_flag(void)
@@ -283,6 +293,45 @@ static int test_ring_shutdown(void)
 	return 0;
 }
 
+static int test_drain(void)
+{
+	struct io_uring ring;
+	int ret, i, fd[2];
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct iovec iovecs[128];
+	char buff[ARRAY_SIZE(iovecs)];
+
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER |
+					    IORING_SETUP_DEFER_TASKRUN |
+					    IORING_SETUP_TASKRUN_FLAG);
+	CHECK(!ret);
+
+	for (i = 0; i < ARRAY_SIZE(iovecs); i++) {
+		iovecs[i].iov_base = &buff[i];
+		iovecs[i].iov_len = 1;
+	}
+
+	ret = t_create_socket_pair(fd, true);
+	CHECK(!ret);
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_writev(sqe, fd[1], &iovecs[0], ARRAY_SIZE(iovecs), 0);
+	sqe->flags |= IOSQE_IO_DRAIN;
+	io_uring_submit(&ring);
+
+	for (i = 0; i < ARRAY_SIZE(iovecs); i++)
+		iovecs[i].iov_base = NULL;
+
+	CHECK(io_uring_wait_cqe(&ring, &cqe) == 0);
+	CHECK(cqe->res == 128);
+
+	close(fd[0]);
+	close(fd[1]);
+	io_uring_queue_exit(&ring);
+	return 0;
+}
+
 int main(int argc, char *argv[])
 {
 	int ret;
@@ -309,7 +358,7 @@ int main(int argc, char *argv[])
 	}
 
 	ret = test_exec(filename);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test_exec failed\n");
 		return T_EXIT_FAIL;
 	}
@@ -332,5 +381,11 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
+	ret = test_drain();
+	if (ret) {
+		fprintf(stderr, "test_drain failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	return T_EXIT_PASS;
 }

test/defer-tw-timeout.c

@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test waiting for more events than what will be posted with
+ *		a timeout with DEFER_TASKRUN. All kernels should time out,
+ *		but a non-buggy kernel will end up with one CQE available
+ *		for reaping. Buggy kernels will not have processed the
+ *		task_work and will have 0 events.
+ *
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+struct d {
+	int fd;
+};
+
+static void *thread_fn(void *data)
+{
+	struct d *d = data;
+	int ret;
+
+	usleep(100000);
+	ret = write(d->fd, "Hello", 5);
+	if (ret < 0)
+		perror("write");
+	return NULL;
+}
+
+static int test_poll(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	struct __kernel_timespec ts;
+	int ret, fds[2], i;
+	pthread_t thread;
+	char buf[32];
+	struct d d;
+	void *tret;
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return 1;
+	}
+	d.fd = fds[1];
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0);
+
+	pthread_create(&thread, NULL, thread_fn, &d);
+
+	ts.tv_sec = 1;
+	ts.tv_nsec = 0;
+
+	ret = io_uring_submit_and_wait_timeout(ring, &cqe, 2, &ts, NULL);
+	if (ret != 1) {
+		fprintf(stderr, "unexpected wait ret %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_peek_cqe(ring, &cqe);
+		if (ret)
+			break;
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	if (i != 1) {
+		fprintf(stderr, "Got %d request, expected 1\n", i);
+		return T_EXIT_FAIL;
+	}
+
+	pthread_join(thread, &tret);
+	return T_EXIT_PASS;
+}
+
+static int test_file(struct io_uring *ring, char *__fname)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	struct __kernel_timespec ts;
+	char filename[64], *fname;
+	int fd, ret, i;
+	void *buf;
+
+	if (!__fname) {
+		fname = filename;
+		sprintf(fname, ".defer-tw-timeout.%d", getpid());
+		t_create_file(fname, 128*1024);
+	} else {
+		fname = __fname;
+	}
+
+	fd = open(fname, O_RDONLY | O_DIRECT);
+	if (fd < 0) {
+		if (errno == EINVAL) {
+			if (!__fname)
+				unlink(fname);
+			return T_EXIT_SKIP;
+		}
+		perror("open");
+		if (!__fname)
+			unlink(fname);
+		return T_EXIT_FAIL;
+	}
+
+	if (!__fname)
+		unlink(fname);
+
+	if (posix_memalign(&buf, 4096, 4096)) {
+		close(fd);
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, fd, buf, 4096, 0);
+
+	ts.tv_sec = 1;
+	ts.tv_nsec = 0;
+
+	ret = io_uring_submit_and_wait_timeout(ring, &cqe, 2, &ts, NULL);
+	if (ret != 1) {
+		fprintf(stderr, "unexpected wait ret %d\n", ret);
+		close(fd);
+		return T_EXIT_FAIL;
+	}
+
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_peek_cqe(ring, &cqe);
+		if (ret)
+			break;
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	if (i != 1) {
+		fprintf(stderr, "Got %d request, expected 1\n", i);
+		close(fd);
+		return T_EXIT_FAIL;
+	}
+
+	close(fd);
+	return T_EXIT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	char *fname = NULL;
+	int ret;
+
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN);
+	if (ret == -EINVAL)
+		return T_EXIT_SKIP;
+
+	if (argc > 1)
+		fname = argv[1];
+
+	ret = test_file(&ring, fname);
+	if (ret != T_EXIT_PASS)
+		return ret;
+
+	ret = test_poll(&ring);
+	if (ret != T_EXIT_PASS)
+		return ret;
+
+	return T_EXIT_PASS;
+}

test/defer.c

@@ -57,7 +57,7 @@ static int init_context(struct test_context *ctx, struct io_uring *ring, int nr,
 		case OP_REMOVE_BUFFERS:
 			io_uring_prep_remove_buffers(sqe, 10, 1);
 			break;
-		};
+		}
 		sqe->user_data = i;
 		ctx->sqes[i] = sqe;
 	}
@@ -88,7 +88,7 @@ static int wait_cqes(struct test_context *ctx)
 	return 0;
 }
 
-static int test_cancelled_userdata(struct io_uring *ring)
+static int test_canceled_userdata(struct io_uring *ring)
 {
 	struct test_context ctx;
 	int ret, i, nr = 100;
@@ -276,9 +276,9 @@ int main(int argc, char *argv[])
 	}
 
 
-	ret = test_cancelled_userdata(&poll_ring);
+	ret = test_canceled_userdata(&poll_ring);
 	if (ret) {
-		printf("test_cancelled_userdata failed\n");
+		printf("test_canceled_userdata failed\n");
 		return ret;
 	}
 

test/double-poll-crash.c

@@ -109,7 +109,7 @@ static long syz_open_dev(volatile long a0, volatile long a1, volatile long a2)
   }
 }
 
-uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff};
+static uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff};
 
 int main(int argc, char *argv[])
 {
@@ -121,10 +121,10 @@ int main(int argc, char *argv[])
   if (argc > 1)
     return T_EXIT_SKIP;
 
-  mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul);
+  mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul);
   if (mmap_ret == MAP_FAILED)
     return T_EXIT_SKIP;
-  mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul);
+  mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul);
   if (mmap_ret == MAP_FAILED)
     return T_EXIT_SKIP;
   intptr_t res = 0;

test/eeed8b54e0df.c

@@ -102,13 +102,18 @@ int main(int argc, char *argv[])
 		goto err;
 	}
 
+	ret = T_EXIT_PASS;
 	if (cqe->res != -EAGAIN && cqe->res != 4096) {
-		printf("cqe error: %d\n", cqe->res);
+		if (cqe->res == -EOPNOTSUPP) {
-		goto err;
+			ret = T_EXIT_SKIP;
+		} else {
+			printf("cqe error: %d\n", cqe->res);
+			goto err;
+		}
 	}
 
 	close(fd);
-	return T_EXIT_PASS;
+	return ret;
 err:
 	close(fd);
 	return T_EXIT_FAIL;

test/eploop.c

@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Test that we don't recursively generate completion events if an io_uring
+ * fd is added to an epoll context, and the ring itself polls for events on
+ * the epollfd. Older kernels will stop on overflow, newer kernels will
+ * detect this earlier and abort correctly.
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/epoll.h>
+#include <sys/types.h>
+#include <poll.h>
+#include "liburing.h"
+#include "helpers.h"
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct epoll_event ev = { };
+	int epollfd, ret, i;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Ring init failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	epollfd = epoll_create1(0);
+	if (epollfd < 0) {
+		perror("epoll_create");
+		return T_EXIT_FAIL;
+	}
+
+	ev.events = EPOLLIN;
+	ev.data.fd = ring.ring_fd;
+	ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, ring.ring_fd, &ev);
+	if (ret < 0) {
+		perror("epoll_ctl");
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_multishot(sqe, epollfd, POLLIN);
+	sqe->user_data = 1;
+	io_uring_submit(&ring);
+
+	sqe = io_uring_get_sqe(&ring);
+	sqe->user_data = 2;
+	io_uring_prep_nop(sqe);
+	io_uring_submit(&ring);
+
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait_cqe ret = %d\n", ret);
+			break;
+		}
+		io_uring_cqe_seen(&ring, cqe);
+	}
+
+	ret = io_uring_peek_cqe(&ring, &cqe);
+	if (!ret) {
+		fprintf(stderr, "Generated too many events\n");
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}

test/eventfd-reg.c

@@ -43,7 +43,7 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	/* Check that registrering again will get -EBUSY */
+	/* Check that registering again will get -EBUSY */
 	ret = io_uring_register_eventfd(&ring, evfd[1]);
 	if (ret != -EBUSY) {
 		fprintf(stderr, "unexpected 2nd register: %d\n", ret);

test/eventfd-ring.c

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT */
 /*
- * Description: run various nop tests
+ * Description: test use of eventfds with multiple rings
  *
  */
 #include <errno.h>

test/eventfd.c

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT */
 /*
- * Description: run various nop tests
+ * Description: run various eventfd tests
  *
  */
 #include <errno.h>

test/evloop.c

@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Test that we don't recursively generate completion events if an io_uring
+ * has an eventfd registered that triggers on completions, and we add a poll
+ * request with multishot on the eventfd. Older kernels will stop on overflow,
+ * newer kernels will detect this earlier and abort correctly.
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/eventfd.h>
+#include <sys/types.h>
+#include <poll.h>
+#include <assert.h>
+#include "liburing.h"
+#include "helpers.h"
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, efd, i;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Ring init failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	efd = eventfd(0, 0);
+	if (efd < 0) {
+		perror("eventfd");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_register_eventfd(&ring, efd);
+	if (ret) {
+		fprintf(stderr, "Ring eventfd register failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_multishot(sqe, efd, POLLIN);
+	sqe->user_data = 1;
+	io_uring_submit(&ring);
+
+	sqe = io_uring_get_sqe(&ring);
+	sqe->user_data = 2;
+	io_uring_prep_nop(sqe);
+	io_uring_submit(&ring);
+
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait_cqe ret = %d\n", ret);
+			break;
+		}
+		io_uring_cqe_seen(&ring, cqe);
+	}
+
+	ret = io_uring_peek_cqe(&ring, &cqe);
+	if (!ret) {
+		fprintf(stderr, "Generated too many events\n");
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}

test/exit-no-cleanup.c

@@ -26,7 +26,7 @@ static pthread_barrier_t init_barrier;
 static int sleep_fd, notify_fd;
 static sem_t sem;
 
-void *thread_func(void *arg)
+static void *thread_func(void *arg)
 {
 	struct io_uring ring;
 	int res;

test/fadvise.c

@@ -186,7 +186,7 @@ int main(int argc, char *argv[])
 	}
 
 	/* too hard to reliably test, just ignore */
-	if (0 && bad > good) {
+	if ((0) && bad > good) {
 		fprintf(stderr, "Suspicious timings\n");
 		goto err;
 	}

test/fallocate.c

@@ -12,6 +12,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <fcntl.h>
+#include <signal.h>
 
 #include "liburing.h"
 #include "helpers.h"
@@ -216,14 +217,22 @@ err:
 	return 1;
 }
 
+static void sig_xfsz(int sig)
+{
+}
+
 int main(int argc, char *argv[])
 {
+	struct sigaction act = { };
 	struct io_uring ring;
 	int ret;
 
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
+	act.sa_handler = sig_xfsz;
+	sigaction(SIGXFSZ, &act, NULL);
+
 	ret = io_uring_queue_init(8, &ring, 0);
 	if (ret) {
 		fprintf(stderr, "ring setup failed\n");

test/fc2a85cb02ef.c

@@ -54,7 +54,7 @@ static int inject_fault(int nth)
   return fd;
 }
 
-static int setup_fault()
+static int setup_fault(void)
 {
   static struct {
     const char* file;
@@ -79,13 +79,13 @@ static int setup_fault()
   return 0;
 }
 
-uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff};
+static uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff};
 
 int main(int argc, char *argv[])
 {
   if (argc > 1)
     return T_EXIT_SKIP;
-  mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, 0x32ul, -1, 0);
+  mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, MAP_ANON|MAP_PRIVATE, -1, 0);
   if (setup_fault()) {
     printf("Test needs failslab/fail_futex/fail_page_alloc enabled, skipped\n");
     return T_EXIT_SKIP;

test/fd-install.c

@@ -0,0 +1,500 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test installing a direct descriptor into the regular
+ *		file table
+ *
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static int no_fd_install;
+
+/* test that O_CLOEXEC is accepted, and others are not */
+static int test_flags(struct io_uring *ring, int async)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, fds[2], fd;
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_register_files(ring, &fds[0], 1);
+	if (ret) {
+		fprintf(stderr, "failed register files %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* check that setting an invalid flag fails */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, 1U << 17);
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != -EINVAL) {
+		fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	/* check that IORING_FIXED_FD_NO_CLOEXEC is accepted */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, IORING_FIXED_FD_NO_CLOEXEC);
+	if (async)
+		sqe->flags |= IOSQE_ASYNC;
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res < 0) {
+		fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	fd = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+
+	close(fds[0]);
+	close(fds[1]);
+	close(fd);
+	io_uring_unregister_files(ring);
+	
+	return T_EXIT_PASS;
+}
+
+static int test_linked(struct io_uring *ring)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, fds[2], fd, i;
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_register_files(ring, &fds[0], 1);
+	if (ret) {
+		fprintf(stderr, "failed register files %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->flags |= IOSQE_IO_LINK;
+	sqe->user_data = 1;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, 0);
+	sqe->user_data = 2;
+
+	ret = io_uring_submit(ring);
+	if (ret != 2) {
+		fprintf(stderr, "submit: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	fd = -1;
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait cqe %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+		if (cqe->res < 0) {
+			fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+			return T_EXIT_FAIL;
+		}
+		if (cqe->user_data == 2)
+			fd = cqe->res;
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	close(fds[0]);
+	close(fds[1]);
+	if (fd != -1)
+		close(fd);
+	io_uring_unregister_files(ring);
+	return T_EXIT_PASS;
+}
+
+/* test not setting IOSQE_FIXED_FILE */
+static int test_not_fixed(struct io_uring *ring)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, fds[2];
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_register_files(ring, &fds[0], 1);
+	if (ret) {
+		fprintf(stderr, "failed register files %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, 0);
+	sqe->flags &= ~IOSQE_FIXED_FILE;
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != -EBADF) {
+		fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_cqe_seen(ring, cqe);
+
+	close(fds[0]);
+	close(fds[1]);
+	io_uring_unregister_files(ring);
+	
+	return T_EXIT_PASS;
+}
+
+/* test invalid direct descriptor indexes */
+static int test_bad_fd(struct io_uring *ring, int some_fd)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, some_fd, 0);
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != -EBADF) {
+		fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_cqe_seen(ring, cqe);
+	return T_EXIT_PASS;
+}
+
+/* test basic functionality of shifting a direct descriptor to a normal file */
+static int test_working(struct io_uring *ring)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, fds[2];
+	char buf[32];
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	/* register read side */
+	ret = io_uring_register_files(ring, &fds[0], 1);
+	if (ret) {
+		fprintf(stderr, "failed register files %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* close normal descriptor */
+	close(fds[0]);
+
+	/* normal read should fail */
+	ret = read(fds[0], buf, 1);
+	if (ret != -1) {
+		fprintf(stderr, "unexpected read ret %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (errno != EBADF) {
+		fprintf(stderr, "unexpected read failure %d\n", errno);
+		return T_EXIT_FAIL;
+	}
+
+	/* verify we can read the data */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0);
+	sqe->flags |= IOSQE_FIXED_FILE;
+	io_uring_submit(ring);
+
+	/* put some data in the pipe */
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return T_EXIT_FAIL;
+	} else if (ret != 5) {
+		fprintf(stderr, "short write %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != 5) {
+		fprintf(stderr, "weird pipe read ret %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	/* fixed pipe read worked, now re-install as a regular fd */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, 0);
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res == -EINVAL) {
+		no_fd_install = 1;
+		return T_EXIT_SKIP;
+	}
+	if (cqe->res < 0) {
+		fprintf(stderr, "failed install fd: %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	/* stash new pipe read side fd in old spot */
+	fds[0] = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return T_EXIT_FAIL;
+	} else if (ret != 5) {
+		fprintf(stderr, "short write %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* normal pipe read should now work with new fd */
+	ret = read(fds[0], buf, sizeof(buf));
+	if (ret != 5) {
+		fprintf(stderr, "unexpected read ret %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* close fixed file */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_close_direct(sqe, 0);
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res) {
+		fprintf(stderr, "close fixed fd %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return T_EXIT_FAIL;
+	} else if (ret != 5) {
+		fprintf(stderr, "short write %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* normal pipe read should still work with new fd */
+	ret = read(fds[0], buf, sizeof(buf));
+	if (ret != 5) {
+		fprintf(stderr, "unexpected read ret %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	/* fixed fd pipe read should now fail */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, 0, buf, sizeof(buf), 0);
+	sqe->flags = IOSQE_FIXED_FILE;
+	io_uring_submit(ring);
+
+	/* put some data in the pipe */
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return T_EXIT_FAIL;
+	} else if (ret != 5) {
+		fprintf(stderr, "short write %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != -EBADF) {
+		fprintf(stderr, "weird pipe read ret %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	close(fds[0]);
+	close(fds[1]);
+	io_uring_unregister_files(ring);
+	return T_EXIT_PASS;
+}
+
+static int test_creds(struct io_uring *ring, int async)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int cred_id, ret, fds[2];
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_register_files(ring, &fds[0], 1);
+	if (ret) {
+		fprintf(stderr, "failed register files %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	cred_id = io_uring_register_personality(ring);
+	if (cred_id < 0) {
+		fprintf(stderr, "Failed registering creds: %d\n", cred_id);
+		return T_EXIT_FAIL;
+	}
+
+	/* check that asking for creds fails */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_fixed_fd_install(sqe, 0, 0);
+	if (async)
+		sqe->flags |= IOSQE_ASYNC;
+	sqe->personality = cred_id;
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res > 0) {
+		fprintf(stderr, "install succeeded with creds\n");
+		return T_EXIT_FAIL;
+	}
+	if (cqe->res != -EPERM) {
+		fprintf(stderr, "unexpected cqe res %d\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	close(fds[0]);
+	close(fds[1]);
+	io_uring_unregister_files(ring);
+	io_uring_unregister_personality(ring, cred_id);
+	return T_EXIT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = io_uring_queue_init(4, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_working(&ring);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_working failed\n");
+		return ret;
+	}
+	if (no_fd_install)
+		return T_EXIT_SKIP;
+
+	ret = test_bad_fd(&ring, 0);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_bad_fd 0 failed\n");
+		return ret;
+	}
+
+	ret = test_bad_fd(&ring, 500);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_bad_fd 500 failed\n");
+		return ret;
+	}
+	
+	ret = test_not_fixed(&ring);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_not_fixed failed\n");
+		return ret;
+	}
+
+	ret = test_flags(&ring, 0);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_flags 0 failed\n");
+		return ret;
+	}
+
+	ret = test_flags(&ring, 1);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_flags 1 failed\n");
+		return ret;
+	}
+
+	ret = test_creds(&ring, 0);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_creds 0 failed\n");
+		return ret;
+	}
+
+	ret = test_creds(&ring, 1);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_creds 1 failed\n");
+		return ret;
+	}
+
+	ret = test_linked(&ring);
+	if (ret != T_EXIT_PASS) {
+		if (ret == T_EXIT_FAIL)
+			fprintf(stderr, "test_linked failed\n");
+		return ret;
+	}
+	
+	return T_EXIT_PASS;
+}

test/fd-pass.c

@@ -13,8 +13,9 @@
 #include "liburing.h"
 #include "helpers.h"
 
-#define FSIZE	128
+#define FSIZE		128
-#define PAT	0x9a
+#define PAT		0x9a
+#define USER_DATA	0x89
 
 static int no_fd_pass;
 
@@ -49,7 +50,7 @@ static int verify_fixed_read(struct io_uring *ring, int fixed_fd, int fail)
 	return 0;
 }
 
-static int test(const char *filename)
+static int test(const char *filename, int source_fd, int target_fd)
 {
 	struct io_uring sring, dring;
 	struct io_uring_sqe *sqe;
@@ -79,10 +80,18 @@ static int test(const char *filename)
 		fprintf(stderr, "register files failed %d\n", ret);
 		return T_EXIT_FAIL;
 	}
+	if (target_fd == IORING_FILE_INDEX_ALLOC) {
+		/* we want to test installing into a non-zero slot */
+		ret = io_uring_register_file_alloc_range(&dring, 1, 1);
+		if (ret) {
+			fprintf(stderr, "io_uring_register_file_alloc_range %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+	}
 
 	/* open direct descriptor */
 	sqe = io_uring_get_sqe(&sring);
-	io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, 0);
+	io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, source_fd);
 	io_uring_submit(&sring);
 	ret = io_uring_wait_cqe(&sring, &cqe);
 	if (ret) {
@@ -96,15 +105,19 @@ static int test(const char *filename)
 	io_uring_cqe_seen(&sring, cqe);
 
 	/* verify data is sane for source ring */
-	if (verify_fixed_read(&sring, 0, 0))
+	if (verify_fixed_read(&sring, source_fd, 0))
 		return T_EXIT_FAIL;
 
 	/* send direct descriptor to destination ring */
 	sqe = io_uring_get_sqe(&sring);
-	io_uring_prep_msg_ring(sqe, dring.ring_fd, 0, 0x89, 0);
+	if (target_fd == IORING_FILE_INDEX_ALLOC) {
-	sqe->addr = 1;
+		io_uring_prep_msg_ring_fd_alloc(sqe, dring.ring_fd, source_fd,
-	sqe->addr3 = 0;
+						USER_DATA, 0);
-	sqe->file_index = 1;
+	} else {
+
+		io_uring_prep_msg_ring_fd(sqe, dring.ring_fd, source_fd,
+					  target_fd, USER_DATA, 0);
+	}
 	io_uring_submit(&sring);
 
 	ret = io_uring_wait_cqe(&sring, &cqe);
@@ -112,7 +125,7 @@ static int test(const char *filename)
 		fprintf(stderr, "wait cqe failed %d\n", ret);
 		return T_EXIT_FAIL;
 	}
-	if (cqe->res) {
+	if (cqe->res < 0) {
 		if (cqe->res == -EINVAL && !no_fd_pass) {
 			no_fd_pass = 1;
 			return T_EXIT_SKIP;
@@ -128,19 +141,30 @@ static int test(const char *filename)
 		fprintf(stderr, "wait cqe failed %d\n", ret);
 		return T_EXIT_FAIL;
 	}
-	if (cqe->user_data != 0x89) {
+	if (cqe->user_data != USER_DATA) {
 		fprintf(stderr, "bad user_data %ld\n", (long) cqe->res);
 		return T_EXIT_FAIL;
 	}
+	if (cqe->res < 0) {
+		fprintf(stderr, "bad result %i\n", cqe->res);
+		return T_EXIT_FAIL;
+	}
+	if (target_fd == IORING_FILE_INDEX_ALLOC) {
+		if (cqe->res != 1) {
+			fprintf(stderr, "invalid allocated index %i\n", cqe->res);
+			return T_EXIT_FAIL;
+		}
+		target_fd = cqe->res;
+	}
 	io_uring_cqe_seen(&dring, cqe);
 
 	/* now verify we can read the sane data from the destination ring */
-	if (verify_fixed_read(&dring, 0, 0))
+	if (verify_fixed_read(&dring, target_fd, 0))
 		return T_EXIT_FAIL;
 
 	/* close descriptor in source ring */
 	sqe = io_uring_get_sqe(&sring);
-	io_uring_prep_close_direct(sqe, 0);
+	io_uring_prep_close_direct(sqe, source_fd);
 	io_uring_submit(&sring);
 
 	ret = io_uring_wait_cqe(&sring, &cqe);
@@ -155,13 +179,15 @@ static int test(const char *filename)
 	io_uring_cqe_seen(&sring, cqe);
 
 	/* check that source ring fails after close */
-	if (verify_fixed_read(&sring, 0, 1))
+	if (verify_fixed_read(&sring, source_fd, 1))
 		return T_EXIT_FAIL;
 
 	/* check we can still read from destination ring */
-	if (verify_fixed_read(&dring, 0, 0))
+	if (verify_fixed_read(&dring, target_fd, 0))
 		return T_EXIT_FAIL;
 
+	io_uring_queue_exit(&sring);
+	io_uring_queue_exit(&dring);
 	return T_EXIT_PASS;
 }
 
@@ -176,9 +202,33 @@ int main(int argc, char *argv[])
 	sprintf(fname, ".fd-pass.%d", getpid());
 	t_create_file_pattern(fname, FSIZE, PAT);
 
-	ret = test(fname);
+	ret = test(fname, 0, 1);
 	if (ret == T_EXIT_FAIL) {
-		fprintf(stderr, "test failed\n");
+		fprintf(stderr, "test failed 0 1\n");
+		ret = T_EXIT_FAIL;
+	}
+
+	ret = test(fname, 0, 2);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test failed 0 2\n");
+		ret = T_EXIT_FAIL;
+	}
+
+	ret = test(fname, 1, 1);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test failed 1 1\n");
+		ret = T_EXIT_FAIL;
+	}
+
+	ret = test(fname, 1, 0);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test failed 1 0\n");
+		ret = T_EXIT_FAIL;
+	}
+
+	ret = test(fname, 1, IORING_FILE_INDEX_ALLOC);
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test failed 1 ALLOC\n");
 		ret = T_EXIT_FAIL;
 	}
 

test/file-register.c

@@ -305,7 +305,7 @@ static int test_sparse(struct io_uring *ring)
 	files = open_files(100, 100, 0);
 	ret = io_uring_register_files(ring, files, 200);
 	if (ret) {
-		if (ret == -EBADF) {
+		if (ret == -EBADF || ret == -EINVAL) {
 			fprintf(stdout, "Sparse files not supported, skipping\n");
 			no_update = 1;
 			goto done;
@@ -352,10 +352,14 @@ err:
 static int test_basic(struct io_uring *ring, int fail)
 {
 	int *files;
-	int ret;
+	int ret, i;
 	int nr_files = fail ? 10 : 100;
 
-	files = open_files(nr_files, 0, 0);
+	files = open_files(nr_files, fail ? 90 : 0, 0);
+	if (fail) {
+		for (i = nr_files; i < nr_files + 90; i++)
+			files[i] = -2;
+	}
 	ret = io_uring_register_files(ring, files, 100);
 	if (ret) {
 		if (fail) {
@@ -935,6 +939,59 @@ static int test_zero_range_alloc(struct io_uring *ring, int fds[2])
 	return 0;
 }
 
+static int test_defer_taskrun(void)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring ring;
+	int ret, fds[2];
+	char buff = 'x';
+
+	ret = io_uring_queue_init(8, &ring,
+				  IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER);
+	if (ret) {
+		fprintf(stderr, "ring init\n");
+		return 1;
+	}
+
+	ret = pipe(fds);
+	if (ret) {
+		fprintf(stderr, "bad pipes\n");
+		return 1;
+	}
+
+	ret = io_uring_register_files(&ring, &fds[0], 2);
+	if (ret) {
+		fprintf(stderr, "bad register %d\n", ret);
+		return 1;
+	}
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, 0, &buff, 1, 0);
+	sqe->flags |= IOSQE_FIXED_FILE;
+	ret = io_uring_submit(&ring);
+	if (ret != 1) {
+		fprintf(stderr, "bad submit\n");
+		return 1;
+	}
+
+	ret = write(fds[1], &buff, 1);
+	if (ret != 1) {
+		fprintf(stderr, "bad pipe write\n");
+		return 1;
+	}
+
+	ret = io_uring_unregister_files(&ring);
+	if (ret) {
+		fprintf(stderr, "bad unregister %d\n", ret);
+		return 1;
+	}
+
+	close(fds[0]);
+	close(fds[1]);
+	io_uring_queue_exit(&ring);
+	return 0;
+}
+
 static int test_file_alloc_ranges(void)
 {
 	struct io_uring ring;
@@ -1120,5 +1177,13 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
+	if (t_probe_defer_taskrun()) {
+		ret = test_defer_taskrun();
+		if (ret) {
+			fprintf(stderr, "test_defer_taskrun failed\n");
+			return T_EXIT_FAIL;
+		}
+	}
+
 	return T_EXIT_PASS;
 }

test/file-verify.c

@@ -10,7 +10,6 @@
 #include <string.h>
 #include <fcntl.h>
 #include <assert.h>
-#include <string.h>
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <linux/fs.h>
@@ -29,24 +28,42 @@
 #define MAX_VECS	16
 
 /*
- * Can be anything, let's just do something for a bit of parallellism
+ * Can be anything, let's just do something for a bit of parallelism
  */
 #define READ_BATCH	16
 
+static void verify_buf_sync(void *buf, size_t size, bool registered)
+{
+#if defined(__hppa__)
+	if (registered) {
+		unsigned long off = (unsigned long) buf & 4095;
+		unsigned long p = (unsigned long) buf & ~4095;
+		int i;
+
+		size += off;
+		for (i = 0; i < size; i += 32)
+			asm volatile("fdc 0(%0)" : : "r" (p + i));
+	}
+#endif
+}
+
 /*
  * Each offset in the file has the offset / sizeof(int) stored for every
  * sizeof(int) address.
  */
-static int verify_buf(void *buf, size_t size, off_t off)
+static int verify_buf(void *buf, size_t size, off_t off, bool registered)
 {
 	int i, u_in_buf = size / sizeof(unsigned int);
 	unsigned int *ptr;
 
+	verify_buf_sync(buf, size, registered);
+
 	off /= sizeof(unsigned int);
 	ptr = buf;
 	for (i = 0; i < u_in_buf; i++) {
 		if (off != *ptr) {
-			fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off);
+			fprintf(stderr, "Found %u, wanted %llu\n", *ptr,
+					(unsigned long long) off);
 			return 1;
 		}
 		ptr++;
@@ -197,7 +214,7 @@ again:
 		goto err;
 	}
 
-	if (verify_buf(buf, CHUNK_SIZE / 2, 0))
+	if (verify_buf(buf, CHUNK_SIZE / 2, 0, false))
 		goto err;
 
 	/*
@@ -364,9 +381,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
 			v[i].iov_base = buf[i];
 			v[i].iov_len = CHUNK_SIZE;
 		}
-		ret = io_uring_register_buffers(ring, v, READ_BATCH);
+		ret = t_register_buffers(ring, v, READ_BATCH);
 		if (ret) {
-			fprintf(stderr, "Error buffer reg %d\n", ret);
+			if (ret == T_SETUP_SKIP) {
+				ret = 0;
+				goto free_bufs;
+			}
 			goto err;
 		}
 	}
@@ -445,12 +465,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
 					void *buf = vecs[index][j].iov_base;
 					size_t len = vecs[index][j].iov_len;
 
-					if (verify_buf(buf, len, voff))
+					if (verify_buf(buf, len, voff, registered))
 						goto err;
 					voff += len;
 				}
 			} else {
-				if (verify_buf(buf[index], CHUNK_SIZE, voff))
+				if (verify_buf(buf[index], CHUNK_SIZE, voff, registered))
 					goto err;
 			}
 		}
@@ -460,6 +480,7 @@ static int test(struct io_uring *ring, const char *fname, int buffered,
 done:
 	if (registered)
 		io_uring_unregister_buffers(ring);
+free_bufs:
 	if (vectored) {
 		for (j = 0; j < READ_BATCH; j++)
 			for (i = 0; i < nr_vecs; i++)

test/files-exit-hang-timeout.c

@@ -21,9 +21,9 @@
 
 #define PORT 9100
 
-struct io_uring ring;
+static struct io_uring ring;
 
-struct __kernel_timespec ts = {
+static struct __kernel_timespec ts = {
 	.tv_sec		= 300,
 	.tv_nsec	= 0,
 };

test/fixed-buf-merge.c

@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Test fixed buffer merging/skipping
+ *
+ * Taken from: https://github.com/axboe/liburing/issues/994
+ *
+ */
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+int main(int argc, char *argv[])
+{
+	int ret, i, fd, initial_offset = 4096, num_requests = 3;
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct iovec iov;
+	char *buffer, *to_free;
+	unsigned head;
+	char filename[64];
+
+	ret = io_uring_queue_init(4, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue_init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	sprintf(filename, ".fixed-buf-%d", getpid());
+	t_create_file(filename, 4 * 4096);
+
+	fd = open(filename, O_RDONLY | O_DIRECT, 0644);
+	if (fd < 0) {
+		if (errno == EINVAL) {
+			unlink(filename);
+			return T_EXIT_SKIP;
+		}
+		perror("open");
+		goto err_unlink;
+	}
+
+	to_free = buffer = aligned_alloc(4096, 128 * 4096);
+	if (!buffer) {
+		perror("aligned_alloc");
+		goto err_unlink;
+	}
+
+	/* Register buffer */
+	iov.iov_base = buffer;
+	iov.iov_len = 128 * 4096;
+
+	ret = io_uring_register_buffers(&ring, &iov, 1);
+	if (ret) {
+		fprintf(stderr, "buf register: %d\n", ret);
+		goto err_unlink;
+	}
+
+	/* Prepare read requests */
+	buffer += initial_offset;
+	for (i = 0; i < num_requests; i++) {
+		sqe = io_uring_get_sqe(&ring);
+		io_uring_prep_read_fixed(sqe, fd, buffer, 4096, 4096 * i, 0);
+		buffer += 4096;
+	}
+
+	/* Submit requests and reap completions */
+	ret = io_uring_submit_and_wait(&ring, num_requests);
+	if (ret != num_requests) {
+		fprintf(stderr, "Submit and wait: %d\n", ret);
+		goto err_unlink;
+	}
+
+	i = 0;
+	io_uring_for_each_cqe(&ring, head, cqe) {
+		if (cqe->res != 4096) {
+			fprintf(stderr, "cqe: %d\n", cqe->res);
+			goto err_unlink;
+		}
+		i++;
+	}
+
+	if (i != num_requests) {
+		fprintf(stderr, "Got %d completions\n", i);
+		goto err_unlink;
+	}
+
+	io_uring_cq_advance(&ring, i);
+	io_uring_queue_exit(&ring);
+	close(fd);
+	free(to_free);
+	unlink(filename);
+	return T_EXIT_PASS;
+err_unlink:
+	unlink(filename);
+	return T_EXIT_FAIL;
+}

test/fixed-hugepage.c

@@ -0,0 +1,411 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Test fixed buffers consisting of hugepages.
+ */
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+/*
+ * Before testing
+ * echo (>=4) > /proc/sys/vm/nr_hugepages
+ * echo madvise > /sys/kernel/mm/transparent_hugepage/enabled
+ * echo always > /sys/kernel/mm/transparent_hugepage/hugepages-16kB/enabled
+ *
+ * Not 100% guaranteed to get THP-backed memory, but in general it does.
+ */
+#define MTHP_16KB	(16UL * 1024)
+#define HUGEPAGE_SIZE	(2UL * 1024 * 1024)
+#define NR_BUFS		1
+#define IN_FD		"/dev/urandom"
+#define OUT_FD		"/dev/zero"
+
+static int open_files(char *fname_in, int *fd_in, int *fd_out)
+{
+	*fd_in = open(fname_in, O_RDONLY, 0644);
+	if (*fd_in < 0) {
+		printf("open %s failed\n", fname_in);
+		return -1;
+	}
+
+	*fd_out = open(OUT_FD, O_RDWR, 0644);
+	if (*fd_out < 0) {
+		printf("open %s failed\n", OUT_FD);
+		return -1;
+	}
+
+	return 0;
+}
+
+static void unmap(struct iovec *iov, int nr_bufs, size_t offset)
+{
+	int i;
+
+	for (i = 0; i < nr_bufs; i++)
+		munmap(iov[i].iov_base - offset, iov[i].iov_len + offset);
+}
+
+static int mmap_hugebufs(struct iovec *iov, int nr_bufs, size_t buf_size, size_t offset)
+{
+	int i;
+
+	for (i = 0; i < nr_bufs; i++) {
+		void *base = NULL;
+
+		base = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
+		if (base == MAP_FAILED) {
+			printf("Unable to map hugetlb page. Try increasing the "
+				"value in /proc/sys/vm/nr_hugepages\n");
+			unmap(iov, i, offset);
+			return -1;
+		}
+
+		memset(base, 0, buf_size);
+		iov[i].iov_base = base + offset;
+		iov[i].iov_len = buf_size - offset;
+	}
+
+	return 0;
+}
+
+/* map a hugepage and smaller page to a contiguous memory */
+static int mmap_mixture(struct iovec *iov, int nr_bufs, size_t buf_size, bool huge_on_left)
+{
+	int i;
+	void *small_base = NULL, *huge_base = NULL, *start = NULL,
+	     *huge_start = NULL, *small_start = NULL;
+	size_t small_size = buf_size - HUGEPAGE_SIZE;
+	size_t seg_size = ((buf_size / HUGEPAGE_SIZE) + 1) * HUGEPAGE_SIZE;
+
+	start = mmap(NULL, seg_size * nr_bufs, PROT_NONE,
+			MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+	if (start == MAP_FAILED) {
+		printf("Unable to preserve the page mixture memory. "
+			"Try increasing the RLIMIT_MEMLOCK resource limit\n");
+		return -1;
+	}
+
+	for (i = 0; i < nr_bufs; i++) {
+		if (huge_on_left) {
+			huge_start = start;
+			small_start = start + HUGEPAGE_SIZE;
+		} else {
+			huge_start = start + HUGEPAGE_SIZE;
+			small_start = start + HUGEPAGE_SIZE - small_size;
+		}
+
+		huge_base = mmap(huge_start, HUGEPAGE_SIZE, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_FIXED, -1, 0);
+		if (huge_base == MAP_FAILED) {
+			printf("Unable to map hugetlb page in the page mixture. "
+				"Try increasing the value in /proc/sys/vm/nr_hugepages\n");
+			unmap(iov, nr_bufs, 0);
+			return -1;
+		}
+
+		small_base = mmap(small_start, small_size, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+		if (small_base == MAP_FAILED) {
+			printf("Unable to map small page in the page mixture. "
+				"Try increasing the RLIMIT_MEMLOCK resource limit\n");
+			unmap(iov, nr_bufs, 0);
+			return -1;
+		}
+
+		if (huge_on_left) {
+			iov[i].iov_base = huge_base;
+			memset(huge_base, 0, buf_size);
+		}
+		else {
+			iov[i].iov_base = small_base;
+			memset(small_base, 0, buf_size);
+		}
+		iov[i].iov_len = buf_size;
+		start += seg_size;
+	}
+
+	return 0;
+}
+
+static void free_bufs(struct iovec *iov, int nr_bufs, size_t offset)
+{
+	int i;
+
+	for (i = 0; i < nr_bufs; i++)
+		free(iov[i].iov_base - offset);
+}
+
+static int get_mthp_bufs(struct iovec *iov, int nr_bufs, size_t buf_size,
+		size_t alignment, size_t offset)
+{
+	int i;
+
+	for (i = 0; i < nr_bufs; i++) {
+		void *base = NULL;
+
+		if (posix_memalign(&base, alignment, buf_size)) {
+			printf("Unable to allocate mthp pages. "
+				"Try increasing the RLIMIT_MEMLOCK resource limit\n");
+			free_bufs(iov, i, offset);
+			return -1;
+		}
+
+		memset(base, 0, buf_size);
+		iov[i].iov_base = base + offset;
+		iov[i].iov_len = buf_size - offset;
+	}
+
+	return 0;
+}
+
+static int do_read(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int i, ret;
+
+	for (i = 0; i < nr_bufs; i++) {
+		sqe = io_uring_get_sqe(ring);
+		if (!sqe) {
+			fprintf(stderr, "Could not get SQE.\n");
+			return -1;
+		}
+
+		io_uring_prep_read_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
+		io_uring_submit(ring);
+
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret < 0) {
+			fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
+			return -1;
+		}
+
+		if (cqe->res < 0) {
+			fprintf(stderr, "Error in async read operation: %s\n", strerror(-cqe->res));
+			return -1;
+		}
+		if (cqe->res != iov[i].iov_len) {
+			fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
+			return -1;
+		}
+
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	return 0;
+}
+
+static int do_write(struct io_uring *ring, int fd, struct iovec *iov, int nr_bufs)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int i, ret;
+
+	for (i = 0; i < nr_bufs; i++) {
+		sqe = io_uring_get_sqe(ring);
+		if (!sqe) {
+			fprintf(stderr, "Could not get SQE.\n");
+			return -1;
+		}
+
+		io_uring_prep_write_fixed(sqe, fd, iov[i].iov_base, iov[i].iov_len, 0, i);
+		io_uring_submit(ring);
+
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret < 0) {
+			fprintf(stderr, "Error waiting for completion: %s\n", strerror(-ret));
+			return -1;
+		}
+
+		if (cqe->res < 0) {
+			fprintf(stderr, "Error in async write operation: %s\n", strerror(-cqe->res));
+			return -1;
+		}
+		if (cqe->res != iov[i].iov_len) {
+			fprintf(stderr, "cqe res: %d, expected: %lu\n", cqe->res, (unsigned long) iov[i].iov_len);
+			return -1;
+		}
+
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	return 0;
+}
+
+static int register_submit(struct io_uring *ring, struct iovec *iov,
+						int nr_bufs, int fd_in, int fd_out)
+{
+	int ret;
+
+	ret = io_uring_register_buffers(ring, iov, nr_bufs);
+	if (ret) {
+		fprintf(stderr, "Error registering buffers: %s\n", strerror(-ret));
+		return ret;
+	}
+
+	ret = do_read(ring, fd_in, iov, nr_bufs);
+	if (ret) {
+		fprintf(stderr, "Read test failed\n");
+		return ret;
+	}
+
+	ret = do_write(ring, fd_out, iov, nr_bufs);
+	if (ret) {
+		fprintf(stderr, "Write test failed\n");
+		return ret;
+	}
+
+	ret = io_uring_unregister_buffers(ring);
+	if (ret) {
+		fprintf(stderr, "Error unregistering buffers for one hugepage test: %s", strerror(-ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+static int test_one_hugepage(struct io_uring *ring, int fd_in, int fd_out)
+{
+	struct iovec iov[NR_BUFS];
+	size_t buf_size = HUGEPAGE_SIZE;
+	int ret;
+
+	if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
+		return T_EXIT_SKIP;
+
+	ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
+	unmap(iov, NR_BUFS, 0);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+}
+
+static int test_multi_hugepages(struct io_uring *ring, int fd_in, int fd_out)
+{
+	struct iovec iov[NR_BUFS];
+	size_t buf_size = 4 * HUGEPAGE_SIZE;
+	int ret;
+
+	if (mmap_hugebufs(iov, NR_BUFS, buf_size, 0))
+		return T_EXIT_SKIP;
+
+	ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
+	unmap(iov, NR_BUFS, 0);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+}
+
+static int test_unaligned_hugepage(struct io_uring *ring, int fd_in, int fd_out)
+{
+	struct iovec iov[NR_BUFS];
+	size_t buf_size = 3 * HUGEPAGE_SIZE;
+	size_t offset = 0x1234;
+	int ret;
+
+	if (mmap_hugebufs(iov, NR_BUFS, buf_size, offset))
+		return T_EXIT_SKIP;
+
+	ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
+	unmap(iov, NR_BUFS, offset);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+}
+
+static int test_multi_unaligned_mthps(struct io_uring *ring, int fd_in, int fd_out)
+{
+	struct iovec iov[NR_BUFS];
+	int ret;
+	size_t buf_size = 3 * MTHP_16KB;
+	size_t offset = 0x1234;
+
+	if (get_mthp_bufs(iov, NR_BUFS, buf_size, MTHP_16KB, offset))
+		return T_EXIT_SKIP;
+
+	ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
+	free_bufs(iov, NR_BUFS, offset);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+}
+
+/* Should not coalesce */
+static int test_page_mixture(struct io_uring *ring, int fd_in, int fd_out, int huge_on_left)
+{
+	struct iovec iov[NR_BUFS];
+	size_t buf_size = HUGEPAGE_SIZE + MTHP_16KB;
+	int ret;
+
+	if (mmap_mixture(iov, NR_BUFS, buf_size, huge_on_left))
+		return T_EXIT_SKIP;
+
+	ret = register_submit(ring, iov, NR_BUFS, fd_in, fd_out);
+	unmap(iov, NR_BUFS, 0);
+	return ret ? T_EXIT_FAIL : T_EXIT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	int ret, fd_in, fd_out;
+	char *fname_in;
+
+	if (argc > 1)
+		fname_in = argv[1];
+	else
+		fname_in = IN_FD;
+
+	if (open_files(fname_in, &fd_in, &fd_out))
+		return T_EXIT_SKIP;
+
+	ret = t_create_ring(8, &ring, 0);
+	if (ret == T_SETUP_SKIP)
+		return T_EXIT_SKIP;
+	else if (ret < 0)
+		return T_EXIT_FAIL;
+
+	ret = test_one_hugepage(&ring, fd_in, fd_out);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test one hugepage failed.\n");
+		return ret;
+	}
+
+	ret = test_multi_hugepages(&ring, fd_in, fd_out);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test multi hugepages failed.\n");
+		return ret;
+	}
+
+	ret = test_unaligned_hugepage(&ring, fd_in, fd_out);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test unaligned hugepage failed.\n");
+		return ret;
+	}
+
+	ret = test_multi_unaligned_mthps(&ring, fd_in, fd_out);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test unaligned multi-size'd THPs failed.\n");
+		return ret;
+	}
+
+	ret = test_page_mixture(&ring, fd_in, fd_out, true);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test huge small page mixture (start with huge) failed.\n");
+		return ret;
+	}
+
+	ret = test_page_mixture(&ring, fd_in, fd_out, false);
+	if (ret != T_EXIT_PASS) {
+		if (ret != T_EXIT_SKIP)
+			fprintf(stderr, "Test huge small page mixture (start with small) failed.\n");
+		return ret;
+	}
+
+	io_uring_queue_exit(&ring);
+	return T_EXIT_PASS;
+}