ice: Add a failure injection facility

Add compile-time optional, non-sleeping fail points around every VF
creation resource boundary, before VF VSI reconstruction, and in the
GET_STATS validation path.

Provide an ICE-wide wrapper and device selector so other driver
subsystems can add scoped points without duplicating the failpoint
plumbing.  Keep the current SR-IOV points and VF selector in an iov
child namespace.

Compile the facility only with options DRIVER_FAILPOINTS.  This shared
option avoids a separate kernel option for every driver that provides
test-only injection hooks.  Ordinary kernels contain no ICE failpoint
objects or sysctl nodes.  Require an exact PF device name and
optionally a VF index before any point can fire.  This prevents a stale
test setting from affecting another PF.

The hooks exposed two reset-lifetime defects while validating the
existing SR-IOV review series.  A PF reset could discard a firmware VSI
before teardown, and a rebuilt sibling could leave stale switch-filter
state after IOV destroy.

Tested on an E810-XXV with INVARIANTS and WITNESS.  All twelve creation
checkpoints rolled back and permitted immediate resource reuse.  Forced
reconstruction and malformed GET_STATS failures also preserved sibling
operation and reply cardinality.

Reviewed by:	gallatin, nprice, ziaee
MFC after:	2 weeks
Sponsored by:	BBOX.io
Differential Revision:	https://reviews.freebsd.org/D58940

share/man/man4/ice.4

@@ -32,7 +32,7 @@
 .\"
 .\" * Other names and brands may be claimed as the property of others.
 .\"
-.Dd August 19, 2026
+.Dd August 27, 2026
 .Dt ICE 4
 .Os
 .Sh NAME
@@ -124,6 +124,8 @@ The following topics are covered in this manual:
 .It
 .Sx IOVCTL OPTIONS
 .It
+.Sx FAILURE INJECTION
+.It
 .Sx SUPPORT
 .It
 .Sx SEE ALSO
@@ -1161,6 +1163,45 @@ option.
 .Pp
 For more information on standard and mandatory parameters, see
 .Xr iovctl.conf 5 .
+.Sh FAILURE INJECTION
+The driver provides destructive, test-only failure injection through the
+.Xr fail 9
+interface.
+Kernels compiled with
+.Cd options DRIVER_FAILPOINTS
+expose the driver fail points beneath the
+.Va debug.fail_point.ice
+sysctl node.
+Ordinary kernels omit these controls.
+.Pp
+Set
+.Va debug.fail_point.ice.device
+to the exact PF device name before a fail point can fire.
+An empty device selector disables every ICE fail point even if an individual
+point remains armed.
+For SR-IOV tests,
+.Va debug.fail_point.ice.iov.vf
+selects a PF-local VF index, and a value of -1 selects every VF on the chosen
+PF.
+.Pp
+List the available selectors and fail points with:
+.Bd -literal -offset indent
+sysctl -aN debug.fail_point.ice
+.Ed
+.Pp
+The individual points accept the syntax described by
+.Xr fail 9 .
+For example, the following injects one
+.Er EIO
+after allocating VF 0's VSI on
+.Li ice0 :
+.Bd -literal -offset indent
+sysctl debug.fail_point.ice.device=ice0
+sysctl debug.fail_point.ice.iov.vf=0
+sysctl debug.fail_point.ice.iov.add_after_vsi_alloc='1*return(5)'
+.Ed
+.Pp
+Clear the individual point and device selector after each destructive test.
 .Sh SUPPORT
 For general information and support, go to the Intel support website at:
 .Lk https://www.intel.com/support/ .
@@ -1173,7 +1214,8 @@ email all the specific information related to the issue to
 .Xr led 4 ,
 .Xr vlan 4 ,
 .Xr ifconfig 8 ,
-.Xr sysctl 8
+.Xr sysctl 8 ,
+.Xr fail 9
 .Sh HISTORY
 The
 .Nm

sys/conf/NOTES

@@ -438,6 +438,13 @@ options 	QUEUE_MACRO_DEBUG_TRASH
 #
 options 	SYSCTL_DEBUG
 
+#
+# Compile test-only failure-injection hooks in supporting device drivers.
+# These expose destructive controls below debug.fail_point and are omitted
+# by default.
+#
+options 	DRIVER_FAILPOINTS
+
 #
 # Enable textdump by default, this disables kernel core dumps.
 #

sys/conf/options

@@ -52,6 +52,7 @@ DDB_CAPTURE_DEFAULTBUFSIZE	opt_ddb.h
 DDB_CAPTURE_MAXBUFSIZE	opt_ddb.h
 DDB_CTF		opt_ddb.h
 DDB_NUMSYM	opt_ddb.h
+DRIVER_FAILPOINTS
 EARLY_PRINTF	opt_global.h
 EXTERR_STRINGS	opt_global.h
 FULL_BUF_TRACKING	opt_global.h

sys/dev/ice/ice_fault.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2026 BBOX.io
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _ICE_FAULT_H_
+#define _ICE_FAULT_H_
+
+#include "ice_opts.h"
+
+#ifdef DRIVER_FAILPOINTS
+
+#include <sys/fail.h>
+
+struct ice_softc;
+
+SYSCTL_DECL(_debug_fail_point_ice);
+
+bool ice_fail_point_device_matches(struct ice_softc *sc);
+
+#define ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, _cond, _flags, _code...) \
+	KFAIL_POINT_CODE_COND(_parent, _name, \
+	    ice_fail_point_device_matches((_sc)) && (_cond), _flags, _code)
+#define ICE_FAIL_POINT_CODE(_sc, _parent, _name, _flags, _code...) \
+	ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, true, _flags, _code)
+
+#else /* !DRIVER_FAILPOINTS */
+
+#define ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, _cond, _flags, _code...) \
+	do { } while (0)
+#define ICE_FAIL_POINT_CODE(_sc, _parent, _name, _flags, _code...) \
+	do { } while (0)
+
+#endif /* DRIVER_FAILPOINTS */
+
+#endif /* _ICE_FAULT_H_ */

sys/dev/ice/ice_iov.c

@@ -38,7 +38,17 @@
  */
 
 #include "ice_iov.h"
+#include "ice_fault.h"
 
+#ifdef DRIVER_FAILPOINTS
+static SYSCTL_NODE(_debug_fail_point_ice, OID_AUTO, iov,
+    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ice SR-IOV fail points");
+
+static int ice_iov_fail_vf = -1;
+SYSCTL_INT(_debug_fail_point_ice_iov, OID_AUTO, vf,
+    CTLFLAG_RW | CTLFLAG_MPSAFE, &ice_iov_fail_vf, 0,
+    "VF eligible for ice SR-IOV fail points (-1 selects every VF)");
+#endif /* DRIVER_FAILPOINTS */
 static struct ice_vf *ice_iov_get_vf(struct ice_softc *sc, int vf_num);
 static void ice_iov_ready_vf(struct ice_softc *sc, struct ice_vf *vf);
 static void ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf,
@@ -84,6 +94,28 @@ static int ice_vc_select_vlans(struct ice_vf *vf, u16 *vids, u16 count,
 static enum virtchnl_status_code ice_iov_err_to_virt_err(int ice_err);
 static int ice_vf_validate_mac(struct ice_vf *vf, const uint8_t *addr);
 
+#ifdef DRIVER_FAILPOINTS
+static bool
+ice_iov_fail_vf_matches(uint16_t vfnum)
+{
+	return (ice_iov_fail_vf == -1 || ice_iov_fail_vf == vfnum);
+}
+#endif
+
+#define	ICE_IOV_FAIL_POINT(_sc, _vfnum, _name, _error, _label) do { \
+	ICE_FAIL_POINT_CODE_COND(_sc, _debug_fail_point_ice_iov, _name, \
+	    ice_iov_fail_vf_matches((_vfnum)), \
+	    FAIL_POINT_NONSLEEPABLE, { \
+		(_error) = RETURN_VALUE; \
+		if ((_error) <= 0) \
+			(_error) = EIO; \
+		device_printf((_sc)->dev, \
+		    "injecting VF %u failure at %s: %d\n", \
+		    (unsigned int)(_vfnum), #_name, (_error)); \
+		goto _label; \
+	}); \
+} while (0)
+
 /**
  * ice_iov_attach - Initialize SR-IOV PF host support
  * @sc: device softc structure
@@ -237,6 +269,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		return (ENOMEM);
 	vf->vsi = vsi;
 	vsi->vf_num = vfnum;
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_alloc, error,
+	    release_vsi);
 
 	vf_num_queues = nvlist_get_number(params, "num-queues");
 	/* Validate and clamp value if invalid */
@@ -256,6 +290,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 	/* Reserve VF queue allocation from PF queues */
 	ice_alloc_vsi_qmap(vsi, vf_num_queues, vf_num_queues);
 	vsi->num_tx_queues = vsi->num_rx_queues = vf_num_queues;
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_queue_maps, error,
+	    release_vsi);
 
 	/* Assign Tx queues from PF space */
 	error = ice_resmgr_assign_scattered(&sc->tx_qmgr, vsi->tx_qmap,
@@ -265,6 +301,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 			      ice_err_str(error));
 		goto release_vsi;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_reservation, error,
+	    release_vsi);
 
 	/* Assign Rx queues from PF space */
 	error = ice_resmgr_assign_scattered(&sc->rx_qmgr, vsi->rx_qmap,
@@ -274,6 +312,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 			      ice_err_str(error));
 		goto release_vsi;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_reservation, error,
+	    release_vsi);
 
 	vsi->max_frame_size = ICE_MAX_FRAME_SIZE;
 
@@ -291,6 +331,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		txq->me = i;
 		txq->vsi = vsi;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_queue_memory, error,
+	    free_txqs);
 
 	/* Allocate queue structure memory */
 	vsi->rx_queues = (struct ice_rx_queue *)
@@ -306,6 +348,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		rxq->me = i;
 		rxq->vsi = vsi;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_queue_memory, error,
+	    free_rxqs);
 
 	/* Allocate space to store the IRQ vector data */
 	vf->num_irq_vectors = vf_num_queues + 1;
@@ -319,6 +363,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		error = ENOMEM;
 		goto free_rxqs;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_irq_memory, error,
+	    free_txirqvs);
 	vf->rx_irqvs = (struct ice_irq_vector *)
 	    malloc(sizeof(struct ice_irq_vector) * (vf->num_irq_vectors),
 		   M_ICE, M_NOWAIT);
@@ -329,6 +375,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		error = ENOMEM;
 		goto free_txirqvs;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_irq_memory, error,
+	    free_rxirqvs);
 
 	/* Assign VF interrupts from PF space */
 	if (!(vf->vf_imap =
@@ -338,12 +386,16 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 		error = ENOMEM;
 		goto free_rxirqvs;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_memory, error,
+	    free_imap);
 	error = ice_resmgr_assign_contiguous(&sc->dev_imgr, vf->vf_imap, vf->num_irq_vectors);
 	if (error) {
 		device_printf(dev, "Unable to assign VF-%d interrupt mapping: %s\n",
 			      vfnum, ice_err_str(error));
 		goto free_imap;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_reservation, error,
+	    release_imap);
 
 	if (nvlist_exists_binary(params, "mac-addr")) {
 		mac = nvlist_get_binary(params, "mac-addr", &size);
@@ -378,6 +430,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 			      vfnum, ice_err_str(error));
 		goto release_imap;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_init, error,
+	    release_imap);
 
 	/* Add the broadcast address */
 	error = ice_add_vsi_mac_filter(vsi, broadcastaddr);
@@ -386,6 +440,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
 			      vfnum, ice_err_str(error));
 		goto release_imap;
 	}
+	ICE_IOV_FAIL_POINT(sc, vfnum, add_after_broadcast_filter, error,
+	    release_imap);
 
 	atomic_set_32(&vf->vf_flags, VF_FLAG_ENABLED);
 	ice_iov_ready_vf(sc, vf);
@@ -602,6 +658,8 @@ ice_iov_rebuild_vf(struct ice_softc *sc, struct ice_vsi *vsi)
 	vf = ice_iov_get_vf(sc, vsi->vf_num);
 	atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED);
 	atomic_set_32(&vf->vf_flags, VF_FLAG_REBUILD_FAILED);
+	ICE_IOV_FAIL_POINT(sc, vf->vf_num, rebuild_before_initialize, error,
+	    fail);
 
 	/* A new hardware VSI starts a new raw statistics epoch. */
 	accumulated_stats = vsi->hw_stats.cur;
@@ -626,6 +684,11 @@ ice_iov_rebuild_vf(struct ice_softc *sc, struct ice_vsi *vsi)
 	atomic_clear_32(&vf->vf_flags, VF_FLAG_REBUILD_FAILED);
 	ice_iov_ready_vf(sc, vf);
 	return (0);
+
+#ifdef DRIVER_FAILPOINTS
+fail:
+	return (error);
+#endif /* DRIVER_FAILPOINTS */
 }
 
 /**
@@ -1816,6 +1879,14 @@ ice_vc_get_stats_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
 	struct ice_hw *hw = &sc->hw;
 
 	vqs = (struct virtchnl_queue_select *)msg_buf;
+	ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov,
+	    get_stats_bad_vsi, ice_iov_fail_vf_matches(vf->vf_num),
+	    FAIL_POINT_NONSLEEPABLE, {
+		vqs->vsi_id = vsi->idx + 1;
+		device_printf(sc->dev,
+		    "injecting invalid GET_STATS VSI ID for VF %u\n",
+		    (unsigned int)vf->vf_num);
+	});
 
 	if (vqs->vsi_id != vsi->idx) {
 		device_printf(sc->dev,

sys/dev/ice/ice_lib.c

@@ -42,6 +42,7 @@
 
 #include "ice_lib.h"
 #include "ice_iflib.h"
+#include "ice_fault.h"
 #ifdef PCI_IOV
 #include "ice_iov.h"
 #endif
@@ -62,6 +63,33 @@
  */
 MALLOC_DEFINE(M_ICE, "ice", "Intel(R) 100Gb Network Driver lib allocations");
 
+#ifdef DRIVER_FAILPOINTS
+
+/*
+ * ICE fail points are global, but only the selected PF may trigger them.  An
+ * empty selector disables every point even if a stale failpoint setting
+ * remains armed.
+ */
+SYSCTL_NODE(_debug_fail_point, OID_AUTO, ice,
+    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ice driver fail points");
+
+static char ice_fail_device[32];
+SYSCTL_STRING(_debug_fail_point_ice, OID_AUTO, device,
+    CTLFLAG_RW | CTLFLAG_MPSAFE, ice_fail_device,
+    sizeof(ice_fail_device), "device eligible for ice fail points");
+
+bool
+ice_fail_point_device_matches(struct ice_softc *sc)
+{
+	const char *nameunit;
+
+	nameunit = device_get_nameunit(sc->dev);
+	return (ice_fail_device[0] != '\0' && nameunit != NULL &&
+	    strcmp(nameunit, ice_fail_device) == 0);
+}
+
+#endif /* DRIVER_FAILPOINTS */
+
 /*
  * Helper function prototypes
  */

sys/dev/ice/ice_opts.h

@@ -43,5 +43,6 @@
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_rss.h"
+#include "opt_driver_failpoints.h"
 
 #endif

sys/modules/ice/Makefile

@@ -8,6 +8,7 @@ SRCS	+= irdma_di_if.h irdma_if.h
 
 # Option headers
 SRCS    += opt_inet.h opt_inet6.h opt_rss.h opt_iflib.h
+SRCS    += opt_driver_failpoints.h
 
 # Core source
 SRCS    += ice_lib.c ice_osdep.c ice_resmgr.c ice_strings.c