ixgbe: Quiesce VFs across PF reset

Stop VF transmit and receive in hardware, clear PF-side mailbox CTS,
and notify active VFs before resetting a PF. A PF reset invalidates VF
queue state, so the no-CTS control message makes cooperative VFs
discard stale state and renegotiate after the PF returns.

The hardware queue gates synchronously prevent further VF DMA. Do not
hold the exclusive iflib context lock for a fixed VF-watchdog interval
after the reset. Report the PF link transition directly instead of
dispatching mailbox work from the stop path, which could otherwise
re-enable VF I/O mid-reset.

The CTS, PF-control, and VF queue controls follow the reset mechanisms
used by DPDK.

MFC after:	2 weeks

sys/dev/ixgbe/if_ix.c

@@ -4757,8 +4757,10 @@ ixgbe_if_stop(if_ctx_t ctx)
 
 	INIT_DEBUGOUT("ixgbe_if_stop: begin\n");
 
-	if (sc->feat_en & IXGBE_FEATURE_SRIOV)
+	if (sc->feat_en & IXGBE_FEATURE_SRIOV) {
 		ixgbe_disable_mdd(hw);
+		ixgbe_quiesce_vfs(sc);
+	}
 	ixgbe_reset_hw(hw);
 	hw->adapter_stopped = false;
 	ixgbe_stop_adapter(hw);
@@ -4767,7 +4769,12 @@ ixgbe_if_stop(if_ctx_t ctx)
 
 	/* Update the stack */
 	sc->link_up = false;
-	ixgbe_if_update_admin_status(ctx);
+	if (sc->link_active) {
+		if (bootverbose)
+			device_printf(sc->dev, "Link is Down\n");
+		iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+		sc->link_active = false;
+	}
 
 	/* reprogram the RAR[0] in case user changed it. */
 	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);

sys/dev/ixgbe/if_sriov.c

@@ -283,6 +283,40 @@ ixgbe_ping_all_vfs(struct ixgbe_softc *sc)
 	}
 } /* ixgbe_ping_all_vfs */
 
+/*
+ * Stop VF DMA before resetting the PF.  A PF reset invalidates the VF queue
+ * state, so allowing an active VF to resume with its old rings can strand
+ * descriptors in both the VF and PF.  Clearing CTS makes a cooperative VF
+ * renegotiate its state after the PF comes back; it is deliberately separate
+ * from IXGBE_VF_IO_DISABLED, which records a persistent administrative or
+ * recovery decision.
+ */
+void
+ixgbe_quiesce_vfs(struct ixgbe_softc *sc)
+{
+	struct ixgbe_hw *hw;
+	struct ixgbe_vf *vf;
+	uint32_t index, mask, vfre, vfte;
+	int i;
+
+	hw = &sc->hw;
+	for (i = 0; i < sc->num_vfs; i++) {
+		vf = &sc->vfs[i];
+		if (!(vf->flags & IXGBE_VF_ACTIVE))
+			continue;
+
+		vf->flags &= ~IXGBE_VF_CTS;
+		index = IXGBE_VF_INDEX(vf->pool);
+		mask = IXGBE_VF_BIT(vf->pool);
+		vfte = IXGBE_READ_REG(hw, IXGBE_VFTE(index));
+		vfre = IXGBE_READ_REG(hw, IXGBE_VFRE(index));
+		IXGBE_WRITE_REG(hw, IXGBE_VFTE(index), vfte & ~mask);
+		IXGBE_WRITE_REG(hw, IXGBE_VFRE(index), vfre & ~mask);
+		ixgbe_send_vf_msg(hw, vf, IXGBE_PF_CONTROL_MSG);
+	}
+	IXGBE_WRITE_FLUSH(hw);
+} /* ixgbe_quiesce_vfs */
+
 
 static bool
 ixgbe_pf_owns_vlan(struct ixgbe_softc *sc, uint16_t tag)

sys/dev/ixgbe/ixgbe_sriov.h

@@ -89,6 +89,7 @@ int  ixgbe_iov_validate(struct ixgbe_softc *, u16);
 void ixgbe_if_iov_uninit(if_ctx_t);
 void ixgbe_initialize_iov(struct ixgbe_softc *);
 void ixgbe_activate_vfs(struct ixgbe_softc *);
+void ixgbe_quiesce_vfs(struct ixgbe_softc *);
 void ixgbe_recalculate_max_frame(struct ixgbe_softc *);
 void ixgbe_ping_all_vfs(struct ixgbe_softc *);
 void ixgbe_init_iov_recovery(struct ixgbe_softc *);
@@ -109,6 +110,7 @@ u32  ixgbe_get_mrqc(int);
 #define ixgbe_uninit_iov(_a)
 #define ixgbe_initialize_iov(_a)
 #define ixgbe_activate_vfs(_a)
+#define ixgbe_quiesce_vfs(_a)
 #define ixgbe_recalculate_max_frame(_a)
 #define ixgbe_ping_all_vfs(_a)
 #define ixgbe_init_iov_recovery(_a)