diff --git a/sys/dev/iavf/iavf_lib.c b/sys/dev/iavf/iavf_lib.c index 01d14a58f0b9..f4f14f799f7d 100644 --- a/sys/dev/iavf/iavf_lib.c +++ b/sys/dev/iavf/iavf_lib.c @@ -315,8 +315,11 @@ iavf_reset(struct iavf_sc *sc) int error = 0; /* Ask the PF to reset us if we are initiating */ - if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) - iavf_request_reset(sc); + if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) { + error = iavf_request_reset(sc); + if (error != 0) + return (error); + } iavf_msec_pause(100); error = iavf_reset_complete(hw); @@ -325,6 +328,12 @@ iavf_reset(struct iavf_sc *sc) __func__); return (error); } + /* + * The completed reset stopped the old queues. Record that fact so + * stop and initialization need not issue DISABLE_QUEUES again. + */ + iavf_set_state(&sc->state, IAVF_STATE_RESET_RECOVERY); + atomic_store_rel_32(&sc->queues_enabled, 0); pci_enable_busmaster(dev); error = iavf_shutdown_adminq(hw); @@ -1488,8 +1497,9 @@ iavf_mark_del_vlan_filter(struct iavf_sc *sc, u16 vtag) * to IAVF_MAX_DIS_Q_RETRY times if the response says that it wasn't * successful. This is intended to workaround a bug that can appear on the PF. * - * @returns zero on success, or an error code if the request could not be sent - * or acknowledged. + * @returns zero on success, EBUSY when the PF replied but did not confirm + * that the queues were disabled, or another error if the request could not + * be sent or acknowledged. */ int iavf_disable_queues_with_retries(struct iavf_sc *sc) @@ -1498,9 +1508,9 @@ iavf_disable_queues_with_retries(struct iavf_sc *sc) int max_attempts = IAVF_MAX_DIS_Q_RETRY; int error = 0, msg_count = 0; - /* While the driver is detaching, it doesn't care if the queue - * disable finishes successfully or not. Just send one message - * to just notify the PF driver. + /* + * The detach path cannot wait for a reply. Send one request and let + * the caller attempt a VF reset if the queues remain unconfirmed. */ if (in_detach) max_attempts = 1; @@ -1519,13 +1529,13 @@ iavf_disable_queues_with_retries(struct iavf_sc *sc) iavf_dbg_vc(sc, "DISABLE_QUEUES messages sent: %d\n", msg_count); - if (!in_detach && msg_count >= max_attempts && + if (msg_count >= max_attempts && atomic_load_acq_32(&sc->queues_enabled)) { - if (iavf_mbx_log_allowed(sc)) + if (!in_detach && iavf_mbx_log_allowed(sc)) device_printf(sc->dev, "%s: DISABLE_QUEUES may have failed\n", __func__); if (error == 0) - error = EIO; + error = EBUSY; } return (error); } diff --git a/sys/dev/iavf/iavf_lib.h b/sys/dev/iavf/iavf_lib.h index c56e55b555a0..fd707051aef8 100644 --- a/sys/dev/iavf/iavf_lib.h +++ b/sys/dev/iavf/iavf_lib.h @@ -313,6 +313,8 @@ enum iavf_state { IAVF_STATE_INITIALIZED, IAVF_STATE_RESET_REQUIRED, IAVF_STATE_RESET_PENDING, + IAVF_STATE_RESET_RECOVERY, + IAVF_STATE_VC_POLLING, IAVF_STATE_RUNNING, /* This entry must be last */ IAVF_STATE_LAST, diff --git a/sys/dev/iavf/iavf_vc_common.c b/sys/dev/iavf/iavf_vc_common.c index 2a11f02db59b..94fdb402bc30 100644 --- a/sys/dev/iavf/iavf_vc_common.c +++ b/sys/dev/iavf/iavf_vc_common.c @@ -638,19 +638,32 @@ iavf_del_ether_filters(struct iavf_sc *sc) * * Request that the PF reset this VF. No response is expected. * - * @returns zero + * @returns zero on success, or an error code if the request could not be + * submitted. */ int iavf_request_reset(struct iavf_sc *sc) { + struct iavf_hw *hw; + u32 old_rstat; + int error; + + hw = &sc->hw; + old_rstat = rd32(hw, IAVF_VFGEN_RSTAT) & + IAVF_VFGEN_RSTAT_VFR_STATE_MASK; /* - ** Set the reset status to "in progress" before - ** the request, this avoids any possibility of - ** a mistaken early detection of completion. - */ - wr32(&sc->hw, IAVF_VFGEN_RSTAT, VIRTCHNL_VFR_INPROGRESS); - iavf_send_pf_msg(sc, VIRTCHNL_OP_RESET_VF, NULL, 0); - return (0); + * The VF must publish INPROGRESS before asking the PF to reset it. This + * also prevents a stale VFACTIVE value from looking like completion. + */ + wr32(hw, IAVF_VFGEN_RSTAT, VIRTCHNL_VFR_INPROGRESS); + iavf_flush(hw); + error = iavf_send_pf_msg(sc, VIRTCHNL_OP_RESET_VF, NULL, 0); + if (error != 0) { + /* No reset was submitted; do not leave a synthetic reset behind. */ + wr32(hw, IAVF_VFGEN_RSTAT, old_rstat); + iavf_flush(hw); + } + return (error); } /** diff --git a/sys/dev/iavf/if_iavf_iflib.c b/sys/dev/iavf/if_iavf_iflib.c index e761ac86c9b0..5b077a4bdfac 100644 --- a/sys/dev/iavf/if_iavf_iflib.c +++ b/sys/dev/iavf/if_iavf_iflib.c @@ -56,6 +56,7 @@ static const struct timeval iavf_mbx_log_interval = { 60, 0 }; #define IAVF_MBX_RECOVERY_ASQ_RETRIES 10 #define IAVF_MBX_RECOVERY_VERSION_RETRIES 3 #define IAVF_MBX_RECOVERY_CONFIG_RETRIES 10 +#define IAVF_QUEUE_RESET_TIMEOUT (60 * SBT_1S) /********************************************************************* * Function prototypes @@ -92,6 +93,7 @@ static void iavf_if_stop(if_ctx_t ctx); static bool iavf_if_needs_restart(if_ctx_t, enum iflib_restart_event); static void iavf_mbx_lost(struct iavf_sc *); +static void iavf_enter_reset_recovery(struct iavf_sc *); static void iavf_mbx_retry_detach(struct iavf_sc *); static void iavf_mbx_retry_failed(if_ctx_t); static void iavf_mbx_retry_prepare(struct iavf_sc *); @@ -99,6 +101,8 @@ static void iavf_mbx_retry_stop(struct iavf_sc *); static void iavf_mbx_retry_succeeded(struct iavf_sc *); static int iavf_reestablish_vc(struct iavf_sc *); static void iavf_replay_filters(struct iavf_sc *); +static void iavf_hold_vc_polling(struct iavf_sc *); +static int iavf_quiesce_queues(struct iavf_sc *); static int iavf_wait_asq(struct iavf_sc *, u32); static int iavf_allocate_pci_resources(struct iavf_sc *); @@ -502,7 +506,7 @@ iavf_setup_vc_tq(struct iavf_sc *sc) static int iavf_if_attach_post(if_ctx_t ctx) { -#ifdef IXL_DEBUG +#ifdef IAVF_DEBUG device_t dev = iflib_get_dev(ctx); #endif struct iavf_sc *sc; @@ -529,6 +533,12 @@ iavf_if_attach_post(if_ctx_t ctx) bzero(&sc->vsi.eth_stats, sizeof(struct iavf_eth_stats)); iavf_add_device_sysctls(sc); + /* + * attach_pre completed the initial reset and resource handshake. Do + * not carry the runtime reset-recovery state into iflib's first + * stop/init cycle. + */ + iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY); atomic_store_rel_32(&sc->queues_enabled, 0); atomic_store_rel_32(&sc->mbx_ready, 1); atomic_store_rel_32(&sc->vc_reinit_required, 0); @@ -822,6 +832,135 @@ iavf_mbx_retry_stop(struct iavf_sc *sc) sc->mbx_retry_stage = 0; } +static void +iavf_enter_reset_recovery(struct iavf_sc *sc) +{ + + iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING); + iavf_set_state(&sc->state, IAVF_STATE_RESET_RECOVERY); +} + +/* + * Record that a reset has stopped the old LAN queues and invalidated the + * current virtchnl connection. The next initialization must rediscover the + * VF resources before it can rebuild and enable the LAN queues. + */ +static void +iavf_reset_queues_stopped(struct iavf_sc *sc) +{ + + iavf_enter_reset_recovery(sc); + atomic_store_rel_32(&sc->queues_enabled, 0); + atomic_store_rel_32(&sc->mbx_ready, 0); + atomic_store_rel_32(&sc->vc_reinit_required, 1); +} + +/* Give synchronous reset/discovery code exclusive ownership of the AdminQ. */ +static void +iavf_hold_vc_polling(struct iavf_sc *sc) +{ + + iavf_set_state(&sc->state, IAVF_STATE_VC_POLLING); + iavf_disable_adminq_irq(&sc->hw); + taskqueue_drain(sc->vc_tq, &sc->vc_task); + /* A task already running when interrupts were masked can re-enable it. */ + iavf_disable_adminq_irq(&sc->hw); +} + +/* + * Wait for a reset which has already started. A completed reset proves the + * old queues are stopped; a timeout does not establish quiescence. + * Allow longer than the retryable attach wait: E830/E825C PFs delay 20 seconds + * during EMPR before checking completion and rebuilding VF state. + */ +static int +iavf_wait_for_queue_reset(struct iavf_sc *sc) +{ + sbintime_t deadline; + + deadline = sbinuptime() + IAVF_QUEUE_RESET_TIMEOUT; + while (!iavf_reset_is_complete(&sc->hw)) { + if (sbinuptime() >= deadline) + return (EBUSY); + iavf_msec_pause(100); + } + iavf_reset_queues_stopped(sc); + return (EAGAIN); +} + +/* + * Attempt to quiesce the hardware before queue resources are reused. + * Success means the queues are stopped and the current AdminQ is still + * usable. EAGAIN means a reset stopped the queues but virtchnl must be + * rediscovered. Other errors do not prove that queue DMA has stopped. + */ +static int +iavf_quiesce_queues(struct iavf_sc *sc) +{ + struct iavf_hw *hw; + bool asq_alive; + int error; + + hw = &sc->hw; + if (iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) { + if (!iavf_reset_is_complete(hw)) { + iavf_hold_vc_polling(sc); + return (iavf_wait_for_queue_reset(sc)); + } + atomic_store_rel_32(&sc->queues_enabled, 0); + return (EAGAIN); + } + if (!iavf_reset_is_complete(hw)) { + iavf_enter_reset_recovery(sc); + iavf_hold_vc_polling(sc); + return (iavf_wait_for_queue_reset(sc)); + } + if (atomic_load_acq_32(&sc->queues_enabled) == 0) + return (0); + + asq_alive = iavf_check_asq_alive(hw); + if (!asq_alive) { + /* + * A dead AdminQ does not prove that LAN DMA stopped. Rebuild + * the virtchnl connection so queue disable or VF reset remains + * available before iflib releases the queue resources. + */ + error = iavf_reestablish_vc(sc); + if (error == 0) { + /* A PF reset may have discarded the VF's filters. */ + atomic_store_rel_32(&sc->vc_reinit_required, 1); + atomic_store_rel_32(&sc->mbx_ready, 1); + asq_alive = true; + } + } + if (atomic_load_acq_32(&sc->mbx_ready) != 0 && asq_alive) + error = iavf_disable_queues_with_retries(sc); + else if (asq_alive) + error = ENXIO; + if (atomic_load_acq_32(&sc->queues_enabled) == 0) + return (0); + + /* The PF might have started its advertised reset while we waited. */ + if (!iavf_reset_is_complete(hw)) { + iavf_enter_reset_recovery(sc); + iavf_hold_vc_polling(sc); + return (iavf_wait_for_queue_reset(sc)); + } + + if (iavf_mbx_log_allowed(sc)) + device_printf(sc->dev, + "Queue disable was not confirmed (%d); requesting a VF reset\n", + error); + iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING); + iavf_hold_vc_polling(sc); + error = iavf_request_reset(sc); + if (error != 0) + return (error); + /* RESET_VF has no reply; allow the PF to begin VFR before polling. */ + iavf_msec_pause(100); + return (iavf_wait_for_queue_reset(sc)); +} + static void iavf_mbx_retry_failed(if_ctx_t ctx) { @@ -873,6 +1012,8 @@ iavf_mbx_retry_succeeded(struct iavf_sc *sc) sc->mbx_last_log.tv_usec = 0; iavf_clear_state(&sc->state, IAVF_STATE_RESET_REQUIRED); iavf_clear_state(&sc->state, IAVF_STATE_RESET_PENDING); + iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY); + iavf_clear_state(&sc->state, IAVF_STATE_VC_POLLING); if (recovered) device_printf(sc->dev, "PF mailbox communication restored\n"); } @@ -927,10 +1068,9 @@ iavf_reestablish_vc(struct iavf_sc *sc) hw = &sc->hw; vsi = &sc->vsi; - iavf_disable_adminq_irq(hw); - taskqueue_drain(sc->vc_tq, &sc->vc_task); - /* A task already running when interrupts were masked can re-enable it. */ - iavf_disable_adminq_irq(hw); + /* Keep the ordinary AdminQ task from consuming polled replies. */ + iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING); + iavf_hold_vc_polling(sc); pci_enable_busmaster(sc->dev); status = iavf_shutdown_adminq(hw); @@ -980,12 +1120,18 @@ iavf_reestablish_vc(struct iavf_sc *sc) } /* - * RESET_PENDING prevents the ordinary AdminQ task from consuming - * messages while the queue may still belong to the pre-reset device. - * The successful VERSION and GET_VF_RESOURCES exchange above proves - * that the reset has completed and this is the replacement AdminQ. - * Clear the stale indication before normal virtchnl requests resume. + * VC_POLLING prevents the ordinary AdminQ task from consuming the + * replies above. The successful exchange proves that the reset has + * completed and this is the replacement AdminQ. + * If a reset stopped the old queues, consume that fact now that the + * replacement AdminQ is established. This permits ordinary AdminQ + * processing while making the following quiesce check a no-op. */ + if (iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) { + atomic_store_rel_32(&sc->queues_enabled, 0); + iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY); + } + iavf_clear_state(&sc->state, IAVF_STATE_VC_POLLING); iavf_clear_state(&sc->state, IAVF_STATE_RESET_PENDING); iavf_enable_adminq_irq(hw); return (0); @@ -1042,6 +1188,7 @@ iavf_if_init(if_ctx_t ctx) replay_filters = atomic_load_acq_32(&sc->vc_reinit_required) != 0; if (!iavf_reset_is_complete(hw)) { + iavf_enter_reset_recovery(sc); atomic_store_rel_32(&sc->vc_reinit_required, 1); if (iavf_mbx_log_allowed(sc)) device_printf(dev, @@ -1063,10 +1210,12 @@ iavf_if_init(if_ctx_t ctx) } } - /* Make sure queues are disabled */ - error = iavf_disable_queues_with_retries(sc); - if (error != 0) - goto fail; + if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) { + /* Make sure queues are disabled. */ + error = iavf_quiesce_queues(sc); + if (error != 0) + goto fail; + } bcopy(if_getlladdr(ifp), tmpaddr, ETHER_ADDR_LEN); if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && @@ -1141,6 +1290,7 @@ iavf_rss_query_status(struct iavf_sc *sc) if (!iavf_test_state(&sc->state, IAVF_STATE_RUNNING) || iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING) || iavf_test_state(&sc->state, IAVF_STATE_RESET_REQUIRED) || + iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY) || atomic_load_acq_32(&sc->mbx_ready) == 0 || atomic_load_acq_32(&sc->vc_reinit_required) != 0 || sc->rss_configured != (IAVF_FLAG_AQ_CONFIG_RSS_KEY | @@ -1599,9 +1749,10 @@ iavf_process_adminq(struct iavf_sc *sc, u16 *pending) int error = 0, loop = 0; u32 reg; - if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) { - status = IAVF_ERR_ADMIN_QUEUE_ERROR; - goto reenable_interrupt; + if (iavf_test_state(&sc->state, IAVF_STATE_VC_POLLING) || + iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) { + *pending = 0; + return (IAVF_ERR_ADMIN_QUEUE_ERROR); } error = iavf_check_aq_errors(sc); @@ -1783,6 +1934,8 @@ iavf_if_timer(if_ctx_t ctx, uint16_t qid) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_hw *hw = &sc->hw; + bool asq_alive; + bool reset_active; u32 val; if (qid != 0) @@ -1791,16 +1944,25 @@ iavf_if_timer(if_ctx_t ctx, uint16_t qid) /* Check for a PF-triggered VF reset or a dead admin send queue. */ val = rd32(hw, IAVF_VFGEN_RSTAT) & IAVF_VFGEN_RSTAT_VFR_STATE_MASK; - if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING) || - !iavf_check_asq_alive(hw) || - (val != VIRTCHNL_VFR_VFACTIVE && - val != VIRTCHNL_VFR_COMPLETED)) { + reset_active = val != VIRTCHNL_VFR_VFACTIVE && + val != VIRTCHNL_VFR_COMPLETED; + asq_alive = iavf_check_asq_alive(hw); + if (reset_active) + iavf_enter_reset_recovery(sc); + if (!asq_alive || reset_active) { iavf_dbg_info(sc, "PF mailbox unavailable (reset state %d)\n", val); iavf_mbx_lost(sc); return; } + /* + * RESET_IMPENDING is only a warning. Preserve the running queues until + * RSTAT proves that the PF started the reset. + */ + if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) + return; + /* Fire off the adminq task */ iflib_admin_intr_deferred(ctx); @@ -2208,6 +2370,7 @@ iavf_stop(struct iavf_sc *sc) { struct iavf_vsi *vsi; bool mailbox_ready; + int error; vsi = &sc->vsi; iavf_mbx_retry_stop(sc); @@ -2216,12 +2379,16 @@ iavf_stop(struct iavf_sc *sc) iavf_disable_intr(vsi); - mailbox_ready = atomic_load_acq_32(&sc->mbx_ready) != 0; - if (mailbox_ready && iavf_reset_is_complete(&sc->hw) && - iavf_disable_queues_with_retries(sc) != 0) - mailbox_ready = false; - atomic_store_rel_32(&sc->mbx_ready, 0); + error = iavf_quiesce_queues(sc); + /* IFDI_STOP cannot report failure; failed shutdown remains best effort. */ + if (error != 0 && error != EAGAIN && iavf_mbx_log_allowed(sc)) + device_printf(sc->dev, + "Unable to confirm VF queue shutdown: %d\n", error); + mailbox_ready = error == 0 && + atomic_load_acq_32(&sc->mbx_ready) != 0 && + !iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY); if (!mailbox_ready) { + atomic_store_rel_32(&sc->mbx_ready, 0); atomic_store_rel_32(&sc->vc_reinit_required, 1); iavf_dbg_vc(sc, "PF mailbox unavailable while stopping\n"); }