cpuctl, x86: claim IBS NMIs orphaned by control MSR writes

A cpuctl(4) write that stops or reprograms IBS can clear the valid bit
of a sample whose NMI is already raised.  hwpmc then declines the NMI,
no other handler claims it, and with the default machdep.panic_on_nmi
the system panics.

Have nmi_handle_intr() claim, after all handlers declined it, one NMI
per cpuctl write to an IBS control MSR.  Claiming last, rather than
from a cpuctl NMI handler, cannot take an NMI away from hwpmc.

cpuctl.ko now needs the new kernel symbols; an MFC has to update both.

Reproduced on an EPYC 9654 by toggling IBS op through /dev/cpuctlN:
about one unclaimed NMI per 2000 toggles.

Reviewed by:	kib
MFC after:	2 weeks
Sponsored by:	AMD
Signed-off-by: Osvaldo Janeri Filho <[email protected]>
Differential Revision:	https://reviews.freebsd.org/D60765

sys/dev/cpuctl/cpuctl.c

@@ -54,6 +54,7 @@
 #include <machine/cpufunc.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
+#include <x86/cputypes.h>
 #include <x86/ucode.h>
 
 static d_open_t cpuctl_open;
@@ -245,6 +246,39 @@ cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data, struct thread *td)
 	return (error);
 }
 
+static bool cpuctl_ibs_present;
+
+static void
+cpuctl_ibs_detect(void)
+{
+	cpuctl_ibs_present = (cpu_vendor_id == CPU_VENDOR_AMD ||
+	    cpu_vendor_id == CPU_VENDOR_HYGON) &&
+	    (amd_feature2 & AMDID2_IBS) != 0;
+}
+
+/* Let nmi_handle_intr() claim an IBS NMI orphaned by the write. */
+static int
+cpuctl_wrmsr(u_int msr, uint64_t val)
+{
+	int ret;
+
+	MPASS(curthread->td_critnest > 0);
+	if (!cpuctl_ibs_present)
+		return (wrmsr_safe(msr, val));
+	switch (msr) {
+	case MSR_AMD_IBS_FETCH_CTL:
+	case MSR_AMD_IBS_OP_CTL:
+	case MSR_AMD_IBS_OP_CTL2:
+	case MSR_AMD_IBS_FETCH_CTL2:
+		x86_ibs_ctl_write_begin();
+		ret = wrmsr_safe(msr, val);
+		x86_ibs_ctl_write_end();
+		return (ret);
+	default:
+		return (wrmsr_safe(msr, val));
+	}
+}
+
 /*
  * Actually perform MSR operations.
  */
@@ -276,18 +310,20 @@ cpuctl_do_msr(int cpu, cpuctl_msr_args_t *data, u_long cmd, struct thread *td)
 		data->data = 0;
 		ret = rdmsr_safe(data->msr, &data->data);
 	} else if (cmd == CPUCTL_WRMSR) {
-		ret = wrmsr_safe(data->msr, data->data);
+		critical_enter();
+		ret = cpuctl_wrmsr(data->msr, data->data);
+		critical_exit();
 	} else if (cmd == CPUCTL_MSRSBIT) {
 		critical_enter();
 		ret = rdmsr_safe(data->msr, &reg);
 		if (ret == 0)
-			ret = wrmsr_safe(data->msr, reg | data->data);
+			ret = cpuctl_wrmsr(data->msr, reg | data->data);
 		critical_exit();
 	} else if (cmd == CPUCTL_MSRCBIT) {
 		critical_enter();
 		ret = rdmsr_safe(data->msr, &reg);
 		if (ret == 0)
-			ret = wrmsr_safe(data->msr, reg & ~data->data);
+			ret = cpuctl_wrmsr(data->msr, reg & ~data->data);
 		critical_exit();
 	} else
 		panic("[cpuctl,%d]: unknown operation requested: %lu",
@@ -574,6 +610,7 @@ cpuctl_modevent(module_t mod __unused, int type, void *data __unused)
 
 	switch(type) {
 	case MOD_LOAD:
+		cpuctl_ibs_detect();
 		if (bootverbose)
 			printf("cpuctl: access to MSR registers/cpuid info.\n");
 		cpuctl_devs = malloc(sizeof(*cpuctl_devs) * (mp_maxid + 1), M_CPUCTL,

sys/x86/include/specialreg.h

@@ -1317,6 +1317,10 @@
 #define	MSR_LS_CFG	0xc0011020
 #define	MSR_IC_CFG	0xc0011021	/* Instruction Cache Configuration */
 #define	MSR_DE_CFG	0xc0011029	/* Decode Configuration */
+#define	MSR_AMD_IBS_FETCH_CTL	0xc0011030
+#define	MSR_AMD_IBS_OP_CTL	0xc0011033
+#define	MSR_AMD_IBS_OP_CTL2	0xc001103e
+#define	MSR_AMD_IBS_FETCH_CTL2	0xc001103f
 
 /* MSR_AMDK8_IPM */
 #define	AMDK8_SMIONCMPHALT	(1ULL << 27)

sys/x86/include/x86_var.h

@@ -156,6 +156,8 @@ void	nmi_call_kdb_smp(u_int type, struct trapframe *frame);
 void	nmi_register_handler(int (*handler)(struct trapframe *));
 void	nmi_remove_handler(int (*handler)(struct trapframe *));
 void	nmi_handle_intr(struct trapframe *frame);
+void	x86_ibs_ctl_write_begin(void);
+void	x86_ibs_ctl_write_end(void);
 void	pagecopy(void *from, void *to);
 void	printcpuinfo(void);
 int	pti_get_default(void);

sys/x86/x86/cpu_machdep.c

@@ -1073,6 +1073,60 @@ nmi_remove_handler(int (*handler)(struct trapframe *))
 	    __func__, handler);
 }
 
+/*
+ * A cpuctl(4) write to an IBS control MSR can clear the valid bit of a
+ * sample whose NMI is already raised, so no handler claims that NMI.
+ * Claim one such NMI per write if it arrives during the write or within
+ * IBS_CTL_WRITE_WINDOW_US after it; on Zen 4 it arrives within 2us.
+ */
+#define	IBS_CTL_WRITE_WINDOW_US	20
+
+#define	IBS_CTL_WRITE_IDLE	0
+#define	IBS_CTL_WRITE_BUSY	1
+#define	IBS_CTL_WRITE_DONE	2
+
+DPCPU_DEFINE_STATIC(u_int, ibs_ctl_write_state);
+DPCPU_DEFINE_STATIC(uint64_t, ibs_ctl_write_deadline);
+
+static u_long ibs_ctl_write_nmis;
+SYSCTL_ULONG(_machdep, OID_AUTO, ibs_ctl_write_nmis, CTLFLAG_RD,
+    &ibs_ctl_write_nmis, 0,
+    "NMIs claimed after an IBS control MSR write through cpuctl(4)");
+
+void
+x86_ibs_ctl_write_begin(void)
+{
+	MPASS(curthread->td_critnest > 0);
+	DPCPU_SET(ibs_ctl_write_state, IBS_CTL_WRITE_BUSY);
+}
+
+void
+x86_ibs_ctl_write_end(void)
+{
+	MPASS(curthread->td_critnest > 0);
+	DPCPU_SET(ibs_ctl_write_deadline, rdtsc() +
+	    tsc_freq / (1000000 / IBS_CTL_WRITE_WINDOW_US));
+	/* An NMI taken during the write may have used the claim already. */
+	atomic_cmpset_int(DPCPU_PTR(ibs_ctl_write_state), IBS_CTL_WRITE_BUSY,
+	    IBS_CTL_WRITE_DONE);
+}
+
+static bool
+x86_ibs_ctl_write_nmi(void)
+{
+	u_int state;
+
+	state = DPCPU_GET(ibs_ctl_write_state);
+	if (state == IBS_CTL_WRITE_IDLE)
+		return (false);
+	DPCPU_SET(ibs_ctl_write_state, IBS_CTL_WRITE_IDLE);
+	if (state == IBS_CTL_WRITE_DONE &&
+	    rdtsc() > DPCPU_GET(ibs_ctl_write_deadline))
+		return (false);
+	atomic_add_long(&ibs_ctl_write_nmis, 1);
+	return (true);
+}
+
 void
 nmi_handle_intr(struct trapframe *frame)
 {
@@ -1105,6 +1159,8 @@ nmi_handle_intr(struct trapframe *frame)
 	}
 	if (handled)
 		return;
+	if (x86_ibs_ctl_write_nmi())
+		return;
 #ifdef SMP
 	if (nmi_is_broadcast) {
 		nmi_call_kdb_smp(T_NMI, frame);