nvme: Add reservation, format and features command building support

Add command (struct nvme_command) building functions for the four
NVMe reservation opcodes (Report, Register, Acquire and Release),
Format NVM, and Get Features / Set Features, along with the
enumerations and data structures that go with them.  These are
shared between the kernel and userland, and are the underpinnings
for new camcontrol(8) subcommands (nvres, nvformat, nvfeature) that
will be added separately.

The reservation type names are intentionally similar to the SCSI
names for the same reservation types.

sys/dev/nvme/nvme.h:
  Add NVMe command building functions for the four NVMe
  reservation opcodes: nvme_ns_resv_report_cmd(),
  nvme_ns_resv_register_cmd(), nvme_ns_resv_acquire_cmd() and
  nvme_ns_resv_release_cmd().

  Add enumerated types for the various NVMe reservation
  subcommand actions (acquire, register, release), reservation
  types, and Change Persist Through Power Loss State values.

  Add struct nvme_resv_acq_data and struct nvme_resv_reg_data,
  the data payloads for reservation Acquire and Register.

  Add a new NVMe Format NVM command building function,
  nvme_admin_format_cmd(), and an enumeration for the Secure
  Erase Setting parameter for format.

  Add nvme_admin_get_feature_cmd() and
  nvme_admin_set_feature_cmd() functions to fill a struct
  nvme_command for Get Features / Set Features.

  Add macros for bitfields in the ctratt and mic dwords of the
  NVMe Identify Controller data structure.

  Add a prototype for nvme_hostid_sbuf().

  Stop byte swapping the hostid fields in
  nvme_resv_status_swapbytes() and
  nvme_resv_status_ext_swapbytes(), and document the fields as
  little endian.  The hostid is an opaque 8 or 16 byte value;
  keep it in the byte order the device returns it in instead of
  interpreting it at this level.

  Widen the nvme_ctrlr_cmd_get_feature() and
  nvme_ctrlr_cmd_set_feature() prototypes as described below.

  While here, fix the entry count clamp in the reservation status
  swapbytes functions to use sizeof(*s) rather than sizeof(s); using
  the pointer size made the clamp one entry too permissive.

sys/dev/nvme/nvme_util.c:
  Add nvme_hostid_sbuf(), which formats an 8 or 16 byte hostid
  (from the Get Features hostid feature, in the little-endian
  byte order the device returns it in) into an sbuf as an
  integer value in base 16, most significant byte first.

sbin/nvmecontrol/resv.c:
  Decode the now little-endian hostid fields explicitly when
  printing, and zero pad the printed 64-bit values.  Previously
  the extended hostid printed in a different byte order on big
  and little endian machines, and without zero padding between
  the two halves.

sys/dev/nvme/nvme_ctrlr_cmd.c:
  Refactor nvme_ctrlr_cmd_get_feature() and
  nvme_ctrlr_cmd_set_feature() to use the new command building
  functions from nvme.h and accept a namespace ID parameter.

  Refactor nvme_ctrlr_cmd_set_feature() to also accept the sv
  (save) parameter to persist a feature value across reset.

sys/dev/nvme/nvme_ctrlr.c:
  Update the temperature threshold, autonomous power state
  transition and host memory buffer callers for the new
  nvme_ctrlr_cmd_get_feature() and nvme_ctrlr_cmd_set_feature()
  arguments.

Co-authored-by:	Reid Linnemann <[email protected]>
Co-authored-by:	Dave Baukus <[email protected]>
Co-authored-by:	Claude Fable 5 <[email protected]>
Reviewed by:	imp
MFC after:	1 week
Sponsored by:	Spectra Logic
Differential Revision:	https://reviews.freebsd.org/D59531

sbin/nvmecontrol/resv.c

@@ -417,8 +417,9 @@ resvreport(const struct cmd *f, int argc, char *argv[])
 			    e->ctrlr[i].rcsts);
 			printf("  Reservation Key:                0x%08jx\n",
 			    e->ctrlr[i].rkey);
-			printf("  Host Identifier:                0x%08jx%08jx\n",
-			    e->ctrlr[i].hostid[0], e->ctrlr[i].hostid[1]);
+			printf("  Host Identifier:                0x%016jx%016jx\n",
+			    le64toh(e->ctrlr[i].hostid[1]),
+			    le64toh(e->ctrlr[i].hostid[0]));
 		}
 	} else {
 		n = MIN(n, (sizeof(data) - sizeof(s)) / sizeof(s->ctrlr[0]));
@@ -427,8 +428,8 @@ resvreport(const struct cmd *f, int argc, char *argv[])
 			    s->ctrlr[i].ctrlr_id);
 			printf("  Reservation Status:             %u\n",
 			    s->ctrlr[i].rcsts);
-			printf("  Host Identifier:                0x%08jx\n",
-			    s->ctrlr[i].hostid);
+			printf("  Host Identifier:                0x%016jx\n",
+			    le64toh(s->ctrlr[i].hostid));
 			printf("  Reservation Key:                0x%08jx\n",
 			    s->ctrlr[i].rkey);
 		}

sys/dev/nvme/nvme.h

@@ -1205,6 +1205,68 @@ _Static_assert(sizeof(struct nvme_timestamp_data) == 8,
 #define NVME_TIMESTAMP_ORIGIN_RESET			(0x0)
 #define NVME_TIMESTAMP_ORIGIN_SET_FEATURES		(0x1)
 
+enum nvme_format_ses {
+	NVME_FORMAT_SES_NONE			= 0x00,
+	NVME_FORMAT_SES_USER			= 0x01,
+	NVME_FORMAT_SES_CRYPTO			= 0x02
+};
+
+/*
+ * Ignore Existing Key
+ */
+#define NVME_RESV_IEKEY_MASK 0x01
+
+/*
+ * Actions for Reservation Acquire
+ */
+#define NVME_RESV_ACQUIRE_MASK 0x07
+enum nvme_resv_acquire_action {
+	NVME_RESV_ACQUIRE_ACQUIRE		= 0x00,
+	NVME_RESV_ACQUIRE_PREEMPT		= 0x01,
+	NVME_RESV_ACQUIRE_PREEMPT_ABT		= 0x02
+};
+
+/*
+ * Reservation types for Reservation Acquire.
+ */
+enum nvme_resv_type {
+	NVME_RESV_TYPE_WR_EX			= 0x01,
+	NVME_RESV_TYPE_EX_AC			= 0x02,
+	NVME_RESV_TYPE_WR_EX_RO			= 0x03,
+	NVME_RESV_TYPE_EX_AC_RO			= 0x04,
+	NVME_RESV_TYPE_WR_EX_AR			= 0x05,
+	NVME_RESV_TYPE_EX_AC_AR			= 0x06
+};
+
+
+/*
+ * Values for Reservation Register Change Persist Through Power Loss State
+ */
+#define NVME_RESV_CPTPL_MASK 0x03
+enum nvme_resv_cptpl {
+	NVME_RESV_CPTPL_NOCHANGE		= 0x00,
+	NVME_RESV_CPTPL_0			= 0x02,
+	NVME_RESV_CPTPL_1			= 0x03
+};
+
+/*
+ * Actions for Reservation Register
+ */
+#define NVME_RESV_REG_MASK 0x07
+enum nvme_resv_register_action {
+	NVME_RESV_REG_REGISTER			= 0x00,
+	NVME_RESV_REG_UNREGISTER		= 0x01,
+	NVME_RESV_REG_REPLACE			= 0x02
+};
+
+/*
+ * Actions for Reservation Release
+ */
+enum nvme_resv_release_action {
+	NVME_RESV_REL_RELEASE			= 0x00,
+	NVME_RESV_REL_CLEAR			= 0x01
+};
+
 struct nvme_power_state {
 	/** Maximum Power */
 	uint16_t	mp;			/* Maximum Power */
@@ -1259,6 +1321,10 @@ struct nvme_controller_data {
 	uint8_t			ieee[3];
 
 	/** multi-interface capabilities */
+#define NVME_CD_MIC_MPORT	0x01
+#define NVME_CD_MIC_MCTRL	0x02
+#define NVME_CD_MIC_SRIOV	0x04
+#define NVME_CD_MIC_ANAR	0x08
 	uint8_t			mic;
 
 	/** maximum data transfer size */
@@ -1280,7 +1346,17 @@ struct nvme_controller_data {
 	uint32_t		oaes;	/* bitfield really */
 
 	/** Controller Attributes */
-	uint32_t		ctratt;	/* bitfield really */
+#define NVME_CD_CTRATT_EXHID	0x001
+#define NVME_CD_CTRATT_NOPSPM	0x002
+#define NVME_CD_CTRATT_NVMSETS	0x004
+#define NVME_CD_CTRATT_RRLVLS	0x008
+#define NVME_CD_CTRATT_ENDGRPS	0x010
+#define NVME_CD_CTRATT_PREDLM	0x020
+#define NVME_CD_CTRATT_TBKAS	0x040
+#define NVME_CD_CTRATT_NSGRAN	0x080
+#define NVME_CD_CTRATT_SQASSOC	0x100
+#define NVME_CD_CTRATT_UUIDLIST	0x200
+	uint32_t		ctratt;
 
 	/** Read Recovery Levels Supported */
 	uint16_t		rrls;
@@ -1865,11 +1941,25 @@ struct intel_log_temp_stats {
 
 _Static_assert(sizeof(struct intel_log_temp_stats) == 13 * 8, "bad size for intel_log_temp_stats");
 
+struct nvme_resv_acq_data {
+	uint64_t	crkey;
+	uint64_t	prkey;
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_acq_data) == 2 * 8, "bad size for nvme_resv_acq_data");
+
+struct nvme_resv_reg_data {
+	uint64_t	crkey;
+	uint64_t	nrkey;
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_reg_data) == 2 * 8, "bad size for nvme_resv_reg_data");
+
 struct nvme_resv_reg_ctrlr {
 	uint16_t		ctrlr_id;	/* Controller ID */
 	uint8_t			rcsts;		/* Reservation Status */
 	uint8_t			reserved3[5];
-	uint64_t		hostid;		/* Host Identifier */
+	uint64_t		hostid;		/* Host Identifier (little endian) */
 	uint64_t		rkey;		/* Reservation Key */
 } __packed __aligned(4);
 
@@ -1880,7 +1970,7 @@ struct nvme_resv_reg_ctrlr_ext {
 	uint8_t			rcsts;		/* Reservation Status */
 	uint8_t			reserved3[5];
 	uint64_t		rkey;		/* Reservation Key */
-	uint64_t		hostid[2];	/* Host Identifier */
+	uint64_t		hostid[2];	/* Host Identifier (little endian) */
 	uint8_t			reserved32[32];
 } __packed __aligned(4);
 
@@ -1998,6 +2088,7 @@ struct nvme_hmb_desc {
 	(NVME_STATUS_GET_SC((cpl)->status) != 0 || NVME_STATUS_GET_SCT((cpl)->status) != 0)
 
 void	nvme_cpl_sbuf(const struct nvme_completion *cpl, struct sbuf *sbuf);
+void	nvme_hostid_sbuf(struct sbuf *sb, const uint8_t *hostid, bool exhid);
 void	nvme_opcode_sbuf(bool admin, uint8_t opc, struct sbuf *sb);
 void	nvme_sc_sbuf(const struct nvme_completion *cpl, struct sbuf *sbuf);
 void	nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen);
@@ -2031,13 +2122,13 @@ int	nvme_ctrlr_linux_passthru_cmd(struct nvme_controller *ctrlr,
 				      bool is_admin);
 
 /* Admin functions */
-void	nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
-				   uint8_t feature, uint32_t cdw11,
+void	nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint32_t nsid,
+				   int sv, uint8_t feature, uint32_t cdw11,
 				   uint32_t cdw12, uint32_t cdw13,
 				   uint32_t cdw14, uint32_t cdw15,
 				   void *payload, uint32_t payload_size,
 				   nvme_cb_fn_t cb_fn, void *cb_arg);
-void	nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
+void	nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint32_t nsid,
 				   uint8_t feature, uint32_t cdw11,
 				   void *payload, uint32_t payload_size,
 				   nvme_cb_fn_t cb_fn, void *cb_arg);
@@ -2177,6 +2268,83 @@ nvme_ns_data_format_index(const struct nvme_namespace_data *nsdata)
 	return (fmt);
 }
 
+static inline
+void	nvme_ns_resv_report_cmd(struct nvme_command *cmd, uint32_t nsid,
+    uint32_t length, uint32_t report_type)
+{
+	cmd->opc = NVME_OPC_RESERVATION_REPORT;
+	cmd->nsid = htole32(nsid);
+	cmd->cdw10 = htole32(length);
+	cmd->cdw11 = htole32(report_type);
+}
+
+static inline
+void	nvme_ns_resv_register_cmd(struct nvme_command *cmd, uint32_t nsid,
+    int cptpl, int iekey, int rrega)
+{
+	cmd->opc = NVME_OPC_RESERVATION_REGISTER;
+	cmd->nsid = htole32(nsid);
+	cmd->cdw10 = htole32((rrega & NVME_RESV_REG_MASK) | ((iekey & NVME_RESV_IEKEY_MASK) << 3) |
+	    ((cptpl & NVME_RESV_CPTPL_MASK) << 30));
+}
+
+static inline
+void	nvme_ns_resv_acquire_cmd(struct nvme_command *cmd, uint32_t nsid,
+     int rtype, int iekey, int racqa)
+{
+	cmd->opc = NVME_OPC_RESERVATION_ACQUIRE;
+	cmd->nsid = htole32(nsid);
+	cmd->cdw10 = htole32(((rtype & 0x0f) << 8) | ((iekey & NVME_RESV_IEKEY_MASK) << 3) |
+	    (racqa & NVME_RESV_ACQUIRE_MASK));
+}
+
+static inline
+void	nvme_ns_resv_release_cmd(struct nvme_command *cmd, uint32_t nsid,
+     int rtype, int iekey, int rrela)
+{
+	cmd->opc = NVME_OPC_RESERVATION_RELEASE;
+	cmd->nsid = htole32(nsid);
+	cmd->cdw10 = htole32(((rtype & 0x0f) << 8) | ((iekey & NVME_RESV_IEKEY_MASK) << 3) |
+	    (rrela & 0x07));
+}
+
+
+static inline
+void	nvme_admin_format_cmd(struct nvme_command *cmd, uint32_t nsid,
+    int ses, int pil, int pi, int mset, int lbaf)
+{
+	cmd->opc = NVME_OPC_FORMAT_NVM;
+	cmd->nsid = htole32(nsid);
+	cmd->cdw10 = htole32((ses << 9) | (pil << 8) | (pi << 5) | (mset << 4) |
+			      lbaf);
+}
+
+static inline
+void	nvme_admin_get_feature_cmd(struct nvme_command *cmd, uint32_t nsid,
+    int sel, uint8_t feature, uint32_t cdw11, int uuid_idx) {
+	cmd->opc = NVME_OPC_GET_FEATURES;
+	cmd->nsid = nsid;
+	cmd->cdw10 = htole32(((sel & 0x7) << 8) | feature);
+	cmd->cdw11 = htole32(cdw11);
+	cmd->cdw14 = htole32(uuid_idx & 0x7f);
+}
+
+
+static inline
+void	nvme_admin_set_feature_cmd(struct nvme_command *cmd, uint32_t nsid,
+    int sv, uint8_t feature, uint32_t cdw11, uint32_t cdw12,
+    uint32_t cdw13, uint32_t cdw14, uint32_t cdw15)
+{
+	cmd->opc = NVME_OPC_SET_FEATURES;
+	cmd->nsid = nsid;
+	cmd->cdw10 = htole32(((sv & 0x1) << 31) | feature);
+	cmd->cdw11 = htole32(cdw11);
+	cmd->cdw12 = htole32(cdw12);
+	cmd->cdw13 = htole32(cdw13);
+	cmd->cdw14 = htole32(cdw14);
+	cmd->cdw15 = htole32(cdw15);
+}
+
 /* Endianess conversion functions for NVMe structs */
 static inline
 void	nvme_completion_swapbytes(struct nvme_completion *s __unused)
@@ -2413,10 +2581,9 @@ void	nvme_resv_status_swapbytes(struct nvme_resv_status *s __unused,
 
 	s->gen = le32toh(s->gen);
 	n = (s->regctl[1] << 8) | s->regctl[0];
-	n = MIN(n, (size - sizeof(s)) / sizeof(s->ctrlr[0]));
+	n = MIN(n, (size - sizeof(*s)) / sizeof(s->ctrlr[0]));
 	for (i = 0; i < n; i++) {
 		s->ctrlr[i].ctrlr_id = le16toh(s->ctrlr[i].ctrlr_id);
-		s->ctrlr[i].hostid = le64toh(s->ctrlr[i].hostid);
 		s->ctrlr[i].rkey = le64toh(s->ctrlr[i].rkey);
 	}
 #endif
@@ -2431,11 +2598,10 @@ void	nvme_resv_status_ext_swapbytes(struct nvme_resv_status_ext *s __unused,
 
 	s->gen = le32toh(s->gen);
 	n = (s->regctl[1] << 8) | s->regctl[0];
-	n = MIN(n, (size - sizeof(s)) / sizeof(s->ctrlr[0]));
+	n = MIN(n, (size - sizeof(*s)) / sizeof(s->ctrlr[0]));
 	for (i = 0; i < n; i++) {
 		s->ctrlr[i].ctrlr_id = le16toh(s->ctrlr[i].ctrlr_id);
 		s->ctrlr[i].rkey = le64toh(s->ctrlr[i].rkey);
-		nvme_le128toh((void *)s->ctrlr[i].hostid);
 	}
 #endif
 }

sys/dev/nvme/nvme_ctrlr.c

@@ -852,7 +852,7 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
 			NVME_ASYNC_EVENT_FW_ACTIVATE);
 
 	status.done = 0;
-	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
+	nvme_ctrlr_cmd_get_feature(ctrlr, 0, NVME_FEAT_TEMPERATURE_THRESHOLD,
 	    0, NULL, 0, nvme_completion_poll_cb, &status);
 	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl) ||
@@ -895,7 +895,7 @@ nvme_ctrlr_configure_apst(struct nvme_controller *ctrlr)
 			data[i] = htole64(data[i]);
 	} else {
 		status.done = 0;
-		nvme_ctrlr_cmd_get_feature(ctrlr,
+		nvme_ctrlr_cmd_get_feature(ctrlr, 0,
 		    NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION, 0,
 		    data, data_size, nvme_completion_poll_cb, &status);
 		nvme_completion_poll(&status);
@@ -904,7 +904,7 @@ nvme_ctrlr_configure_apst(struct nvme_controller *ctrlr)
 	}
 
 	status.done = 0;
-	nvme_ctrlr_cmd_set_feature(ctrlr,
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0,
 	    NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION, enable, 0, 0,
 	    0, 0, data, data_size, nvme_completion_poll_cb, &status);
 	nvme_completion_poll(&status);
@@ -932,8 +932,8 @@ nvme_ctrlr_set_timestamp(struct nvme_controller *ctrlr)
 		ts->tstmp[i] = (ms >> (i * 8)) & 0xff;
 
 	status.done = 0;
-	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_TIMESTAMP, 0, 0, 0, 0, 0,
-	    ts, sizeof(*ts), nvme_completion_poll_cb, &status);
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0, NVME_FEAT_TIMESTAMP, 0, 0, 0,
+	    0, 0, ts, sizeof(*ts), nvme_completion_poll_cb, &status);
 	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl) && bootverbose)
 		nvme_printf(ctrlr, "failed to set the timestamp\n");
@@ -1179,11 +1179,10 @@ nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)
 	if (memret)
 		cdw11 |= 2;
 	status.done = 0;
-	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,
-	    ctrlr->hmb_nchunks * ctrlr->hmb_chunk / ctrlr->page_size,
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0, NVME_FEAT_HOST_MEMORY_BUFFER,
+	    cdw11, ctrlr->hmb_nchunks * ctrlr->hmb_chunk / 4096,
 	    ctrlr->hmb_desc_paddr, ctrlr->hmb_desc_paddr >> 32,
-	    ctrlr->hmb_nchunks, NULL, 0,
-	    nvme_completion_poll_cb, &status);
+	    ctrlr->hmb_nchunks, NULL, 0, nvme_completion_poll_cb, &status);
 	nvme_completion_poll(&status);
 	if (nvme_completion_is_error(&status.cpl))
 		nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");

sys/dev/nvme/nvme_ctrlr_cmd.c

@@ -158,9 +158,9 @@ nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
 }
 
 void
-nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
-    uint32_t cdw11, uint32_t cdw12, uint32_t cdw13, uint32_t cdw14,
-    uint32_t cdw15, void *payload, uint32_t payload_size,
+nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint32_t nsid, int sv,
+    uint8_t feature, uint32_t cdw11, uint32_t cdw12, uint32_t cdw13,
+    uint32_t cdw14, uint32_t cdw15, void *payload, uint32_t payload_size,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
@@ -173,20 +173,15 @@ nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
 		req = nvme_allocate_request_null(M_WAITOK, cb_fn, cb_arg);
 
 	cmd = &req->cmd;
-	cmd->opc = NVME_OPC_SET_FEATURES;
-	cmd->cdw10 = htole32(feature);
-	cmd->cdw11 = htole32(cdw11);
-	cmd->cdw12 = htole32(cdw12);
-	cmd->cdw13 = htole32(cdw13);
-	cmd->cdw14 = htole32(cdw14);
-	cmd->cdw15 = htole32(cdw15);
+	nvme_admin_set_feature_cmd(cmd, nsid, sv, feature, cdw11, cdw12, cdw13,
+	    cdw14, cdw15);
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
-nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
-    uint32_t cdw11, void *payload, uint32_t payload_size,
+nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint32_t nsid,
+    uint8_t feature, uint32_t cdw11, void *payload, uint32_t payload_size,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
@@ -199,9 +194,7 @@ nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
 		req = nvme_allocate_request_null(M_WAITOK, cb_fn, cb_arg);
 
 	cmd = &req->cmd;
-	cmd->opc = NVME_OPC_GET_FEATURES;
-	cmd->cdw10 = htole32(feature);
-	cmd->cdw11 = htole32(cdw11);
+	nvme_admin_get_feature_cmd(cmd, nsid, 0, feature, cdw11, 0);
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
@@ -213,8 +206,8 @@ nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
 	uint32_t cdw11;
 
 	cdw11 = ((num_queues - 1) << 16) | (num_queues - 1);
-	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_NUMBER_OF_QUEUES, cdw11,
-	    0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0, NVME_FEAT_NUMBER_OF_QUEUES,
+	    cdw11, 0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
 }
 
 void
@@ -224,7 +217,7 @@ nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
 	uint32_t cdw11;
 
 	cdw11 = state;
-	nvme_ctrlr_cmd_set_feature(ctrlr,
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0,
 	    NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, 0, 0, 0, 0, NULL, 0,
 	    cb_fn, cb_arg);
 }
@@ -250,8 +243,8 @@ nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
 	}
 
 	cdw11 = ((microseconds/100) << 8) | threshold;
-	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_INTERRUPT_COALESCING, cdw11,
-	    0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0, NVME_FEAT_INTERRUPT_COALESCING,
+	    cdw11, 0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
 }
 
 void
@@ -259,8 +252,8 @@ nvme_ctrlr_cmd_set_arbitration(struct nvme_controller *ctrlr,
     uint32_t cdw11, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 
-	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_ARBITRATION, cdw11,
-	    0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
+	nvme_ctrlr_cmd_set_feature(ctrlr, 0, 0, NVME_FEAT_ARBITRATION,
+	    cdw11, 0, 0, 0, 0, NULL, 0, cb_fn, cb_arg);
 }
 
 void

sys/dev/nvme/nvme_util.c

@@ -268,6 +268,21 @@ nvme_cpl_sbuf(const struct nvme_completion *cpl, struct sbuf *sb)
 		sbuf_printf(sb, " DNR");
 }
 
+/*
+ * Append an 8 or 16 byte hostid as an integer value in base 16 to an sbuf.
+ * The hostid is in the little-endian byte order the device returns it in,
+ * so print it most significant byte first.
+ */
+void
+nvme_hostid_sbuf(struct sbuf *sb, const uint8_t *hostid, bool exhid)
+{
+	int i;
+
+	sbuf_cat(sb, "0x");
+	for (i = (exhid ? 16 : 8) - 1; i >= 0; i--)
+		sbuf_printf(sb, "%02x", hostid[i]);
+}
+
 void
 nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen)
 {