camcontrol: Support NVMe namespace formatting via the format subcommand

Support formatting NVMe namespaces through the existing format
subcommand.  The format subcommand now dispatches on the device
protocol, in the same way the identify and persist subcommands do,
and issues the NVMe FORMAT NVM command for NVMe devices.

The SCSI syntax and behavior are unchanged.  The NVMe-specific
options are -e (secure erase setting), -G (format all namespaces),
-f (LBA format), -l (protection information location), -m (metadata
setting) and -p (protection information type).  The -q and -y options
work for both protocols; NVMe formats gain the same confirmation
question that SCSI formats have always had.  The -r and -w options
are SCSI only; -r returns an error for NVMe devices and -w has no
effect, since the NVMe format command does not complete until the
format is finished.

The default timeout for NVMe is 30 seconds for a cryptographic erase
and 30 minutes otherwise; the generic -t option overrides it.

sbin/camcontrol/nvformat.c:
  New file, the NVMe format backend for the format subcommand.

sbin/camcontrol/camcontrol.c:
  Dispatch the format subcommand to scsiformat() or
  nvmeformat() based on the device protocol.

sbin/camcontrol/camcontrol.h:
  Add the nvmeformat() prototype.

sbin/camcontrol/Makefile:
  Add nvformat.c to the build.

sbin/camcontrol/camcontrol.8:
  Document NVMe support in the format subcommand section, and
  note the differences from SCSI where they occur.

Co-authored-by:	Claude Fable 5 <[email protected]>
Reviewed by:	imp
MFC after:	1 week
Sponsored by:	Spectra Logic
Differential Revision:	https://reviews.freebsd.org/D59535

sbin/camcontrol/Makefile

@@ -18,7 +18,7 @@ CFLAGS+= -I${SRCTOP}/sbin/nvmecontrol
 SRCS+=	identify_ext.c
 SRCS+=	nc_util.c
 .PATH:	${SRCTOP}/sys/dev/nvme
-SRCS+=	nvme_util.c nvres.c
+SRCS+=	nvme_util.c nvres.c nvformat.c
 # This is verboten
 .if ${MACHINE_CPUARCH} == "arm"
 WARNS?= 3

sbin/camcontrol/camcontrol.8

@@ -223,6 +223,12 @@
 .Op Fl r
 .Op Fl w
 .Op Fl y
+.Op Fl e Ar ses
+.Op Fl G
+.Op Fl f Ar lbaf
+.Op Fl l Ar pil
+.Op Fl m Ar mset
+.Op Fl p Ar pi
 .Nm
 .Ic sanitize
 .Op device id
@@ -1229,7 +1235,17 @@ negotiation parameters will take effect.
 .It Ic format
 Issue the
 .Tn SCSI
-FORMAT UNIT command to the named device.
+FORMAT UNIT command or the
+.Tn NVMe
+FORMAT NVM command to the named device;
+.Nm
+issues the command appropriate for the device given.
+For
+.Tn NVMe
+devices, the format applies to the namespace addressed by the command,
+unless the
+.Fl G
+option is given.
 .Pp
 .Em WARNING! WARNING! WARNING!
 .Pp
@@ -1259,6 +1275,17 @@ This is often because the drive
 does not really support the FORMAT UNIT command -- it just accepts the
 command, waits a few minutes and then returns it.
 .Pp
+For
+.Tn NVMe
+devices, the default timeout is 30 seconds for a cryptographic erase, and
+30 minutes otherwise.
+Most
+.Tn NVMe
+devices format very quickly, but for drives that take longer, you can see
+format progress in the
+.Tn NVMe
+namespace Identify data if the drive supports it.
+.Pp
 The
 .Sq format
 subcommand takes several arguments that modify its default behavior.
@@ -1280,6 +1307,9 @@ Run in
 .Dq report only
 mode.
 This will report status on a format that is already running on the drive.
+Not supported for
+.Tn NVMe
+devices.
 .It Fl w
 Issue a non-immediate format command.
 By default,
@@ -1301,6 +1331,10 @@ argument is specified,
 will issue a non-immediate format command, and will be unable to print any
 information to let the user know what percentage of the disk has been
 formatted.
+For
+.Tn NVMe
+devices, the format command does not complete until the format is
+finished, so this option has no effect.
 .It Fl y
 Do not ask any questions.
 By default,
@@ -1310,6 +1344,85 @@ and also if the default format command timeout is acceptable.
 The user
 will not be asked about the timeout if a timeout is specified on the
 command line.
+For
+.Tn NVMe
+devices, only the first question is asked.
+.It Fl e Ar ses
+Specify the
+.Tn NVMe
+Secure Erase Setting.
+Only supported for
+.Tn NVMe
+devices.
+.Bl -tag -width 8n
+.It none
+This is the default.
+It indicates that no specific erase option is requested.
+.It user
+This requests that all user data be erased.
+The controller can use any method to erase the user data.
+.It crypto
+This requests that the controller erase the data cryptographically by
+deleting the encryption key.
+.It num
+Specify a numeric (decimal or hexadecimal) value
+.El
+.It Fl G
+Some
+.Tn NVMe
+controllers only allow formatting all namespaces attached to the
+controller.
+By default,
+.Nm
+will format the
+.Tn NVMe
+namespace addressed by the command.
+If you get an error message saying that the controller doesn't support
+per-NS format, you can try this option.
+Only supported for
+.Tn NVMe
+devices.
+.It Fl f Ar lbaf
+Use the specified LBA format number.
+Only supported for
+.Tn NVMe
+devices.
+You can use this option to change the sector size of the drive.
+To see a list of allowed LBA formats (combination of sector size and
+metadata size), and the current LBA format, look at
+the NVMe Namespace Identify data.
+You can see it with:
+.Pp
+.Dl camcontrol identify ndaX -N
+.It Fl l Ar pil
+Specify the protection information location.
+Only supported for
+.Tn NVMe
+devices.
+If set to 1, and protection information is enabled (see the
+.Fl p
+option), protection information is transferred as the first 8 bytes of
+metadata.
+If set to 0 (the default) and protection information is enabled, protection
+information is transferred as the last 8 bytes of metadata.
+.It Fl m Ar mset
+If set to 1, metadata is transferred as part of an extended LBA.
+If set to 0 (the default), metadata is transferred as part of a separate
+buffer.
+Only supported for
+.Tn NVMe
+devices.
+.It Fl p Ar pi
+This specifies whether protection information is enabled, and if so, the
+type of protection information.
+Only supported for
+.Tn NVMe
+devices.
+The default is 0, no protection information enabled.
+You may specify Type 1, 2 or 3 protection with a numeric argument of 1, 2
+or 3 respectively.
+Other types are allowed up to 7, but the only values currently defined in
+the NVMe 1.4 specification are 0-3.
 .El
 .It Ic sanitize
 Issue the SANITIZE command to the named device.

sbin/camcontrol/camcontrol.c

@@ -216,7 +216,7 @@ static struct camcontrol_opts option_table[] = {
 	{"negotiate", CAM_CMD_RATE, CAM_ARG_NONE, negotiate_opts},
 	{"rate", CAM_CMD_RATE, CAM_ARG_NONE, negotiate_opts},
 	{"debug", CAM_CMD_DEBUG, CAM_ARG_NONE, "IPTSXcp"},
-	{"format", CAM_CMD_FORMAT, CAM_ARG_NONE, "qrwy"},
+	{"format", CAM_CMD_FORMAT, CAM_ARG_NONE, "e:f:Gl:m:p:qrwy"},
 	{"sanitize", CAM_CMD_SANITIZE, CAM_ARG_NONE, "a:c:IP:qrUwy"},
 	{"idle", CAM_CMD_IDLE, CAM_ARG_NONE, "t:"},
 	{"standby", CAM_CMD_STANDBY, CAM_ARG_NONE, "t:"},
@@ -10124,6 +10124,8 @@ usage(int printlong)
 "                              [-q][-R syncrate][-v][-T <enable|disable>]\n"
 "                              [-U][-W bus_width]\n"
 "        camcontrol format     [dev_id][generic args][-q][-r][-w][-y]\n"
+"                              [-e <user|crypto|num>][-G][-f format][-l pil]\n"
+"                              [-m mset][-p pi]\n"
 "        camcontrol sanitize   [dev_id][generic args]\n"
 "                              [-a overwrite|block|crypto|exitfailure]\n"
 "                              [-c passes][-I][-P pattern][-q][-U][-r][-w]\n"
@@ -10199,7 +10201,7 @@ usage(int printlong)
 "debug       turn debugging on/off for a bus, target, or lun, or all devices\n"
 "tags        report or set the number of transaction slots for a device\n"
 "negotiate   report or set device negotiation parameters\n"
-"format      send the SCSI FORMAT UNIT command to the named device\n"
+"format      send the SCSI FORMAT UNIT or NVMe FORMAT NVM command\n"
 "sanitize    send the SCSI SANITIZE command to the named device\n"
 "idle        send the ATA IDLE command to the named device\n"
 "standby     send the ATA STANDBY command to the named device\n"
@@ -10309,9 +10311,15 @@ usage(int printlong)
 "-v                also print a Path Inquiry CCB for the controller\n"
 "format arguments:\n"
 "-q                be quiet, don't print status messages\n"
-"-r                run in report only mode\n"
-"-w                don't send immediate format command\n"
+"-r                run in report only mode (SCSI only)\n"
+"-w                don't send immediate format command (SCSI only)\n"
 "-y                don't ask any questions\n"
+"-e ses            secure erase setting: user, crypto (NVMe only)\n"
+"-G                global format, formats all Namespaces (NVMe only)\n"
+"-f num            specify format number, see Namespace Identify (NVMe only)\n"
+"-l pil            Protection Information Location, 1 = start, 0 = end (NVMe)\n"
+"-m mset           metadata setting, 0 = separate buffer, 1 ext. LBA (NVMe)\n"
+"-p pi             Protection Information type (NVMe only)\n"
 "sanitize arguments:\n"
 "-a operation      operation mode: overwrite, block, crypto or exitfailure\n"
 "-c passes         overwrite passes to perform (1 to 31)\n"
@@ -10789,11 +10797,25 @@ main(int argc, char **argv)
 		error = ratecontrol(cam_dev, task_attr, retry_count,
 				    timeout, argc, argv, combinedopt);
 		break;
-	case CAM_CMD_FORMAT:
-		error = scsiformat(cam_dev, argc, argv,
-				   combinedopt, task_attr, retry_count,
-				   timeout);
+	case CAM_CMD_FORMAT: {
+		struct ccb_pathinq cpi;
+
+		if (get_cpi(cam_dev, &cpi) != 0) {
+			warnx("couldn't get CPI");
+			error = 1;
+			break;
+		}
+		if (cpi.protocol == PROTO_NVME)
+			error = nvmeformat(cam_dev, argc, argv, combinedopt,
+			    retry_count, timeout,
+			    arglist & CAM_ARG_VERBOSE,
+			    arglist & CAM_ARG_ERR_RECOVER);
+		else
+			error = scsiformat(cam_dev, argc, argv,
+					   combinedopt, task_attr, retry_count,
+					   timeout);
 		break;
+	}
 	case CAM_CMD_REPORTLUNS:
 		error = scsireportluns(cam_dev, argc, argv,
 				       combinedopt, task_attr,

sbin/camcontrol/camcontrol.h

@@ -80,6 +80,9 @@ int nvme_get_data(struct cam_device *dev, int nvme_data_type, void *data,
 int nvmepersist(struct cam_device *device, int argc, char **argv,
 		char *combinedopt, int retry_count, int timeout,
 		int verbosemode, int err_recover);
+int nvmeformat(struct cam_device *device, int argc, char **argv,
+	       char *combinedopt, int retry_count, int timeout,
+	       int verbosemode, int err_recover);
 int camxferrate(struct cam_device *device);
 int fwdownload(struct cam_device *device, int argc, char **argv,
 	       char *combinedopt, int printerrors, int task_attr,

sbin/camcontrol/nvformat.c

@@ -0,0 +1,313 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (C) 2018-2019 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (C) 2021 Spectra Logic Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*
+ * NVMe format support for camcontrol(8).
+ */
+
+#include <sys/ioctl.h>
+#include <sys/stdint.h>
+#include <sys/types.h>
+#include <sys/endian.h>
+#include <sys/sbuf.h>
+#include <sys/queue.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <limits.h>
+#include <err.h>
+
+#include <cam/cam.h>
+#include <cam/cam_debug.h>
+#include <cam/cam_ccb.h>
+#include <cam/scsi/scsi_all.h>
+#include <cam/scsi/scsi_pass.h>
+#include <dev/nvme/nvme.h>
+#include <camlib.h>
+#include "camcontrol.h"
+
+static struct scsi_nv nvme_ses_settings[] = {
+	{ "none", NVME_FORMAT_SES_NONE },
+	{ "user", NVME_FORMAT_SES_USER },
+	{ "crypto", NVME_FORMAT_SES_CRYPTO },
+};
+
+int
+nvmeformat(struct cam_device *device, int argc, char **argv, char *combinedopt,
+	   int retry_count, int timeout, int verbosemode, int err_recover)
+{
+	union ccb *ccb = NULL;
+	struct nvme_controller_data cdata;
+	struct nvme_namespace_data nsdata;
+	char *end;
+	int global_op = 0;
+	int lbaf = -1, mset = -1, pi = -1, pil = -1, ses = -1;
+	int quiet = 0, ycount = 0;
+	int error = 0;
+	int c;
+
+	ccb = cam_getccb(device);
+	if (ccb == NULL) {
+		warnx("%s: error allocating CCB", __func__);
+		error = 1;
+		goto bailout;
+	}
+
+	CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->nvmeio);
+
+	while ((c = getopt(argc, argv, combinedopt)) != -1) {
+		switch (c) {
+		case 'e': {
+			scsi_nv_status status;
+			int table_entry = 0;
+
+			status = scsi_get_nv(nvme_ses_settings,
+			    sizeof(nvme_ses_settings) /
+			    sizeof(nvme_ses_settings[0]), optarg,
+			    &table_entry, SCSI_NV_FLAG_IG_CASE);
+			if (status == SCSI_NV_FOUND)
+				ses = nvme_ses_settings[table_entry].value;
+			else if (status == SCSI_NV_AMBIGUOUS) {
+				warnx("%s: ambiguous Secure Erase Setting %s",
+				    __func__, optarg);
+				error = 1;
+				goto bailout;
+			} else {
+				ses = strtol(optarg, &end, 0);
+				if ((*end != '\0')
+				 || ((ses < 0) || (ses > 0x7))) {
+					warnx("%s: invalid Secure Erase "
+					    "setting %s", __func__, optarg);
+					error = 1;
+					goto bailout;
+				}
+			}
+			break;
+		}
+		case 'G':
+			global_op = 1;
+			break;
+		case 'f':
+			lbaf = strtol(optarg, &end, 0);
+			if (*end != '\0') {
+				warnx("%s: invalid format argument %s",
+				    __func__, optarg);
+				error = 1;
+				goto bailout;
+			}
+			break;
+		case 'l':
+			pil = strtol(optarg, &end, 0);
+			if ((*end != '\0') || ((pil < 0) || (pil > 1))) {
+				warnx("%s: invalid location argument %s",
+				    __func__, optarg);
+				error = 1;
+				goto bailout;
+			}
+			break;
+		case 'm':
+			mset = strtol(optarg, &end, 0);
+			if ((*end != '\0') || ((mset < 0) || (mset > 1))) {
+				warnx("%s: invalid metadata setting %s",
+				    __func__, optarg);
+				error = 1;
+				goto bailout;
+			}
+			break;
+		case 'p':
+			pi = strtol(optarg, &end, 0);
+			if ((*end != '\0') || ((pi < 0) || (pi > 0x7))) {
+				warnx("%s: invalid Protection Information "
+				    "type %s", __func__, optarg);
+				error = 1;
+				goto bailout;
+			}
+			break;
+		case 'q':
+			quiet = 1;
+			break;
+		case 'r':
+			warnx("%s: option -r is not supported for NVMe "
+			    "devices", __func__);
+			error = 1;
+			goto bailout;
+		case 'w':
+			/*
+			 * The NVMe format command does not complete until
+			 * the format is finished, so there is nothing extra
+			 * to wait for.
+			 */
+			break;
+		case 'y':
+			ycount++;
+			break;
+		}
+	}
+
+	bzero(&cdata, sizeof(cdata));
+	error = nvme_get_data(device, CDAI_TYPE_NVME_CNTRL, &cdata,
+	    sizeof(cdata));
+	if (error != 0)
+		goto bailout;
+
+	if (((cdata.oacs >> NVME_CTRLR_DATA_OACS_FORMAT_SHIFT) &
+	    NVME_CTRLR_DATA_OACS_FORMAT_MASK) == 0) {
+		warnx("%s: controller doesn't support format", __func__);
+		error = 1;
+		goto bailout;
+	}
+
+	if ((((cdata.fna >> NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_SHIFT) &
+	    NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_MASK) == 0) && (ses == 2)){
+		warnx("%s: controller doesn't support crypto erase", __func__);
+		error = 1;
+		goto bailout;
+	}
+
+	if (global_op == 0) {
+
+		if ((cdata.fna >> NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT) &
+		     NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK) {
+			warnx("%s: controller does not support per-NS format, "
+			    "use -G", __func__);
+			error = 1;
+			goto bailout;
+		}
+                if ((cdata.fna >> NVME_CTRLR_DATA_FNA_ERASE_ALL_SHIFT) &
+                    NVME_CTRLR_DATA_FNA_ERASE_ALL_MASK) {
+                        warnx("%s: controller does not support per-NS erase, "
+			    "use -G", __func__);
+			error = 1;
+			goto bailout;
+		}
+
+		bzero(&nsdata, sizeof(nsdata));
+		error = nvme_get_data(device, CDAI_TYPE_NVME_NS, &nsdata,
+		    sizeof(nsdata));
+		if (error != 0)
+			goto bailout;
+
+		if (lbaf < 0)
+			lbaf = nvme_ns_data_format_index(&nsdata);
+
+		if (lbaf > nsdata.nlbaf) {
+			warnx("%s: LBA format %d is out of range", __func__,
+			    lbaf);
+			error = 1;
+			goto bailout;
+		}
+		if (mset < 0) {
+			mset = (nsdata.flbas >>
+			    NVME_NS_DATA_FLBAS_EXTENDED_SHIFT)
+			    & NVME_NS_DATA_FLBAS_EXTENDED_MASK;
+		}
+		if (pi < 0) {
+			pi = (nsdata.dps >> NVME_NS_DATA_DPS_MD_START_SHIFT)
+			    & NVME_NS_DATA_DPS_MD_START_MASK;
+		}
+		if (pil < 0) {
+			pil = (nsdata.dps >> NVME_NS_DATA_DPS_PIT_SHIFT) &
+			    NVME_NS_DATA_DPS_PIT_MASK;
+		}
+		if (ses < 0)
+			ses = 0;
+	} else {
+		if (ses < 0)
+			ses = 0;
+		if (lbaf < 0)
+			lbaf = 0;
+		if (mset < 0)
+			mset = 0;
+		if (pi < 0)
+			pi = 0;
+		if (pil < 0)
+			pil = 0;
+	}
+
+	if (quiet == 0 && ycount == 0) {
+		char product[41];
+
+		nvme_strvis((uint8_t *)product, cdata.mn, sizeof(product),
+		    sizeof(cdata.mn));
+		fprintf(stdout, "You are about to REMOVE ALL DATA from %s on "
+		    "the following device:\n<%s>\n", (global_op != 0) ?
+		    "ALL NAMESPACES" : "the namespace", product);
+	}
+
+	if (ycount == 0) {
+		if (!get_confirmation()) {
+			error = 1;
+			goto bailout;
+		}
+	}
+
+	nvme_format(/*nvmeio*/ &ccb->nvmeio,
+		    /*retries*/ retry_count,
+		    /*cbfcnp*/ NULL,
+		    /*nsid*/ global_op ? NVME_GLOBAL_NAMESPACE_TAG :
+					 device->target_lun,
+		    /*ses*/ ses,
+		    /*pil*/ pil,
+		    /*pi*/ pi,
+		    /*mset*/ mset,
+		    /*lbaf*/ lbaf,
+		    /*
+		     * A cryptographic erase just changes the media
+		     * encryption key, so it should be quick.  A user data
+		     * erase or a plain format can take a long time.
+		     */
+		    /*timeout*/ timeout ? timeout :
+		    (ses == NVME_FORMAT_SES_CRYPTO ?
+		    (30 * 1000) : (30 * 60 * 1000)));
+
+	if (err_recover != 0)
+		ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
+
+	if (cam_send_ccb(device, ccb) < 0) {
+		warn("error sending NVMe Format command");
+		error = 1;
+		goto bailout;
+	}
+
+	if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+		if (verbosemode != 0) {
+			cam_error_print(device, ccb, CAM_ESF_ALL,
+					CAM_EPF_ALL, stderr);
+		}
+		error = 1;
+		goto bailout;
+	}
+
+bailout:
+	return (error);
+}