bhyve: Generalize the IPC thread

Move the code for the snapshotting IPC thread into a separate file
and define macros for adding new IPC commands.
No functional change intended.

Reviewed by:	rew
Differential Revision:	https://reviews.freebsd.org/D54650

usr.sbin/bhyve/Makefile

@@ -32,6 +32,7 @@ SRCS=	\
 	ctl_util.c		\
 	hda_codec.c		\
 	iov.c			\
+	ipc.c			\
 	mem.c			\
 	mevent.c		\
 	net_backend_netmap.c	\

usr.sbin/bhyve/aarch64/bhyverun_machdep.c

@@ -49,6 +49,7 @@
 #include "debug.h"
 #include "fdt.h"
 #include "mem.h"
+#include "ipc.h"
 #include "pci_emul.h"
 #include "pci_irq.h"
 #include "rtc_pl031.h"
@@ -87,6 +88,7 @@ bhyve_init_config(void)
 	set_config_bool("acpi_tables", false);
 	set_config_bool("acpi_tables_in_memory", false);
 	set_config_value("memory.size", "256M");
+	set_config_value("rundir", BHYVE_RUN_DIR);
 }
 
 void

usr.sbin/bhyve/amd64/bhyverun_machdep.c

@@ -46,6 +46,7 @@
 #include "e820.h"
 #include "fwctl.h"
 #include "ioapic.h"
+#include "ipc.h"
 #include "inout.h"
 #include "kernemu_dev.h"
 #include "mptbl.h"
@@ -68,9 +69,7 @@ bhyve_init_config(void)
 	set_config_bool("x86.strictmsr", true);
 	set_config_bool("x86.verbosemsr", false);
 	set_config_value("lpc.fwcfg", "bhyve");
-#ifdef BHYVE_SNAPSHOT
 	set_config_value("rundir", BHYVE_RUN_DIR);
-#endif
 }
 
 void

usr.sbin/bhyve/bhyverun.c

@@ -82,6 +82,7 @@
 #ifdef BHYVE_GDB
 #include "gdb.h"
 #endif
+#include "ipc.h"
 #include "mem.h"
 #include "mevent.h"
 #include "pci_emul.h"
@@ -1107,13 +1108,11 @@ main(int argc, char *argv[])
 	 */
 	setproctitle("%s", vmname);
 
-#ifdef BHYVE_SNAPSHOT
 	/*
-	 * checkpointing thread for communication with bhyvectl
+	 * Thread for handling bhyvectl commands.
 	 */
-	if (init_checkpoint_thread(ctx, get_config_value("rundir")) != 0)
-		errx(EX_OSERR, "Failed to start checkpoint thread");
-#endif
+	if (init_ipc_thread(ctx, get_config_value("rundir")) != 0)
+		errx(EX_OSERR, "Failed to start IPC thread");
 
 #ifndef WITHOUT_CAPSICUM
 	caph_cache_catpages();

usr.sbin/bhyve/ipc.c

@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2016 Flavius Anton
+ * Copyright (c) 2016 Mihai Tiganus
+ * Copyright (c) 2016-2019 Mihai Carabas
+ * Copyright (c) 2017-2019 Darius Mihai
+ * Copyright (c) 2017-2019 Elena Mihailescu
+ * Copyright (c) 2018-2019 Sergiu Weisz
+ * Copyright (c) 2025 Bojan Novković <bnovkov@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "ipc.h"
+
+#ifndef WITHOUT_CAPSICUM
+#include <sys/capsicum.h>
+#endif
+#include <sys/types.h>
+#include <sys/linker_set.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifndef WITHOUT_CAPSICUM
+#include <capsicum_helpers.h>
+#endif
+#include <sys/cpuset.h>
+#include <sys/ioctl.h>
+
+#include <machine/vmm.h>
+#include <machine/vmm_snapshot.h>
+
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <pthread.h>
+#include <pthread_np.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+#include <unistd.h>
+#include <vmmapi.h>
+
+#include "bhyverun.h"
+#include "debug.h"
+
+#define IPC_COMMAND_FOREACH(pvar, set) SET_FOREACH(pvar, set)
+
+SET_DECLARE(ipc_cmd_set, struct ipc_command);
+
+static struct ipc_thread_ctx {
+	struct vmctx *vmctx;
+	int sockfd;
+} thr_ctx;
+
+static nvlist_t *
+handle_message(struct vmctx *ctx, nvlist_t *nvl)
+{
+	const char *cmd;
+	struct ipc_command **ipc_cmd;
+
+	cmd = nvlist_get_string(nvl, "cmd");
+	IPC_COMMAND_FOREACH(ipc_cmd, ipc_cmd_set)
+	{
+		if (strcmp(cmd, (*ipc_cmd)->name) == 0)
+			return ((*ipc_cmd)->handler(ctx, nvl));
+	}
+
+	return (NULL);
+}
+
+/*
+ * Listen for commands from bhyvectl.
+ */
+static void *
+ipc_thread(void *param)
+{
+	int fd;
+	nvlist_t *nvl, *reply;
+	const char *cmdname;
+	struct ipc_thread_ctx *ctx;
+
+	pthread_set_name_np(pthread_self(), "IPC thread");
+	ctx = (struct ipc_thread_ctx *)param;
+	while ((fd = accept(ctx->sockfd, NULL, NULL)) != -1) {
+		nvl = nvlist_recv(fd, 0);
+		if (nvl == NULL) {
+			EPRINTLN("%s: nvlist_recv() failed: %s", __func__,
+			    strerror(errno));
+			close(fd);
+			continue;
+		}
+
+		cmdname = nvlist_get_string(nvl, "cmd");
+		if (cmdname == NULL) {
+			nvlist_add_string(nvl, "error", "missing command name");
+			nvlist_send(fd, nvl);
+			nvlist_destroy(nvl);
+			close(fd);
+			continue;
+		}
+		reply = handle_message(ctx->vmctx, nvl);
+		if (reply == NULL) {
+			reply = nvlist_create(0);
+			nvlist_add_stringf(reply, "error", "command '%s' not found",
+			    cmdname);
+		}
+		/* The handler should set the error message if need be. */
+		nvlist_send(fd, reply);
+
+		nvlist_destroy(nvl);
+		nvlist_destroy(reply);
+		close(fd);
+	}
+
+	return (NULL);
+}
+
+/*
+ * Create the listening socket for IPC with bhyvectl.
+ */
+int
+init_ipc_thread(struct vmctx *ctx, const char *bhyve_run_dir)
+{
+	struct sockaddr_un addr;
+	pthread_t ipc_pthread;
+	int socket_fd;
+	int err, ret;
+#ifndef WITHOUT_CAPSICUM
+	cap_rights_t rights;
+#endif
+
+	memset(&addr, 0, sizeof(addr));
+
+	socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+	if (socket_fd < 0) {
+		EPRINTLN("Socket creation failed: %s", strerror(errno));
+		err = -1;
+		goto fail;
+	}
+
+	addr.sun_family = AF_UNIX;
+
+	ret = snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s",
+	    bhyve_run_dir, vm_get_name(ctx));
+	if ((ret < 0) || ((size_t)ret >= sizeof(addr.sun_path))) {
+		EPRINTLN("%s: error setting socket path (%d)", __func__, ret);
+		err = -1;
+		goto fail;
+	}
+
+	addr.sun_len = SUN_LEN(&addr);
+	unlink(addr.sun_path);
+
+	if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
+		EPRINTLN("Failed to bind socket \"%s\": %s\n",
+		    addr.sun_path, strerror(errno));
+		err = -1;
+		goto fail;
+	}
+
+	if (listen(socket_fd, 10) < 0) {
+		EPRINTLN("ipc socket listen: %s\n", strerror(errno));
+		err = errno;
+		goto fail;
+	}
+
+#ifndef WITHOUT_CAPSICUM
+	cap_rights_init(&rights, CAP_ACCEPT, CAP_READ, CAP_RECV, CAP_WRITE,
+	    CAP_SEND, CAP_GETSOCKOPT);
+
+	if (caph_rights_limit(socket_fd, &rights) == -1)
+		errx(EX_OSERR, "Unable to apply rights for sandbox");
+#endif
+	memset(&thr_ctx, 0, sizeof(thr_ctx));
+	thr_ctx.vmctx = ctx;
+	thr_ctx.sockfd = socket_fd;
+
+	err = pthread_create(&ipc_pthread, NULL, ipc_thread,
+		&thr_ctx);
+	if (err != 0)
+		goto fail;
+
+	return (0);
+fail:
+	if (socket_fd > 0)
+		close(socket_fd);
+	unlink(addr.sun_path);
+
+	return (err);
+}

usr.sbin/bhyve/ipc.h

@@ -33,18 +33,19 @@
 #include <sys/linker_set.h>
 #include <sys/nv.h>
 
+#define BHYVE_RUN_DIR "/var/run/bhyve/"
+
+struct vmctx;
+
 struct ipc_command {
 	const char *name;
-	int (*handler)(struct vmctx *ctx, const nvlist_t *nvl);
+	nvlist_t *(*handler)(struct vmctx *ctx, const nvlist_t *nvl);
 };
 
-#define IPC_COMMAND(set, name, function)			\
-	static struct ipc_command name ## _ipc_command =	\
-	{ #name, function };					\
-	DATA_SET(set, name ## _ipc_command)
+#define IPC_COMMAND(name, function)                                         \
+	static struct ipc_command name##_ipc_command = { #name, function }; \
+	DATA_SET(ipc_cmd_set, name##_ipc_command)
 
-#define IPC_COMMAND_FOREACH(pvar, set)	SET_FOREACH(pvar, set)
-
-SET_DECLARE(ipc_cmd_set, struct ipc_command);
+int init_ipc_thread(struct vmctx *ctx, const char *bhyve_run_dir);
 
 #endif /* _IPC_H_ */

usr.sbin/bhyve/riscv/bhyverun_machdep.c

@@ -51,6 +51,7 @@
 #include "config.h"
 #include "debug.h"
 #include "fdt.h"
+#include "ipc.h"
 #include "mem.h"
 #include "pci_emul.h"
 #include "pci_irq.h"
@@ -82,6 +83,7 @@ bhyve_init_config(void)
 	set_config_bool("acpi_tables", false);
 	set_config_bool("acpi_tables_in_memory", false);
 	set_config_value("memory.size", "256M");
+	set_config_value("rundir", BHYVE_RUN_DIR);
 }
 
 void

usr.sbin/bhyve/snapshot.c

@@ -1331,56 +1331,13 @@ done:
 	return (error);
 }
 
-static int
-handle_message(struct vmctx *ctx, nvlist_t *nvl)
-{
-	const char *cmd;
-	struct ipc_command **ipc_cmd;
-
-	if (!nvlist_exists_string(nvl, "cmd"))
-		return (EINVAL);
-
-	cmd = nvlist_get_string(nvl, "cmd");
-	IPC_COMMAND_FOREACH(ipc_cmd, ipc_cmd_set) {
-		if (strcmp(cmd, (*ipc_cmd)->name) == 0)
-			return ((*ipc_cmd)->handler(ctx, nvl));
-	}
-
-	return (EOPNOTSUPP);
-}
-
-/*
- * Listen for commands from bhyvectl
- */
-void *
-checkpoint_thread(void *param)
-{
-	int fd;
-	struct checkpoint_thread_info *thread_info;
-	nvlist_t *nvl;
-
-	pthread_set_name_np(pthread_self(), "checkpoint thread");
-	thread_info = (struct checkpoint_thread_info *)param;
-
-	while ((fd = accept(thread_info->socket_fd, NULL, NULL)) != -1) {
-		nvl = nvlist_recv(fd, 0);
-		if (nvl != NULL)
-			handle_message(thread_info->ctx, nvl);
-		else
-			EPRINTLN("nvlist_recv() failed: %s", strerror(errno));
-
-		close(fd);
-		nvlist_destroy(nvl);
-	}
-
-	return (NULL);
-}
-
-static int
+static nvlist_t *
 vm_do_checkpoint(struct vmctx *ctx, const nvlist_t *nvl)
 {
 	int error;
+	nvlist_t *reply;
 
+	reply = nvlist_create(0);
 	if (!nvlist_exists_string(nvl, "filename") ||
 	    !nvlist_exists_bool(nvl, "suspend") ||
 	    !nvlist_exists_descriptor(nvl, "fddir"))
@@ -1390,86 +1347,12 @@ vm_do_checkpoint(struct vmctx *ctx, const nvlist_t *nvl)
 		    nvlist_get_descriptor(nvl, "fddir"),
 		    nvlist_get_string(nvl, "filename"),
 		    nvlist_get_bool(nvl, "suspend"));
+	if (error != 0)
+		nvlist_add_string(reply, "error", strerror(error));
 
-	return (error);
-}
-IPC_COMMAND(ipc_cmd_set, checkpoint, vm_do_checkpoint);
-
-/*
- * Create the listening socket for IPC with bhyvectl
- */
-int
-init_checkpoint_thread(struct vmctx *ctx, const char *bhyve_run_dir)
-{
-	struct checkpoint_thread_info *checkpoint_info = NULL;
-	struct sockaddr_un addr;
-	int socket_fd;
-	pthread_t checkpoint_pthread;
-	int err, ret;
-#ifndef WITHOUT_CAPSICUM
-	cap_rights_t rights;
-#endif
-
-	memset(&addr, 0, sizeof(addr));
-
-	socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
-	if (socket_fd < 0) {
-		EPRINTLN("Socket creation failed: %s", strerror(errno));
-		err = -1;
-		goto fail;
-	}
-
-	addr.sun_family = AF_UNIX;
-
-	ret = snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s",
-	    bhyve_run_dir, vm_get_name(ctx));
-	if ((ret < 0) || ((size_t)ret >= sizeof(addr.sun_path))) {
-		EPRINTLN("%s: error setting socket path (%d)", __func__, ret);
-		err = -1;
-		goto fail;
-	}
-
-	addr.sun_len = SUN_LEN(&addr);
-	unlink(addr.sun_path);
-
-	if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
-		EPRINTLN("Failed to bind socket \"%s\": %s\n",
-		    addr.sun_path, strerror(errno));
-		err = -1;
-		goto fail;
-	}
-
-	if (listen(socket_fd, 10) < 0) {
-		EPRINTLN("ipc socket listen: %s\n", strerror(errno));
-		err = errno;
-		goto fail;
-	}
-
-#ifndef WITHOUT_CAPSICUM
-	cap_rights_init(&rights, CAP_ACCEPT, CAP_READ, CAP_RECV, CAP_WRITE,
-	    CAP_SEND, CAP_GETSOCKOPT);
-
-	if (caph_rights_limit(socket_fd, &rights) == -1)
-		errx(EX_OSERR, "Unable to apply rights for sandbox");
-#endif
-	checkpoint_info = calloc(1, sizeof(*checkpoint_info));
-	checkpoint_info->ctx = ctx;
-	checkpoint_info->socket_fd = socket_fd;
-
-	err = pthread_create(&checkpoint_pthread, NULL, checkpoint_thread,
-		checkpoint_info);
-	if (err != 0)
-		goto fail;
-
-	return (0);
-fail:
-	free(checkpoint_info);
-	if (socket_fd > 0)
-		close(socket_fd);
-	unlink(addr.sun_path);
-
-	return (err);
+	return (reply);
 }
+IPC_COMMAND(checkpoint, vm_do_checkpoint);
 
 void
 vm_snapshot_buf_err(const char *bufname, const enum vm_snapshot_op op)

usr.sbin/bhyve/snapshot.h

@@ -40,7 +40,6 @@
 #include <libxo/xo.h>
 #include <ucl.h>
 
-#define BHYVE_RUN_DIR "/var/run/bhyve/"
 #define MAX_SNAPSHOT_FILENAME PATH_MAX
 
 struct vmctx;