feat(core): Add push-based reload endpoint for custom node development (#36837)

CONTRIBUTING.md

@@ -349,6 +349,20 @@ When developing custom nodes or credentials, you can enable hot reload to automa
 N8N_DEV_RELOAD=true pnpm dev:be
 ```
 
+This enables two mechanisms:
+
+- a **file watcher** over the loaded node directories, and
+- `POST /rest/dev/reload`, an unauthenticated (rate-limited) endpoint that re-reads the node
+  files already on disk. `@n8n/node-cli`'s `dev` command uses this to push a
+  reload after each successful compile, because a container cannot watch a bind
+  mount and the Alpine image has no `@parcel/watcher` prebuild.
+
+The variable is honoured regardless of `NODE_ENV`, so it also applies to
+production builds and the published Docker image. Never set it on an instance
+reachable by anyone you would not give a shell to. It also disables the crash
+journal, so a dev container that was killed rather than shut down gracefully
+does not pay the 10 second crash-loop penalty on its next boot.
+
 **Performance considerations:**
 - File watching adds overhead to your system, especially on slower machines
 - The watcher monitors potentially thousands of files, which can impact CPU and memory usage

packages/cli/src/__tests__/crash-journal.test.ts

@@ -0,0 +1,69 @@
+import { Service } from '@n8n/di';
+import { sleep } from '@n8n/utils/sleep';
+import { existsSync } from 'fs';
+import { mkdir, utimes } from 'fs/promises';
+
+// Hoisted so the assertions can reach the spy without importing the mocked
+// module here, which would run the factory below before `@n8n/di` initialises.
+const { loggerError } = vi.hoisted(() => ({ loggerError: vi.fn() }));
+
+// `inProduction` is derived from NODE_ENV at import time, which is `test` here,
+// so force it on — otherwise `init()` bails before reaching the guard under test.
+vi.mock('@n8n/backend-common', () => {
+	@Service()
+	class Logger {
+		error = loggerError;
+	}
+
+	return { inProduction: true, Logger };
+});
+
+vi.mock('n8n-core', () => {
+	@Service()
+	class InstanceSettings {
+		n8nFolder = '/tmp/n8n-crash-journal-test';
+	}
+
+	return { InstanceSettings };
+});
+
+vi.mock('fs');
+vi.mock('fs/promises');
+vi.mock('@n8n/utils/sleep');
+
+import { init } from '../crash-journal';
+
+describe('crash journal init()', () => {
+	const originalDevReload = process.env.N8N_DEV_RELOAD;
+
+	beforeEach(() => {
+		vi.mocked(existsSync).mockReturnValue(true); // a journal from a previous session
+	});
+
+	afterEach(() => {
+		if (originalDevReload === undefined) delete process.env.N8N_DEV_RELOAD;
+		else process.env.N8N_DEV_RELOAD = originalDevReload;
+		vi.clearAllMocks();
+	});
+
+	it('skips the crash-loop pause and the journal write when N8N_DEV_RELOAD is enabled', async () => {
+		process.env.N8N_DEV_RELOAD = 'true';
+
+		await init();
+
+		expect(sleep).not.toHaveBeenCalled();
+		expect(mkdir).not.toHaveBeenCalled();
+		expect(utimes).not.toHaveBeenCalled();
+		expect(loggerError).not.toHaveBeenCalled();
+	});
+
+	it('pauses and rewrites the journal when N8N_DEV_RELOAD is not set', async () => {
+		delete process.env.N8N_DEV_RELOAD;
+
+		await init();
+
+		expect(loggerError).toHaveBeenCalledWith('Last session crashed');
+		expect(sleep).toHaveBeenCalledWith(10_000);
+		expect(utimes).toHaveBeenCalled();
+	});
+});

packages/cli/src/__tests__/load-nodes-and-credentials.test.ts

@@ -5,13 +5,14 @@ import { join } from 'node:path';
 import { Service } from '@n8n/di';
 import watcher from '@parcel/watcher';
 import fs from 'fs/promises';
-import { CUSTOM_NODES_PACKAGE_NAME, DirectoryLoader } from 'n8n-core';
+import { CUSTOM_NODES_PACKAGE_NAME, CustomDirectoryLoader, DirectoryLoader } from 'n8n-core';
 import type {
 	ICredentialType,
 	INodeProperties,
 	INodeTypeDescription,
 	NodeLoader,
 } from 'n8n-workflow';
+import { UserError } from 'n8n-workflow';
 import type { Mock } from 'vitest';
 import { mock } from 'vitest-mock-extended';
 
@@ -700,6 +701,160 @@ describe('LoadNodesAndCredentials', () => {
 			expect(mockLoader.loadAll).toHaveBeenCalled();
 			expect(postProcessSpy).toHaveBeenCalled();
 		});
+
+		it('should serialize a watcher reload against a concurrent endpoint reload', async () => {
+			vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			let active = 0;
+			let maxActive = 0;
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/some/custom/path',
+				isLazyLoaded: false,
+				reset: vi.fn(),
+				loadAll: vi.fn(async () => {
+					active++;
+					maxActive = Math.max(maxActive, active);
+					await new Promise((resolve) => setTimeout(resolve, 5));
+					active--;
+				}),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+			instance.loaders = { CUSTOM: customLoader };
+
+			await instance.setupHotReload();
+			const [, onFileUpdate] = vi.mocked(watcher.subscribe).mock.calls[0];
+
+			// Both fire on the same save with `pnpm dev:be` + `n8n-node dev`.
+			await Promise.all([
+				onFileUpdate(null, [{ type: 'update', path: '/some/custom/path/X.node.js' }]),
+				instance.reloadCustomNodes(),
+			]);
+
+			expect(customLoader.loadAll).toHaveBeenCalledTimes(2);
+			expect(maxActive).toBe(1);
+		});
+
+		it('should not reject into the watcher callback when a reload fails', async () => {
+			vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/some/custom/path',
+				isLazyLoaded: false,
+				reset: vi.fn(),
+				loadAll: vi.fn().mockRejectedValue(new Error('broken node constructor')),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+			instance.loaders = { CUSTOM: customLoader };
+
+			await instance.setupHotReload();
+			const [, onFileUpdate] = vi.mocked(watcher.subscribe).mock.calls[0];
+
+			await expect(
+				onFileUpdate(null, [{ type: 'update', path: '/some/custom/path/X.node.js' }]),
+			).resolves.not.toThrow();
+		});
+	});
+
+	describe('reloadCustomNodes', () => {
+		it('should reload only custom-directory loaders', async () => {
+			const instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock(), mock(), mock());
+			const postProcessSpy = vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/home/node/.n8n/custom',
+				reset: vi.fn(),
+				loadAll: vi.fn(),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+
+			const baseLoader = mock<DirectoryLoader>({
+				packageName: 'n8n-nodes-base',
+				directory: '/app/nodes-base',
+				reset: vi.fn(),
+				loadAll: vi.fn(),
+			} as never);
+			Object.setPrototypeOf(baseLoader, DirectoryLoader.prototype);
+
+			instance.loaders = { CUSTOM: customLoader, 'n8n-nodes-base': baseLoader };
+
+			await expect(instance.reloadCustomNodes()).resolves.toEqual([CUSTOM_NODES_PACKAGE_NAME]);
+
+			// reset() drops the require cache for the loader's own files — covered by
+			// DirectoryLoader's own tests, since `reset` is mocked out here.
+			expect(customLoader.reset).toHaveBeenCalled();
+			expect(customLoader.loadAll).toHaveBeenCalled();
+			expect(baseLoader.reset).not.toHaveBeenCalled();
+			expect(postProcessSpy).toHaveBeenCalled();
+		});
+
+		it('should throw a UserError when a loader fails, so the endpoint reports failure', async () => {
+			const instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock(), mock(), mock());
+			vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/home/node/.n8n/custom',
+				reset: vi.fn(),
+				loadAll: vi.fn().mockRejectedValue(new Error('broken node constructor')),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+			instance.loaders = { CUSTOM: customLoader };
+
+			await expect(instance.reloadCustomNodes()).rejects.toThrow(UserError);
+			await expect(instance.reloadCustomNodes()).rejects.toThrow(
+				`Hot reload failed for ${CUSTOM_NODES_PACKAGE_NAME}`,
+			);
+		});
+
+		it('should keep serializing after a failed reload', async () => {
+			const instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock(), mock(), mock());
+			vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/home/node/.n8n/custom',
+				reset: vi.fn(),
+				loadAll: vi.fn().mockRejectedValueOnce(new Error('broken')).mockResolvedValue(undefined),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+			instance.loaders = { CUSTOM: customLoader };
+
+			await expect(instance.reloadCustomNodes()).rejects.toThrow(UserError);
+			await expect(instance.reloadCustomNodes()).resolves.toEqual([CUSTOM_NODES_PACKAGE_NAME]);
+		});
+
+		it('should serialize concurrent reloads instead of interleaving them', async () => {
+			const instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock(), mock(), mock());
+			vi.spyOn(instance, 'postProcessLoaders').mockResolvedValue(undefined);
+
+			let active = 0;
+			let maxActive = 0;
+			const customLoader = mock<CustomDirectoryLoader>({
+				packageName: CUSTOM_NODES_PACKAGE_NAME,
+				directory: '/home/node/.n8n/custom',
+				reset: vi.fn(),
+				loadAll: vi.fn(async () => {
+					active++;
+					maxActive = Math.max(maxActive, active);
+					await new Promise((resolve) => setTimeout(resolve, 5));
+					active--;
+				}),
+			} as never);
+			Object.setPrototypeOf(customLoader, CustomDirectoryLoader.prototype);
+			instance.loaders = { CUSTOM: customLoader };
+
+			await Promise.all([
+				instance.reloadCustomNodes(),
+				instance.reloadCustomNodes(),
+				instance.reloadCustomNodes(),
+			]);
+
+			expect(customLoader.loadAll).toHaveBeenCalledTimes(3);
+			expect(maxActive).toBe(1);
+		});
 	});
 
 	describe('postProcessLoaders', () => {

packages/cli/src/controllers/dev.controller.ts

@@ -0,0 +1,28 @@
+import { Post, RestController } from '@n8n/decorators';
+
+import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
+
+/**
+ * Dev-only endpoints. Registered only when `N8N_DEV_RELOAD=true`.
+ *
+ * The published image cannot watch the filesystem (no musl prebuild of
+ * `@parcel/watcher`, and inotify does not cross a bind mount), so reload is a
+ * push from whoever owns the compiler — see `@n8n/node-cli`'s `dev` command.
+ */
+@RestController('/dev')
+export class DevController {
+	constructor(private readonly loadNodesAndCredentials: LoadNodesAndCredentials) {}
+
+	/**
+	 * Re-read custom node files already on disk. Takes no payload.
+	 * Unauthenticated but rate limited: a reload is CPU-expensive
+	 * (postProcessLoaders rebuilds the full type registry), and this may run on
+	 * an instance where the var was set by mistake. 100/min never throttles a
+	 * human dev loop, but caps abuse.
+	 */
+	@Post('/reload', { skipAuth: true, ipRateLimit: { limit: 100, windowMs: 60_000 } })
+	async reload() {
+		const reloaded = await this.loadNodesAndCredentials.reloadCustomNodes();
+		return { reloaded };
+	}
+}

packages/cli/src/crash-journal.ts

@@ -21,7 +21,7 @@ const { n8nFolder } = Container.get(InstanceSettings);
 const journalFile = join(n8nFolder, 'crash.journal');
 
 export const init = async () => {
-	if (!inProduction) return;
+	if (!inProduction || process.env.N8N_DEV_RELOAD === 'true') return;
 
 	if (existsSync(journalFile)) {
 		// Crash detected

packages/cli/src/load-nodes-and-credentials.ts

@@ -691,13 +691,69 @@ export class LoadNodesAndCredentials {
 		throw new UnrecognizedCredentialTypeError(credentialType);
 	}
 
+	private reloadQueue: Promise<unknown> = Promise.resolve();
+
+	/**
+	 * Re-read the files already on disk for a loader and push the updated
+	 * descriptions to open editors. Touches no native module, so it works
+	 * inside the published image where the file watcher cannot run.
+	 *
+	 * Serialized here rather than at the call sites, because the endpoint and
+	 * the file watcher both reload and can fire on the same save. Concurrent
+	 * reloads would interleave reset()/loadAll(), leaving nodes unresolvable.
+	 */
+	private async reloadLoader(loader: DirectoryLoader) {
+		const run = this.reloadQueue.then(async () => {
+			this.logger.info(`Hot reload triggered for ${loader.packageName}`);
+			try {
+				loader.reset();
+				await loader.loadAll();
+				await this.postProcessLoaders();
+				const { Push } = await import('@/push/index.js');
+				Container.get(Push).broadcast({ type: 'nodeDescriptionUpdated', data: {} });
+			} catch (error) {
+				this.logger.error(`Hot reload failed for ${loader.packageName}`, {
+					error: ensureError(error),
+				});
+				throw new UserError(`Hot reload failed for ${loader.packageName}`, { cause: error });
+			}
+		});
+		this.reloadQueue = run.catch(() => {});
+		await run;
+	}
+
+	/**
+	 * Reload nodes from the custom directories on demand, for the dev reload
+	 * endpoint. Returns the package names that were reloaded. Throws if any
+	 * loader fails, so the endpoint does not report a broken node as reloaded.
+	 */
+	async reloadCustomNodes() {
+		const loaders = Object.values(this.loaders).filter(
+			(loader) => loader instanceof CustomDirectoryLoader,
+		);
+
+		for (const loader of loaders) {
+			await this.reloadLoader(loader);
+		}
+
+		return loaders.map((loader) => loader.packageName);
+	}
+
 	async setupHotReload() {
 		const { default: debounce } = await import('lodash/debounce.js');
 
-		const { subscribe } = await import('@parcel/watcher');
-
-		const { Push } = await import('@/push/index.js');
-		const push = Container.get(Push);
+		let subscribe: typeof ParcelWatcher.subscribe;
+		try {
+			({ subscribe } = await import('@parcel/watcher'));
+		} catch (error) {
+			// No prebuild for this platform (e.g. musl in the official image). File
+			// watching is unavailable; POST /rest/dev/reload still works.
+			this.logger.warn(
+				'File watching for hot reload is unavailable on this platform. Use POST /rest/dev/reload to reload nodes.',
+				{ error: ensureError(error) },
+			);
+			return;
+		}
 
 		for (const loader of Object.values(this.loaders)) {
 			if (!(loader instanceof DirectoryLoader)) continue;
@@ -709,17 +765,9 @@ export class LoadNodesAndCredentials {
 				continue;
 			}
 
-			const reloader = debounce(async () => {
-				this.logger.info(`Hot reload triggered for ${loader.packageName}`);
-				try {
-					loader.reset();
-					await loader.loadAll();
-					await this.postProcessLoaders();
-					push.broadcast({ type: 'nodeDescriptionUpdated', data: {} });
-				} catch (error) {
-					this.logger.error(`Hot reload failed for ${loader.packageName}`);
-				}
-			}, 100);
+			// Already logged inside reloadLoader; swallow so a broken node does not
+			// reject into the watcher callback as an unhandled rejection.
+			const reloader = debounce(async () => await this.reloadLoader(loader).catch(() => {}), 100);
 
 			// For lazy loaded packages, we need to watch the dist directory
 			const watchPaths = loader.isLazyLoaded ? [path.join(directory, 'dist')] : [directory];

packages/cli/src/server.ts

@@ -117,7 +117,7 @@ export class Server extends AbstractServer {
 		await super.start();
 		this.logger.debug(`Server ID: ${this.instanceSettings.hostId}`);
 
-		if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') {
+		if (process.env.N8N_DEV_RELOAD === 'true') {
 			void this.loadNodesAndCredentials.setupHotReload();
 		}
 
@@ -135,6 +135,10 @@ export class Server extends AbstractServer {
 			await import('@/controllers/e2e.controller.js');
 		}
 
+		if (process.env.N8N_DEV_RELOAD === 'true') {
+			await import('@/controllers/dev.controller.js');
+		}
+
 		if (isMfaFeatureEnabled()) {
 			await Container.get(MfaService).init();
 			await import('@/controllers/mfa.controller.js');

packages/core/src/nodes-loader/__tests__/directory-loader.test.ts

@@ -496,6 +496,25 @@ describe('DirectoryLoader', () => {
 			expect(loader.known.nodes).toEqual({});
 			expect(loader.known.credentials).toEqual({});
 		});
+
+		it('should drop the require cache under the directory, but leave other entries', () => {
+			mockFs.readFileSync.calledWith(`${directory}/package.json`).mockReturnValue(packageJson);
+			mockFs.readdirSync.mockReturnValue([]);
+
+			const loader = new PackageDirectoryLoader(directory);
+
+			const ownModule = `${directory}/dist/Node1/Node1.node.js`;
+			const foreignModule = '/somewhere/else/dist/Other.node.js';
+			require.cache[ownModule] = mock<NodeJS.Module>({ filename: ownModule });
+			require.cache[foreignModule] = mock<NodeJS.Module>({ filename: foreignModule });
+
+			loader.reset();
+
+			expect(require.cache[ownModule]).toBeUndefined();
+			expect(require.cache[foreignModule]).toBeDefined();
+
+			delete require.cache[foreignModule];
+		});
 	});
 
 	describe('getVersionedNodeTypeAll', () => {