opfs speed optimization

README.md

@@ -0,0 +1,5 @@
+# node
+
+Frontend for [node-worker](https://github.com/HeyPuter/node-worker), a Node.js-compatible runtime running transformed code in a Worker with node globals.
+
+Includes the code for the Puter terminal app, a local dev testbed, and the demo at developer.puter.com.

src/legacy-testbed.ts

@@ -8,6 +8,7 @@
 
 import { NodeWorker } from "node-worker";
 import workerURL from "node-worker/worker?url";
+import swURL from "node-worker/sw?url";
 
 import { mountEditor } from "./monaco";
 import { mountTerminal, type TerminalController } from "./terminal";
@@ -384,7 +385,7 @@ export async function mountApp(root: Element, runtimeStore: RuntimeStore) {
 
 		try {
 			worker?.terminate();
-			worker = new NodeWorker(workerURL, token, cwd);
+			worker = new NodeWorker(workerURL, token, cwd, { swURL });
 			terminal?.attachConsole(worker.console);
 			await worker.ready;
 			workerReady = true;

src/project/handle-store.ts

@@ -57,6 +57,25 @@ export interface HandleStoreOptions {
 /** How many of the biggest directories an over-limit folder names. */
 const BLAME_COUNT = 3;
 
+/**
+ * How many files `writeMany` has in flight at once.
+ *
+ * Writing one is four round-trips to the browser's filesystem thread — resolve its directory, open
+ * the file, open a writable, close it — and the react-ts template is 4,927 files, most of them
+ * small and several directories deep inside `node_modules`. Done strictly one at a time that is
+ * some twenty thousand sequential round-trips: it measured 10s on a fast desktop, and it is the
+ * kind of cost that scales with somebody else's disk.
+ *
+ * Almost all of it is latency rather than disk or CPU, so overlapping is nearly pure win: the same
+ * template writes in 2.6-3.0s this way. 16 rather than more because a sweep over a tree this shape
+ * stopped improving there and was slower by 32 — the filesystem backend is saturated by then, and
+ * the only thing further concurrency buys is more buffers held in flight.
+ */
+const WRITE_CONCURRENCY = 16;
+
+/** How often a bulk write reports, in entries. Matches `hydrate`, for the same reason. */
+const PROGRESS_EVERY = 50;
+
 function segments(path: string): string[] {
 	return path.split("/").filter((s) => s.length > 0 && s !== ".");
 }
@@ -103,6 +122,33 @@ async function dirFor(
 	return dir;
 }
 
+/**
+ * Run `each` over every item, at most `limit` at a time.
+ *
+ * Stops at the first failure and rethrows it once the runners already in flight have finished,
+ * which is what the serial loop this replaced did — and the reason not to lean on `Promise.all`
+ * alone, whose rejection would surface with a dozen writes still running behind it.
+ */
+async function inParallel<T>(
+	items: readonly T[],
+	limit: number,
+	each: (item: T) => Promise<void>
+): Promise<void> {
+	let next = 0;
+	let failure: unknown;
+	let runner = async () => {
+		while (next < items.length && failure === undefined) {
+			try {
+				await each(items[next++]);
+			} catch (err) {
+				failure = err;
+			}
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runner));
+	if (failure !== undefined) throw failure;
+}
+
 export function createHandleStore(
 	handle: FileSystemDirectoryHandle,
 	existing: boolean,
@@ -197,19 +243,54 @@ export function createHandleStore(
 			return out;
 		},
 
-		async writeMany(entries) {
+		async writeMany(entries, onProgress) {
-			for (const entry of entries) {
+			const all = [...entries];
-				const parts = segments(entry.path);
+
-				if (parts.length === 0) continue;
+			/**
-				if (entry.data === undefined) {
+			 * Directory handles by path, so a directory is opened once however many files go
-					await dirFor(handle, parts, true);
+			 * into it.
-					continue;
+			 *
+			 * The win is twofold. Resolving `node_modules/@babel/parser/lib` used to cost four
+			 * round-trips *per file written into it* — a template's 5,000 files sit in a few
+			 * hundred directories, so nearly all of that was re-answering the same question.
+			 * And the entries are cached as **promises**: the sixteen writes that arrive at a
+			 * brand-new directory together then share one `create: true` rather than racing to
+			 * make it.
+			 *
+			 * Per call, not for the store's lifetime. A directory can go away underneath us —
+			 * `remove`, a program the runtime is running, or the user in their own file manager
+			 * — and a handle cached across calls would keep failing every write into it.
+			 */
+			const dirs = new Map<string, Promise<FileSystemDirectoryHandle | undefined>>();
+
+			const dirOnce = (parts: string[]): Promise<FileSystemDirectoryHandle | undefined> => {
+				if (parts.length === 0) return Promise.resolve(handle);
+				const path = parts.join("/");
+				let found = dirs.get(path);
+				if (!found) {
+					// Resolved from its own parent rather than walked from the root, so a new
+					// directory costs one round-trip instead of one per segment above it.
+					const name = parts[parts.length - 1];
+					found = dirOnce(parts.slice(0, -1)).then((parent) =>
+						parent
+							?.getDirectoryHandle(name, { create: true })
+							.catch(() => undefined)
+					);
+					dirs.set(path, found);
 				}
-				const dir = await dirFor(handle, parts.slice(0, -1), true);
+				return found;
-				if (!dir) continue;
+			};
-				const file = await dir.getFileHandle(parts[parts.length - 1], {
+
-					create: true,
+			const writeOne = async (entry: MountEntry) => {
-				});
+				const parts = segments(entry.path);
+				if (parts.length === 0) return;
+				if (entry.data === undefined) {
+					await dirOnce(parts);
+					return;
+				}
+				const dir = await dirOnce(parts.slice(0, -1));
+				if (!dir) return;
+				const file = await dir.getFileHandle(parts[parts.length - 1], { create: true });
 				const writable = await file.createWritable({ keepExistingData: false });
 				try {
 					await writable.write(entry.data as unknown as ArrayBufferView<ArrayBuffer>);
@@ -218,7 +299,18 @@ export function createHandleStore(
 					await writable.abort().catch(() => undefined);
 					throw err;
 				}
-			}
+			};
+
+			let done = 0;
+			await inParallel(all, WRITE_CONCURRENCY, async (entry) => {
+				await writeOne(entry);
+				// Counted even when the entry was skipped, so the total the caller is shown
+				// against is the one it handed over.
+				done += 1;
+				if (done % PROGRESS_EVERY === 0 || done === all.length) {
+					onProgress?.(done, all.length);
+				}
+			});
 		},
 
 		async remove(paths) {

src/project/mirror.ts

@@ -77,10 +77,17 @@ interface FileNode {
  * absent from the next run.
  */
 export interface MirrorTarget {
-	writeMany(entries: Iterable<MountEntry>): Promise<void>;
+	writeMany(entries: Iterable<MountEntry>, onProgress?: WriteProgress): Promise<void>;
 	remove(paths: Iterable<string>): Promise<void>;
 }
 
+/**
+ * How far a bulk write has got. Only the one caller that writes thousands of entries at once —
+ * unpacking a template, in `./source` — passes this; an edit from the editor is one file and has
+ * nothing to report.
+ */
+export type WriteProgress = (done: number, total: number) => void;
+
 /** The store the project lives in: a mirror target that can also be read back. */
 export interface ProjectStore extends MirrorTarget {
 	read(path: string): Promise<Uint8Array | undefined>;

src/project/source.ts

@@ -179,8 +179,7 @@ async function resolveLocal(
 		entries = await loadTemplate(info, (p) => {
 			report.phase(p.phase, p.fraction ?? 0, `${phaseVerb(p.phase)} ${p.detail}`);
 		});
-		report.phase("extract", 0.9, "Saving the project…");
+		await saveProject(store, entries, report);
-		await store.writeMany(entries);
 		summary = `${info.id} template unpacked into “${handle.name}” (${info.packages} packages)`;
 	} else {
 		entries = [];
@@ -222,7 +221,15 @@ async function resolveTemplate(
 	if (store.existing) {
 		report.phase("extract", 0, "Opening your project…");
 		entries = await store.hydrate((count) => {
-			report.phase("extract", 0, `Reading ${count} files…`);
+			// Against the manifest's count, because it is this template that was written here —
+			// a moving fraction rather than the flat 0 a reload used to sit on for the whole
+			// read. An estimate, not a total: `hydrate` counts directories as well as files, so
+			// it runs out of bar slightly early and waits there. Clamped for that reason.
+			report.phase(
+				"extract",
+				info.files > 0 ? Math.min(1, count / info.files) : 0,
+				`Reading ${count} files…`
+			);
 		});
 		report.log(`project: ${entries.length} entries already on disk`);
 	} else {
@@ -230,8 +237,7 @@ async function resolveTemplate(
 			report.phase(p.phase, p.fraction ?? 0, `${phaseVerb(p.phase)} ${p.detail}`);
 		});
 		report.log(`template: ${entries.length} entries, ${info.packages} packages`);
-		report.phase("extract", 0.9, "Saving the project…");
+		await saveProject(store, entries, report);
-		await store.writeMany(entries);
 	}
 
 	return {
@@ -249,6 +255,31 @@ async function resolveTemplate(
 
 // ------------------------------------------------------------------ helpers
 
+/**
+ * Write a freshly unpacked template out to the store, reporting as it goes.
+ *
+ * Its own splash phase rather than the tail end of "extract". This is the slowest thing a first
+ * visit does — thousands of individual file creates, which no amount of batching turns into one
+ * operation — and it used to be a single "Saving the project…" in front of a bar parked on one
+ * number for the duration, which is indistinguishable from a hang.
+ */
+async function saveProject(
+	store: HandleStore,
+	entries: MountEntry[],
+	report: SourceReporter
+): Promise<void> {
+	const count = entries.length.toLocaleString();
+	report.phase("save", 0, `Saving ${count} files…`);
+	await store.writeMany(entries, (done, total) => {
+		report.phase(
+			"save",
+			total > 0 ? done / total : 1,
+			`Saved ${done.toLocaleString()} of ${total.toLocaleString()} files…`
+		);
+	});
+	report.log(`project: ${count} entries written`);
+}
+
 function phaseVerb(phase: string): string {
 	return phase === "download" ? "Downloading" : "Unpacking";
 }

src/runtime/drive-command.ts

@@ -8,6 +8,7 @@
 
 import { NodeWorker, WorkerExitError, type Console as NodeWorkerConsole } from "node-worker";
 import workerURL from "node-worker/worker?url";
+import swURL from "node-worker/sw?url";
 
 import { createDriveTarget } from "../install/drive-target";
 import type { CommandTarget } from "./programs";
@@ -35,7 +36,10 @@ export function createDriveCommandTarget(opts: DriveCommandOptions): DriveComman
 
 	let acquire = async (): Promise<NodeWorker> => {
 		if (worker) return worker;
-		let next = new NodeWorker(workerURL, opts.token, opts.cwd, { keepalive: true });
+		// `swURL` for the same reason the workspace passes it: without the service worker there
+		// is no synchronous filesystem, and the runtime refuses to start rather than failing
+		// later inside whatever needed a sync read.
+		let next = new NodeWorker(workerURL, opts.token, opts.cwd, { keepalive: true, swURL });
 		worker = next;
 		opts.attach(next.console as NodeWorkerConsole);
 		await next.ready;

src/ui/shell.ts

@@ -175,7 +175,10 @@ const MARKUP = `
 										stroke="currentColor" stroke-width="1.3" stroke-linejoin="round" />
 								</svg>
 							</button>
-							<span class="select-pill">
+							<!-- The pill, not the select, carries the locked look and its tooltip: a
+							     disabled select takes no pointer events, so a title on it would never
+							     show. -->
+							<span class="select-pill" id="presetPill">
 								<select id="presetSelect" aria-label="Project source"></select>
 							</span>
 						</div>
@@ -306,6 +309,7 @@ export interface ShellElements {
 	exportBtn: HTMLButtonElement;
 	restartBtn: HTMLButtonElement;
 	presetSelect: HTMLSelectElement;
+	presetPill: HTMLElement;
 	openFolderBtn: HTMLButtonElement;
 	tree: HTMLElement;
 	tabs: HTMLElement;
@@ -370,6 +374,7 @@ export function buildShell(root: Element): ShellElements {
 		exportBtn: pick("exportBtn"),
 		restartBtn: pick("restartBtn"),
 		presetSelect: pick("presetSelect"),
+		presetPill: pick("presetPill"),
 		openFolderBtn: pick("openFolderBtn"),
 		tree: pick("tree"),
 		tabs: pick("tabs"),

src/ui/splash.ts

@@ -2,8 +2,14 @@
 //
 // Its percentages are real, unlike the mock's timed script: the download reports bytes
 // against the archive's known size, the extract reports entries against the manifest's
-// file count, and the populate reports batches. Weighted so the bar advances at a
+// file count, and the save reports files written. Weighted so the bar advances at a
 // roughly even rate rather than sitting at 90% through the slowest phase.
+//
+// A phase nothing reports is worse than no phase at all — it is a slice of bar the run
+// silently skips, which is exactly what "Populating filesystem" became once the worker
+// started mounting the project instead of being filled with it. So the list below is kept
+// to phases that something actually calls; see the `.phase(` callers in `src/workspace.ts`
+// and `src/project/source.ts`.
 
 import type { ShellElements } from "./shell";
 
@@ -18,8 +24,11 @@ export interface SplashPhase {
 const PHASES: SplashPhase[] = [
 	{ key: "download", label: "Downloading project image", weight: 3 },
 	{ key: "extract", label: "Unpacking project", weight: 2 },
+	// Weighted like the download because it costs like the download: a first visit writes
+	// every one of the template's ~5,000 files out to disk here.
+	{ key: "save", label: "Saving project", weight: 3 },
+	{ key: "auth", label: "Connecting", weight: 1 },
 	{ key: "worker", label: "Starting worker", weight: 1 },
-	{ key: "populate", label: "Populating filesystem", weight: 3 },
 	{ key: "probe", label: "Checking the runtime", weight: 1 },
 ];
 

src/ui/styles.css

@@ -490,6 +490,59 @@ select {
   pointer-events: none;
 }
 
+/* Locked while a program holds the worker, because switching source reloads the page.
+   The native disabled `<select>` is a text shade nobody notices, so the pill goes flat
+   and grey with a padlock where the chevron was — and the pill takes the cursor and the
+   `title`, since the disabled control inside it gets no pointer events. */
+.select-pill[data-locked="true"] {
+  cursor: not-allowed;
+}
+
+/* `--ink-mid` rather than the fainter `--ink-soft`, because which project is loaded stays
+   worth reading while it runs — the flat fill and the padlock carry the unavailability. */
+.select-pill select:disabled {
+  border-color: var(--line-soft);
+  background: var(--track);
+  color: var(--ink-mid);
+  cursor: not-allowed;
+}
+
+.select-pill select:disabled:hover {
+  border-color: var(--line-soft);
+}
+
+/* The padlock, drawn from the pill's two pseudo-elements — `::after` is the body (the
+   chevron's borders traded for a fill), `::before` the shackle sitting on top of it.
+   Both keep the chevron's footprint, so locking does not resize the pill.
+
+   `box-sizing` is set here because the `*` reset does not reach pseudo-elements: on
+   content-box the shackle's borders would sit outside its width and shift it off the
+   body's centre. With both boxes measured the same way, the two `right` offsets put
+   their centres on the same 12.5px. */
+.select-pill[data-locked="true"]::after {
+  box-sizing: border-box;
+  width: 9px;
+  height: 7px;
+  border: none;
+  border-radius: 1.5px;
+  background: var(--ink-mid);
+  transform: translateY(1.5px);
+}
+
+.select-pill[data-locked="true"]::before {
+  content: "";
+  box-sizing: border-box;
+  position: absolute;
+  right: 10px;
+  width: 5px;
+  height: 5px;
+  border: 1.4px solid var(--ink-mid);
+  border-bottom: none;
+  border-radius: 3px 3px 0 0;
+  transform: translateY(-4px);
+  pointer-events: none;
+}
+
 /* --------------------------------------------------------- file tree */
 
 .tree {

src/workspace.ts

@@ -146,9 +146,22 @@ async function boot(
 	// of it back would double the boot cost for no effect — and against a folder somebody picked,
 	// it would rewrite every file in their project on open. The target is set once the pool
 	// exists, because that is what an edit has to travel through — see `pool.projectTarget`.
+	splash.phase(
+		"save",
+		1,
+		`Building the file tree from ${project.entries.length.toLocaleString()} entries…`
+	);
 	mirror.seed(project.entries);
 
 	// --- token ------------------------------------------------------------
+	//
+	// Reported, because it is not free and it used to run under whatever the project step had
+	// last said: an anonymous run mints a peer token, waits for that write to land, then fetches
+	// a relay URL. Measured at 360 ms through the dev proxy and worse as a Puter app, where the
+	// write is a real round trip and `waitForPuterKV` will sit for up to five seconds waiting for
+	// the SDK to show up. However long it takes, the splash used to spend it insisting it was
+	// still saving files — which is how a slow connection came to look like a slow save.
+	splash.phase("auth", 0, "Connecting…");
 	let auth = await opts.resolveAuth(() => splash.requestToken());
 	let anonymous = !auth.token;
 	if (anonymous) {
@@ -180,6 +193,7 @@ async function boot(
 		else void pool.stop();
 	}
 
+	splash.phase("auth", 1, "Preparing the terminal…");
 	let terminal = mountTerminal(el.shell, {
 		onCommand: (line) => void submit(line),
 		onInterrupt: interrupt,
@@ -264,6 +278,9 @@ async function boot(
 
 	renderSources(el, manifest.templates, project, rememberedFolder, idleLabel);
 	if (!localDirSupported()) {
+		// Marked as well as disabled so `setBusy` can tell a browser that will never do this
+		// from a run that is only holding it for now, and leave the former alone.
+		el.openFolderBtn.dataset.unsupported = "true";
 		el.openFolderBtn.disabled = true;
 		el.openFolderBtn.title =
 			"this browser cannot open a folder from your device — Chrome, Edge and other Chromium browsers can";
@@ -803,6 +820,19 @@ function setBusy(el: ShellElements, busy: boolean, idleLabel: string) {
 	el.exportBtn.disabled = busy;
 	el.restartBtn.disabled = busy;
 	el.presetSelect.disabled = busy;
+	// Says *why* it is unavailable, which the greying alone cannot: a switch reloads the page,
+	// so it has to wait for whatever holds the worker. See `.select-pill[data-locked]`.
+	el.presetPill.dataset.locked = busy ? "true" : "false";
+	if (busy) {
+		el.presetPill.title = "Stop what is running first — switching project reloads the page";
+	} else {
+		el.presetPill.removeAttribute("title");
+	}
+	// Opening a folder swaps the project the same way the select does, so it waits for the same
+	// reason — `swapSource` already declines while busy, and this stops the click that goes
+	// nowhere. The button keeps its own title: a disabled one shows no tooltip in every browser,
+	// which is why the reason sits on the pill beside it.
+	el.openFolderBtn.disabled = busy || el.openFolderBtn.dataset.unsupported === "true";
 }
 
 function showPreview(el: ShellElements, port: number, peerToken?: string) {

vite.config.ts

@@ -25,6 +25,24 @@ export default defineConfig({
 	// `/labs/node`) — relative URLs resolve against the *document*, so a missing slash drops a
 	// segment. Dev is unaffected: Vite forces base to `/` when serving.
 	base: "./",
+	server: {
+		proxy: {
+			// The relay endpoint is origin-gated and does not know `localhost`, so anonymous mode
+			// cannot be exercised under `pnpm dev` at all — it fails before the workspace boots.
+			// This forwards it under an origin the endpoint does recognize, which is the only thing
+			// it checks. Dev-only (Vite ignores `server` in a build), and reached by pointing
+			// `VITE_WISP_URL_ENDPOINT` at `/__wisp/`.
+			"/__wisp": {
+				target: "https://sensible-ship-8305.puter.work",
+				changeOrigin: true,
+				rewrite: (path: string) => path.replace(/^\/__wisp\/?/, "/"),
+				headers: {
+					Origin: "https://developer.puter.com",
+					Referer: "https://developer.puter.com/labs/node/",
+				},
+			},
+		},
+	},
 	build: {
 		chunkSizeWarningLimit: Infinity,
 		rolldownOptions: {