From c363f3e27b485d3f6330c3634d5d9c8f80d98d72 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 5 Nov 2026 18:01:25 -0800 Subject: [PATCH] feat(desktop): pick seed and end by checkpoint, and keep the capture host out of bundles (#9037) * feat(desktop): capture a Linux desktop application in use Add `capture-desktop-application`, a CLI-only, Linux and X11 capture that launches an installed desktop application, watches it while it is used and writes capture Evidence plus SHA-256 addressed content: - inventory: desktop entry, launch command, executable, package and its apt sources (or snap, flatpak, AppImage), Electron markers and candidate state directories; - session: the launched process tree, client windows from xprop with the first application window time and a window match, and files held open; - state: a stat walk of the home directory before launch, at checkpoints read from stdin, at stop and after exit, with created, modified, deleted and retyped paths and the content of changed files and state directories. It refuses non-Linux hosts, missing DISPLAY or xprop, a non-empty output folder and an application whose window is already open, and signals only processes it started. Co-Authored-By: Claude Opus 5.5 * fix(desktop): own launched processes, confirm walk coverage and keep cancelled captures - Launch through the owned provider path with a run token; release with SIGTERM, then the token-verified group cleanup, after every session. Processes without the token are never signalled; each records whether it carries it, and the session records whether cleanup was verified. - Report creations and deletions only where a walk listed the nearest existing directory; count the rest as unconfirmed. - Retain a file only when it is still the walked version, through owned no-follow reads; charge the budget only for new content; create files exclusively in a private output folder owned by the caller. - Walk content roots outside the home directory, compare paths in their physical spelling, and run executables from the caller's directory. - Refuse when the window list cannot be read, re-evaluate window ownership each sample, handle one queued request per sample with a bounded stdin queue, and write the capture with end reason `cancelled` on interruption. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep owned cleanup failures and fresh digests - Allow a capture cancelled before any requested checkpoint to hold only the baseline. - Retry an incomplete stop once; if cleanup is still unverified, write the capture and fail with the run token and process group as cleanup resources. - Classify path changes after a walk (ENOTDIR and other selected-file errors) as changed_since_walk instead of failing the capture. - Treat a non-directory ancestor as proof that nothing existed beneath it. - Report a before digest only for the file version it was read from. - Derive stderr_truncated from stderr's own byte counts. Co-Authored-By: Claude Opus 5.5 * wip(desktop): state view over a capture folder Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep failed file-handle closes owned for retry Content reads pass retainFailedClose, so a handle whose close fails stays owned by the content store; the capture retries those closes after it is written and fails with the output folder as the cleanup resource if one still cannot be closed. Co-Authored-By: Claude Opus 5.5 * feat(desktop): read a captured application's state Add inspect-desktop-application-state, a CLI command that reads a capture folder into the application's state: every retained or changed file with its role and reasons, a typed JSON view of each version (SQLite through the shared worker, dconf through a private dconf dump, JSON, XML, key files, text) and the kinds of change between versions (lines, values, keys, rows). Co-Authored-By: Claude Opus 5.5 * fix(desktop): report process and handle cleanup failures together When both the launched processes and a retained file's handle remain unresolved after the capture is written, one cleanup error now lists both reasons and both resource sets. Co-Authored-By: Claude Opus 5.5 * feat(desktop): export a captured application as an agentenv computer app Add export-desktop-application, which writes an agentenv bundle with one computer_app environment from a capture folder and its state view: the apt package, a boot launch matched on the captured window class with a window ready check, owned paths from the state roles, a seed of the baseline files, data hooks that call agentenv's add/get/reset helpers with a typed JSON view on get, a skill, a README and a deploy smoke task. The desktop CLI commands now load their services lazily. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep state views faithful to the capture - Read key files, XML attributes, dconf groups and SQLite tables into dictionaries without inherited properties, so captured keys such as __proto__ stay data. - Version databases with their write-ahead log as of each checkpoint, and stage only digest-verified bytes for the SQLite and dconf readers. - Detect the format per version, report format changes, keep JSON null and empty containers as values, keep XML text and preserved whitespace, and mark truncated views (including XML depth) and comparisons of them. - Number versions and changes by checkpoint index, decide baseline presence from the captured type, and keep tagged SQLite integers, text bytes and blobs distinct. - Carry the capture's limitations and omitted content forward, match state directories by their home path, index changes once per capture, keep failed handle closes owned, and finish the CLI through the shared owner. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep coverage, logs and handles honest in the state view - Read coverage flags only from formats whose readers write them, so JSON fields named truncated stay data. - Fail a SQLite version whose write-ahead log existed but was not retained or does not match its digest, instead of reading the database alone. - Close retained content on every exit, including cancellation, and report a close that still fails together with the primary failure. - Finish the CLI with the canonical operation name. Co-Authored-By: Claude Opus 5.5 * fix(desktop): export only what the computer can install and seed - Refuse packages not from the Ubuntu 24.04 archive, since the computer installs by name from that archive. - Start the launch command in the captured working directory. - Respell the home path only in files the state view read as text. - Report files that were not retained instead of dropping them silently. - Load captured dconf keys for apps that own no files. - Keep home-level files as owned paths; rename reserved Python class names. - Mark truncated SQLite and text views in the class's get view, and read file types the way the state view does. - Release the output tree through a resource scope, and read seeds through an owner closed on every exit. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep staged folders owned, symlinked state roots and XML values - Stage database bytes in a private runtime root released through a resource scope, so a failed removal keeps its owner and is reported with the read's outcome. - Report cleanup facts and owners from desktop failures instead of wrapping them as a plain adapter error. - Record the state directories as walked, in their physical spelling, and match roles against both spellings, so files in a symlinked state directory are not mistaken for selected documents. - Keep whitespace-only XML leaf values; drop only formatting between elements. Co-Authored-By: Claude Opus 5.5 * fix(desktop): read XML in the exported app the way the state view does Keep whitespace-only leaf values and xml:space="preserve" text in the generated get view; drop only formatting between elements. Co-Authored-By: Claude Opus 5.5 * fix(desktop): release retained handles with their owners, read older captures - Release retained-content handles whose close failed through a resource scope on every exit; a release that fails again reports the exact handle owners. A read that finished keeps its bytes when only its close failed. - Accept captures written before state directories were recorded, and say in the state view's limitations that symlinked state directories may be misclassed for them. - Compute each XML element's whitespace context once instead of scanning siblings for every text run. Co-Authored-By: Claude Opus 5.5 * fix(desktop): boot without the working folder, bound XML, accept mirrors - Try the captured working directory only when an argument is relative, and start the app even when that folder is missing. - Cut the generated XML view at REA's node and depth limits, and keep one unreadable file from failing data/get. - Accept Ubuntu 24.04 mirrors, not only the canonical hosts; PPAs and other repositories are still refused. - Read seeds through the retained-content owner's new signature. Co-Authored-By: Claude Opus 5.5 * fix(desktop): state view review round - Stage a retained -journal beside its database so the SQLite worker accepts an inactive journal and refuses a hot one; a -wal or -journal that existed but was not retained (including one omitted at the baseline) makes the version unreadable. - Keep tables that read when another fails, and let cancellation and unfinished cleanup propagate instead of becoming view errors. - Read text as a key file only when every line parses as one. - Rank a file named in a window title or selected by the caller as a document even when the application holds it open; match titles on file-name boundaries; leave the display name out of dconf matching. - Bound JSON and key file views (4 MiB, 500 levels, exact numbers) and report too-large, missing and altered retained content separately. - Add no version or change while a database stays absent. - Read staged databases in place instead of copying them again. - Give Evidence the same limitations as the result. Co-Authored-By: Claude Opus 5.5 * fix(desktop): export review round - Load captured dconf keys without D-Bus: install dconf-cli, compile the existing and captured key files into the user database, and warn on failure instead of stopping the server. - Start relative-argument launches in the captured working directory, the home included, since the framework launches from the app folder; respell the home itself and option values under it. - Read views in a thread, skip files over 64 MiB, align the generated readers with the state view's, and close SQLite connections. - Never own the home or a shared base folder; map a symlinked state directory's files back to their home spelling. - Report omitted files and unretained opened folders. - Name the module so it cannot shadow another, and keep the class off the module's own names; derive names the bundle check accepts. - Write seeds as they are read, and remove an unpublished bundle on failure. Co-Authored-By: Claude Opus 5.5 * fix(desktop): capture review round - Ask the launch process group and token carriers to exit before killing them, on errors too; report incomplete cleanup with its facts. - Count descendants of the launch as started by the capture even when their environment cannot be read; treat a reused pid as a new process. - Follow symlinked desktop entries; name a flatpak's state after its application id; match flatpak windows by name only. - Stop on end of input; refuse empty, duplicate and reserved checkpoint names; require whitespace after the checkpoint keyword. - Stay on each walk root's device, and never walk or retain `/` or a directory containing the home; say so in the limitations. - Refuse a window manager that does not publish _NET_CLIENT_LIST. - Remove retained content when a capture fails before it is written. - Skip reading a file larger than the remaining content budget. - Record apt's archive origin for each package source. Co-Authored-By: Claude Opus 5.5 * fix(desktop): read an unset client list as no windows when supported xfwm4 leaves _NET_CLIENT_LIST unset while it manages no windows, so the capture refused an ordinary desktop with nothing open. Read _NET_SUPPORTED with the list: an unset list means no windows when the window manager supports it, and the capture is refused only when it does not. `xprop`'s "no such atom" output now reads as unset too. Co-Authored-By: Claude Opus 5.5 * fix(desktop): accept packages by apt's archive origin Accept a package source when apt reports its origin as Ubuntu for a noble suite, so mirrors count and PPAs or other repositories that merely have "ubuntu" in their URL do not. Captures made before origins were recorded fall back to Canonical's own archive hosts. Seed reads report why retained content could not be used. Co-Authored-By: Claude Opus 5.5 * feat(desktop): capture and export apps from vendor apt repositories A closed-source app installed from its vendor's apt repository, such as Typora, can now be captured and exported as a computer app. - The capture records each non-Ubuntu repository listing the installed version (uri, suite, components from apt's sources files, one-line or deb822) and the key that signs it: the signed-by key, or the trusted key that gpgv shows verifies the repository's signed index. The key is retained with the capture. - The capture records the packages owning the libraries the executable loads (DT_NEEDED with RUNPATH and the linker cache, through REA's ELF readers), since a vendor package may not depend on them. - The export declares a verified vendor repository as an AptSource with its key, installs a vendor package's library packages, and adds --no-sandbox to an Electron app's launch, since a container denies Chromium's sandbox. Co-Authored-By: Claude Opus 5.5 * fix(desktop): report a locked database at once in data/get A database the running app holds locked made each data/get wait sqlite's five-second busy timeout before reporting it. Record the Typora run in an agentenv computer in the verification. Co-Authored-By: Claude Opus 5.5 * perf(desktop): detect key files without splitting the whole text Read lines one at a time and stop at the first that a key file view would drop, instead of splitting a file of up to 64 MiB into arrays before its format is known. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep symlinked homes and state folders under the home in exports - Respell a path under its state folder's logical spelling only when that spelling is under the captured home, so a symlinked HOME keeps its files; pair walked and logical state folders only when the capture lists the same number, and match the innermost folder first. - When two files map to one home path, keep the one present before the capture and name the other in the limitations. - Read key files in the generated data/get with the state view's rule, checked against it by running the helper under python3. Co-Authored-By: Claude Opus 5.5 * fix(desktop): end captures on zombies and keep cleanup owners - A zombie or dead process no longer keeps the session open. - Remove only the partial capture file this run created. - Keep the exact owners of handles and processes whose cleanup failed on the returned error, so a retry can release them; test the published capture's cleanup-failure branch with mixed outcomes and retries. - Release the application once even when the exit checkpoint fails. - A cancellation before the capture is written removes what it retained. Co-Authored-By: Claude Opus 5.5 * fix(desktop): vendor package review round - Match apt sources through `apt-get indextargets`, which covers flat `./` and `/` suites and gives each release file and sources entry. - Accept a key whose signature verifies even when the release carries another signature without a key here; keep only the verifying key from a keyring holding several (OpenPGP packet splitter). - Install the package from its repository's release (`name/`), declare one source per URI and suite, support only the capture host's architecture, and refuse Ubuntu Pro repositories. - List only library packages the computer can install; walk libraries only for vendor installs, with RPATH inherited as ld.so does, normalized paths, and DT_NEEDED bounded while reading. - Put --no-sandbox after the program an env wrapper launches; keep the gpgv folder's owner when its removal fails; respect --max-file-bytes for keys; wait 0.25 s on a busy SQLite database in data/get. Co-Authored-By: Claude Opus 5.5 * docs(desktop): keep the vendor spec's gpgv command on one line A code span broken across lines left `` for VitePress to read as an unclosed HTML tag, so the docs build failed. Co-Authored-By: Claude Opus 5.5 * feat(desktop): add a validated task to exported desktop applications Each capture can now yield one agentenv task in its exported bundle. An agent writes the task (a prompt plus declarative checks over the app's data/get views); REA validates it and stays model-free. - `describe-desktop-application-task` summarises a capture's net change per file (role, format, views before and after) and drafts checks. - Checks: file presence, lines present/absent/counted, JSON-pointer values, SQLite rows and counts. REA keeps a task only if a check fails on the seed and every check passes on the captured end state; a file that was not retained at the baseline counts as absent, as the computer starts. - `export-desktop-application --task` writes tasks/NAME.json (deploy, agent, skills, recording, prompt, verifier), a compiled verify.py that reads data/get, and an eval. A boundary test runs the verifier under python3 and checks it agrees with REA's evaluator on every check kind. - Docs, spec and skill guidance, with the solve-rate recipe. Co-Authored-By: Claude Opus 5.5 * test(desktop): keep the seed test in its own block Co-Authored-By: Claude Opus 5.5 * docs(desktop): record the task verifier driven in an agentenv computer Co-Authored-By: Claude Opus 5.5 * docs(desktop): record the Typora task run in an agentenv computer Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep task checks bounded, complete and attributable - line_count counts lines holding a substring (contains) instead of matching a regular expression, which could stall the verifier on agent-written text and read differently in Python. - value_equals of a whole object or array is undecided on a truncated view. - The draft keeps a database whose rows changed under an unchanged hash, and a file whose truncated views match while it changed. - A task file that cannot be read keeps the reader's detail, and a failed cleanup keeps the read failure as its primary. Co-Authored-By: Claude Opus 5.5 * fix(desktop): leave a file with one version out of the task draft Co-Authored-By: Claude Opus 5.5 * docs(desktop): update the task verification's test counts Co-Authored-By: Claude Opus 5.5 * docs(desktop): record the task solve rate with Claude Code and Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 * feat(desktop): capture and export apps installed from a downloaded .deb capture-desktop-application --package-file reads the file's control fields with dpkg-deb and records its SHA-256, size and dependencies when it holds the installed package; the file itself is never kept. The library walk now also runs for local installs. export-desktop-application --package-url declares agentenv's Download(url, sha256) for that file, its dependencies and library packages as apt packages, and a run script that installs it with dpkg. Co-Authored-By: Claude Opus 5.5 * fix(desktop): seed app-named dotfiles and respell UTF-16 seeds The capture now treats dotfiles in the home named after the application (.varicad-cfg.ini, .notesrc) as state roots, so a file the app only reads during the capture is still seeded. The exported seeding respells the captured home in UTF-16 text files with a byte-order mark, keeping the mark. Co-Authored-By: Claude Opus 5.5 * fix(desktop): hold readiness until the app's own window shows An exported app is ready only once a window of its class that the taskbar lists is shown, so a splash screen of the class, which skips the taskbar, no longer counts as the app being up. Co-Authored-By: Claude Opus 5.5 * docs(desktop): record VariCAD captured, exported and run in an agentenv computer Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep exported views strict JSON for infinite and inexact numbers The exported data/get spells infinite and negative-zero SQLite reals as REA's SQLite reader does ("Infinity", "-Infinity", "-0"), and refuses a JSON file holding a number that reading would change (1e999, integers past 2^53, -0, rounded fractions) with the state view's own error, instead of a view with an infinity in it. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep package-file facts tied to one file and URLs unechoed - A --package-url refusal never repeats the URL, and credentials are checked before the scheme. - The package file's identity must hold between dpkg-deb's read and the hash. - A failed close keeps its cleanup owner even when the capture is cancelled. - Readiness also ignores windows typed as splash screens. Co-Authored-By: Claude Opus 5.5 * fix(desktop): record database logs and keep state changes distinct - A selected path that is not a directory brings its -wal and -journal as content roots, so the capture records whether they exist and retains them. - The state view treats a file as absent only under recorded content roots and refuses a database whose log presence is unknown. - JSON changes list containers too, so an array that becomes an object with the same members is a change. - Key file entries escape the key as a JSON pointer segment, so [a/b] c and [a] b/c stay apart. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep watching through unread samples and late workers - An unreadable window list is unknown, not zero windows, and is named in the limitations. - Before ending the session, live run-token carriers no parent link reached are tracked; an unreadable process table leaves the exit unknown. - An env prefix's assignments find the program and name its XDG folders. - flatpak run options that take a value are skipped with it, -- is honoured and a /arch/branch suffix is dropped from the id. - State directories include the entry's full id. - Path options refuse empty values; roots count toward the entry limit. - A capture file whose close failed stays owned and is retried on discard. Co-Authored-By: Claude Opus 5.5 * fix(desktop): bound apt sources, keep older captures and cleanup owners - Sources files parse into one stanza per line or deb822 paragraph with its URIs and suites kept as lists, so no URI x suite product is built. - Repository architectures and release are optional, so captures written before they were recorded parse; the export treats them as unrestricted and unknown. - Library sources are matched against every index of a declared repository's URI, suite and components. - Two failures side by side keep both causes and both cleanup owners, and a host file whose close fails reports a lease on its handle. Co-Authored-By: Claude Opus 5.5 * fix(desktop): match escaped key file entries and suggest containers through members - The draft's key file lookup uses the state view's escaped `group/key`, so a key holding `/` or `~` still gets its suggestion. - JSON and XML changes now list containers too; the draft suggests an added or removed container through its listed members, and a container only when none of its members is listed. Co-Authored-By: Claude Opus 5.5 * fix(desktop): read relative PATH entries from the launch folder A program, or an empty or relative PATH entry, is resolved from the folder the program starts in, as the launch does, so `env PATH=. app` with the entry's Path still finds the app. Co-Authored-By: Claude Opus 5.5 * perf(desktop): compare JSON views on one shared path JSON and XML changes are found by walking both trees with one shared pointer path, building a pointer only for a listed value, so memory follows depth rather than every container's full path. A pointer over 4096 characters is counted but not listed. A 256-level document with 4000-character keys went from about 251 MiB of heap to under 1 MiB. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep JSON Lines as text in the exported reader The exported reader parses the whole text before an inexact number makes it JSON with an error, so JSON Lines with an integer above 2^53 stay text as in REA. Over-long integer spellings are unsafe without converting them. The guide no longer claims large integers keep their value. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep the state-path help free of dash-led words The CLI alias check reads `-wal` and `-journal` in help text as short flags, so the help names the write-ahead log and rollback journal instead. Co-Authored-By: Claude Opus 5.5 * fix(desktop): never record or export apt source credentials - Source URIs lose any user name and password as they are parsed, the way apt splits them, and so do index-target URIs and descriptions and the sources apt-cache policy lists, so matching works whether or not apt prints them. - A repository whose entry held credentials is marked `uri_credentials_removed`, and the export refuses it, or an older capture whose URI still holds them, with a reason that names no URL. Co-Authored-By: Claude Opus 5.5 * fix(desktop): look programs up without normalizing their paths Program paths and PATH entries are joined rather than resolved, so `..` after a symbolic link means what it means to the launch. The lookup moves to its own module. Co-Authored-By: Claude Opus 5.5 * fix(desktop): note inexact JSON numbers with a flag, not a list The exported reader only needs to know whether any number was inexact, so it keeps a boolean instead of every rejected spelling. Co-Authored-By: Claude Opus 5.5 * fix(desktop): refuse older captures that still hold apt credentials A capture written before credentials were removed counted the credentialed repository's libraries as installable, so the export refuses it whole and asks for a new capture rather than declaring packages no source offers. Recorded policy sources are redacted in export messages too. Co-Authored-By: Claude Opus 5.5 * test(desktop): keep the credential fixture on an example host A login on a real host reads as a secret to the commit scanner, so the older capture's fixture uses apt.example.com like the other credential tests. Co-Authored-By: Claude Opus 5.5 * fix(desktop): tidy looked-up paths and search relative entries only for env Empty and `.` segments are dropped from a looked-up program path, which keeps `/snap/bin/` entries recognisable as snaps, while `..` is still left for the kernel. Relative and empty PATH entries are searched only in the PATH an `env` prefix sets, which travels with the exported launch; the host's own PATH does not. Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep excluded and unwalked database logs unknown The capture records the paths every walk left out, and the state view treats a log as absent only under content roots the baseline walked completely and outside those exclusions, so an excluded or unvisited log leaves the database unread rather than read without it. Co-Authored-By: Claude Opus 5.5 * fix(desktop): judge JSON numbers after duplicate keys settle The exported reader marks an inexact number instead of flagging it during the parse, then looks for marks in the parsed value, so a number a later duplicate key overwrote no longer refuses the file, as in REA. Co-Authored-By: Claude Opus 5.5 * fix(desktop): redact a repository's policy source when it names it An older capture whose policy source matched no sources entry kept that text whole, and the export named the repository by it; the export now names it without any user name or password. Co-Authored-By: Claude Opus 5.5 * fix(desktop): never signal the launch group on unknown membership When the current read cannot observe a carrier's process group, the carrier is signalled on its own instead of letting an older process-table row grant a group-wide SIGTERM. Co-Authored-By: Claude Opus 5.5 * fix(desktop): look programs up in the launch's order and refuse host-relative finds The lookup searches PATH as the launch does, an empty entry meaning the launch folder, so inspection and launch name the same program. A program found only through a relative or empty entry of the host's own PATH is refused, since the exported computer would not have it; an env prefix's PATH travels with the command. Co-Authored-By: Claude Opus 5.5 * fix(desktop): count unchecked paths and keep logs behind them unknown Each walk counts the paths it could not stat and the directories on another device it did not enter, and the checkpoint records it as `unchecked_paths`. The state view treats a log as absent only when the baseline left nothing unchecked, so a log whose stat failed stays unknown. Co-Authored-By: Claude Opus 5.5 * perf(desktop): scan parsed JSON with one iterator per level The check for marked numbers walks the parsed value with an iterator per nesting level instead of copying each container's children; a million-entry array went from about 15.7 MiB to 8.1 MiB of peak traced memory. Co-Authored-By: Claude Opus 5.5 * fix(desktop): start a relative program from its captured folder An exported launch whose program path is relative, such as `./app` from an entry's Path, now changes to the captured working directory first, as it already did for relative arguments. Co-Authored-By: Claude Opus 5.5 * feat(desktop): read zipped documents and name XML elements in task checks - A file that starts as a zip archive gets a `zip` view listing its entries in archive order, each with its own format and view, within limits on entries, bytes and expansion; the exported Python reader matches it. - A pointer segment applied to an array may be a selector, `[KEY=VALUE]` or `[@KEY=VALUE]` for XML attributes, matching exactly one member, so checks name elements instead of positions that shift between saves and versions. - `value_equals` takes an optional numeric `tolerance`, comparing numbers and numeric strings. - Changes inside zip views are listed by selector, and the task draft suggests name-addressed checks. FreeCAD's `.FCStd` is the first case: a hole cut into a plate is verified from the zipped `Document.xml` on files from both FreeCAD 0.21 and 1.0. Co-Authored-By: Claude Opus 5.5 * fix(desktop): record the desktop entry's package and app profiles in any case - A launch from a desktop entry records the package that owns the entry, so the export installs the app the entry starts (Writer, not only libreoffice-common); the program's own package is kept as executable_package when it differs. - State folders match the app's names ignoring letter case, as direct children of the XDG folders and the home, so GIMP's ~/.config/GIMP profile is seeded and reset; roles follow the same rule. Co-Authored-By: Claude Opus 5.5 * feat(desktop): read namespaced XML alike in REA and the exported verifier The exported Python reader now names XML elements and attributes as the document spells them (svg, text:p, sodipodi:docname, xml:space), with namespace declarations kept as attributes, which is what REA's view already produced. Export therefore accepts checks into Inkscape SVGs, LibreOffice content.xml and namespaced XML inside zips, and the draft suggests them. expat replaces ElementTree: one pass with namespace processing refuses what REA's parser refuses, and a streaming pass builds the view within the same node and depth limits. Text splits at CDATA, comments and processing instructions as REA's DOM splits it. Fixtures are real Inkscape 1.2.2 and LibreOffice 24.2 saves. Co-Authored-By: Claude Opus 5.5 * fix(desktop): install the program's own package and keep unknown owners unknown - When the desktop entry's package does not own the program it runs, the export installs the program's package too. The capture records why it cannot (executable_package_reason), and the export then refuses. - The export owns a state folder in the letter case the app wrote it, as the state roles already did, so data/get and data/reset cover a profile folder the app created during the capture. - A dpkg-query that fails or exits 2 or higher leaves the package unknown instead of reading as "no owner". Co-Authored-By: Claude Opus 5.5 * fix(desktop): keep selector checks undecided on partial views and pair members one to one - A selector check is undecided while an array it searched may be missing members: a cut zip listing, or a truncated XML or entry view. Both evaluators agree. - Diffs pair the members of two versions one to one, so two new elements can no longer both match the same old one and hide an addition. - The task draft descends into JSON values and readable entries added whole, and never suggests an element by its position, last step included. - The export's namespace refusal also checks XML entries inside the value a pointer names, such as the whole listing. - The exported reader keeps REA's 500-level JSON depth limit, so one deep entry no longer hides the rest of the archive, and treats an entry whose data runs past its declared size as damaged. Co-Authored-By: Claude Opus 5.5 * fix(desktop): check approved packages against the repositories the export keeps The capture approved library packages and the program's package against every repository with a verified key, but the export can still leave a repository out: its key dropped by the content budget, an architecture restriction, or a conflicting key. Each repository now records the packages only it supplies (`supplies`). The export drops stranded library packages with a limitation, and refuses when the program's own package is stranded. Captures without `supplies` behave as before. The install limitations move to appExportInstallLimitations.ts to keep appExportInstall.ts under the line limit. Co-Authored-By: Claude Opus 5.5 * fix(desktop): compare type changes in place, keep empty values and unread entries honest - When an array becomes another type, its members are compared with what their pointers name there, so values that are still present no longer look removed. - The draft checks an added empty array or object by equality. - The exported reader's JSON depth check keeps one iterator per level and copies no wide list, back to the memory peak before the check. - An entry whose view could not be read is no `null` value: the draft suggests no check on it, and a check into its view fails in both evaluators with the read error. Co-Authored-By: Claude Opus 5.5 * fix(desktop): decide entry view checks only on entries with a view, and walk JSON depth by level - A check into an archive entry's view needs an entry read in a format with a view. A binary or unread entry's `view` of `null` no longer passes a `null` or absence check, in either evaluator, and the draft suggests no absence inside an entry that ends without a view. - REA's JSON depth check keeps one iterator per level instead of queuing a pair per member, so 800,000 nulls need 18 MiB instead of more than 64. The helper came with the state view; its Python twin was fixed here. Co-Authored-By: Claude Opus 5.5 * fix(desktop): refuse an export whose desktop entry runs a program no package owns When a package owns the desktop entry but no package owns the program it runs, such as one in ~/.local/bin first on PATH, the capture keeps the entry's package and records why the program cannot be installed. Export refuses instead of installing a package that does not bring that program. Co-Authored-By: Claude Opus 5.5 * fix(desktop): bound zip entry reads, export metadata checks, and diff by the evaluator's pointer rules - The exported reader inflates a zip entry at most 64 KiB at a time and stops a byte past its declared size, so a forged small size no longer holds the whole stream in memory (77.5 MiB to 0.1 MiB on a 32 KiB bomb). - The namespace refusal applies only to checks that depend on an entry's XML, so checks on an entry's name, size or digest, or on its absence, export. - Diffs find a member's counterpart by the evaluator's pointer rules, so a selector step across a type change no longer reports a present value removed. Co-Authored-By: Claude Opus 5.5 * fix(desktop): read at most 1,000 zip directory records, and index arrays once per diff - Both zip readers now read the central directory a record at a time, keep the first 1,000 file records and count the rest, then hand the library an archive whose directory holds only those. Directories are told apart as zip.js does, so the Python reader stops listing attribute-marked directories as files. 600,000 entries now peak at 1.3 MiB in Python (was 293.6 MiB) and need 24 MiB of heap in REA. - The exported Python zip reader moves to appExportPythonZip.ts. - Diffs look members up through an index built once per compared array, so 20,000 selector keys against 20,000 members take 31 ms, not 5.4 s. Co-Authored-By: Claude Opus 5.5 * feat(desktop): pick seed and end by checkpoint, and keep the capture host out of bundles - Export, task export and the task draft take --seed-checkpoint and --end-checkpoint, so one capture can make the seed in the app and then the edit; the defaults stay the baseline and the last version. - Capture reads apt's lists folder and trusted keys from apt-config, so a user-space apt-get update under APT_CONFIG is found, and starts the app without AGENTENV_* variables. - Capture records the host name and user. Export rewrites user@host, the host name as a whole name and the captured home in text, UTF-16 and dconf seeds, ships SEED_HOME=/home/user instead of the capture user's home, lists scrubbed and unscrubbable seeds, and refuses checks that name the host. Co-Authored-By: Claude Opus 5.5 * fix(desktop): check tasks against the shipped seed and scrub JSON and XML by value - Task validation reads the seed as the bundle ships it, after the scrub, so a task whose end value the scrub writes into the seed is refused. - JSON strings and XML text, attribute values, comments and CDATA are scrubbed as their decoded values and rewritten only when they change, so syntax stays valid and escaped identities are found. - One leftover search covers every shipped file's path and contents, generated module and dconf keys included; the strict whole-name rule is kept for replacing only. - A host named localhost is never replaced, but user@localhost is. Co-Authored-By: Claude Opus 5.5 * fix(desktop): resolve a record's selector keys together, and follow Zip64 locators without sentinels - The diff resolves all of a record's selector-shaped keys in one lookup: conditions are normalized per field, selectors grouped by field set, and only the values asked for are kept, so retained memory grows with the keys, not members times field sets (68 MB to 5.5 MB, 76 MB to 4.9 MB on the reported cases). - Both zip readers follow a Zip64 locator just before the end record whatever the classic record holds, as zipfile does, so valid Zip64 archives with ordinary classic values read again. The exported module's synthetic tail carries a real Zip64 end record, so bytes that only look like a locator no longer make zipfile refuse it. Co-Authored-By: Claude Opus 5.5 * fix(desktop): scrub JSON values only, with a scanner whose stack does not grow with string length JSON seeds are walked one string at a time, finding each closing quote with indexOf and a backslash count instead of a backtracking regex, so a 16,000,000-character string no longer overflows the stack. Member names are never rewritten, so two keys cannot collide; a name holding the host leaves the file in unscrubbed_files. Co-Authored-By: Claude Opus 5.5 * fix(desktop): scrub dconf and UTF-16 seeds by format, search check values for the host, and gate checks on the selected ends - dconf: the host scrub ran over the whole key file, so on a host named `true` `wrap=true` became an invalid `wrap=localhost`. Only decoded GVariant string values are scrubbed now; keys, groups, bytestrings, dictionary keys and object paths ship as captured and are reported. - Task identity: the host search read the serialized check, whose field names (`file`, `kind`) and default ids matched hosts of those names. It now reads the check's string values only. - Value checks: the text-file refusal read every recorded version; it now reads only the shipped seed and the end. - UTF-16 seeds: the plain scrubber renamed JSON keys and XML names; the decoded text's format now picks the scrubber, keeping the byte order mark. Co-Authored-By: Claude Opus 5.5 * fix(desktop): pair members by identity first, build selectors lazily, and refuse oversized and deep text before building a view - Pairing: an element with an identifier (name, id, xid, *:id) pairs only by it, so an unchanged id still names an edited element instead of a removal and an addition, and two objects of one type are never paired as a rename. Selectors keep the tag apart from each attribute condition and spell only the chosen one, so a long tag with many attributes no longer multiplies. - Size: REA's detection and the exported reader decide JSON with a scanner that builds no value, and refuse a view over the parsed-view limit before building it, in files and zip entries alike. - Deep text: the exported reader confirms the whole text is JSON before reporting a nesting depth error, as REA reads it. Co-Authored-By: Claude Opus 5.5 * fix(desktop): count a selector's matches over every sibling while identified elements choose only their identifiers Identity-first pairing also dropped an identified sibling's other attributes from the counts that prove a selector names one member, so a shared attribute such as `color` or `fill` could be chosen though it matched two siblings: the diff reported present elements removed and drafts lost their checks. Counting now covers every attribute of every sibling; only the choice is limited to an identified element's tag and identifiers. Co-Authored-By: Claude Opus 5.5 * fix(desktop): tell a UTF-16 seed's format without building it, and scrub XML without one piece per token Choosing the scrubber for a decoded UTF-16 seed ran the full state detection, which built the whole XML document (or JSON value), so a large seed within the capture limit could exhaust memory at export. A syntax pass now decides the format (the JSON token scan, and the XML parser's events keeping only the open elements), and the XML scrubber copies out only the spans it changes. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../references/environments.md | 8 +- docs/desktop-application-capture.md | 72 ++- docs/desktop-application-export.md | 158 +++++- docs/desktop-application-state.md | 77 ++- docs/desktop-application-tasks.md | 151 +++++- docs/specs/desktop-app-environments/README.md | 9 + .../capture-for-building/plan.md | 23 + .../capture-for-building/spec.md | 74 +++ .../capture-for-building/verification.md | 186 +++++++ .../desktop-app-environments/capture/spec.md | 22 +- .../desktop-app-environments/capture/tasks.md | 2 + .../capture/verification.md | 70 +++ .../document-views/plan.md | 25 + .../document-views/spec.md | 106 ++++ .../document-views/verification.md | 220 ++++++++ .../desktop-app-environments/export/spec.md | 13 +- .../state-view/spec.md | 10 +- .../desktop-app-environments/tasks/spec.md | 11 +- .../tasks/verification.md | 5 +- .../desktop/DesktopApplicationCapture.test.ts | 5 + .../desktop/DesktopApplicationCapture.ts | 18 +- .../desktop/DesktopApplicationExport.test.ts | 77 +++ .../desktop/DesktopApplicationExport.ts | 359 +++++++------ .../desktop/DesktopApplicationState.ts | 40 +- .../desktop/DesktopArchiveTask.test.ts | 273 ++++++++++ .../desktop/DesktopCheckpointEnds.test.ts | 251 +++++++++ src/application/desktop/DesktopExportFiles.ts | 182 +++++++ .../desktop/DesktopExportIdentity.test.ts | 348 ++++++++++++ .../desktop/DesktopSigningKeys.test.ts | 77 ++- .../desktop/DesktopStateVersions.ts | 62 ++- src/application/desktop/DesktopTaskDraft.ts | 35 +- src/application/desktop/DesktopTaskExport.ts | 92 ++-- src/application/desktop/DesktopZipView.ts | 131 +++++ .../desktop/desktopCapture.fixture.ts | 59 +- src/artifacts/ZipArtifactReader.ts | 30 +- src/cli/desktopCaptureCommands.ts | 25 +- src/desktop/ApplicationStateDirectories.ts | 106 ++++ src/desktop/AptRepositoryInventory.test.ts | 42 ++ src/desktop/AptRepositoryInventory.ts | 39 ++ src/desktop/DebianPackageInventory.test.ts | 303 +++++++++++ src/desktop/DebianPackageInventory.ts | 239 ++++++--- src/desktop/DesktopApplicationSession.test.ts | 29 +- src/desktop/DesktopApplicationSession.ts | 16 +- src/desktop/LibraryPackageInventory.ts | 38 +- src/desktop/LinuxApplicationInventory.test.ts | 34 ++ src/desktop/LinuxApplicationInventory.ts | 95 +--- src/domain/desktop/appExport.test.ts | 16 +- src/domain/desktop/appExport.ts | 70 ++- src/domain/desktop/appExportInstall.test.ts | 161 +++++- src/domain/desktop/appExportInstall.ts | 153 +++--- .../desktop/appExportInstallLimitations.ts | 81 +++ src/domain/desktop/appExportPython.ts | 211 +++++--- src/domain/desktop/appExportPythonXml.ts | 101 ++++ src/domain/desktop/appExportPythonZip.ts | 237 +++++++++ src/domain/desktop/appTask.ts | 12 +- src/domain/desktop/appTaskChecks.fixture.ts | 416 +++++++++++++++ src/domain/desktop/appTaskChecks.test.ts | 158 ++++++ src/domain/desktop/appTaskChecks.ts | 124 +++-- src/domain/desktop/appTaskDraft.test.ts | 503 ++++++++++++++++++ src/domain/desktop/appTaskDraft.ts | 153 +++++- src/domain/desktop/appTaskPython.ts | 167 +++++- src/domain/desktop/aptRepositories.ts | 27 + src/domain/desktop/desktopCapture.ts | 10 + src/domain/desktop/desktopState.ts | 14 + src/domain/desktop/hostIdentity.test.ts | 110 ++++ src/domain/desktop/hostIdentity.ts | 106 ++++ src/domain/desktop/libraryPackages.test.ts | 1 + src/domain/desktop/libraryPackages.ts | 30 +- src/domain/desktop/seedScrub.test.ts | 333 ++++++++++++ src/domain/desktop/seedScrub.ts | 418 +++++++++++++++ src/domain/desktop/stateChanges.test.ts | 428 +++++++++++++++ src/domain/desktop/stateChanges.ts | 109 +++- src/domain/desktop/stateFormats.test.ts | 9 + src/domain/desktop/stateFormats.ts | 230 +++++++- src/domain/desktop/stateRoles.test.ts | 25 + src/domain/desktop/stateRoles.ts | 23 +- src/domain/desktop/viewPointer.test.ts | 240 +++++++++ src/domain/desktop/viewPointer.ts | 400 ++++++++++++++ src/domain/desktop/zipArchive.fixture.ts | 172 ++++++ src/domain/desktop/zipDirectory.ts | 207 +++++++ src/domain/desktop/zipView.ts | 200 +++++++ src/domain/xmlDocument.ts | 37 +- tests/boundary/cli/desktopCaptureCli.test.ts | 55 ++ .../providers/desktopExportPython.test.ts | 164 +++++- .../providers/desktopStateHeap.test.ts | 142 +++++ .../providers/desktopXmlPython.test.ts | 198 +++++++ .../boundary/providers/desktopZipHeap.test.ts | 50 ++ .../providers/desktopZipPython.test.ts | 477 +++++++++++++++++ .../fixtures/freecad/0.21/after/Document.xml | 222 ++++++++ .../fixtures/freecad/0.21/before/Document.xml | 119 +++++ tests/fixtures/freecad/1.0/after/Document.xml | 272 ++++++++++ .../fixtures/freecad/1.0/before/Document.xml | 151 ++++++ tests/fixtures/freecad/README.md | 17 + tests/fixtures/inkscape/README.md | 13 + tests/fixtures/inkscape/after/poster.svg | 57 ++ tests/fixtures/inkscape/before/poster.svg | 57 ++ tests/fixtures/libreoffice/README.md | 13 + tests/fixtures/libreoffice/after/content.xml | 2 + tests/fixtures/libreoffice/before/content.xml | 2 + tests/support/desktopExportPython.ts | 149 ++++++ 100 files changed, 11304 insertions(+), 795 deletions(-) create mode 100644 docs/specs/desktop-app-environments/capture-for-building/plan.md create mode 100644 docs/specs/desktop-app-environments/capture-for-building/spec.md create mode 100644 docs/specs/desktop-app-environments/capture-for-building/verification.md create mode 100644 docs/specs/desktop-app-environments/document-views/plan.md create mode 100644 docs/specs/desktop-app-environments/document-views/spec.md create mode 100644 docs/specs/desktop-app-environments/document-views/verification.md create mode 100644 src/application/desktop/DesktopArchiveTask.test.ts create mode 100644 src/application/desktop/DesktopCheckpointEnds.test.ts create mode 100644 src/application/desktop/DesktopExportFiles.ts create mode 100644 src/application/desktop/DesktopExportIdentity.test.ts create mode 100644 src/application/desktop/DesktopZipView.ts create mode 100644 src/desktop/ApplicationStateDirectories.ts create mode 100644 src/desktop/AptRepositoryInventory.test.ts create mode 100644 src/domain/desktop/appExportInstallLimitations.ts create mode 100644 src/domain/desktop/appExportPythonXml.ts create mode 100644 src/domain/desktop/appExportPythonZip.ts create mode 100644 src/domain/desktop/hostIdentity.test.ts create mode 100644 src/domain/desktop/hostIdentity.ts create mode 100644 src/domain/desktop/seedScrub.test.ts create mode 100644 src/domain/desktop/seedScrub.ts create mode 100644 src/domain/desktop/viewPointer.test.ts create mode 100644 src/domain/desktop/viewPointer.ts create mode 100644 src/domain/desktop/zipArchive.fixture.ts create mode 100644 src/domain/desktop/zipDirectory.ts create mode 100644 src/domain/desktop/zipView.ts create mode 100644 tests/boundary/providers/desktopStateHeap.test.ts create mode 100644 tests/boundary/providers/desktopXmlPython.test.ts create mode 100644 tests/boundary/providers/desktopZipHeap.test.ts create mode 100644 tests/boundary/providers/desktopZipPython.test.ts create mode 100644 tests/fixtures/freecad/0.21/after/Document.xml create mode 100644 tests/fixtures/freecad/0.21/before/Document.xml create mode 100644 tests/fixtures/freecad/1.0/after/Document.xml create mode 100644 tests/fixtures/freecad/1.0/before/Document.xml create mode 100644 tests/fixtures/freecad/README.md create mode 100644 tests/fixtures/inkscape/README.md create mode 100644 tests/fixtures/inkscape/after/poster.svg create mode 100644 tests/fixtures/inkscape/before/poster.svg create mode 100644 tests/fixtures/libreoffice/README.md create mode 100644 tests/fixtures/libreoffice/after/content.xml create mode 100644 tests/fixtures/libreoffice/before/content.xml create mode 100644 tests/support/desktopExportPython.ts diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md index 3157c81c99..331c0e9e69 100644 --- a/.agents/skills/reverse-engineer-anything/references/environments.md +++ b/.agents/skills/reverse-engineer-anything/references/environments.md @@ -337,7 +337,13 @@ Kinds: `file_present` and `file_absent`; `lines_present` and `lines_absent` `at_most`); `value_equals` and `value_absent` (a JSON `pointer` into the view, such as `/groups/GROUP/KEY` for a key file); `row_present` and `row_absent` (`table`, `where` by column) and `row_count`. Each names its `file` as a path -under the home. The +under the home. A document saved as a zip archive, such as FreeCAD's +`.FCStd`, reads as its entries: point into an entry's view, and name array +members by selector rather than position, as +`/entries/[name=Document.xml]/view/root/children/[name=Object][@name=Hole]`, +so the check survives an app version that writes them in another order. Give +`value_equals` a `tolerance` to compare a number the app writes as text, such +as `"5.0000000000000000"`, with a number. The [desktop application tasks](https://github.com/morluto/rea/blob/main/docs/desktop-application-tasks.md) guide has the exact rules. diff --git a/docs/desktop-application-capture.md b/docs/desktop-application-capture.md index b9f56a3db9..f759fb8532 100644 --- a/docs/desktop-application-capture.md +++ b/docs/desktop-application-capture.md @@ -52,7 +52,10 @@ application is already open, or if the window list cannot be read. A launch beside a running instance can hand off to it, and that instance's unsaved state would be missing from the baseline. It then records the starting state, launches the application in its own process group with a run token in its -environment, and reads commands from stdin, one per line: +environment, and reads commands from stdin, one per line. The application gets +REA's environment without the `AGENTENV_*` variables an agent environment sets, +such as its control token: an app the exported computer launches never has +them, and an app captured inside a computer must not hold a credential to it. | Line | Effect | | ------------------------------------------- | ----------------------------------------------------------- | @@ -65,6 +68,15 @@ session. A name that repeats an earlier checkpoint, or is one of the capture's own names (`baseline`, `stop`, `time-limit`, `exit`), is refused with a progress line on stderr; no checkpoint is taken and the session continues. +Name the checkpoints a later step reads by what they mark. The +[export](desktop-application-export.md) seeds the bundle from the baseline +unless `--seed-checkpoint` names another, and a +[task](desktop-application-tasks.md) is checked against the last checkpoint +unless `--end-checkpoint` names another. So one capture can build a seed and +then the change: launch the app, create and save the document, send +`checkpoint seed`, make the edit and save it, send `checkpoint end`, then +`stop`. + The session also finishes when the launched process has exited and the application's processes and windows are all gone in two consecutive samples, or when `--max-duration-seconds` passes. Before deciding the processes are @@ -122,7 +134,13 @@ command makes it private (mode 0700). It holds: distinct content. The Evidence operation is `capture_desktop_application` and its predicate type -is `rea.desktop-application-capture`. Its result has four parts. +is `rea.desktop-application-capture`. Its `host` names the platform, +architecture, display and home directory, and the host's name (`hostname`) +and the user's (`user`), which captured state can hold, as in a mail +template's `ada@e81232be457d`; the export removes them from the bundle. A +capture written before they were recorded has neither, and one is left out +when the system cannot say, as for a user with no passwd entry. The rest of +the result has four parts. **Application.** The desktop entry (id, path, name, `Exec`, icon, `StartupWMClass`, categories, MIME types), the launch command and working @@ -132,7 +150,39 @@ architecture and the apt sources that list the installed version, with the origin (`o=`) apt reports for each source in `source_origins`, such as `Ubuntu` for the Ubuntu archive and its mirrors or `LP-PPA-` for a PPA; an empty source list means the version came from a local `.deb` or the package lists -are missing, which `package_lists_present` tells apart. +are missing, which `package_lists_present` tells apart. apt's lists folder, +and the trusted keys below, are where `apt-config` resolves +`Dir::State::Lists`, `Dir::Etc::TrustedParts` and `Dir::Etc::Trusted` under +`Dir`, so an `APT_CONFIG` file that moves them is followed: on a computer +without lists, an ordinary user can run `apt-get update` with an `APT_CONFIG` +naming writable `Dir::State::Lists` and `Dir::Cache` folders and capture with +the same `APT_CONFIG`. Without `apt-config`, or when it fails, they are +`/var/lib/apt/lists`, `/etc/apt/trusted.gpg.d` and `/etc/apt/trusted.gpg`. + +The Debian package is the one `dpkg-query -S` names as the owner of the +desktop entry the capture launched, or else of the executable, since the +entry's package is what installs the application: LibreOffice Writer's entry +belongs to `libreoffice-writer`, while the `/usr/bin/libreoffice` it runs +belongs to `libreoffice-common`, which does not bring Writer with it. When the +two differ, `executable_package` names the executable's package, which the +export installs beside the entry's, since the entry's package need not depend +on it (`vim-common` owns Vim's entry but does not depend on `vim`). +`executable_package_reason` says why the computer could not install it by +name, by the rule for library packages below: neither the Ubuntu 24.04 archive +nor a vendor repository the export declares offers it, or `apt-cache policy` +did not say. When a package owns the entry but none owns its program, such as +a `~/.local/bin/` found first on `PATH`, the entry's package is recorded +without `executable_package`, and `executable_package_reason` says no package +owns the program, so the export refuses rather than install a package that +does not bring it. A program link that dpkg does not own counts as owned when +dpkg owns the file it resolves to, as an alternatives link does. An +`--executable` launch, or an entry no package owns, such as +one in `~/.local/share/applications`, records the executable's package. When +`dpkg-query` does not finish or exits with an error for the executable or the +entry, the package is `unknown` and the limitations say which path, since a +failed lookup is not proof that no package owns it. The sources, vendor +repositories and `--package-file` match below are the recorded package's; the +library walk still reads the executable. Each source whose origin is not `Ubuntu` is also recorded in `apt_repositories`. `apt-get indextargets` names the sources file entry apt @@ -148,8 +198,8 @@ and in `apt-get indextargets` and `apt-cache policy` output alike, so entries match whether or not apt prints it, and sets `uri_credentials_removed` on a repository whose URI held one (a capture written before this was recorded omits the field). The key is found by verification, not by name: the candidates are the -entry's `signed-by` keyrings, or else every `.gpg` and `.asc` key in -`/etc/apt/trusted.gpg.d` and `/etc/apt/trusted.gpg`, and each is checked with +entry's `signed-by` keyrings, or else every `.gpg` and `.asc` key in apt's trusted +parts folder and its trusted keyring, and each is checked with `gpgv` against the release's `InRelease` (or `Release` with `Release.gpg`) beside its index in apt's lists folder. A key counts when gpgv reports a valid signature by it and no bad, expired, revoked or unverifiable signature @@ -187,7 +237,12 @@ listed, and neither is the application's own package. One batched recorded vendor repository with a verified key offers, in any component its sources entry lists, since the export declares them all; the rest, such as packages from a PPA or a local `.deb`, are listed in -`uninstallable_library_packages`. Sonames found nowhere are listed in +`uninstallable_library_packages`. The same query checks `executable_package`. +Each recorded repository's `supplies` lists the kept packages, and the +executable's package, that the archive does not offer and its package files +list, so the export can leave out those only a repository it does not declare +supplies, such as one whose key the content budget later dropped. Sonames +found nowhere are listed in `missing_libraries`, and libraries no package owns in `unpackaged_libraries`. `library_packages_reason` says why the lists are incomplete: the executable is not an ELF image (a launcher script, for example), `ldconfig`, `dpkg-query` or @@ -223,7 +278,10 @@ the fuse sentinel in their binary or a `resources/app.asar` beside it. State directories are the XDG config, data, state and cache folders and the dot-folder named after the package, the executable, the desktop entry's full id and its last part, and the entry's name, with whether each existed at the -start. Dotfiles in the home named after the application (`.rc`, or +start. A folder in those places whose name is one of these in another letter +case joins them, such as GIMP's `~/.config/GIMP` for the `gimp` package; only +the whole name is compared, so `~/.config/GIMPshop` does not. Dotfiles in the +home named after the application (`.rc`, or `.` followed by `-`, `.` or `_`, for a name of three or more characters) that exist at the start join them, since an application may only read such a file during the capture. A `flatpak run` launch is named after its application diff --git a/docs/desktop-application-export.md b/docs/desktop-application-export.md index ab687e849a..207abfac95 100644 --- a/docs/desktop-application-export.md +++ b/docs/desktop-application-export.md @@ -22,10 +22,16 @@ the capture, once REA has validated it against the capture; see [desktop application tasks](desktop-application-tasks.md). `--package-url` says where the computer downloads the package of an application installed from a `.deb` file; see [Applications from a .deb file](#applications-from-a-deb-file). +`--seed-checkpoint` names the checkpoint whose state seeds the bundle, the +baseline by default, and `--end-checkpoint` the one a task must pass on, the +capture's last by default; see [The seed checkpoint](#the-seed-checkpoint). The command returns Evidence with operation `export_desktop_application` and predicate type `rea.desktop-application-export`, listing the environment name, -owned paths, seed files and written files. A failed export, cancellation +the seed and end `checkpoints`, owned paths, seed files, the seeds it +rewrote to remove the capture host's name and user (`scrubbed_files`), the +bundle files that still hold them (`unscrubbed_files`), and written files, +the last three by their path in the bundle. A failed export, cancellation included, removes the bundle it had not yet published, so no partial copy of the user's data remains. If the bundle was published but its output handles could not be closed, the command reports a cleanup error that carries the export @@ -38,7 +44,7 @@ Evidence; the bundle itself is complete. | `envs//env.toml` | `type = "computer_app"`, the module and class, `computer = "ubuntu-24.04"`, `driver = "reference"` | | `envs//.py` | The `ComputerAppEnvironment` class | | `envs//SKILL.md` | What the app is, its window class and where its files live | -| `envs//seed/` | The application's files from before the capture, by home-relative path | +| `envs//seed/` | The application's files at the seed checkpoint, by home-relative path | | `envs//apt-keys/` | The signing key of each vendor apt repository the app installs from | | `envs//install-package.sh` | With `--package-url`: the run script that installs the downloaded package with dpkg | | `tasks/smoke.json` | A task that deploys the app | @@ -63,8 +69,18 @@ The class declares: leaves out dependencies on the libraries it loads, as Typora's Electron package does, still starts; library packages the capture found neither the Ubuntu 24.04 archive nor a declared repository offers are left out, with a - limitation. An archive package's declared dependencies are trusted as they - are. The package must come from the Ubuntu 24.04 archive or a mirror of it, + limitation, and so are those only repositories the export leaves out supply + (a repository's `supplies`), such as one whose key the capture could not + retain. An archive package's declared dependencies are trusted as they + are. When the desktop entry's package is not the one that owns the program + it runs, as `vim-common`'s entry runs `vim`'s program, the program's package + (`executable_package`) follows the app's by name, since the entry's package + need not depend on it; a capture that found no archive or declared + repository offering it records why, and the export refuses it with that + reason, as it does when only repositories it leaves out supply it, or when + no package owns the program at all (`executable_package_reason` without + `executable_package`). The package must come from the Ubuntu 24.04 archive + or a mirror of it, or from a vendor apt repository whose signing key the capture verified and retained. A source counts as the archive when apt reports its origin as `Ubuntu` for a noble suite; a capture made before origins were recorded @@ -111,7 +127,11 @@ The class declares: computer's container does not grant, and Electron exits without them. The README and the result's limitations say the sandbox is off. - `paths`: the state directories holding the application's state files and the - folders holding its documents, nested folders collapsed. The home itself and + folders holding its documents, nested folders collapsed. A folder beside a + state directory whose name differs from it only in letter case is owned + whole in the files' spelling, so a `~/.config/GIMP` profile GIMP created + during the capture beside the candidate `~/.config/gimp` is owned as + `.config/GIMP`, not as the folder of its first file. The home itself and the shared `.config`, `.local`, `.local/share`, `.local/state` and `.cache` folders are never owned whole: a file directly in one is owned as itself. Files the capture recorded under a symlinked state directory's physical @@ -127,18 +147,14 @@ The class declares: with a typed JSON view of each, read in a worker thread. An application with no observed state declares no paths and no hooks. -Seeded text files (text, JSON, key files and XML in the state view) have the -captured home path respelled to the computer's home wherever it is followed by -`/`, the end of the text, or a character that cannot continue a file name. A -UTF-16 file that starts with a byte-order mark and holds the captured home is -respelled the same way, keeping its mark; other files are copied as captured. A file under the owned paths whose starting -bytes the capture did not retain is not seeded: one in the state view without -retained baseline content, or one the capture omitted (for example too large -or past the content budget). The result lists them in `unseeded_files` and the -README names them. When the application held files open in folders whose -starting content the capture did not retain, a limitation names those folders. +A file under the owned paths whose bytes at the seed the capture did not +retain is not seeded: one in the state view without retained content at that +checkpoint, or one the capture omitted (for example too large or past the +content budget). The result lists them in `unseeded_files` and the README +names them. When the application held files open in folders whose starting +content the capture did not retain, a limitation names those folders. -The application's dconf keys at the baseline, if any, are written once when the +The application's dconf keys at the seed, if any, are written once when the app first starts, also for an application that owns no files, and the class adds `dconf-cli` to its apt packages for that. The server starts before the desktop's D-Bus session, so the class does not use `dconf load`: it dumps any @@ -148,6 +164,85 @@ win, and replaces `~/.config/dconf/user`. If that fails, the server prints a one-line warning and the app starts without the captured keys. `data/reset` does not restore them. +## The seed checkpoint + +By default the seed is the state before the capture, its `baseline`, and a +task's end state is the capture's last checkpoint. `--seed-checkpoint` and +`--end-checkpoint` name others, so one capture can make a seed and then a +change: the user creates and saves a document and takes `checkpoint seed`, +then edits and saves it and takes `checkpoint end`. Exported with +`--seed-checkpoint seed --end-checkpoint end`, the bundle seeds the document +as it was saved at `seed`, owned paths, dconf keys and unseeded files +included, and a task must fail on that seed and pass on the state at `end`. +A file's state at a checkpoint is its last version recorded at or before it. +The export refuses a name the capture does not hold exactly once, and an end +checkpoint taken before the seed checkpoint; `inspect-desktop-application-state` +lists the names in `capture.checkpoints`. + +## Host name and user + +A seed can hold the capture host's name and the user's: Geany saved its mail +template as `pref_template_mail=ada@e81232be457d`, the user `ada` on a +container named `e81232be457d`, and many files hold the home path +`/home/ada`. The capture records `host.hostname` and `host.user`, and the +export removes them from what the bundle ships: + +- In seeded text files (text, JSON, key files and XML in the state view), in + UTF-16 files that start with a byte-order mark (keeping the mark and byte + order), and in the dconf keys, `user@host` becomes `user@localhost`, the + host name standing alone becomes `localhost`, and the captured home becomes + `/home/user` wherever it is followed by `/`, the end of the text, or a + character that cannot continue a file name. The host name counts only as a + whole name: not inside a longer word, before a `-`, or as a label of a + domain such as `e81232be457d.local`, so a host named `ubuntu` leaves + `ports.ubuntu.com/ubuntu-ports` alone. A host named `localhost` is left + alone, but its `user@localhost` still takes the computer's user. The user + name alone is never replaced, since a short one such as `ada` or `user` is + also ordinary text. A rewritten seed is written with its own SHA-256, and + `scrubbed_files` and the README list it. +- JSON and XML keep their syntax. In JSON only string values change, each + read as its decoded value, so `\u0065` escapes cannot hide the host, and is + written back as JSON only when it changed; numbers and `true`, `false` and + `null` stay as they are even for a host named `true`. Member names are never + rewritten, since `{"ubuntu": 1, "localhost": 2}` on a host named `ubuntu` + would get two `localhost` keys and lose a setting; a file whose names hold + the host is listed in `unscrubbed_files` instead. In XML only character + data, attribute values, comments and CDATA change, character references + decoded the same way; element and attribute names and processing + instructions stay. A UTF-16 file is scrubbed by the format its text would + have in UTF-8, so JSON or XML in UTF-16 follows these rules too. +- In the dconf keys only GVariant strings change, each read as its decoded + value and written back in its own quotes only when it changed. Group names, + keys, dictionary keys, bytestrings and everything outside quotes stay, so a + host named `true` leaves `wrap=true` alone. A value whose string does not + parse, or that holds an object path or signature, ships as captured. +- `SEED_HOME` in the class is `/home/user`, the home the seed spells, which + the class respells to the computer's own home when it seeds the home at + boot, as it did the captured home before. The replacements read as the + computer's own: its user is `user`, with its home in `/home/user`, and + `localhost` is the name every host has for itself. +- Binary files, SQLite databases among them, and text files over 256 MiB are + copied as captured. Then every file the bundle ships, the class module with + its dconf keys and the README included, is searched by a looser rule than + the replacement's: the host name and `user@host` anywhere, as in + `e81232be457d.local` or a seed named `e81232be457d-notes.txt`, and the + captured home as a whole path, in its path and its content, JSON and dconf + strings and XML references decoded and the replacements themselves set + aside. Each one found is listed in `unscrubbed_files`, and in the README + unless it is the README, to check before sharing the bundle. Bytes that are not text are + searched as they are, so a binary file that holds the host in another + encoding is not listed. +- A task check whose values name `user@host` or the host name anywhere, by + that rule, is refused, as one naming the captured home is: the bundle does + not hold them. Its field names, kind and id are not values, so a host named + `file` or `kind` refuses no check. A task's checks must fail on the seed as + the bundle ships it, scrubbed, so a check the scrub itself satisfies, such + as a mail setting the user changed from `ada@e81232be457d` to + `user@localhost`, does not count as failing there. + +A capture written before the host's name and user were recorded has only its +home removed, with a limitation saying so; capture again to remove the rest. + ## Applications from a .deb file A closed-source application you installed from a `.deb` file you downloaded, @@ -199,24 +294,35 @@ shapes: SQLite tables (at most 64 ordinary tables, `tables_truncated` past that, and per table `columns`, `primary_key`, at most 500 `rows` and `truncated`, with blobs, unsafe integers and text that is not UTF-8 tagged as REA tags them), JSON, XML as `{root, truncated}` with DOCTYPEs refused and REA's -node, depth and whitespace rules, key file groups, and text as -`{lines, line_count, truncated}` split on `\n` and `\r\n`. A JSON file +node, depth and whitespace rules, key file groups, text as +`{lines, line_count, truncated}` split on `\n` and `\r\n`, and zip archives +as their entries, read with Python's `zipfile` from the file in place, with +REA's entry, size, compression ratio and archive limits and its wording for an +entry it does not read. The module reads the central directory itself, as REA +does, and shows `zipfile` only the records it lists, reading each entry in +bounded chunks no further than a byte past its declared size. A JSON file holding a number JavaScript cannot read exactly, such as an integer above 2^53, -gets an `error` instead of a view, as in REA, while text that is not one JSON +or nested more than 500 levels deep, gets an `error` instead of a view, as in +REA, while text that is not one JSON document, such as JSON Lines, stays text. Format detection follows REA's rules. Files larger than 64 MiB get an `error` instead of a view -and are not read. Known differences remain: +and are not read. XML is read with Python's `expat` twice: with namespace +processing, which refuses what REA's parser refuses, such as an undeclared +prefix, then without it, so element and attribute names keep the prefixes the +document spells and namespace declarations stay attributes, as in REA's view; +text splits into nodes where REA's splits it, at CDATA sections, comments and +processing instructions. Known differences remain: -- XML names: element and attribute names are Python's `{namespace}local` - spelling rather than REA's prefixed names, namespace declarations are not - listed as attributes, and CDATA sections merge with the text around them, so - they count as one node. -- dconf databases are reported as binary, since REA reads them with `dconf`. +- dconf databases are reported as binary, since REA reads them with `dconf`; + inside an archive, one is `dconf` with an error, as in REA. +- Zip entry names: a name in an Info-ZIP Unicode Path extra field is used only + where Python's `zipfile` reads it (3.12 and later), and an entry whose local + header names it differently from the central directory is damaged rather + than read. - SQLite databases are read in place, read-only, rather than from a staged copy; a write lock the running app holds is waited out for a quarter of a second and then gives the file an `error`, and so does a table or column name that is not UTF-8. -- JSON nested deeper than Python's recursion limit gives the file an `error`. The seed is the user's real data. Keep the bundle local and do not publish it. The class was checked against agentenv-framework 0.9.1431 with diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md index 75c2423e8f..605d43c728 100644 --- a/docs/desktop-application-state.md +++ b/docs/desktop-application-state.md @@ -30,6 +30,11 @@ changed later. Each path records the evidence the capture holds about it: | `dconf_application_keys` | It is a dconf database holding groups whose path names the application | | `desktop_service` | It lies where desktop services and toolkits keep shared files, or caches | +A folder beside a state directory whose name differs from it only in letter +case counts as that state directory, so a profile the application first +creates during the capture, such as GIMP's `~/.config/GIMP` beside the +candidate `~/.config/gimp`, is its state too. + The role follows from the reasons, in this order: `application_state` for application keys in dconf, or a state directory outside shared desktop locations; `document` for a file named in a window title or selected by the @@ -48,13 +53,20 @@ would claim other applications' groups. Each path lists its versions: the baseline, then its state after each checkpoint where it, or a database's `-wal` or `-journal` file, changed. A -checkpoint that leaves an absent file absent adds no version. A version records +checkpoint that leaves an absent file absent adds no version. A file's state +at any checkpoint is its last version recorded at or before it, which is how +the [export](desktop-application-export.md#the-seed-checkpoint) and the +[task draft](desktop-application-tasks.md) read it at the seed and end +checkpoints they are given; `capture.checkpoints` lists the names in order. +Views hold the captured bytes as they are, the capture host's name and user +included; the export removes those from the bundle. A version records whether the file was present, the SHA-256 of its retained bytes, and a view: | Format | Detected by | View | | --------- | ---------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | | `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows | | `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups | +| `zip` | A zip local file header, `PK\x03\x04`, whatever the name | Its file entries in archive order, each with its own format and view | | `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels | | `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DOCTYPEs refused | | `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB | @@ -64,6 +76,16 @@ whether the file was present, the SHA-256 of its retained bytes, and a view: A JSON value holding a number JavaScript would round, such as an integer above 2^53, is not read rather than shown rounded. +An XML view names elements and attributes as the document spells them, prefix +included: a saved Inkscape drawing's root is `svg`, in its default namespace, +with `sodipodi:docname` and the declarations `xmlns` and `xmlns:sodipodi` +among its attributes, and a Writer paragraph is `text:p`. An element's +`children` hold its elements and text in order, text split where a CDATA +section, comment or processing instruction falls; whitespace-only text in an +element that holds elements is dropped unless `xml:space="preserve"` applies. +The view keeps up to 5,000 nodes, comments and processing instructions +counted, nested up to 64 levels, and sets `truncated` when it leaves some out. + SQLite databases are read through REA's SQLite worker from a private copy, with any retained `-wal` or `-journal` file staged beside it; the worker accepts an inactive rollback journal and refuses a hot one. A version is not @@ -84,6 +106,52 @@ A version has a `view_error` when its bytes were not retained, are over the 64 MiB read limit, are missing or no longer match their digest, exceed a format's limit above, or its reader failed. +## Archives + +Many applications save a document as a zip archive of XML and other files: +FreeCAD's `.FCStd` and `.FCBak`, OpenDocument (`.odt`, `.ods`, whose +`content.xml` is the document), Office Open XML, EPUB and JAR. Such a file is +`zip`, and its view lists its file entries in archive order, leaving out +directories: an entry whose name ends in `/`, or whose attributes mark a Unix +directory or, from an MS-DOS host, the directory attribute: + +```json +{ + "entries": [ + { + "name": "Document.xml", + "size": 11923, + "sha256": "…", + "format": "xml", + "view": { + "root": { "name": "Document", "attributes": {}, "children": [] }, + "truncated": false + }, + "view_error": null + } + ], + "entry_count": 15, + "truncated": false +} +``` + +Each entry's bytes are read from the archive's verified bytes in memory, +checked against its CRC-32 and declared size, and given a `format` and `view` +by the rules and limits above, with the entry's name in place of a file name. +Nothing is extracted to disk. An entry has a `view_error`, with `sha256`, +`format` and `view` null, when it is encrypted, compressed other than stored or +deflated, larger than 16 MiB, more than 100 times its compressed size, past the +64 MiB read from one archive, or its data is damaged. A SQLite or dconf +database, or an archive, inside an archive keeps its format and `sha256` and +has a `view_error` instead of a view: it is not read. The view lists at most +1,000 entries; `entry_count` counts them all, and `truncated` is true when some +are not listed. REA reads the central directory itself, a record at a time, +through the Zip64 end record whenever a Zip64 locator precedes the end record, +as Python's `zipfile` does, and hands the zip reader only the records it lists, +so an archive of hundreds of thousands of entries costs no more to read +than one of a thousand. An archive whose central directory cannot be read has a +`view_error` of its own. + ## Kinds of change Between consecutive versions a change is `created`, `deleted`, or a @@ -94,6 +162,13 @@ each, with counts): - `values` for JSON and XML: JSON pointers to values, containers included, so an array that becomes an object with the same members is changed. A pointer longer than 4096 characters is counted but not listed. +- `values` for zip archives too, with array members named by + [selectors](desktop-application-tasks.md#pointers-and-selectors) rather than + positions wherever a selector names one member in both versions: entries by + name, as `/entries/[name=Document.xml]`, and XML elements by tag and an + attribute, as `/view/root/children/[name=Object][@name=Hole]`, so an object + inserted before another does not shift it. A comparison is incomplete when + either view leaves out entries, or an entry's view is missing or truncated. - `keys` for key files and dconf: `group/key`, with `~` and `/` in the key written `~0` and `~1` as in a JSON pointer, so `[a/b]` `c` and `[a]` `b/c` stay apart. diff --git a/docs/desktop-application-tasks.md b/docs/desktop-application-tasks.md index 100a34a07a..11480c5ba8 100644 --- a/docs/desktop-application-tasks.md +++ b/docs/desktop-application-tasks.md @@ -27,9 +27,15 @@ operation `describe_desktop_application_task` and predicate type window class. - `owned_paths`: the paths the export owns, the only ones whose files the app's `data/get` serves and checks can read. -- `files`: each file under them whose state differs between the capture's - first and last version, with its home path, role, both versions and the net - change between them, from the state view's own diff (`lines`, `values`, +- `checkpoints`: the two checkpoints compared, `baseline` for the seed and + `final` for the end. They are the capture's first and last unless + `--seed-checkpoint` and `--end-checkpoint` name others, as for the + [export](desktop-application-export.md#the-seed-checkpoint), which takes + the same options; a name the capture does not hold, or an end before the + seed, is refused. +- `files`: each file under them whose state differs between those + checkpoints, with its home path, role, its versions at both (`baseline` and + `final`, the last recorded at or before each) and the net change between them, from the state view's own diff (`lines`, `values`, `keys` or `rows` with what was added, removed and changed, or `created`, `deleted`, `format`, `content`, `unreadable`; `content` also when the views hold only part of a file and match while the file changed). A @@ -46,8 +52,13 @@ operation `describe_desktop_application_task` and predicate type lines it lost (when either is at most ten), and `line_count` for each line holding a letter whose count changed; JSON, XML and key files gain `value_equals` and `value_absent` by pointer; SQLite tables gain rows added, - changed and removed, by primary key, and their new row count. Values that - look like a time the app stamps are left out. + changed and removed, by primary key, and their new row count. A zip archive + gains checks inside the JSON, key file and XML views of its entries, by + [selectors](#pointers-and-selectors) and never through an element by + position: an element added whole that it exists, one removed whole that it + is absent, then values changed, added and removed, an entry or a JSON value + added whole through its members, and an empty array or object as it is. + Values that look like a time the app stamps are left out. The suggestions are a starting point. Keep the ones that say what the user did; drop the ones that only follow from how the app saves its file. A LibreCAD @@ -86,7 +97,7 @@ file, in the shapes of the state view. | `lines_present` | `lines` | Every listed line is a whole line of the text view | | `lines_absent` | `lines` | None of the listed lines is | | `line_count` | `exact` or `contains`; `equals`, `at_least` or `at_most` | The number of lines equal to `exact`, or holding `contains`, compares so | -| `value_equals` | `pointer`, `value` | The JSON pointer names a value equal to `value` | +| `value_equals` | `pointer`, `value`; optional `tolerance` | The pointer names a value equal to `value`, or within `tolerance` of it | | `value_absent` | `pointer` | The pointer names no value | | `row_present` | `table`, `where` | A row of the SQLite table has every column in `where` equal | | `row_absent` | `table`, `where` | No row does | @@ -102,9 +113,16 @@ The rules both evaluators follow: - Equality is JSON equality: numbers by value, never a boolean for a number, objects by their keys whatever their order, arrays in order. - Pointers follow RFC 6901 (`~1` for `/`, `~0` for `~`) and must start with - `/`. Key files read as `{"groups": {"GROUP": {"KEY": "VALUE"}}}`, so a - setting is `/groups/GROUP/KEY`; XML as - `{"root": {"name", "attributes", "children"}}`; SQLite as `{"tables": {"TABLE": {"columns", "rows"}}}`. + `/`; on an array a segment may be a selector, below. Key files read as + `{"groups": {"GROUP": {"KEY": "VALUE"}}}`, so a setting is + `/groups/GROUP/KEY`; XML as `{"root": {"name", "attributes", "children"}}`; + SQLite as `{"tables": {"TABLE": {"columns", "rows"}}}`; a zip archive as + `{"entries": [{"name", "format", "view", …}]}`. +- With `tolerance`, a non-negative number, `value` must be a number, and the + value found passes when it is a number, or a string that is a decimal + numeral such as `"5.0000000000000000"`, at most `tolerance` from `value`. + `tolerance: 0` compares numerically, so `"1.2500"` equals `1.25`. Anything + else fails as not a number. - `contains` matches a line that holds it anywhere, case and all. Checks take no regular expressions: a pattern run on what the agent wrote could stall the verifier, and JavaScript and Python read some patterns differently. @@ -114,17 +132,118 @@ The rules both evaluators follow: is decided on it. The export refuses, naming the check, a file outside the owned paths, a file -spelled with a leading slash, `.` or `..`, a pointer into a text file or into -XML that uses namespaces (the verifier's reader names them differently), the -empty pointer (the whole file), a value that names the captured home, and a -value that looks like an ISO 8601 date and time. Other volatile content, such -as a counter the app bumps on every save, is for the author to leave out. +spelled with a leading slash, `.` or `..`, a pointer into a file that is text +at the seed, as the bundle ships it, or at the end checkpoint, the empty +pointer (the whole file), a value that names the captured home, or the +capture host's name or `user@host` anywhere (which the +[export removes from the seed](desktop-application-export.md#host-name-and-user)), a +`tolerance` with a value that is not a number, and a value that looks like an +ISO 8601 date and time. Other volatile content, such as a counter the app +bumps on every save, is for the author to leave out. + +## Pointers and selectors + +An app that rewrites a document moves things: FreeCAD 1.0 records more +properties than 0.21, so a hole's radius sits at another position among its +siblings, and a plate's entry moves once a hole is added before it. A pointer +segment applied to an array may therefore name a member by what it holds +rather than by its position: + +- `[KEY=VALUE]` names the member whose string field `KEY` is `VALUE`, such as + a zip entry by `[name=Document.xml]` or an XML element by its tag, + `[name=Object]`. +- `[@KEY=VALUE]` names the member whose `attributes` hold `KEY="VALUE"`, as + the XML view keeps an element's attributes. +- Conditions can follow one another, `[name=Object][@name=Hole]`, and all must + hold. They compare strings only, the selector is escaped like any pointer + segment (`~1` for `/`), and a value cannot hold `][`. + +A selector must name exactly one member. When several match, or none does +before the last segment, the check fails and says which selector matched what. +When the last segment's selector matches none, the pointer names no value: +`value_absent` passes, which is how a check says an element was removed, and +`value_equals` fails. On an object, a bracketed segment is an ordinary key. + +A FreeCAD task that checks the user cut a 10 mm hole through the plate, in +FreeCAD 0.21 and 1.0 alike: + +```json +{ + "name": "cut-hole", + "prompt": "Cut a 10 mm hole through the middle of the plate and save it.", + "checks": [ + { + "id": "cut", + "kind": "value_equals", + "file": "Documents/plate.FCStd", + "pointer": "/entries/[name=Document.xml]/view/root/children/[name=Objects]/children/[name=Object][@type=Part::Cut]/attributes/type", + "value": "Part::Cut" + }, + { + "id": "radius", + "kind": "value_equals", + "file": "Documents/plate.FCStd", + "pointer": "/entries/[name=Document.xml]/view/root/children/[name=ObjectData]/children/[name=Object][@name=Hole]/children/[name=Properties]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value", + "value": 5, + "tolerance": 0.01 + } + ] +} +``` + +Before the edit the archive has no `Part::Cut` and no `Hole`, so both checks +fail on the seed; after it, `Radius` is `"5.0000000000000000"` and both pass. +A check decided by what a view leaves out depends, in an archive, on the entry +its pointer goes into: it fails when that entry's view is missing or +truncated, or when the entry is not listed and the listing is truncated. A +selector must see every member to know it names one, so a check whose pointer +searches an array the view may hold only part of fails as undecided: any array +of a truncated XML view, of a truncated entry's view, or the listing when it +is cut. A position is decided on what the view holds. Like a file, an entry +that has no view (binary, not read, or a database or archive inside the +archive) fails every check into its `view`; its `null` there is no value, while +an entry holding the JSON document `null` reads as `null`. Checks on its `name`, +`size`, `sha256` and `format` still read. + +XML that uses namespaces is named as the document spells it, in both +evaluators: an element or attribute by its prefixed name, such as `text:p` +or `inkscape:label`, an element in a default namespace by its bare name, such +as `svg`, and namespace declarations as the attributes `xmlns` and +`xmlns:PREFIX`. Text is a string among an element's `children`. These checks +hold a poster's headline in an Inkscape SVG and a date in a LibreOffice Writer +document: + +```json +[ + { + "id": "headline", + "kind": "value_equals", + "file": "Pictures/poster.svg", + "pointer": "/root/children/[name=g][@inkscape:label=Layer 1]/children/[name=text][@id=headline]/children/[name=tspan]/children/0", + "value": "Summer Bake Sale" + }, + { + "id": "date", + "kind": "value_equals", + "file": "Documents/flyer.odt", + "pointer": "/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:text]/children/[name=text:p][@text:style-name=Subtitle]/children/0", + "value": "Monday 24 November" + } +] +``` + +Inkscape writes a line of text as a `tspan` inside its `text` element, and +Writer keeps a paragraph's style in `text:style-name`. A prefix is the one the +document uses, so a check names the prefixes the app writes. ## Validation and export `rea export-desktop-application --task ` -evaluates the checks on the capture's state view: on the first version of -each file, which is the seed, and on its last version, the captured end state. +evaluates the checks on the capture's state view: on each file as it was at +the seed checkpoint, which is the seed, as the bundle ships it with the +capture host's name and user removed, and at the end checkpoint, the +captured end state. Those are the capture's first and last checkpoints unless +`--seed-checkpoint` and `--end-checkpoint` name others. It refuses a task whose every check passes on the seed, and a check that fails on the end state, with that check's reason. A capture yields one task, so the option is taken once. The result's `task` names the task, its number of checks diff --git a/docs/specs/desktop-app-environments/README.md b/docs/specs/desktop-app-environments/README.md index f778c90e26..2da35c2485 100644 --- a/docs/specs/desktop-app-environments/README.md +++ b/docs/specs/desktop-app-environments/README.md @@ -30,6 +30,15 @@ The work lands as one PR per step: installed from a downloaded `.deb` file, which the computer downloads from an operator's URL and checks by its captured SHA-256, starting with VariCAD. +10. [Document views](document-views/spec.md): documents saved as zip + archives (FreeCAD, OpenDocument, Office Open XML, EPUB) read as their + entries, checked by name rather than position, namespaced XML included, + and numbers compared within a tolerance, starting with FreeCAD. +11. [Capture for building](capture-for-building/spec.md): one capture makes + the seed and then the change, at checkpoints the export and the task + draft are told; apt lists from a user-space `apt-get update` are found; + the application does not inherit the computer's control token; and the + bundle holds neither the capture host's name nor its user's. Version 1 covers Linux with X11, applications whose state lives in files or SQLite under the home directory, and real applications from public packages. diff --git a/docs/specs/desktop-app-environments/capture-for-building/plan.md b/docs/specs/desktop-app-environments/capture-for-building/plan.md new file mode 100644 index 0000000000..b05887eb01 --- /dev/null +++ b/docs/specs/desktop-app-environments/capture-for-building/plan.md @@ -0,0 +1,23 @@ +# Plan + +1. Checkpoints: `versionAt` and `StateEnds` beside the state view's schema + in `src/domain/desktop/desktopState.ts`, since the draft is domain code; + `selectStateEnds` in `DesktopStateVersions.ts` turns names into positions + and refuses under the calling operation. The export's plan, its task + validation and the draft read every file through `versionAt`. +2. apt folders: `parseAptConfigShell` in `aptRepositories.ts` and + `resolveAptLocations` in `AptRepositoryInventory.ts`, called once per + Debian package with the host's folders as the fallback. +3. Launch: `applicationEnvironment` in `DesktopApplicationSession.ts` filters + the environment the owned process is spawned with. +4. Identity: `host.hostname` and `host.user` in the capture schema and + record; `src/domain/desktop/hostIdentity.ts` owns the matching rules, the + replacements and the per-seed decision, used by the export and by the task + refusal. The bundle files move to `DesktopExportFiles.ts`, which rewrites + seeds as it copies them and writes the README last, listing them. +5. CLI options, the guides, this spec and its verification. +6. Tests first: domain rules, the seed and end scenario through the export + and the draft, the export's scrub with a dconf database and a binary seed, + apt folders from a stub and from real `apt-config`, the launch environment + and the recorded host on Linux, and the CLI options; then Mousepad in the + Ubuntu 24.04 container. diff --git a/docs/specs/desktop-app-environments/capture-for-building/spec.md b/docs/specs/desktop-app-environments/capture-for-building/spec.md new file mode 100644 index 0000000000..acf5dcaa71 --- /dev/null +++ b/docs/specs/desktop-app-environments/capture-for-building/spec.md @@ -0,0 +1,74 @@ +# Capture for building + +## Outcome + +REA's capture, state view, task draft and export fit a build in which an +agent drives the application inside an agentenv computer, makes the seed and +then the change in one capture, and exports a bundle that can be shared. The +seed is the state at a checkpoint the agent names rather than the state before +launch; the capture finds the package's sources from lists an ordinary user +downloaded; the application does not inherit the computer's control token; +and the bundle holds neither the capture host's name nor its user's. + +## Requirements + +- Checkpoints: the export takes `--seed-checkpoint` and `--end-checkpoint`, + and the task draft the same two options. The seed checkpoint chooses the + seed files, their text formats, the unseeded files and the dconf keys; the + end checkpoint is where a task's checks must all pass, the seed where at + least one must fail; the draft compares the two. A file's state at a + checkpoint is its last version recorded at or before it. Without the + options they are the baseline and the capture's last checkpoint, so earlier + captures export and draft as before. A name the capture does not hold + exactly once, and an end before the seed, are refused naming the option. + The export reports both names in `checkpoints`, and the draft in its own + `checkpoints` as `baseline` and `final`. +- apt folders: the lists folder, the trusted parts folder and the trusted + keyring are where `apt-config shell` resolves `Dir::State::Lists/d`, + `Dir::Etc::TrustedParts/d` and `Dir::Etc::Trusted/f`, so an `APT_CONFIG` + file that moves them is followed; each is apt's default when `apt-config` is + missing, fails or names no absolute path. +- Launch environment: the application gets REA's environment without any + variable whose name starts with `AGENTENV_`, plus the run token. +- Host identity: the capture records `host.hostname` and `host.user`, each + optional, so captures written before parse; one is left out when the system + cannot say. +- Scrub: the export replaces `user@host` with `user@localhost`, the host name + as a whole name with `localhost`, and the captured home where a whole path + segment ends after it with `/home/user`, in seeded text files read as + UTF-8, in UTF-16 seeds with a byte-order mark, and in the dconf keys. A + host named `localhost` keeps that name, but its captured `user@localhost` + is still replaced. The user name alone is never replaced. JSON is scrubbed + by its string values, decoded, never its member names, and XML by its + decoded character data and attribute values, plus comments and CDATA, so + neither's syntax or names change and escapes hide nothing; a UTF-16 seed is + scrubbed by the format its text would have in UTF-8. dconf keys are + scrubbed by their decoded GVariant strings, never group names, keys, + dictionary keys or other values, and a value that does not parse or holds + an object path or signature ships as captured. A rewritten seed is a + generated file with its own SHA-256. The class's `SEED_HOME` is `/home/user`, which it respells + to the computer's home at boot as before. A capture whose home is already + `/home/user`, as inside a computer, has no home to replace. A capture + without the host's name and user has only its home replaced, with a + limitation. +- Leftovers: every shipped file, generated ones included, is searched in its + path and content, escapes decoded, for the host name and `user@host` + anywhere and the captured home as a whole path, with the replacements set + aside. `scrubbed_files` lists the rewritten seeds and `unscrubbed_files` + every file found, by bundle path; the README lists both. +- Tasks: checks whose values name `user@host` or the host name by that rule + are refused, like those that name the captured home, but not for their + field names, kind or id; and the checks that must fail on the seed are + evaluated on the seed as the bundle ships it. A value check is refused as + pointing into text only when its file is text at the seed or the end + checkpoint, so versions before or after them do not count. + +## Non-goals + +- No MCP tools, no build computer, no control channel with acknowledgements + and no witness; later steps own those. +- No change to captured Evidence or the state view's views: they keep the + captured bytes, and only the bundle is scrubbed. +- No rewriting of binary files, SQLite databases or file names, and no + replacement of the user name standing alone. +- No change to the draft's field names, `baseline` and `final`. diff --git a/docs/specs/desktop-app-environments/capture-for-building/verification.md b/docs/specs/desktop-app-environments/capture-for-building/verification.md new file mode 100644 index 0000000000..0adce414ee --- /dev/null +++ b/docs/specs/desktop-app-environments/capture-for-building/verification.md @@ -0,0 +1,186 @@ +# Verification + +Run on 2026-11-05 on top of the namespaced XML and capture fixes branches: +gates and source tests on macOS 26.5 with Node v22.23.1 and Python 3.11.14; +Linux tests and the end-to-end capture in an Ubuntu 24.04.5 arm64 container +with Node v24.22.1, Python 3.12.3, apt 2.8.3, Xvfb, xfwm4 and Mousepad +0.6.1-1build2, run as the ordinary user `ada` on a container whose host name +is its id. + +## Gates + +- `npm run typecheck`, `format:check`, `build:cached`, `knip`, `jscpd`, + `docs:check`, `docs:build`, `verify:test-discovery` and + `verify:architecture-guards`: passed. `lint` and `lint:typed` fail only on + main's own `tests/acceptance/webSourceLocationCli.test.ts`, with no + diagnostics in the touched files. Of the two clones jscpd lists in touched + files, one is a new test's nine-line import block and the other a pair of + cancellation tests already in `DesktopApplicationCapture.test.ts`. +- `npx vitest run src/domain/desktop src/application/desktop src/desktop` + with `tests/boundary/providers/desktop*.test.ts` and the desktop CLI + boundary tests `cliStartupImports`, `desktopCaptureCli`, + `desktopApplicationCli`, `cliInputSchemaRequired` and `cliToolCoverage`: + 468 passed and 20 skipped on macOS (the Linux-only cases); 485 passed and 3 + skipped in the container. + +## Tests + +Each new test fails on the code before this change, run from that code with +only the new tests and fixtures added: on macOS for the source and CLI tests, +in the container for the three Linux-only ones. + +- The scenario: a capture where the app saves a new note before `seed`, edits + it before `end`, and writes its settings as it exits. Exported with + `--seed-checkpoint seed --end-checkpoint end`, the bundle seeds the note as + saved at `seed` and the settings as they were then, and a task whose check + of the edit fails at `seed` and passes at `end` is kept. A check that holds + only at `exit` is refused with the end at `end` and kept with the end left + to default. Without the options the seed is the baseline, without the note. + The draft between the two lists only the note, with `eggs` added. An + unknown name and an end before the seed are refused naming the option, + under the export's and the draft's own operations, and write no bundle. + The same options through the public CLI, with a refusal's structured + issue. +- The scrub's rules: Geany's `pref_template_mail=ada@e81232be457d` becomes + `user@localhost` while `pref_template_developer=ada` stays; the host name + alone becomes `localhost` but not inside `xe81232be457d`, before `-2` or in + `e81232be457d.local`, and a host named `ubuntu` leaves + `ports.ubuntu.com/ubuntu-ports` alone; the home where a path segment ends, + not `/home/adam` or `/home/ada.bak`; only the home for a capture without + the host's name and user; a UTF-8 byte-order mark kept; UTF-16 seeds in + both byte orders; leftovers found in a binary database and in + `e81232be457d.local`; nothing to replace in a capture whose home is + `/home/user`. +- Through the export: a capture whose settings file, SQLite database and + dconf keys name the host and user. No bundle file but the database holds + the host name, `ada@` or `/home/ada`; the settings seed is rewritten, the + class holds `SEED_HOME = "/home/user"` and the scrubbed dconf keys, and the + result and README list the rewritten settings and the database. Checks + naming `ada@e81232be457d` or the host name are refused; a capture written + before the host was recorded still parses and has its home replaced. +- apt folders: lists in the folder a stubbed `apt-config` names, with a + quoted `'` in its name, are found, and the default folder is used when + `apt-config` fails or is missing (adapter test); on Linux, real + `apt-config` under an `APT_CONFIG` file resolves `Dir::State::Lists` + relative to `Dir::State`, and a missing `apt-config` gives the defaults. +- Linux: a launched shell's environment has no `AGENTENV_*` variable while + it keeps the others and the run token; a capture of the fake application + records the host's name and user as Node reads them. + +## Mousepad end to end + +As `ada`, with no apt lists in `/var/lib/apt/lists`, an `apt-get update` +under an `APT_CONFIG` naming `/tmp/apt-user/lists` and `/tmp/apt-user/cache` +wrote 15 Packages lists there. With that `APT_CONFIG` and +`AGENTENV_CONTROL_TOKEN` set, `capture-desktop-application --desktop-entry +org.xfce.mousepad` ran while `xdotool` typed `milk` and +`from ada@`, saved the file as `~/Documents/list.txt`, sent +`checkpoint seed`, typed `eggs`, saved, sent `checkpoint end` and then +`stop`. + +- Mousepad's environment held the run token and no `AGENTENV_*` variable. +- The package is `mousepad` 0.6.1-1build2 from + `http://ports.ubuntu.com/ubuntu-ports noble/universe arm64 Packages`, origin + `Ubuntu`, with `package_lists_present` true and no apt limitation. +- `host` records `/home/ada`, the container's host name and `ada`; the + checkpoints are `baseline`, `seed`, `end`, `stop` and `exit`. +- The draft between `seed` and `end` lists `Documents/list.txt` with `eggs` + added and suggests `lines_present` of `eggs`. +- The export with `--seed-checkpoint seed --end-checkpoint end` and that check + kept the task (`failing_on_seed: ["eggs"]`), seeded `list.txt` as + `milk\nfrom user@localhost\n` and listed it in `scrubbed_files`; + `unscrubbed_files` is empty and no bundle file holds the host name, `ada@` + or `/home/ada`. The dconf keys' recent file is + `file:///home/user/Documents/list.txt`. A task checking the line + `from ada@` was refused, and the export without options seeded + nothing, since the note did not exist at the baseline. +- The exported module's own `_seed_home` wrote `list.txt` unchanged into + another home and respelled the dconf keys' path to it. + `python -m agentenv_protocol.computer._declare` of agentenv-framework + 0.9.1431 declared the class, and `agent-env run --task smoke --dry-run` + planned the deploy. + +## Review fixes + +Run on 2026-11-05 on the same toolchains, after the first review. Each new +test failed on the reviewed head, with only the new tests added, and passes +now. + +- A task's seed checks read the seed as the bundle ships it. A mail setting + changed from `ada@e81232be457d` to `user@localhost` already reads + `user@localhost` in the scrubbed seed, so its check is no longer counted as + failing there: `failing_on_seed` went from `mail` and `note` to `note`. +- JSON is scrubbed by its strings' decoded values, so a host named `true` + leaves `{"enabled": true}` valid and a `\u0065` escape no longer hides the + host; an untouched string keeps its escapes. XML is scrubbed in its decoded + character data and attribute values, comments and CDATA, so a host named + `item` leaves `` alone and `e` references hide nothing; the + output is still read as XML. +- One search for what is left covers every shipped file, in its path and its + content: a clean seed named `e81232be457d-notes.txt`, the class module + whose dconf keys hold `ada@e81232be457d.local` and the README that lists + them are listed in `unscrubbed_files`, which now names bundle paths and + equals an independent search of the bundle. A task check holding + `e81232be457d.local` is refused by the same rule. The replacements are set + aside, so a host named `host` is not found in `localhost`. +- A host named `localhost` keeps that name, but `ada@localhost` becomes + `user@localhost`, and a binary file holding it is reported. +- A second review found member names rewritten too: on a host named + `ubuntu`, `{"ubuntu": 1, "localhost": 2}` got two `localhost` keys. Names + are now kept, an escaped one before a colon on the next line included, and + a seed whose JSON key holds the host ships as captured and is listed in + `unscrubbed_files`. +- The same review found that the regular expression that found JSON strings + threw `RangeError: Maximum call stack size exceeded` on one string of + 16,000,000 characters, which a 64 MiB capture can retain, so such a seed + stopped the export. The scrub now walks the document one string at a + time, and that seed is scrubbed in about 20 ms. The XML expressions, run on + a comment, CDATA section, processing instruction, attribute value and text + of the same length, finished within 6 ms each. + +The gates passed as before, `lint` and `lint:typed` again failing only on +main's `webSourceLocationCli.test.ts` with no diagnostics in this change's +files; the desktop tests ran 483 passed and 20 skipped on macOS, and 500 +passed and 3 skipped in the container, where the Mousepad run above gave the +same result, its rewritten seed now listed by bundle path. + +A third review found four more. Each new test failed on the reviewed head and +passes now; `src/domain/desktop`, `src/application/desktop` and +`tests/boundary/providers/desktop` ran 404 passed and 11 skipped on macOS. + +- dconf keys were scrubbed as one text, so on a host named `true` the setting + `wrap=true` became `wrap=localhost`, which `dconf compile` refuses, and + keys and group names could be renamed. Only decoded GVariant strings change + now; a value that does not parse ships as captured and the class module is + listed in `unscrubbed_files`. +- A task's check was searched as serialized JSON, so on a host named `file` + or `kind` every check was refused for its field names. Only its values are + searched now, not its field names, kind or id. +- A value check was refused when its file was text at any recorded version; + an empty `.json` before `seed` or after `end` refused a task whose file is + JSON at both. Only the seed, as shipped, and the end are read now. +- UTF-16 JSON and XML were scrubbed as plain text, so on a host named + `ubuntu` `{"ubuntu": 1, "localhost": 2}` got two `localhost` keys. They are + scrubbed by the format their text would have in UTF-8, in their own byte + order. + +## UTF-16 format detection + +- A decoded UTF-16 seed's scrubber is chosen by a syntax pass that builds + no value or document (`textSyntaxFormat`: the JSON token scan, and the + XML parser's events with only the open elements kept), and the XML + scrubber copies out only changed spans. A UTF-16 XML seed of a million + elements is scrubbed in a 24 MiB heap, where the previous code needed + more than 768 MiB; `desktopStateHeap.test.ts` runs it in 32 MiB and failed + on the previous code. On 200,000 random XML-like inputs and 80 UTF-16 + cases the output is byte-identical to the previous code's. + +## Limits of this verification + +- The bundle was not run in an agentenv computer, and REA was not run inside + one; the build computer is a later step. +- Geany's leak is covered by a fixture with its line, not by capturing + Geany. No real application was captured with a UTF-16 seed or a binary + file holding the host name. +- The apt-config fallback for a missing command and a failing one is covered + by the adapter test, and the real command only on Ubuntu 24.04. diff --git a/docs/specs/desktop-app-environments/capture/spec.md b/docs/specs/desktop-app-environments/capture/spec.md index 7e20044794..586b3abffe 100644 --- a/docs/specs/desktop-app-environments/capture/spec.md +++ b/docs/specs/desktop-app-environments/capture/spec.md @@ -44,12 +44,28 @@ folder to build the environment; this step builds nothing. folders also name the state directories), and identifies the package: snap, flatpak and AppImage from paths (and the AppImage magic, and a `flatpak run` launch), otherwise - the dpkg owner with its version, architecture, the apt sources listing the - installed version and the `o=` origin apt reports for each source. It marks + the dpkg owner of the launched desktop entry, or else of the executable, + with its version, architecture, the apt sources listing the + installed version and the `o=` origin apt reports for each source. The + entry's package is the one that installs the application: LibreOffice + Writer's entry belongs to `libreoffice-writer`, while the `libreoffice` it + runs belongs to `libreoffice-common`, which does not depend on Writer. When + the two differ, `executable_package` names the executable's, with + `executable_package_reason` when neither the Ubuntu 24.04 archive nor a + declared vendor repository offers it or apt could not say; each vendor + repository records in `supplies` the library packages and executable's + package only vendor repositories offer that its package files list. When + a package owns the entry but none owns the program, nor the file it resolves + to, `executable_package_reason` says so without `executable_package`. A + `dpkg-query` lookup that does not finish or exits with an error leaves the + package + `unknown`, with a limitation naming the path. It marks Electron applications by the fuse sentinel or `resources/app.asar`, and lists candidate state directories under the XDG folders and dot-folders named after the package, executable, desktop entry's full id and last part, - and name; a `flatpak run` launch uses the application id (its first operand + and name, and every folder present there whose name is one of those in + another letter case, such as GIMP's `~/.config/GIMP`; a `flatpak run` + launch uses the application id (its first operand after options and their values) instead of the launcher's name and adds `~/.var/app/`. An `--executable` runs from the caller's directory, which also resolves a relative path to it. diff --git a/docs/specs/desktop-app-environments/capture/tasks.md b/docs/specs/desktop-app-environments/capture/tasks.md index a4ac5e4747..05b57f9db5 100644 --- a/docs/specs/desktop-app-environments/capture/tasks.md +++ b/docs/specs/desktop-app-environments/capture/tasks.md @@ -10,3 +10,5 @@ - [x] Add unit, adapter, service and Linux capture tests. - [x] Capture Mousepad end to end in a container. - [x] Run the gates and record verification. +- [x] Take a Debian package from the launched desktop entry's owner (LibreOffice Writer). +- [x] Find state folders named after the application in another letter case (GIMP). diff --git a/docs/specs/desktop-app-environments/capture/verification.md b/docs/specs/desktop-app-environments/capture/verification.md index a4530430f2..1a1aea1553 100644 --- a/docs/specs/desktop-app-environments/capture/verification.md +++ b/docs/specs/desktop-app-environments/capture/verification.md @@ -58,6 +58,76 @@ nothing to the output folder. Before that refusal existed, the same setup showed Mousepad handing the launch to the running instance, which opened a new tab in its existing window while the launched process exited at once. +## Fixes after the five-application test + +Run on 2026-11-05 on macOS 26.5 with Node v22.23.1. A test that captured and +exported five applications found two inventory gaps: + +- LibreOffice Writer, captured with `--desktop-entry libreoffice-writer`, + recorded `libreoffice-common`, the owner of the `/usr/bin/libreoffice` its + entry runs, which does not depend on Writer. The exported computer installed + only that package and failed its readiness check with + `no window with class 'libreoffice-writer'`. The package is now the entry's + owner, with the executable's in `executable_package`. Adapter tests on a + fake dpkg host whose entry and program have different owners cover an entry + launch, an `--executable` launch, an entry no package owns and a + `--package-file` matched against the entry's package. +- GIMP 2.10 keeps its profile in `~/.config/GIMP/2.10`, which no candidate + name (`gimp`) matched, so it was neither seeded nor reset. A folder in the + XDG folders or the home whose name is one of the candidate names in another + letter case is now a state directory, and the state view counts a path in a + folder beside a state directory that differs from it only in case as in it. + An adapter test with GIMP's layout and a domain test cover both, and leave + out `GIMPshop`, `MyGIMP` and a nested `Other/GIMP`. + +Review of those fixes found three more gaps, each now covered by a test that +failed on the first fix: + +- The entry's package need not depend on the program's (`vim-common` owns + Vim's entry, `vim` its program), so the export now installs + `executable_package` by name beside the app's package, for archive, vendor + repository and package-file installs alike. The capture checks it by the + library packages' rule and records `executable_package_reason` when neither + the archive nor a declared repository offers it, or apt could not say; the + export then refuses. Adapter, domain and export service tests cover the + record, the three install paths, the refusal and the rendered + `Packages(apt=[...])`. +- A profile folder created during the capture in another letter case than its + candidate was application state but owned as the folder of its first file + (`.config/GIMP/2.10`), so later files elsewhere in it escaped `data/get` and + `data/reset`. Owned paths now use the same match as roles and own + `.config/GIMP` in the files' spelling; a domain test and an export service + test of a created profile cover it. +- A `dpkg-query -S` that timed out or exited with an error counted as "no + owner", so a failed entry lookup quietly recorded the program's package. It + now leaves the package `unknown` with a limitation naming the path, for the + entry and the executable, after a timeout or an exit status of 2. + +A later review found that both the program's package and library packages +were approved at capture against every repository with a verified key, while +the content budget can then drop a repository's key and the export leaves that +repository out. Repositories now record what they `supply`, and the export +leaves out library packages, and refuses a program's package, that only +repositories it leaves out supply. A capture adapter test, two export domain +tests and a key-retention service test that drops one of two keys under the +per-file limit cover it. + +A third review found that an entry owned by a package whose program no +package owns, such as a `~/.local/bin` program found first on `PATH`, was +recorded as the entry's package alone, which the export then installed though +it does not bring that program. The capture now records that no package owns +the program, and the export refuses it. An adapter test, an export domain test +and an export service test cover it, and an adapter test keeps a program link +whose resolved file dpkg owns counted as owned. + +Each new test failed on the code before its fix, run in a copy of that commit +with only the new tests added. `npm run typecheck`, `lint`, `lint:typed` +(no diagnostics in the touched files), `format:check`, `build:cached`, +`knip`, `jscpd`, `docs:check`, `docs:build`, `verify:test-discovery` and +`verify:architecture-guards` passed, and +`npx vitest run src/domain/desktop src/application/desktop src/desktop tests/boundary/cli/cliStartupImports.test.ts` +ran 351 passed and 18 skipped. Neither application was captured again. + ## Limits of this verification - Only Mousepad was captured end to end, on arm64 in a container. GnuCash, diff --git a/docs/specs/desktop-app-environments/document-views/plan.md b/docs/specs/desktop-app-environments/document-views/plan.md new file mode 100644 index 0000000000..73e31d6bf9 --- /dev/null +++ b/docs/specs/desktop-app-environments/document-views/plan.md @@ -0,0 +1,25 @@ +# Plan + +1. Format and reader: `zip` detected by its local file header; the text, JSON, + key file and XML readers shared as `textualView`, so a file and an entry + read alike; a domain module for the entry limits, the entry view and the + archive view, and an application reader that walks the verified bytes with + zip.js under the options and name decoding `ZipArtifactReader` uses. +2. Selectors: `locatePointer` in `src/domain/desktop/viewPointer.ts`, used by + REA's evaluator and the draft, and `_locate` in the exported verifier, with + the same messages; `tolerance` in the task schema and both evaluators. +3. Changes and draft: member keys chosen per array from the candidate + selectors of both versions, a keyed walk for zip views in `compareTrees`, + and a walk that notes a member added or removed whole once for the draft's + archive suggestions. +4. Exported module: `_zip` and `_entry_view` beside `_view`, with + `zipfile`, `zlib` and `hashlib` imported at the top. +5. Docs and tests: real FreeCAD 0.21 and 1.0 `Document.xml` files zipped into + `.FCStd` archives and checked by both evaluators, an `.odt` built in the + test, archives crafted for each limit, and the shared check cases run + through both evaluators. +6. Namespaced XML: the exported module reads XML with `expat` rather than + `ElementTree`, one pass with namespace processing to check the document and + one without to build the view as REA's DOM does, names and text nodes + alike; the export's namespace refusal and the draft's namespace gate go. + Parity tests read real Inkscape and LibreOffice Writer saves. diff --git a/docs/specs/desktop-app-environments/document-views/spec.md b/docs/specs/desktop-app-environments/document-views/spec.md new file mode 100644 index 0000000000..9b06107c1d --- /dev/null +++ b/docs/specs/desktop-app-environments/document-views/spec.md @@ -0,0 +1,106 @@ +# Documents saved as zip archives + +## Outcome + +A document an application saves as a zip archive, first FreeCAD's `.FCStd`, +is read into a view of its entries, so a task can check what the user did in +it. Checks name the XML elements they read by tag and attribute rather than by +position, so the same check holds across application versions that write the +document in another order, and compare numbers written as text within a +tolerance. XML that uses namespaces, as OpenDocument's `content.xml` and +Inkscape's SVG do, is named by the prefixes the document spells. Both +evaluators, REA's and the exported verifier's, agree. + +## Requirements + +- Format: a file starting with a zip local file header (`PK\x03\x04`) is + `zip`, whatever its name. Its view is + `{entries, entry_count, truncated}`: the file entries in archive order, + directories left out, each `{name, size, sha256, format, view, view_error}`. +- Reading: from the retained bytes already checked against their digest, in + memory, with REA's zip reader options (CRC-32 and overlap checks, CP437 or + UTF-8 names); nothing is extracted. Each entry's format and view come from + the state view's own readers and limits, with the entry name in place of the + file name. A SQLite or dconf database, or an archive, inside an archive is + not read and says so. +- Limits: at most 1,000 entries listed (`entry_count` counts all); an entry is + read only when it is unencrypted, stored or deflated, at most 16 MiB + uncompressed and at most 100 times its compressed size, and while the + archive has read at most 64 MiB. An entry over a limit, or whose data fails + its checks, has `sha256`, `format` and `view` null and a `view_error`. +- Exported module: `data/get` gives the same view, read with Python's + `zipfile` from the file in place with the same limits, order of checks and + wording; parity tests compare the two readers on the same archives. +- XML names: both readers name elements and attributes as the document spells + them, prefix included (`text:p`, `sodipodi:docname`), an element in a + default namespace by its bare name (`svg`), and keep namespace declarations + as the attributes `xmlns` and `xmlns:PREFIX`, in document order. The + exported module reads XML with `expat` twice: with namespace processing, to + refuse what REA's parser refuses, then without, for the names. Text splits + into nodes as REA's DOM splits it: at CDATA sections, comments and + processing instructions, an empty CDATA section joining the text around it. +- Parsed views: a JSON or key file entry or file over the 4 MiB limit is + refused before its value is built, in both readers. Its format is told + apart from text by a token scan that builds no value and holds a byte per + open container, which the exported reader also uses when Python's parser + reaches its recursion limit, so deeply nested text is JSON only when all of + it is. +- Selectors: on an array, a pointer segment `[KEY=VALUE]` names the member + whose string field `KEY` is `VALUE`, `[@KEY=VALUE]` the member whose + `attributes` hold it, and conditions can follow one another. A selector must + match exactly one member; several, or none before the last segment, fail the + check with a message naming the selector. None at the last segment names no + value, so `value_absent` passes and `value_equals` fails. +- Tolerance: `value_equals` takes an optional non-negative `tolerance`, with a + number `value`, and passes when the value found is a number, or a decimal + numeral string, at most `tolerance` from it. +- Truncation: a check decided by what a zip view leaves out depends on the + entry its pointer goes into: that entry's view missing or truncated, or the + listing truncated when the entry is not listed. A pointer whose selector + searches an array the view may hold only part of (any array of a truncated + XML view or entry view, or a cut listing) fails as undecided in both + evaluators. A check into the view of an entry without one (binary, or with a + `view_error`) fails in both evaluators, while checks on its metadata read; + the draft checks only entries with a view, and an absence only where the + entry still has one. +- Changes: `values` between zip versions pair array members one to one, by + the first selector naming a member alone in its version and one unpaired + member of the other (entries by name, XML elements by tag and attribute), + even when a selector before it names nothing there, then by position; a + member left unpaired is keyed by a selector naming it alone and nothing in + the other version. Identity comes first: an element with identifying + attributes (`name`, `id`, `xid`, or a prefixed `id` such as `xml:id`) is + keyed by its tag and those alone, so one whose identifiers name nothing in + the other version is added or removed; its other attributes (`type`, + `color`) key only elements without one. A selector is used only when it + matches one sibling, counted over every attribute of every sibling, + identified ones included. Selectors are spelled only once chosen, so a + long tag is not copied per attribute. A comparison is incomplete when + either view leaves something out. +- Draft: for a zip document, checks inside its entries' JSON, key file and + XML views, through no object or array by position and no array a view may + cut: an element added whole exists, one removed whole is absent, then values + changed, added and removed, with an entry or a JSON value added whole checked + through its members. +- Export refusals: a `tolerance` whose value is not a number. A pointer into + XML that uses namespaces is accepted, in a file or an archive entry. +- Reading: the exported reader reads an entry in bounded chunks, at most one + byte past its declared size, so a forged small size cannot make it expand + the rest of the stream. +- Directory: both readers read the central directory themselves, a record at a + time through the Zip64 end record whenever a Zip64 locator precedes the end + record, whatever the end record holds, count the file entries (directories told + apart as zip.js does: a final slash, a Unix directory mode, or the MS-DOS + attribute), and give zip.js or `zipfile` only the first 1,000 records, so + memory does not grow with the number of entries. + +## Non-goals + +- No reading of an archive inside an archive, or of a database inside one. +- No names by namespace URI: a check names the prefix the document uses, so + one written for a document that binds another prefix to the same namespace + does not match it. +- No other compression methods (bzip2, LZMA, Zstandard) and no encrypted + entries. +- No selectors over non-string values, nested fields other than `attributes`, + or values holding `][`. diff --git a/docs/specs/desktop-app-environments/document-views/verification.md b/docs/specs/desktop-app-environments/document-views/verification.md new file mode 100644 index 0000000000..1e4a4bb2f8 --- /dev/null +++ b/docs/specs/desktop-app-environments/document-views/verification.md @@ -0,0 +1,220 @@ +# Verification + +Run on 2026-11-05 on top of the vendor .deb files branch, on macOS with Node +v22.23.1 and Python 3.11.14. + +## Gates + +- `npm run typecheck`, `lint`, `lint:typed`, `format:check`, `knip` (after + `build:cached`), `jscpd`, `docs:check`, `docs:build`, + `verify:test-discovery` and `verify:architecture-guards`: passed, with no + lint or typed-lint diagnostics in the touched files. The one clone jscpd + lists in a touched file, `DesktopApplicationState.ts` against + `GpgvKeyCheck.ts`, was there before. After main was merged in, `lint` and + `lint:typed` fail only on main's own `tests/acceptance/webSourceLocationCli.test.ts`. +- `npx vitest run src/domain/desktop src/application/desktop src/desktop` with + `tests/boundary/providers/desktopExportPython.test.ts`, + `desktopTaskPython.test.ts` and `tests/boundary/cli/cliStartupImports.test.ts`: + 377 passed and 18 skipped (Linux-only cases). The new + `tests/boundary/providers/desktopZipPython.test.ts`: 13 passed, under + Python 3.9.6, 3.10, 3.11, 3.12, 3.13 and 3.14 alike; + `desktopStateHeap.test.ts`: 2 passed, and `desktopZipHeap.test.ts`: 1. The desktop + CLI boundary tests `desktopApplicationCli`, `cliInputSchemaRequired` and + `cliToolCoverage`: 28 passed. +- Namespaced XML, on the same day and toolchain: the same gates passed, with + no lint, typed-lint or jscpd findings in the touched files, and the desktop + tests with the four provider parity files, `desktopXmlPython.test.ts` among + them: 370 passed and 18 skipped. + +## Tests + +Each new test fails on the code before this change, run from a copy of that +commit with only the new tests and fixtures added; the tests already in the +touched files still pass there. + +- Parity of the two zip readers, REA's and the exported module's, on the same + archives: entries in order with JSON, key file, XML, text, UTF-8-named and + binary entries, a directory left out, and an archive, a SQLite database and + an inexact JSON number inside, each not read for its reason; an encrypted + entry, bzip2, a declared 17 MiB entry, a 943-times expansion, a bad CRC-32 + and a short declared size; a damaged archive; 1,002 entries; and the 64 MiB + archive budget through both readers' refusal functions. +- FreeCAD: the real `Document.xml` of FreeCAD 0.21 and 1.0, before and after + the hole, zipped into `.FCStd` archives. Both readers give the same views, + and both evaluators give the same results and descriptions: a `Part::Cut` + exists, the `Hole` radius is 5 ± 0.01 and the cut's tool links `Hole` all + fail before the edit and pass after it, in both versions, by the same + pointers. +- OpenDocument: an `.odt` built in the test reads as `mimetype` text and + `content.xml` and `META-INF/manifest.xml` as XML, with the paragraph's text + in the view; the readers give the same view, namespaced names included, and + the export keeps a check of the paragraph by `[name=text:p]`. +- Namespaced XML, in `desktopXmlPython.test.ts`: the real saves of an + Inkscape 1.2.2 poster and a LibreOffice 24.2 Writer flyer, before and after + one edit, read alike by both readers, the SVG as a file and `content.xml` + in an `.odt` built in the test. Both evaluators give the same results and + descriptions: the headline `[name=text][@id=headline]` under the layer + `[@inkscape:label=Layer 1]` and the `[@text:style-name=Subtitle]` date fail + before the edit and pass after it, and `inkscape:document-units` passes in + both. Built documents cover `xmlns` declarations in document order, an + undeclared default namespace, `xml:` attributes, CDATA, comments and + processing instructions, what both parsers refuse (an undeclared prefix, + one attribute twice under two prefixes, a prefix bound to `""`, two colons + in a name, a colon in a processing instruction's target, a DOCTYPE), + and the node and depth limits on prefixed names. +- Selectors and tolerance in the shared check cases both evaluators run: one + member matched, several, none before or at the last segment, a malformed + selector, an escaped `/` in an entry name, an entry by index, a selector + over strings in an array, per-entry truncation, an entry the listing left + out; numbers and numeral strings within a tolerance, outside it, a string + and a boolean that are not numbers, and `tolerance: 0`. +- `locatePointer` and the member keys: the most general selector unique among + siblings, passed over when it names several members of the other version, + and values that hold `][` or look like a time left out. +- The keyed zip diff: an object inserted before another shifts nothing, and a + comparison is incomplete when an entry is unread, truncated or not listed. +- The draft: an archive's added and removed elements and changed values by + selector, inside namespaced XML too, where a paragraph is named by + `[name=text:p][@text:style-name=Subtitle]`; and through the application, a + capture with a FreeCAD document and an `.odt` whose draft, with a check of + the list's paragraph, the export then accepts. + +## Review fixes + +Each fix has a regression test that fails on the reviewed commit and passes +after it: + +- A selector over an array a view may hold only part of is undecided in both + evaluators: a truncated XML view, a cut listing searched by entry name, and + a truncated entry's XML, beside positions into the same views that stay + decided. +- The zip diff pairs members one to one: an element whose attributes split + across two new siblings leaves one paired and the other added. +- The draft checks an entry or a JSON value added whole through its members, + and suggests nothing that reaches an element by position, the last step + included. +- The export accepts a check on the whole listing whose value holds + namespaced XML, since both readers name it alike. +- The exported reader keeps a JSON entry nested 1,100 deep on that entry, + with its siblings read, and refuses JSON nested more than 500 levels as REA + does, 501 levels read and 502 refused, in a file or an entry. +- The exported reader reports an entry damaged when its declared size is + short and its checksum is the prefix's, deflated or stored, as REA does. +- An array that becomes an object with the same keys, or the reverse, is a + changed value, with nothing in it added or removed, so the draft suggests no + absence the end state contradicts. +- The draft checks an empty array or object added to a JSON entry as it is. +- An entry REA could not read, whose `view` is therefore `null`, gets no + drafted check, and both evaluators fail `value_equals null` into its view, + while a JSON entry holding `null` still passes it. +- The exported reader's depth check holds one iterator per level: on a + 4,000,001-byte array of 800,000 nulls, `_text_view` peaks at 6.7 MB of + traced allocations, as before the check, against 58.2 MB when it queued + every member. +- A binary entry fails `value_equals null` and `value_absent` into its view in + both evaluators, its `size` still checked, and the draft suggests no absence + in an entry that ends binary. +- REA's own depth check (`deeperThan`, from the state view) also holds one + iterator per level: on 800,000 nulls it grows the heap by 7.3 MB of + short-lived iterator results, against 72.7 MB when it queued a pair per + member, and runs under an 18 MiB old-space limit where the queue needed more + than 64 MiB. A test runs it under a 32 MiB limit. +- The exported reader reads an entry in 64 KiB chunks up to a byte past its + declared size: a 32,731-byte archive holding 32 MiB of zeros declared as one + byte peaks at 0.1 MiB of traced allocations, against 77.5 MiB when it read + the stream whole before clipping it, both reporting the entry damaged as REA + does. An 8.7 MiB text entry peaks at 48.5 MiB against 47.6 MiB. REA's zip.js + reader already stops at the first chunk past the declared size: reading that + archive grows the peak resident size by 0.8 MiB, as a one-byte archive does. +- A check of a namespaced entry's `size`, of its absence, or comparing the + entry whole exports, since both readers name namespaced XML alike. +- The diff finds a counterpart by the evaluator's own step: a bracketed key + against an array is a selector, so `{"[name=x]": 1}` becoming + `[{"name": "x"}]` reports nothing removed, in an archive or a JSON file, and + the draft suggests no absence of a value the selector still names. +- Both readers bound the central directory: an archive of 600,000 empty + entries (52.4 MiB) reads in a 24 MiB V8 old space against more than 48 MiB + when zip.js parsed every record, and grows peak RSS by 37 MiB against + 184 MiB; the exported reader peaks at 1.3 MiB of traced allocations against + 293.6 MiB, and at 1.3 MiB against 25.1 MiB for 50,000 entries. Directories + are told apart as zip.js tells them in both. +- Both readers follow a Zip64 locator whether or not the classic end record + holds sentinels, as `zipfile` does: an archive whose classic record keeps its + own counts, size and offset, which `zipfile` and zip.js both read, was + unreadable to both readers and now lists its entries in both. A last name + ending in bytes that only look like a locator keeps the classic values in + both, and the exported module gives `zipfile` real Zip64 end records for the + directory it shows it, so `zipfile` no longer refuses such an archive; the + parity tests pass under Python 3.9 to 3.14. +- A diff comparing an object of selector-shaped keys with an array indexes the + array once per set of selector fields: 2,000 keys read its members 4,000 + times against 4,002,000, and 8,000 and 20,000 keys diff in 19 and 31 ms + against 980 ms and 5.4 s. +- The keys are resolved together, their conditions sorted and de-duplicated, + holding only the values they ask for rather than an index of the array per + set of fields. 127 field combinations over 5,000 members grow the heap by + 5.5 MB against 68.0 MB, in 138 ms against 174 ms; 160 spellings of one + selector over 800 members by 4.9 MB against 75.9 MB, in 4 ms against + 388 ms, reading the members 1,600 times against 128,800. Both diff in a + 16 MiB V8 old space where they needed more than 48 MiB; a test runs them + under a 32 MiB limit. +- Member pairing keeps an element's tag apart from its attribute conditions + and spells a selector only once it is chosen: an element with a + 38,588-character tag and 2,000 attributes pairs with itself in 4 ms, + growing the heap by 1.1 MiB, against 7.2 s and 148 MiB when the tag was + copied into a selector per attribute. A test pairs two such elements under + a 32 MiB V8 old space. +- An element pairs by its identifiers first, and by a later selector both + versions share before one that names nothing in the other version keys it + as added: `` becoming `` beside another `item` is a changed `color` under + `[name=item][@id=1]`, in the diff and the draft, rather than one element + removed and another added. A FreeCAD `Hole` deleted and a `Hole001` added, + both `Part::Cylinder`, stay one removal and one addition rather than a + rename found by their shared `type`. An identified element's other + attributes still count against its siblings' selectors: `[@color=blue]` is + not reported removed when it moves to the identified `item`, and + `[@role=body]`, `[@role=second]` and an Inkscape group's + `[@inkscape:label=note]` key their elements rather than a `color`, `class` + or `fill` an identified sibling shares, which would match both. The real + FreeCAD 0.21 + and 1.0 documents, and the Inkscape, LibreOffice and AbiWord ones of the + branches built on this one, zipped as entries, give the same diffs, drafts + and member keys as before. +- Both readers refuse a JSON or key file over 4 MiB before building its value, + telling JSON from text by a token scan. REA's reader detects a + 4,200,004-byte array of empty objects in 58 ms, growing the heap by 4.2 MiB + (the decoded text), against 129 ms and 85.5 MiB when detection parsed it, + and a test reads it under a 32 MiB V8 old space. The exported reader's + entry peaks at 8.4 MB of traced allocations against 110.0 MB, a 16 MiB one + at 35.2 MB against 442.5 MB, and a 4.9 MB key file at 10.4 MB against + 86.7 MB; its scan takes 0.9 s and 3.5 s for those entries against 0.7 s and + 2.7 s for the parse. The exported reader now applies the limit to files as + REA does, where it read them whole before. +- The exported reader takes text nested past Python's recursion limit as JSON + only when the token scan finds all of it JSON, as REA does: 1,100 opening + brackets and 1,099 closing ones are text in both readers, where the export + called them JSON nested too deeply, and 1,100 of each are still refused for + their depth in both. + +## FreeCAD captures + +The real `.FCStd` files the FreeCAD investigation captured, from FreeCAD 0.21 +(Ubuntu 24.04 with the FreeCAD PPA) and 1.0 (Ubuntu 25.10), read by the built +CLI: + +- `inspect-desktop-application-state`: `plate.FCStd` is `zip` in every + version, 2 entries before and 15 or 16 after, with a complete `values` + change listing `/entries/[name=Document.xml]/view/root/children/[name=Objects]/children/[name=Object][@name=Hole]` + among the added values; the `.FCBak` backups are `zip` too. +- REA's evaluator and the exported module's, each on its own view of the + retained bytes: identical views, and the hole task's two checks fail before + and pass after in both versions. +- `export-desktop-application --task` on the 0.21 capture keeps the task with + both checks failing on the seed and writes the selectors and the tolerance + into `verify.py`. The 1.0 capture is not exported, as before this change: + its package installs only on Ubuntu 25.10. +- `describe-desktop-application-task` on the 0.21 capture drafts ten checks on + `plate.FCStd`, all by selector (the six objects and dependencies FreeCAD + added, then counts and statuses), and the export keeps that draft with all + ten failing on the seed. diff --git a/docs/specs/desktop-app-environments/export/spec.md b/docs/specs/desktop-app-environments/export/spec.md index fc6109a041..4b6e2722b8 100644 --- a/docs/specs/desktop-app-environments/export/spec.md +++ b/docs/specs/desktop-app-environments/export/spec.md @@ -16,11 +16,18 @@ state through `data/add`, `data/get` and `data/reset`. shared safe output tree. - It refuses applications not installed from a Debian package archive, captures without an application window, and environment names that are not portable. -- The class declares the apt package, the captured launch command with +- The class declares the apt package, and the package of the program the + desktop entry runs when another one owns it (refused when the capture found + no source the computer can install that package from, or only repositories + the export leaves out supply it, as library packages so supplied are left + out; a capture whose entry runs a program no package owns is refused), the + captured launch + command with arguments under the captured home respelled under `$HOME`, an exact window class match with a window ready check, `paths` from the state view's roles - (state directories of application state files and folders of documents, - nested ones collapsed) and the skill file. + (state directories of application state files, or a folder beside one that + differs only in letter case in the files' spelling, and folders of + documents, nested ones collapsed) and the skill file. - With paths, it declares all three data hooks and calls agentenv's helpers: seed the home once per boot then `keep_initial`, `add`, `reset`, and `get` plus a data part holding a typed JSON view of each file. Without paths it diff --git a/docs/specs/desktop-app-environments/state-view/spec.md b/docs/specs/desktop-app-environments/state-view/spec.md index c335a05d75..5a9ccb5846 100644 --- a/docs/specs/desktop-app-environments/state-view/spec.md +++ b/docs/specs/desktop-app-environments/state-view/spec.md @@ -15,6 +15,9 @@ generation reads the kinds of change. valid result; anything else is refused as invalid input. - Retained bytes are read without following links and used only when their SHA-256 matches the name they are stored under. +- A path in a folder beside a state directory whose name differs from it only + in letter case, such as GIMP's `~/.config/GIMP` beside `~/.config/gimp`, + lies in that state directory. - Each path gets the reasons the capture supports (state directory, open file, window title, selected path, dconf application keys, shared desktop location) and one role derived from them. @@ -28,14 +31,17 @@ generation reads the kinds of change. `dconf dump /` on a private file profile, keeping only groups whose path names the application; JSON up to 4 MiB and 500 levels with exact numbers; XML through the shared strict parser with DOCTYPEs - refused; key files up to 4 MiB whose every line parses; text; binary. + refused; key files up to 4 MiB whose every line parses; zip archives by + their local file header, as their entries with each entry's own format and + view ([document views](../document-views/spec.md)); text; binary. - Changes between versions are created, deleted, or lines, values, keys or rows with added, removed and changed lists (100 each, with counts); content for binary files and unreadable when a view is missing. Values are JSON pointers to every value, containers included, compared on one shared path so memory follows depth, and a pointer over 4096 characters is counted but not listed; keys are `group/key` with the key escaped as a JSON pointer - segment, so distinct entries never share a name. + segment, so distinct entries never share a name. Zip values name array + members by selector where one names a member in both versions. - The Evidence has operation `inspect_desktop_application_state`, predicate type `rea.desktop-application-state` and confidence `inferred`. diff --git a/docs/specs/desktop-app-environments/tasks/spec.md b/docs/specs/desktop-app-environments/tasks/spec.md index 1dbe40b2cf..f6d5a5f248 100644 --- a/docs/specs/desktop-app-environments/tasks/spec.md +++ b/docs/specs/desktop-app-environments/tasks/spec.md @@ -26,7 +26,8 @@ never runs code the agent wrote. and 1 to 32 `checks`, parsed by a zod schema in `src/domain/desktop/`. Check kinds: `file_present`, `file_absent`, `lines_present`, `lines_absent`, `line_count` (`exact` or `contains`; `equals`, `at_least` or `at_most`), - `value_equals` and `value_absent` (RFC 6901 pointer), `row_present`, + `value_equals` (optional numeric `tolerance`) and `value_absent` (RFC 6901 + pointer, with selectors on arrays), `row_present`, `row_absent` and `row_count` (SQLite table, `where` by column). Ids default to `check-N` and must be unique. - Semantics: an absent file reads as empty; a file without a view fails every @@ -37,10 +38,10 @@ never runs code the agent wrote. regular expressions: a pattern run on agent-written text can stall the verifier, and the two evaluators would read some differently. - Refusals, each naming the check: a file outside the owned paths or not - spelled as a home path; the empty pointer; a pointer not starting with `/`, - into text, or into XML using namespaces; a repeated line; a value that names the - captured home or looks like an ISO 8601 date and time; the draft's - placeholder prompt. + spelled as a home path; the empty pointer; a pointer not starting with `/` + or into text; a repeated line; a `tolerance` whose value is not a number; a + value that names the captured home or looks like an ISO 8601 date and time; + the draft's placeholder prompt. - `export-desktop-application --task FILE` evaluates the checks on the state view's first versions (the seed) and last versions (the end), refuses a task whose every check passes on the seed and a check that fails at the end, diff --git a/docs/specs/desktop-app-environments/tasks/verification.md b/docs/specs/desktop-app-environments/tasks/verification.md index e2c9777cbb..a5f229a02a 100644 --- a/docs/specs/desktop-app-environments/tasks/verification.md +++ b/docs/specs/desktop-app-environments/tasks/verification.md @@ -127,5 +127,6 @@ end to end, not how hard a task can be. - SQLite and dconf readers were not exercised on the host captures, which hold text files; SQLite checks are covered by synthetic views. -- The XML namespace refusal and the CDATA difference the export guide lists - are covered by code review only. +- Namespaced XML and CDATA, which the export once refused or read + differently, are covered by the document views' parity tests on real + Inkscape and LibreOffice saves, not by a host capture. diff --git a/src/application/desktop/DesktopApplicationCapture.test.ts b/src/application/desktop/DesktopApplicationCapture.test.ts index 5e7aee96b8..daa87bbbeb 100644 --- a/src/application/desktop/DesktopApplicationCapture.test.ts +++ b/src/application/desktop/DesktopApplicationCapture.test.ts @@ -8,6 +8,7 @@ import { writeFile, type FileHandle, } from "node:fs/promises"; +import { hostname, userInfo } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -256,6 +257,10 @@ describe.skipIf(process.platform !== "linux")( window_match: { class: "FakeApp", instance: "fakeapp" }, }); expect(capture.session.first_window_ms).not.toBeNull(); + expect(capture.host).toMatchObject({ + hostname: hostname(), + user: userInfo().username, + }); expect( capture.state.checkpoints.map((checkpoint) => checkpoint.name), ).toEqual(["baseline", "written", "stop", "exit"]); diff --git a/src/application/desktop/DesktopApplicationCapture.ts b/src/application/desktop/DesktopApplicationCapture.ts index 3e5edae98f..43876b9dcc 100644 --- a/src/application/desktop/DesktopApplicationCapture.ts +++ b/src/application/desktop/DesktopApplicationCapture.ts @@ -1,4 +1,4 @@ -import { arch } from "node:os"; +import { arch, hostname, userInfo } from "node:os"; import { dirname } from "node:path"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; @@ -84,6 +84,21 @@ export interface DesktopCaptureErrorOutput extends AnalysisErrorProjection { const unique = (paths: readonly string[]) => [...new Set(paths)]; +/** The host's name and the user's, each left out when the system cannot say, as for a user with no passwd entry. */ +const hostIdentity = (): { hostname?: string; user?: string } => { + const name = hostname(); + let user: string | undefined; + try { + user = userInfo().username; + } catch { + user = undefined; + } + return { + ...(name.length === 0 ? {} : { hostname: name }), + ...(user === undefined || user.length === 0 ? {} : { user }), + }; +}; + const limitationsFor = ( session: DesktopSession, recorder: CheckpointRecorder, @@ -273,6 +288,7 @@ const recordCapture = async ( architecture: arch(), display: admitted.display, home, + ...hostIdentity(), }, application, session, diff --git a/src/application/desktop/DesktopApplicationExport.test.ts b/src/application/desktop/DesktopApplicationExport.test.ts index 2e7dd68f24..bc12ee8c1f 100644 --- a/src/application/desktop/DesktopApplicationExport.test.ts +++ b/src/application/desktop/DesktopApplicationExport.test.ts @@ -306,6 +306,70 @@ const exportVendorCapture = async (local: DesktopApplicationCapture) => { return { result, directory }; }; +describe("exportDesktopApplication of a program in another package", () => { + it("installs the package of the program the desktop entry runs beside the entry's, or refuses when it cannot", async () => { + const local = capture(); + const withProgram = (reason?: string) => + exportCapture({ + ...local, + application: { + ...local.application, + package: { + ...local.application.package, + executable_package: "notes-bin", + ...(reason === undefined + ? {} + : { executable_package_reason: reason }), + }, + }, + }); + + const { result, directory } = await withProgram(); + const refused = await withProgram( + "apt-cache policy did not say where notes-bin comes from", + ); + + if (result instanceof Error) throw result; + expect( + await readFile(join(directory, "envs/notes/notes_app.py"), "utf8"), + ).toContain('packages=Packages(apt=["notes", "notes-bin"])'); + expect(await readFile(join(directory, "README.md"), "utf8")).toContain( + "`notes-bin`, which owns the program the desktop entry runs, is installed with it.", + ); + expect(refused.result).toMatchObject({ + _tag: "AnalysisInputError", + issues: [expect.objectContaining({ path: ["capture"] })], + }); + expect(JSON.stringify(refused.result)).toContain( + "the program it runs belongs to notes-bin, which the computer cannot install: apt-cache policy did not say where notes-bin comes from", + ); + expect(await exists(refused.directory)).toBe(false); + }); + + it("refuses a capture whose desktop entry runs a program no package owns, and writes nothing", async () => { + const local = capture(); + const { result, directory } = await exportCapture({ + ...local, + application: { + ...local.application, + package: { + ...local.application.package, + executable_package_reason: `no package owns ${HOME}/.local/bin/notes`, + }, + }, + }); + + expect(result).toMatchObject({ + _tag: "AnalysisInputError", + issues: [expect.objectContaining({ path: ["capture"] })], + }); + expect(JSON.stringify(result)).toContain( + `the computer cannot install the program it runs: no package owns ${HOME}/.local/bin/notes`, + ); + expect(await exists(directory)).toBe(false); + }); +}); + describe("exportDesktopApplication from a vendor repository", () => { it("copies the verified key, declares the repository and library packages, and turns the sandbox off", async () => { const { result, directory } = await exportVendorCapture( @@ -499,6 +563,19 @@ describe("exportDesktopApplication home paths", () => { }); }); + it("owns a profile folder the application created beside a state directory in another letter case", async () => { + const { result } = await exportWalked({ + stateDirectories: [`${HOME}/.config/notes`, `${HOME}/.local/share/notes`], + baseline: [], + created: [[`${HOME}/.config/Notes/2.10/sessionrc`, "[ui]\nwrap=true\n"]], + }); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + paths: [".config/Notes"], + }); + }); + it("keeps the file present before the capture when two map to one home path and names the other", async () => { const later = `${HOME}/.config/notes/settings.ini`; const { result, directory } = await exportWalked({ diff --git a/src/application/desktop/DesktopApplicationExport.ts b/src/application/desktop/DesktopApplicationExport.ts index d93167c20a..97e450fd74 100644 --- a/src/application/desktop/DesktopApplicationExport.ts +++ b/src/application/desktop/DesktopApplicationExport.ts @@ -1,7 +1,5 @@ -import { createHash } from "node:crypto"; import { mkdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { Readable } from "node:stream"; import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js"; import { @@ -25,18 +23,23 @@ import { renderSmokeTask, type DesktopAppExportPlan, type HomePathCollision, + type SeedScrubReport, } from "../../domain/desktop/appExport.js"; import { INSTALL_SCRIPT, electronLaunchArguments, - installLimitations, } from "../../domain/desktop/appExportInstall.js"; +import { installLimitations } from "../../domain/desktop/appExportInstallLimitations.js"; import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js"; import { desktopApplicationStateSchema, + versionAt, type DesktopApplicationState, } from "../../domain/desktop/desktopState.js"; import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js"; +import type { HostIdentity } from "../../domain/desktop/hostIdentity.js"; +import { scrubDconf } from "../../domain/desktop/seedScrub.js"; +import { isTextualFormat } from "../../domain/desktop/stateFormats.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import type { JsonValue } from "../../domain/jsonValue.js"; @@ -52,18 +55,20 @@ import { import { admitApplication, exportEnvironmentName, - refuse, } from "./DesktopExportInstall.js"; import { - withRetainedContent, - type RetainedContent, -} from "./DesktopRetainedContent.js"; + retainedCopies, + shippedSeedViews, + writeFiles, + type SeedVersion, +} from "./DesktopExportFiles.js"; +import { withRetainedContent } from "./DesktopRetainedContent.js"; +import { selectStateEnds } from "./DesktopStateVersions.js"; import { admitDesktopTask, type AdmittedTask } from "./DesktopTaskExport.js"; const OPERATION = "export_desktop_application"; /** Largest seed file the export copies; the state view reads only 64 MiB. */ const SEED_MAX_BYTES = 1024 * 1024 * 1024; -const TEXT_FORMATS = new Set(["text", "json", "keyfile", "xml"]); /** What the caller asked to export, after CLI parsing. */ export interface DesktopExportRequest { @@ -74,6 +79,10 @@ export interface DesktopExportRequest { readonly tasks: readonly string[]; /** Where the computer downloads the captured package file from, for an application installed from one. */ readonly packageUrl?: string | undefined; + /** The checkpoint the seed is taken at; the baseline when omitted. */ + readonly seedCheckpoint?: string | undefined; + /** The checkpoint a task's end state is read at; the last when omitted. */ + readonly endCheckpoint?: string | undefined; } /** What the export wrote. */ @@ -81,9 +90,14 @@ export interface DesktopExportResult { readonly directory: string; readonly environment: string; readonly framework_version: string; + readonly checkpoints: { readonly seed: string; readonly end: string }; readonly paths: readonly string[]; readonly seed_files: readonly string[]; readonly unseeded_files: readonly string[]; + /** Seed files rewritten to remove the capture host's name and user. */ + readonly scrubbed_files: readonly string[]; + /** Seed files that still hold them, in bytes the export does not rewrite or in their name. */ + readonly unscrubbed_files: readonly string[]; readonly dconf_groups: number; readonly files: readonly string[]; readonly limitations: readonly string[]; @@ -95,29 +109,22 @@ const isRecord = ( ): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); -/** The application's dconf groups at the baseline, as a dconf key file. */ -const baselineDconf = ( +/** The application's dconf groups at the seed, as a dconf key file with the identity scrubbed from its strings. */ +const seedDconf = ( state: DesktopApplicationState, -): { text: string; groups: number } => { + seed: number, + identity: HostIdentity, +): { text: string; groups: number; holdsIdentity: boolean } => { const database = state.paths.find( (path) => path.format === "dconf" && path.role === "application_state", ); - const view = database?.versions.find((version) => version.present)?.view; - if ( - database?.versions[0]?.present !== true || - !isRecord(view) || - !isRecord(view.groups) - ) - return { text: "", groups: 0 }; - const lines: string[] = []; - for (const [group, keys] of Object.entries(view.groups)) { - if (!isRecord(keys)) continue; - lines.push(`[${group}]`); - for (const [key, value] of Object.entries(keys)) - if (typeof value === "string") lines.push(`${key}=${value}`); - lines.push(""); - } - return { text: lines.join("\n"), groups: Object.keys(view.groups).length }; + const version = + database === undefined ? undefined : versionAt(database, seed); + const view = version?.view; + if (version?.present !== true || !isRecord(view) || !isRecord(view.groups)) + return { text: "", groups: 0, holdsIdentity: false }; + const groups = Object.keys(view.groups).length; + return { ...scrubDconf(view.groups, identity), groups }; }; type StatePath = DesktopApplicationState["paths"][number]; @@ -126,20 +133,22 @@ type FailedRelease = Extract< { readonly kind: "failed" } >; -/** Files under the owned paths that existed before the capture without retained bytes, in the state view or omitted. */ +/** Files under the owned paths that existed at the seed without retained bytes, in the state view or omitted. */ const unseededFiles = ( capture: DesktopApplicationCapture, homePath: (path: string) => string | null, files: ReadonlyMap, - paths: readonly string[], + { paths, seed }: { readonly paths: readonly string[]; readonly seed: number }, ): readonly string[] => { const listed = [...files] - .filter( - ([home, path]) => + .filter(([home, path]) => { + const version = versionAt(path, seed); + return ( isWithinAny(home, paths) && - path.versions[0]?.present === true && - path.versions[0].sha256 === null, - ) + version?.present === true && + version.sha256 === null + ); + }) .map(([home]) => home); const omitted = capture.state.omitted_content .map(({ path }) => homePath(path)) @@ -150,18 +159,23 @@ const unseededFiles = ( return [...new Set([...listed, ...omitted])].sort(); }; -/** Decide the bundle from the capture and its state view, with each seed's digest. */ +/** Decide the bundle from the capture and its state view at the seed, with each seed's digest. */ const planExport = ( request: DesktopExportRequest, capture: DesktopApplicationCapture, state: DesktopApplicationState, - evidenceId: string, + context: { + readonly evidenceId: string; + readonly seed: number; + readonly identity: HostIdentity; + }, ): { readonly plan: DesktopAppExportPlan; - readonly seeds: ReadonlyMap; + readonly seeds: ReadonlyMap; readonly limitations: readonly string[]; readonly collisions: readonly HomePathCollision[]; readonly files: ReadonlyMap; + readonly dconf: { readonly groups: number; readonly holdsIdentity: boolean }; } => { const { application, host } = capture; const source = exportEnvironmentName(capture, request.name); @@ -175,14 +189,19 @@ const planExport = ( capture, state, ); - const seeds = new Map(); + const { text: dconf, ...dconfScrub } = seedDconf( + state, + context.seed, + context.identity, + ); + const seeds = new Map(); const textSeeds: string[] = []; for (const [home, path] of files) { - const baseline = path.versions[0]; - if (!isWithinAny(home, paths) || baseline?.present !== true) continue; - if (baseline.sha256 === null) continue; - seeds.set(home, baseline.sha256); - if (TEXT_FORMATS.has(baseline.format ?? "")) textSeeds.push(home); + const version = versionAt(path, context.seed); + if (!isWithinAny(home, paths) || version?.present !== true) continue; + if (version.sha256 === null) continue; + seeds.set(home, { sha256: version.sha256, format: version.format }); + if (isTextualFormat(version.format)) textSeeds.push(home); } const plan: DesktopAppExportPlan = { name: source, @@ -195,6 +214,9 @@ const planExport = ( aptSources: install.sources, architectures: install.architectures, packageFile: install.packageFile, + ...(install.executablePackage === undefined + ? {} + : { executablePackage: install.executablePackage }), electron, command: launchCommand( electronLaunchArguments(application.argv, electron), @@ -204,23 +226,27 @@ const planExport = ( windowClass, paths, capturedHome: host.home, + seedCheckpoint: state.capture.checkpoints[context.seed] ?? "baseline", seedFiles: [...seeds.keys()].sort(), textSeeds: textSeeds.sort(), - unseededFiles: unseededFiles(capture, homePath, files, paths), - dconf: baselineDconf(state).text, - captureEvidenceId: evidenceId, + unseededFiles: unseededFiles(capture, homePath, files, { + paths, + seed: context.seed, + }), + dconf, + captureEvidenceId: context.evidenceId, }; const limitations = installLimitations( install, application.package, electron, ); - return { plan, seeds, limitations, collisions, files }; + return { plan, seeds, limitations, collisions, files, dconf: dconfScrub }; }; const EXPORT_LIMITATIONS = [ - "The seed is the user's real state before the capture; keep the bundle local.", - "Binary state files are seeded as captured; only text files, UTF-8 or UTF-16 with a byte-order mark, have the captured home path respelled.", + "The seed is the user's real state at the seed checkpoint; keep the bundle local.", + "Binary state files are seeded as captured; only text files, UTF-8 or UTF-16 with a byte-order mark, have the capture host's name, user@host and home path replaced.", ]; /** @@ -275,78 +301,76 @@ const writeBundle = async ( return undefined; }; -/** A bundle file copied from retained content: a seed or an apt key. */ -interface RetainedCopy { - readonly path: string; - readonly digest: string; - readonly name: string; -} - -/** The generated files and the retained files copied after them. */ -interface ExportBundle { - readonly generated: readonly { - readonly path: string; - readonly bytes: Buffer; - }[]; - readonly copies: readonly RetainedCopy[]; -} - -const seedPath = (plan: DesktopAppExportPlan, seed: string) => - `envs/${plan.name}/seed/${seed}`; - -/** Each seed by its home path, then each distinct apt key in the environment folder. */ -const retainedCopies = ( +/** The bundle's generated files, the class module flagged when its dconf keys hold the identity, and its README, made once the seeds' scrub is known. */ +const bundleText = ( plan: DesktopAppExportPlan, - seeds: ReadonlyMap, -): readonly RetainedCopy[] => [ - ...[...seeds].map(([seed, digest]) => ({ - path: seedPath(plan, seed), - digest, - name: `~/${seed}`, - })), - ...[ - ...new Map( - plan.aptSources.map((source) => [source.keyFile, source.keySha256]), - ), - ].map(([keyFile, digest]) => ({ - path: `envs/${plan.name}/${keyFile}`, - digest, - name: `the apt key ${keyFile}`, - })), -]; + admitted: AdmittedTask | undefined, + dconfHoldsIdentity: boolean, +) => { + const environment = `envs/${plan.name}`; + const text = (path: string, content: string, holdsIdentity = false) => ({ + path, + bytes: Buffer.from(content), + holdsIdentity, + }); + return { + readme: (scrub: SeedScrubReport) => + text("README.md", renderReadme(plan, admitted?.readme, scrub)), + generated: [ + text(`${environment}/env.toml`, renderEnvToml(plan)), + text( + `${environment}/${appModuleFile(plan.name)}`, + renderAppClass(plan), + dconfHoldsIdentity, + ), + text(`${environment}/SKILL.md`, renderSkill(plan)), + ...(plan.packageFile === null + ? [] + : [ + text( + `${environment}/${INSTALL_SCRIPT}`, + renderInstallScript(plan.packageFile), + ), + ]), + text("tasks/smoke.json", renderSmokeTask(plan)), + ...(admitted?.files ?? []).map((file) => text(file.path, file.content)), + ], + }; +}; -/** Write the generated files, then each retained file as it is read, so one at a time is in memory. */ -const writeFiles = async ( - tree: SafeOutputTree, - content: RetainedContent, - bundle: ExportBundle, - signal: AbortSignal | undefined, -): Promise => { - for (const file of bundle.generated) { - const sha256 = createHash("sha256").update(file.bytes).digest("hex"); - await tree.write( - file.path, - Readable.from([file.bytes]), - { bytes: file.bytes.length, sha256 }, - signal, - ); - } - for (const copy of bundle.copies) { - signal?.throwIfAborted(); - const bytes = await content.read(copy.digest, signal); - if (!bytes.ok) - throw refuse( - `The retained content of ${copy.name} ${bytes.error}.`, - "capture", - "Capture the application again, leaving a seed file out of its state paths if it cannot be retained.", - ); - await tree.write( - copy.path, - Readable.from([bytes.value]), - { bytes: bytes.value.length, sha256: copy.digest }, - signal, - ); - } +const exportLimitations = ( + capture: DesktopApplicationCapture, + plan: DesktopAppExportPlan, + facts: { + readonly install: readonly string[]; + readonly collisions: readonly HomePathCollision[]; + readonly identity: HostIdentity; + }, +): string[] => { + const opened = capture.state.unretained_opened_directories; + return [ + ...EXPORT_LIMITATIONS, + ...facts.install, + ...(plan.unseededFiles.length === 0 + ? [] + : [ + `${String(plan.unseededFiles.length)} file(s) under the owned paths existed at the seed but were not retained, so they are not seeded.`, + ]), + ...(facts.identity.hostname === null || facts.identity.user === null + ? [ + "The capture does not record the host's name and user, so only the captured home is removed from the seed; capture again to remove them too.", + ] + : []), + ...(opened.length === 0 + ? [] + : [ + `The application held files open in ${opened.join(", ")}, whose starting content the capture did not retain, so nothing there is seeded.`, + ]), + ...facts.collisions.map( + ({ home, kept, dropped }) => + `${dropped} and ${kept} both map to ~/${home}; the export uses ${kept} and leaves out ${dropped}.`, + ), + ]; }; const runExport = async ( @@ -367,77 +391,71 @@ const runExport = async ( const state = desktopApplicationStateSchema.parse( stateEvidence.normalized_result, ); - const { plan, seeds, collisions, files, ...install } = planExport( + const { checkpoints } = state.capture; + const ends = selectStateEnds( + checkpoints, + { seed: request.seedCheckpoint, end: request.endCheckpoint }, + OPERATION, + ); + const { home, hostname, user } = capture.host; + const identity = { home, hostname: hostname ?? null, user: user ?? null }; + const { plan, seeds, collisions, files, dconf, ...install } = planExport( request, capture, state, - evidence.evidence_id, - ); - const admitted = await admitDesktopTask( - request.tasks, - { ...plan, files }, - signal, + { evidenceId: evidence.evidence_id, seed: ends.seed, identity }, ); const copies = retainedCopies(plan, seeds); - const environment = `envs/${plan.name}`; - const text = (path: string, content: string) => ({ - path, - bytes: Buffer.from(content), - }); - const generated = [ - text("README.md", renderReadme(plan, admitted?.readme)), - text(`${environment}/env.toml`, renderEnvToml(plan)), - text(`${environment}/${appModuleFile(plan.name)}`, renderAppClass(plan)), - text(`${environment}/SKILL.md`, renderSkill(plan)), - ...(plan.packageFile === null - ? [] - : [ - text( - `${environment}/${INSTALL_SCRIPT}`, - renderInstallScript(plan.packageFile), - ), - ]), - text("tasks/smoke.json", renderSmokeTask(plan)), - ...(admitted?.files ?? []).map((file) => text(file.path, file.content)), - ]; + // A task's seed checks read the seeds as the bundle ships them, scrubbed. + const seedViews = + request.tasks.length === 0 + ? new Map() + : await withRetainedContent( + request.capture, + (content) => shippedSeedViews(content, copies, identity, signal), + SEED_MAX_BYTES, + ); + const admitted = await admitDesktopTask( + request.tasks, + { ...plan, files, identity, ends, seedViews }, + signal, + ); + const { generated, readme } = bundleText(plan, admitted, dconf.holdsIdentity); + let scrub: SeedScrubReport = { scrubbed: [], unscrubbed: [] }; const unclosed = await withRetainedContent( request.capture, (content) => writeBundle( request.directory, - (tree) => writeFiles(tree, content, { generated, copies }, signal), + async (tree) => { + const bundle = { generated, copies, identity, readme }; + scrub = await writeFiles(tree, content, bundle, signal); + }, signal, ), SEED_MAX_BYTES, ); - const opened = capture.state.unretained_opened_directories; - const limitations = [ - ...EXPORT_LIMITATIONS, - ...install.limitations, - ...(plan.unseededFiles.length === 0 - ? [] - : [ - `${String(plan.unseededFiles.length)} file(s) under the owned paths existed before the capture but were not retained, so they are not seeded.`, - ]), - ...(opened.length === 0 - ? [] - : [ - `The application held files open in ${opened.join(", ")}, whose starting content the capture did not retain, so nothing there is seeded.`, - ]), - ...collisions.map( - ({ home, kept, dropped }) => - `${dropped} and ${kept} both map to ~/${home}; the export uses ${kept} and leaves out ${dropped}.`, - ), - ]; + const limitations = exportLimitations(capture, plan, { + install: install.limitations, + collisions, + identity, + }); const result: DesktopExportResult = { directory: request.directory, environment: plan.name, framework_version: DESKTOP_EXPORT_FRAMEWORK_VERSION, + checkpoints: { + seed: checkpoints[ends.seed] ?? "baseline", + end: checkpoints[ends.end] ?? "baseline", + }, paths: plan.paths, seed_files: plan.seedFiles, unseeded_files: plan.unseededFiles, - dconf_groups: baselineDconf(state).groups, + scrubbed_files: [...scrub.scrubbed].sort(), + unscrubbed_files: [...scrub.unscrubbed].sort(), + dconf_groups: dconf.groups, files: [ + "README.md", ...generated.map((file) => file.path), ...copies.map((copy) => copy.path), ], @@ -489,12 +507,17 @@ export const exportDesktopApplication = async ( named: request.name !== undefined, tasks: request.tasks.length, package_url: request.packageUrl !== undefined, + seed_checkpoint: request.seedCheckpoint ?? null, + end_checkpoint: request.endCheckpoint ?? null, }, result: { ...result, + checkpoints: { ...result.checkpoints }, paths: [...result.paths], seed_files: [...result.seed_files], unseeded_files: [...result.unseeded_files], + scrubbed_files: [...result.scrubbed_files], + unscrubbed_files: [...result.unscrubbed_files], files: [...result.files], limitations: [...result.limitations], }, diff --git a/src/application/desktop/DesktopApplicationState.ts b/src/application/desktop/DesktopApplicationState.ts index cd211e26bb..103026eaaf 100644 --- a/src/application/desktop/DesktopApplicationState.ts +++ b/src/application/desktop/DesktopApplicationState.ts @@ -14,14 +14,11 @@ import type { } from "../../domain/desktop/desktopState.js"; import { diffStateViews } from "../../domain/desktop/stateChanges.js"; import { - decodeStateText, detectStateFormat, - jsonView, keyfileView, plainRecord, SQLITE_SIDECARS, - textView, - xmlView, + textualView, } from "../../domain/desktop/stateFormats.js"; import { applicationNames, @@ -37,6 +34,7 @@ import { err, ok, type Result } from "../../domain/result.js"; import { describeValidationFailure } from "../../domain/validationFailure.js"; import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; import { readJsonFile } from "../JsonFiles.js"; +import { readZipView } from "./DesktopZipView.js"; import { withRetainedContent, type RetainedContent, @@ -257,9 +255,9 @@ const dconfView = async ( : failed(dump.error); }); -const MIB = 1024 * 1024; -/** Largest JSON or key file read into a view, whose whole value enters Evidence. */ -const MAX_PARSED_VIEW_BYTES = 4 * MIB; +/** A reader's result as a version's view or view_error. */ +const outcomeOf = (read: Result): ViewOutcome => + read.ok ? { view: read.value, error: null } : failed(read.error); /** Read one retained version into JSON for its own format. */ const viewOf = async ( @@ -267,31 +265,14 @@ const viewOf = async ( context: ViewContext, ): Promise => { const format = input.version.format ?? "binary"; - if ( - (format === "json" || format === "keyfile") && - input.bytes.length > MAX_PARSED_VIEW_BYTES - ) - return failed( - `larger than the ${String(MAX_PARSED_VIEW_BYTES / MIB)} MiB limit for JSON and key file views`, - ); - const text = decodeStateText(input.bytes) ?? ""; switch (format) { case "text": - return { view: textView(text), error: null }; - case "json": { - const parsed = jsonView(text); - return parsed.ok - ? { view: parsed.value, error: null } - : failed(parsed.error); - } + case "json": case "keyfile": - return { view: { groups: keyfileView(text) }, error: null }; - case "xml": { - const tree = xmlView(text); - return tree === undefined - ? failed("not well-formed XML, or it declares a DOCTYPE") - : { view: tree, error: null }; - } + case "xml": + return outcomeOf(textualView(format, input.bytes)); + case "zip": + return outcomeOf(await readZipView(input.bytes, context.signal)); case "dconf": return dconfView(input, context); case "sqlite": @@ -416,6 +397,7 @@ const STATE_LIMITATIONS = [ "SQLite views list at most 500 rows of at most 64 ordinary tables, text views at most 5,000 lines and XML views at most 5,000 nodes; changes between truncated views, or views with an unread table, are marked incomplete.", "JSON and key file views read files of at most 4 MiB, JSON nested at most 500 levels and only numbers JavaScript holds exactly; other files of those formats have a view_error.", "dconf views keep only groups whose path names the application.", + "Zip views list at most 1,000 entries and read an entry only when it is unencrypted, stored or deflated, at most 16 MiB and at most 100 times its compressed size, up to 64 MiB per archive; databases and archives inside an archive are not read.", ]; /** Read a capture folder into the application's state view. */ diff --git a/src/application/desktop/DesktopArchiveTask.test.ts b/src/application/desktop/DesktopArchiveTask.test.ts new file mode 100644 index 0000000000..057f757f2d --- /dev/null +++ b/src/application/desktop/DesktopArchiveTask.test.ts @@ -0,0 +1,273 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js"; +import { desktopApplicationStateSchema } from "../../domain/desktop/desktopState.js"; +import { + odtArchive, + zipArchive, +} from "../../domain/desktop/zipArchive.fixture.js"; +import { exportDesktopApplication } from "./DesktopApplicationExport.js"; +import { inspectDesktopApplicationState } from "./DesktopApplicationState.js"; +import { DESKTOP_OPERATION } from "./DesktopCaptureRefusals.js"; +import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js"; +import { + HOME, + capture, + change, + readers, + retained, + writeCaptureFolder, +} from "./desktopCapture.fixture.js"; + +const FREECAD = join( + import.meta.dirname, + "../../../tests/fixtures/freecad/1.0", +); +const PLATE = "Documents/plate.FCStd"; +const LIST = "Documents/list.odt"; +/** The list's paragraphs, by their prefixed element names. */ +const LIST_TEXT = + "/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:text]/children"; + +/** FreeCAD's save of the plate, before and after the user cut the hole. */ +const plate = async (end: "before" | "after") => + zipArchive([ + { + name: "Document.xml", + data: await readFile(join(FREECAD, end, "Document.xml")), + }, + { name: "thumbnails/Thumbnail.png", data: Buffer.from([0x89, 0x50, 0]) }, + ]); + +/** The notes capture with a FreeCAD document and an OpenDocument list the user also edited, both named in window titles. */ +const archiveCapture = async ( + list: readonly [Buffer, Buffer] = [ + odtArchive("Buy bread"), + odtArchive("Buy milk"), + ], +) => { + const base = capture(); + const [first, second] = base.state.checkpoints; + if (first === undefined || second === undefined) throw new Error("fixture"); + const files = { + [`${HOME}/${PLATE}`]: [await plate("before"), await plate("after")], + [`${HOME}/${LIST}`]: list, + } as const; + const captured: DesktopApplicationCapture = { + ...base, + session: { + ...base.session, + windows: base.session.windows.map((window) => ({ + ...window, + titles: [...window.titles, "plate.FCStd - Notes", "list.odt - Notes"], + })), + }, + state: { + ...base.state, + checkpoints: [ + { + ...first, + retained: [ + ...first.retained, + ...Object.entries(files).map(([path, [before]]) => + retained(path, before), + ), + ], + }, + { + ...second, + changes: [ + ...second.changes, + ...Object.entries(files).map(([path, [before, after]]) => + change(path, { before, after }), + ), + ], + retained: [ + ...second.retained, + ...Object.entries(files).map(([path, [, after]]) => + retained(path, after), + ), + ], + }, + ], + }, + }; + return writeCaptureFolder( + captured, + DESKTOP_OPERATION, + Object.values(files).flat(), + ); +}; + +const HOLE = { + kind: "value_equals", + file: PLATE, + pointer: + "/entries/[name=Document.xml]/view/root/children/[name=ObjectData]/children/[name=Object][@name=Hole]/children/[name=Properties]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value", + value: 5, + tolerance: 0.01, +}; + +/** The entries of the list document's last version in the capture's state view. */ +const endEntries = async (folder: string) => { + const evidence = await inspectDesktopApplicationState(folder, readers); + if (!("normalized_result" in evidence)) throw evidence; + const state = desktopApplicationStateSchema.parse(evidence.normalized_result); + const end = state.paths + .find((entry) => entry.home_path === LIST) + ?.versions.at(-1)?.view; + return ( + (end as { entries: Record[] } | undefined)?.entries ?? [] + ); +}; + +const exportTask = async (folder: string, checks: unknown[]) => { + const scratch = await createTestTempDirectory("desktop-archive-task-"); + const path = join(scratch, "task.json"); + await writeFile( + path, + JSON.stringify({ + name: "cut-hole", + prompt: "Cut a 10 mm hole through the plate and save it.", + checks, + }), + ); + const directory = join(scratch, "bundle"); + const result = await exportDesktopApplication( + { capture: folder, directory, name: undefined, tasks: [path] }, + readers, + ); + return { result, directory }; +}; + +describe("documents saved as zip archives", () => { + it("reads each version into its entries and lists the change by entry and element name", async () => { + const evidence = await inspectDesktopApplicationState( + await archiveCapture(), + readers, + ); + if (!("normalized_result" in evidence)) throw evidence; + const state = desktopApplicationStateSchema.parse( + evidence.normalized_result, + ); + const path = state.paths.find((entry) => entry.home_path === PLATE); + + expect(path).toMatchObject({ + role: "document", + format: "zip", + versions: [ + { format: "zip", view: { entry_count: 2, truncated: false } }, + { format: "zip", view_error: null }, + ], + changes: [{ kind: "values", complete: true }], + }); + expect(path?.changes[0]?.added).toContain( + "/entries/[name=Document.xml]/view/root/children/[name=Objects]/children/[name=Object][@name=Hole]", + ); + }); + + it("exports a task that checks the hole and the list's namespaced XML by name", async () => { + const folder = await archiveCapture(); + const milk = { + kind: "value_equals", + file: LIST, + pointer: `${LIST_TEXT}/[name=text:p]/children/0`, + value: "Buy milk", + }; + + const { result, directory } = await exportTask(folder, [HOLE, milk]); + if (result instanceof Error) throw result; + const verifier = await readFile( + join(directory, "artifacts/cut-hole-check/verify.py"), + "utf8", + ); + expect(verifier).toContain(`"tolerance": 0.01`); + expect(verifier).toContain(JSON.stringify(milk.pointer)); + }); + + it("accepts a check on the whole listing, whose value holds namespaced XML", async () => { + const folder = await archiveCapture(); + const entries = await endEntries(folder); + expect(entries).toHaveLength(3); + + const { result } = await exportTask(folder, [ + { kind: "value_equals", file: LIST, pointer: "/entries", value: entries }, + ]); + + if (result instanceof Error) throw result; + }); + + it("exports checks of a namespaced entry's metadata, its absence, and the entry compared whole", async () => { + const comments = { + name: "comments.xml", + data: '', + }; + const folder = await archiveCapture([ + odtArchive("Buy bread", [comments]), + odtArchive("Buy milk"), + ]); + const content = (await endEntries(folder)).find( + (entry) => entry.name === "content.xml", + ); + + const { result } = await exportTask(folder, [ + { + kind: "value_equals", + file: LIST, + pointer: "/entries/[name=content.xml]/size", + value: content?.size, + }, + { + kind: "value_absent", + file: LIST, + pointer: "/entries/[name=comments.xml]", + }, + ]); + expect( + result instanceof Error + ? (result as Error & { userMessage: string }).userMessage + : undefined, + ).toBeUndefined(); + + const whole = await exportTask(folder, [ + { + kind: "value_equals", + file: LIST, + pointer: "/entries/[name=content.xml]", + value: content, + }, + ]); + if (whole.result instanceof Error) throw whole.result; + }); + + it("drafts name-addressed checks of the archive that the export then accepts", async () => { + const folder = await archiveCapture(); + + const drafted = await draftDesktopApplicationTask(folder, readers); + if (drafted instanceof Error) throw drafted; + const task = ( + drafted.normalized_result as { + task: { checks: { file: string; pointer?: string }[] }; + } + ).task; + + const archived = task.checks.filter(({ file }) => file === PLATE); + expect(archived.length).toBeGreaterThan(0); + expect( + archived.every(({ pointer }) => pointer?.includes("[name=") === true), + ).toBe(true); + expect(task.checks.filter(({ file }) => file === LIST)).toEqual([ + expect.objectContaining({ + kind: "value_equals", + pointer: `${LIST_TEXT}/[name=text:p]/children/0`, + value: "Buy milk", + }), + ]); + const { result } = await exportTask(folder, task.checks); + expect(result).not.toBeInstanceOf(Error); + }); +}); diff --git a/src/application/desktop/DesktopCheckpointEnds.test.ts b/src/application/desktop/DesktopCheckpointEnds.test.ts new file mode 100644 index 0000000000..c5102faa8d --- /dev/null +++ b/src/application/desktop/DesktopCheckpointEnds.test.ts @@ -0,0 +1,251 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js"; +import { exportDesktopApplication } from "./DesktopApplicationExport.js"; +import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js"; +import { + EDITED_NOTE, + HOME, + SAVED_NOTE, + change, + readers, + retained, + seedEndCapture, + writeCaptureFolder, +} from "./desktopCapture.fixture.js"; + +const NOTE = "Documents/today.txt"; + +const folder = () => + writeCaptureFolder(seedEndCapture(), undefined, [EDITED_NOTE]); + +const STATE = `${HOME}/.config/notes/state.json`; +const UNSET = '{"eggs": false}'; +const SET = '{"eggs": true}'; +/** The state file's versions after the baseline, where it is empty, which reads as text. */ +const STATE_VERSIONS: Readonly< + Record +> = { + seed: { before: "", after: UNSET }, + end: { before: UNSET, after: SET }, + exit: { before: SET, after: "" }, +}; + +/** The capture with a JSON state file that is empty before `seed` and again after `end`. */ +const emptiedStateFolder = () => { + const base = seedEndCapture(); + const emptied: DesktopApplicationCapture = { + ...base, + state: { + ...base.state, + checkpoints: base.state.checkpoints.map((point) => { + if (point.name === "baseline") + return { + ...point, + retained: [...point.retained, retained(STATE, "")], + }; + const version = STATE_VERSIONS[point.name]; + return version === undefined + ? point + : { + ...point, + changes: [ + ...point.changes, + change(STATE, version, { state: true }), + ], + retained: [...point.retained, retained(STATE, version.after)], + }; + }), + }, + }; + return writeCaptureFolder(emptied, undefined, [EDITED_NOTE, "", UNSET, SET]); +}; + +const eggs = { id: "eggs", kind: "lines_present", file: NOTE, lines: ["eggs"] }; +const wrapped = { + id: "wrap", + kind: "value_equals", + file: ".config/notes/settings.ini", + pointer: "/groups/ui/wrap", + value: "true", +}; + +/** Export the capture between the named checkpoints, with a task holding these checks when any are given. */ +const exportBetween = async ( + ends: { seedCheckpoint?: string; endCheckpoint?: string }, + checks: readonly unknown[] = [], + captured = folder, +) => { + const scratch = await createTestTempDirectory("desktop-ends-"); + const task = join(scratch, "task.json"); + await writeFile( + task, + JSON.stringify({ + name: "add-eggs", + prompt: "Add eggs to my list.", + checks, + }), + ); + const directory = join(scratch, "bundle"); + const result = await exportDesktopApplication( + { + capture: await captured(), + directory, + name: undefined, + tasks: checks.length === 0 ? [] : [task], + ...ends, + }, + readers, + ); + return { result, directory }; +}; + +const exists = (path: string) => + access(path).then( + () => true, + () => false, + ); + +describe("exportDesktopApplication between named checkpoints", () => { + it("seeds the files as they were at the seed checkpoint and keeps a task that fails there and passes at the end", async () => { + const { result, directory } = await exportBetween( + { seedCheckpoint: "seed", endCheckpoint: "end" }, + [eggs], + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + checkpoints: { seed: "seed", end: "end" }, + seed_files: [".config/notes/settings.ini", NOTE], + task: { name: "add-eggs", failing_on_seed: ["eggs"] }, + }); + const seed = join(directory, "envs/notes/seed"); + expect(await readFile(join(seed, NOTE), "utf8")).toBe(SAVED_NOTE); + expect( + await readFile(join(seed, ".config/notes/settings.ini"), "utf8"), + ).toBe("[ui]\nwrap=false\n"); + }); + + it("keeps the baseline and the last checkpoint when none is named", async () => { + const { result } = await exportBetween({}); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + checkpoints: { seed: "baseline", end: "exit" }, + seed_files: [".config/notes/settings.ini"], + }); + }); + + it("reads the end state at the end checkpoint, not at the capture's last", async () => { + const atEnd = await exportBetween( + { seedCheckpoint: "seed", endCheckpoint: "end" }, + [eggs, wrapped], + ); + const atExit = await exportBetween({ seedCheckpoint: "seed" }, [ + eggs, + wrapped, + ]); + + expect(atEnd.result).toMatchObject({ issues: [{ path: ["task"] }] }); + expect((atEnd.result as { userMessage: string }).userMessage).toContain( + '"wrap"', + ); + if (atExit.result instanceof Error) throw atExit.result; + expect(atExit.result.normalized_result).toMatchObject({ + checkpoints: { seed: "seed", end: "exit" }, + task: { failing_on_seed: ["eggs", "wrap"] }, + }); + }); + + it("reads a value check's format at the seed and end only, so a version read as text before or after them does not refuse it", async () => { + const eggsSet = { + id: "eggs", + kind: "value_equals", + file: ".config/notes/state.json", + pointer: "/eggs", + value: true, + }; + + const between = await exportBetween( + { seedCheckpoint: "seed", endCheckpoint: "end" }, + [eggsSet], + emptiedStateFolder, + ); + const fromBaseline = await exportBetween( + { endCheckpoint: "end" }, + [eggsSet], + emptiedStateFolder, + ); + + if (between.result instanceof Error) throw between.result; + expect(between.result.normalized_result).toMatchObject({ + checkpoints: { seed: "seed", end: "end" }, + task: { failing_on_seed: ["eggs"] }, + }); + expect( + (fromBaseline.result as { userMessage: string }).userMessage, + ).toContain("points into .config/notes/state.json, which is text"); + }); + + it.each([ + [{ seedCheckpoint: "saved" }, "seed_checkpoint"], + [{ endCheckpoint: "Exit" }, "end_checkpoint"], + [{ seedCheckpoint: "end", endCheckpoint: "seed" }, "end_checkpoint"], + ])( + "refuses %j, naming the option, and writes no bundle", + async (ends, field) => { + const { result, directory } = await exportBetween(ends); + + expect(result).toMatchObject({ + operation: "export_desktop_application", + issues: [{ path: [field], reason: "invalid_value" }], + }); + expect(await exists(directory)).toBe(false); + }, + ); +}); + +describe("draftDesktopApplicationTask between named checkpoints", () => { + it("diffs the seed checkpoint against the end checkpoint", async () => { + const result = await draftDesktopApplicationTask( + await folder(), + readers, + undefined, + { seed: "seed", end: "end" }, + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + checkpoints: { baseline: "seed", final: "end" }, + files: [ + { + file: NOTE, + baseline: { checkpoint: "seed" }, + final: { checkpoint: "end" }, + change: { kind: "lines", added: ["eggs"], removed: [] }, + }, + ], + task: { + checks: [{ kind: "lines_present", file: NOTE, lines: ["eggs"] }], + }, + }); + }); + + it("refuses an unknown checkpoint under its own operation", async () => { + const result = await draftDesktopApplicationTask( + await folder(), + readers, + undefined, + { seed: "saved" }, + ); + + expect(result).toMatchObject({ + operation: "describe_desktop_application_task", + issues: [{ path: ["seed_checkpoint"], reason: "invalid_value" }], + }); + }); +}); diff --git a/src/application/desktop/DesktopExportFiles.ts b/src/application/desktop/DesktopExportFiles.ts new file mode 100644 index 0000000000..6479d34908 --- /dev/null +++ b/src/application/desktop/DesktopExportFiles.ts @@ -0,0 +1,182 @@ +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; + +import type { SafeOutputTree } from "../../artifacts/SafeOutputTree.js"; +import type { + DesktopAppExportPlan, + SeedScrubReport, +} from "../../domain/desktop/appExport.js"; +import type { TaskFileView } from "../../domain/desktop/appTaskChecks.js"; +import { + identityLeftIn, + type HostIdentity, +} from "../../domain/desktop/hostIdentity.js"; +import { scrubSeed } from "../../domain/desktop/seedScrub.js"; +import { + detectStateFormat, + isTextualFormat, + textualView, +} from "../../domain/desktop/stateFormats.js"; +import { refuse } from "./DesktopExportInstall.js"; +import type { RetainedContent } from "./DesktopRetainedContent.js"; + +/** A file the export writes from bytes it made. */ +export interface GeneratedFile { + readonly path: string; + readonly bytes: Buffer; + /** Whether what it was made from holds the identity in a spelling its bytes do not show, such as a decoded escape. */ + readonly holdsIdentity?: boolean; +} + +/** A seed file's retained digest and the format the state view read it as. */ +export interface SeedVersion { + readonly sha256: string; + readonly format: string | null; +} + +/** A bundle file copied from retained content: a seed, with its home path and format, or an apt key. */ +export interface RetainedCopy { + readonly path: string; + readonly digest: string; + readonly name: string; + readonly seed?: { readonly home: string; readonly format: string | null }; +} + +/** The files a bundle holds; the README is made last, from what the seeds' scrub found. */ +export interface ExportBundle { + readonly generated: readonly GeneratedFile[]; + readonly copies: readonly RetainedCopy[]; + readonly identity: HostIdentity; + readonly readme: (scrub: SeedScrubReport) => GeneratedFile; +} + +/** Each seed by its home path, then each distinct apt key in the environment folder. */ +export const retainedCopies = ( + plan: DesktopAppExportPlan, + seeds: ReadonlyMap, +): readonly RetainedCopy[] => [ + ...[...seeds].map(([seed, { sha256, format }]) => ({ + path: `envs/${plan.name}/seed/${seed}`, + digest: sha256, + name: `~/${seed}`, + seed: { home: seed, format }, + })), + ...[ + ...new Map( + plan.aptSources.map((source) => [source.keyFile, source.keySha256]), + ), + ].map(([keyFile, digest]) => ({ + path: `envs/${plan.name}/${keyFile}`, + digest, + name: `the apt key ${keyFile}`, + })), +]; + +/** A retained file's bytes, refused when they cannot be read. */ +const retainedBytes = async ( + content: RetainedContent, + copy: RetainedCopy, + signal: AbortSignal | undefined, +): Promise => { + signal?.throwIfAborted(); + const bytes = await content.read(copy.digest, signal); + if (!bytes.ok) + throw refuse( + `The retained content of ${copy.name} ${bytes.error}.`, + "capture", + "Capture the application again, leaving a seed file out of its state paths if it cannot be retained.", + ); + return bytes.value; +}; + +/** + * The view of each text seed the scrub rewrites, as the bundle ships it, for + * a task's checks on the seed: the computer starts from these bytes. + */ +export const shippedSeedViews = async ( + content: RetainedContent, + copies: readonly RetainedCopy[], + identity: HostIdentity, + signal: AbortSignal | undefined, +): Promise> => { + const views = new Map(); + for (const copy of copies) { + const seed = copy.seed; + if (seed === undefined || !isTextualFormat(seed.format)) continue; + const bytes = await retainedBytes(content, copy, signal); + const shipped = scrubSeed(bytes, seed, identity).bytes; + if (shipped === undefined) continue; + const format = detectStateFormat(seed.home, shipped); + const view = isTextualFormat(format) + ? textualView(format, shipped) + : undefined; + views.set( + seed.home, + view === undefined || view.ok + ? { format, view: view?.value ?? null } + : { format, view: null, error: view.error }, + ); + } + return views; +}; + +/** + * Write the generated files, then each retained file as it is read, so one at + * a time is in memory: a seed whose text names the capture host becomes a + * generated file with its own digest. The README follows, listing them. Every + * file is listed as unscrubbed whose path or content still holds the identity. + */ +export const writeFiles = async ( + tree: SafeOutputTree, + content: RetainedContent, + bundle: ExportBundle, + signal: AbortSignal | undefined, +): Promise => { + const scrubbed: string[] = []; + const unscrubbed: string[] = []; + const write = async ( + path: string, + bytes: Buffer, + shipped: { readonly sha256?: string; readonly holdsIdentity: boolean }, + ) => { + await tree.write( + path, + Readable.from([bytes]), + { + bytes: bytes.length, + sha256: + shipped.sha256 ?? createHash("sha256").update(bytes).digest("hex"), + }, + signal, + ); + if ( + shipped.holdsIdentity || + identityLeftIn(path, bundle.identity) !== undefined + ) + unscrubbed.push(path); + }; + const generated = (file: GeneratedFile) => + write(file.path, file.bytes, { + holdsIdentity: + file.holdsIdentity === true || + identityLeftIn(file.bytes.toString("utf8"), bundle.identity) !== + undefined, + }); + for (const file of bundle.generated) await generated(file); + for (const copy of bundle.copies) { + const bytes = await retainedBytes(content, copy, signal); + const scrub = + copy.seed === undefined + ? undefined + : scrubSeed(bytes, copy.seed, bundle.identity); + const holdsIdentity = scrub?.holdsIdentity === true; + if (scrub?.bytes === undefined) + await write(copy.path, bytes, { sha256: copy.digest, holdsIdentity }); + else { + scrubbed.push(copy.path); + await write(copy.path, scrub.bytes, { holdsIdentity }); + } + } + await generated(bundle.readme({ scrubbed, unscrubbed: [...unscrubbed] })); + return { scrubbed, unscrubbed }; +}; diff --git a/src/application/desktop/DesktopExportIdentity.test.ts b/src/application/desktop/DesktopExportIdentity.test.ts new file mode 100644 index 0000000000..f1e26513b7 --- /dev/null +++ b/src/application/desktop/DesktopExportIdentity.test.ts @@ -0,0 +1,348 @@ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join, relative } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js"; +import { exportDesktopApplication } from "./DesktopApplicationExport.js"; +import { + DATABASE, + HOME, + NOTE, + SETTINGS, + capture, + change, + checkpoint, + readers, + retained, + writeCaptureFolder, +} from "./desktopCapture.fixture.js"; + +const HOSTNAME = "e81232be457d"; +/** Geany's leak: its mail template took the capture user and the container's host name. */ +const CONFIG = `[ui]\npref_template_mail=ada@${HOSTNAME}\npref_template_developer=ada\nlast=${HOME}/Documents/today.txt\n`; +const DB = `SQLite format 3\0rows from ${HOSTNAME}`; +const DCONF_DB = "GVariant database"; +const DCONF = `[org/notes]\nmail='ada@${HOSTNAME}'\nrecent='${HOME}/x'\n`; +/** A mail setting the user changed from the captured user@host to the value the scrub writes. */ +const MAIL = { + before: `[ui]\nmail=ada@${HOSTNAME}\n`, + after: "[ui]\nmail=user@localhost\n", +}; +const NAMED = `${HOME}/.config/notes/${HOSTNAME}-notes.txt`; +/** A JSON setting keyed by the host name, which the export must not rename. */ +const KEYED = `{"${HOSTNAME}": {"open": true}, "localhost": {"open": false}}`; +const EXTRA_CONTENT = [MAIL.before, MAIL.after, "clean\n", KEYED]; + +/** A capture whose seed names the host and user in a text file, a database and dconf keys. */ +const identityCapture = ( + host: Partial = { + hostname: HOSTNAME, + user: "ada", + }, +): DesktopApplicationCapture => { + const base = capture(); + return { + ...base, + host: { ...base.host, ...host }, + state: { + ...base.state, + checkpoints: [ + checkpoint( + "baseline", + "baseline", + [], + [ + retained(SETTINGS, CONFIG), + retained(DATABASE, DB), + retained(`${HOME}/.config/dconf/user`, DCONF_DB), + ], + ), + ...base.state.checkpoints.slice(1), + ], + }, + }; +}; + +/** Export the capture, with a task holding these checks when any are given, and these dconf keys. */ +const exportIdentity = async ( + local: DesktopApplicationCapture, + checks: readonly unknown[] = [], + dconf = DCONF, +) => { + const scratch = await createTestTempDirectory("desktop-identity-"); + const task = join(scratch, "task.json"); + await writeFile( + task, + JSON.stringify({ name: "mail", prompt: "Set my mail.", checks }), + ); + const directory = join(scratch, "bundle"); + const result = await exportDesktopApplication( + { + capture: await writeCaptureFolder(local, undefined, [ + CONFIG, + DB, + DCONF_DB, + ...EXTRA_CONTENT, + ]), + directory, + name: undefined, + tasks: checks.length === 0 ? [] : [task], + }, + { ...readers, dconf: async () => ({ ok: true as const, text: dconf }) }, + ); + return { result, directory }; +}; + +/** Every file in a bundle by its path, as text. */ +const bundleFiles = async (directory: string) => { + const entries = await readdir(directory, { + recursive: true, + withFileTypes: true, + }); + const files = new Map(); + for (const entry of entries.filter((candidate) => candidate.isFile())) { + const path = join(entry.parentPath, entry.name); + files.set(relative(directory, path), await readFile(path, "latin1")); + } + return files; +}; + +/** The bundle files whose path or content holds the host name, `ada@` or the captured home, sorted. */ +const leaking = (files: ReadonlyMap) => + [...files] + .filter(([path, text]) => + [HOSTNAME, "ada@", HOME].some( + (spelling) => path.includes(spelling) || text.includes(spelling), + ), + ) + .map(([path]) => path) + .sort(); + +describe("exportDesktopApplication of a capture that names its host", () => { + it("ships no host name, user@host or captured home outside the seeds it cannot rewrite, and lists both", async () => { + const { result, directory } = await exportIdentity(identityCapture()); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + scrubbed_files: ["envs/notes/seed/.config/notes/settings.ini"], + unscrubbed_files: ["envs/notes/seed/.local/share/notes/notes.db"], + }); + const files = await bundleFiles(directory); + const seed = "envs/notes/seed/.config/notes/settings.ini"; + expect(files.get(seed)).toBe( + "[ui]\npref_template_mail=user@localhost\npref_template_developer=ada\nlast=/home/user/Documents/today.txt\n", + ); + expect(leaking(files)).toEqual([ + "envs/notes/seed/.local/share/notes/notes.db", + ]); + const module = files.get("envs/notes/notes_app.py") ?? ""; + expect(module).toContain('SEED_HOME = "/home/user"'); + expect(module).toContain("mail='user@localhost'"); + const readme = files.get("README.md") ?? ""; + expect(readme).toContain("`envs/notes/seed/.config/notes/settings.ini`"); + expect(readme).toContain("`envs/notes/seed/.local/share/notes/notes.db`"); + }); + + it("lists every shipped file that still holds the host name, by its name, a JSON key or content, generated ones included", async () => { + const base = identityCapture(); + const [baseline, ...later] = base.state.checkpoints; + const named = { + ...base, + state: { + ...base.state, + checkpoints: [ + { + ...checkpoint("baseline", "baseline", [], []), + retained: [ + ...(baseline?.retained ?? []), + retained(NAMED, "clean\n"), + retained(`${HOME}/.config/notes/hosts.json`, KEYED), + ], + }, + ...later, + ], + }, + }; + + const { result, directory } = await exportIdentity( + named, + [], + `[org/notes]\nmail='ada@${HOSTNAME}.local'\n`, + ); + + if (result instanceof Error) throw result; + const files = await bundleFiles(directory); + const shipped = leaking(files); + expect(shipped).toEqual([ + "README.md", + "envs/notes/notes_app.py", + `envs/notes/seed/.config/notes/${HOSTNAME}-notes.txt`, + "envs/notes/seed/.config/notes/hosts.json", + "envs/notes/seed/.local/share/notes/notes.db", + ]); + expect(files.get("envs/notes/seed/.config/notes/hosts.json")).toBe(KEYED); + expect(result.normalized_result).toMatchObject({ + unscrubbed_files: shipped, + }); + }); + + it("checks a task's seed as the bundle ships it, so a check the scrub satisfies does not count as failing there", async () => { + const base = capture(); + const mailCapture = { + ...identityCapture(), + state: { + ...base.state, + checkpoints: [ + checkpoint( + "baseline", + "baseline", + [], + [retained(SETTINGS, MAIL.before)], + ), + checkpoint( + "edited", + "request", + [ + change(SETTINGS, MAIL, { state: true }), + change(NOTE, { after: "milk\n" }), + ], + [retained(SETTINGS, MAIL.after), retained(NOTE, "milk\n")], + ), + ], + }, + }; + + const { result } = await exportIdentity(mailCapture, [ + { + id: "mail", + kind: "value_equals", + file: ".config/notes/settings.ini", + pointer: "/groups/ui/mail", + value: "user@localhost", + }, + { + id: "note", + kind: "lines_present", + file: "Documents/today.txt", + lines: ["milk"], + }, + ]); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + task: { failing_on_seed: ["note"] }, + }); + }); + + it.each([ + [`pref_template_mail=ada@${HOSTNAME}`, `ada@${HOSTNAME}`], + [HOSTNAME, HOSTNAME], + [`url=http://${HOSTNAME}.local/`, HOSTNAME], + ])("refuses a check whose value names %s", async (line, named) => { + const { result } = await exportIdentity(identityCapture(), [ + { + id: "mail", + kind: "lines_absent", + file: ".config/notes/settings.ini", + lines: [line], + }, + ]); + + expect(result).toMatchObject({ issues: [{ path: ["task"] }] }); + expect((result as { userMessage: string }).userMessage).toContain(named); + }); + + it("still parses and respells the home of a capture written before it recorded the host", async () => { + const { + hostname: _hostname, + user: _user, + ...older + } = identityCapture().host; + + const { result, directory } = await exportIdentity({ + ...identityCapture(), + host: older, + }); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + scrubbed_files: ["envs/notes/seed/.config/notes/settings.ini"], + unscrubbed_files: [], + }); + expect( + await readFile( + join(directory, "envs/notes/seed/.config/notes/settings.ini"), + "utf8", + ), + ).toBe( + `[ui]\npref_template_mail=ada@${HOSTNAME}\npref_template_developer=ada\nlast=/home/user/Documents/today.txt\n`, + ); + }); +}); + +describe("exportDesktopApplication where names, syntax or an unparsed value spell the host", () => { + it.each(["file", "kind", "check"])( + "keeps a task on a host named %s, which only a check's field names, kind and id spell", + async (hostname) => { + const { result } = await exportIdentity( + identityCapture({ hostname, user: "ada" }), + [ + { + kind: "lines_present", + file: "Documents/today.txt", + lines: ["milk"], + }, + ], + ); + + if (result instanceof Error) throw result; + expect(result.normalized_result).toMatchObject({ + task: { failing_on_seed: ["check-1"] }, + }); + }, + ); + + it("scrubs dconf strings by their decoded values, so a host named true keeps every key, group and literal", async () => { + const dconf = String.raw`[org/notes/true] +wrap=true +true=uint32 1 +mail='ada@true' +hosts={'true': <'true'>, 'b': <"it's true">} +blob=b'true' +`; + + const { result, directory } = await exportIdentity( + identityCapture({ hostname: "true", user: "ada" }), + [], + dconf, + ); + + if (result instanceof Error) throw result; + const module = await readFile(join(directory, "envs/notes/notes_app.py")); + expect(module.toString("utf8")).toContain( + `DCONF = ${JSON.stringify(`[org/notes/true]\nwrap=true\ntrue=uint32 1\nmail='user@localhost'\nhosts={'true': <'localhost'>, 'b': <"it's localhost">}\nblob=b'true'\n`)}`, + ); + }); + + it("keeps a dconf value whose string does not parse as captured, and lists the module holding it", async () => { + const { result, directory } = await exportIdentity( + identityCapture(), + [], + `[org/notes]\nmail='ada@${HOSTNAME}\nrecent='${HOME}/x'\n`, + ); + + if (result instanceof Error) throw result; + const module = await readFile(join(directory, "envs/notes/notes_app.py")); + expect(module.toString("utf8")).toContain( + `DCONF = ${JSON.stringify(`[org/notes]\nmail='ada@${HOSTNAME}\nrecent='/home/user/x'\n`)}`, + ); + expect(result.normalized_result).toMatchObject({ + unscrubbed_files: [ + "envs/notes/notes_app.py", + "envs/notes/seed/.local/share/notes/notes.db", + ], + }); + }); +}); diff --git a/src/application/desktop/DesktopSigningKeys.test.ts b/src/application/desktop/DesktopSigningKeys.test.ts index 32825e9aae..c177739e0b 100644 --- a/src/application/desktop/DesktopSigningKeys.test.ts +++ b/src/application/desktop/DesktopSigningKeys.test.ts @@ -5,7 +5,11 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; -import type { DesktopApplicationInventory } from "../../domain/desktop/desktopCapture.js"; +import { exportInstall } from "../../domain/desktop/appExportInstall.js"; +import type { + AptRepository, + DesktopApplicationInventory, +} from "../../domain/desktop/desktopCapture.js"; import { DesktopContentStore } from "../../desktop/DesktopContentStore.js"; import { capture } from "./desktopCapture.fixture.js"; import { retainSigningKeys } from "./DesktopSigningKeys.js"; @@ -98,3 +102,74 @@ describe("retainSigningKeys", () => { }, ); }); + +describe("retainSigningKeys before an export", () => { + it("leaves out the program's package when only the repository whose key was dropped supplied it", async () => { + const large = Buffer.alloc(64, 1); + const small = Buffer.alloc(16, 2); + const digest = (bytes: Buffer) => + createHash("sha256").update(bytes).digest("hex"); + const signed = ( + host: string, + bytes: Buffer, + supplies: readonly string[], + ): AptRepository => ({ + source: `https://${host}/linux ./ Packages`, + sources_file: `/etc/apt/sources.list.d/${host}.list`, + uri: `https://${host}/linux`, + suite: "./", + components: [], + architectures: [], + release: "stable", + signing_key: { + path: `/etc/apt/trusted.gpg.d/${host}.gpg`, + sha256: digest(bytes), + format: "binary", + fingerprint: host, + }, + supplies: [...supplies], + }); + const { application } = capture(); + const inventory: DesktopApplicationInventory = { + ...application, + package: { + kind: "deb", + name: "typora", + executable_package: "typora-bin", + version: "1.14.9-1", + architecture: "arm64", + sources: ["https://a.example/linux ./ Packages"], + source_origins: ["typora.io"], + package_lists_present: true, + apt_repositories: [ + signed("a.example", large, ["typora-bin"]), + signed("b.example", small, []), + ], + library_packages: [], + }, + }; + const { content } = await store(1024, 32); + + const retained = await retainSigningKeys( + inventory, + new Map([ + [digest(large), large], + [digest(small), small], + ]), + content, + ); + + expect(retained.package.apt_repositories?.[0]).toMatchObject({ + signing_key: null, + supplies: ["typora-bin"], + }); + expect(exportInstall(retained.package, "typora", "arm64")).toMatchObject({ + ok: false, + error: { + reason: expect.stringContaining( + "typora-bin, which the computer cannot install: only apt repositories the export leaves out offer it", + ), + }, + }); + }); +}); diff --git a/src/application/desktop/DesktopStateVersions.ts b/src/application/desktop/DesktopStateVersions.ts index 649d906cfd..5db5854209 100644 --- a/src/application/desktop/DesktopStateVersions.ts +++ b/src/application/desktop/DesktopStateVersions.ts @@ -1,13 +1,73 @@ +import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import type { DesktopApplicationCapture, DesktopStateChange, } from "../../domain/desktop/desktopCapture.js"; -import type { DesktopStatePath } from "../../domain/desktop/desktopState.js"; +import type { + DesktopStatePath, + StateEnds, +} from "../../domain/desktop/desktopState.js"; import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js"; import { SQLITE_SIDECARS } from "../../domain/desktop/stateFormats.js"; type Version = DesktopStatePath["versions"][number]; +/** The checkpoints a caller named for the seed and the end; unnamed ones default. */ +export interface CheckpointSelection { + readonly seed?: string | undefined; + readonly end?: string | undefined; +} + +/** + * The seed and end among a capture's checkpoint names, by position: the + * baseline and the last checkpoint unless named. Refuses a name the capture + * does not hold exactly once, and an end before the seed. + */ +export const selectStateEnds = ( + checkpoints: readonly string[], + selection: CheckpointSelection, + operation: string, +): StateEnds => { + const refuse = (field: string, userMessage: string) => + new AnalysisInputError( + operation, + { + userMessage, + remediationAction: + "Name a checkpoint inspect-desktop-application-state lists, with the end at or after the seed.", + conformsToInputContract: true, + }, + [{ path: [field], reason: "invalid_value", message: userMessage }], + ); + const position = ( + field: string, + name: string | undefined, + fallback: number, + ) => { + if (name === undefined) return fallback; + const found = checkpoints.flatMap((candidate, index) => + candidate === name ? [index] : [], + ); + if (found.length === 1) return found[0] ?? fallback; + const count = + found.length === 0 + ? "no checkpoint" + : `${String(found.length)} checkpoints`; + throw refuse( + field, + `The capture has ${count} named ${JSON.stringify(name)}; its checkpoints are ${checkpoints.join(", ")}.`, + ); + }; + const seed = position("seed_checkpoint", selection.seed, 0); + const end = position("end_checkpoint", selection.end, checkpoints.length - 1); + if (end < seed) + throw refuse( + "end_checkpoint", + `The end checkpoint ${checkpoints[end] ?? ""} was taken before the seed checkpoint ${checkpoints[seed] ?? ""}.`, + ); + return { seed, end }; +}; + /** The changes of each path in checkpoint order. */ type ChangeIndex = ReadonlyMap< string, diff --git a/src/application/desktop/DesktopTaskDraft.ts b/src/application/desktop/DesktopTaskDraft.ts index 1d50a56d62..349b4cd258 100644 --- a/src/application/desktop/DesktopTaskDraft.ts +++ b/src/application/desktop/DesktopTaskDraft.ts @@ -16,11 +16,15 @@ import { desktopOperationFailure, } from "./DesktopCaptureRefusals.js"; import { exportEnvironmentName } from "./DesktopExportInstall.js"; +import { + selectStateEnds, + type CheckpointSelection, +} from "./DesktopStateVersions.js"; const OPERATION = "describe_desktop_application_task"; const DRAFT_LIMITATIONS = [ - "Files are those under the paths the export owns, whose state differs between the capture's first and last version; data/get serves nothing else.", + "Files are those under the paths the export owns, whose state differs between the seed and end checkpoints, by default the capture's first and last; data/get serves nothing else.", "Suggested checks and views cover documents, or the app's own state when no document changed; each check is derived from the net change and is a starting point, not a task.", "Views longer than 200 items or strings longer than 2,000 characters are cut here (view_cut); inspect-desktop-application-state shows every view whole.", ]; @@ -29,6 +33,7 @@ const draft = async ( folder: string, readers: DesktopStateReaders, signal: AbortSignal | undefined, + selection: CheckpointSelection, ): Promise => { const { evidence, capture } = await readDesktopCapture(folder, signal); const stateEvidence = await inspectDesktopApplicationState( @@ -40,10 +45,15 @@ const draft = async ( const state = desktopApplicationStateSchema.parse( stateEvidence.normalized_result, ); + const { checkpoints } = state.capture; + const ends = selectStateEnds(checkpoints, selection, OPERATION); const { files, paths } = exportedStateFiles(capture, state); - const drafted = draftTaskFiles(files, (home) => isWithinAny(home, paths)); + const drafted = draftTaskFiles( + files, + (home) => isWithinAny(home, paths), + ends, + ); const environment = exportEnvironmentName(capture, undefined); - const checkpoints = state.capture.checkpoints; const result = { application: { environment, @@ -51,7 +61,10 @@ const draft = async ( window_class: capture.session.window_match?.class ?? null, }, owned_paths: [...paths], - checkpoints: { baseline: checkpoints[0], final: checkpoints.at(-1) }, + checkpoints: { + baseline: checkpoints[ends.seed], + final: checkpoints[ends.end], + }, files: drafted.map(({ file, role, baseline, final, change }) => ({ file, role, @@ -74,7 +87,11 @@ const draft = async ( return createEvidence(undefined, DESKTOP_PROVIDER, { predicateType: "rea.desktop-application-task-draft", operation: OPERATION, - parameters: { capture_evidence_id: evidence.evidence_id }, + parameters: { + capture_evidence_id: evidence.evidence_id, + seed_checkpoint: selection.seed ?? null, + end_checkpoint: selection.end ?? null, + }, result, confidence: "inferred", authority: "analyst-inference", @@ -82,14 +99,18 @@ const draft = async ( }); }; -/** Describe how a capture's files changed, with a task draft an agent completes and passes to the export. */ +/** + * Describe how a capture's files changed between the seed and end + * checkpoints, with a task draft an agent completes and passes to the export. + */ export const draftDesktopApplicationTask = async ( folder: string, readers: DesktopStateReaders, signal?: AbortSignal, + selection: CheckpointSelection = {}, ): Promise => { try { - return await draft(folder, readers, signal); + return await draft(folder, readers, signal, selection); } catch (cause: unknown) { return desktopOperationFailure( cause, diff --git a/src/application/desktop/DesktopTaskExport.ts b/src/application/desktop/DesktopTaskExport.ts index a3657f461d..fb4b9c4e9d 100644 --- a/src/application/desktop/DesktopTaskExport.ts +++ b/src/application/desktop/DesktopTaskExport.ts @@ -6,7 +6,6 @@ import { } from "../../domain/desktop/appTask.js"; import { evaluateTaskChecks, - namesXmlNamespace, versionView, type TaskFileView, } from "../../domain/desktop/appTaskChecks.js"; @@ -17,8 +16,17 @@ import { taskVerifierArtifact, } from "../../domain/desktop/appTaskPython.js"; import type { DesktopAppExportPlan } from "../../domain/desktop/appExport.js"; -import type { DesktopStatePath } from "../../domain/desktop/desktopState.js"; +import { + versionAt, + type DesktopStatePath, + type StateEnds, +} from "../../domain/desktop/desktopState.js"; +import { + identityLeftIn, + type HostIdentity, +} from "../../domain/desktop/hostIdentity.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; import { EvidenceFileError } from "../../domain/evidenceErrors.js"; @@ -33,12 +41,16 @@ import { refuse } from "./DesktopExportInstall.js"; const OPERATION = "export_desktop_application"; const REWRITE = "Fix the task file and export again."; -/** What the export knows of the app: its name, owned paths, captured home and state files by home path. */ +/** What the export knows of the app: its name, owned paths, captured home, host identity, state files by home path and the checkpoints it reads. */ type TaskTarget = Pick< DesktopAppExportPlan, "name" | "paths" | "capturedHome" > & { readonly files: ReadonlyMap; + readonly identity: HostIdentity; + readonly ends: StateEnds; + /** Views of the seeds the export rewrites, as it ships them. */ + readonly seedViews: ReadonlyMap; }; /** A task the capture validated: the files it adds to the bundle, its README section and its summary. */ @@ -101,44 +113,63 @@ const readTaskFile = async ( throw primary ?? typed(loaded.error); }; +/** The task's views of every file at its seed, as the bundle ships it, and at its end. */ +interface TaskViews { + readonly seed: ReadonlyMap; + readonly end: ReadonlyMap; +} + +/** Every string in a value, member names included. */ +const valueStrings = (value: JsonValue | undefined): string[] => { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(valueStrings); + if (typeof value !== "object" || value === null) return []; + return Object.entries(value).flatMap(([name, member]) => [ + name, + ...valueStrings(member), + ]); +}; + /** Why a check cannot be validated against this app, or undefined. */ const targetProblem = ( check: DesktopTaskCheck, target: TaskTarget, + views: TaskViews, ): string | undefined => { if (!isWithinAny(check.file, target.paths)) return `names ${check.file}, which is not under the app's owned paths (${target.paths.join(", ") || "none"}); data/get serves only those`; - if (JSON.stringify(check).includes(target.capturedHome)) - return `names the captured home ${target.capturedHome}, which the computer's home replaces`; + // What the check compares and names; its field names, kind and id label it. + const { id: _id, kind: _kind, ...fields } = check; + for (const value of Object.values(fields).flatMap(valueStrings)) { + if (value.includes(target.capturedHome)) + return `names the captured home ${target.capturedHome}, which the computer's home replaces`; + const named = identityLeftIn(value, target.identity); + if (named !== undefined) + return `names ${named}, the capture host's name or user, which the export removes from the seed`; + } if (check.kind !== "value_equals" && check.kind !== "value_absent") return undefined; - const versions = target.files.get(check.file)?.versions ?? []; - if (versions.some((version) => version.format === "text")) - return `points into ${check.file}, which is text; use lines_present, lines_absent or line_count`; - return versions.some( - (version) => version.format === "xml" && namesXmlNamespace(version.view), + return [views.seed, views.end].some( + (at) => at.get(check.file)?.format === "text", ) - ? `points into ${check.file}, whose XML uses namespaces the verifier names differently` + ? `points into ${check.file}, which is text; use lines_present, lines_absent or line_count` : undefined; }; -/** Views of every file at one end of the capture: its first or its last version. */ +/** Views of every file at the end checkpoint, or at the seed as the bundle ships it. */ const viewsAt = ( - files: ReadonlyMap, - end: "baseline" | "final", + target: TaskTarget, + end: keyof StateEnds, ): ReadonlyMap => { const views = new Map(); - for (const [home, path] of files) { - const version = - end === "baseline" ? path.versions[0] : path.versions.at(-1); - // A file not retained at the baseline is not seeded, so the computer starts without it. - if ( - end === "baseline" && - version?.present === true && - version.sha256 === null - ) + for (const [home, path] of target.files) { + const version = versionAt(path, target.ends[end]); + // A file not retained at the seed is not seeded, so the computer starts without it. + if (end === "seed" && version?.present === true && version.sha256 === null) continue; - const view = version === undefined ? undefined : versionView(version); + const shipped = end === "seed" ? target.seedViews.get(home) : undefined; + const view = + shipped ?? (version === undefined ? undefined : versionView(version)); if (view !== undefined) views.set(home, view); } return views; @@ -198,8 +229,12 @@ export const admitDesktopTask = async ( REWRITE, ); const task = parsed.value; + const views: TaskViews = { + seed: viewsAt(target, "seed"), + end: viewsAt(target, "end"), + }; for (const check of task.checks) { - const problem = targetProblem(check, target); + const problem = targetProblem(check, target, views); if (problem !== undefined) throw refuse( `Check ${JSON.stringify(check.id)} ${problem}.`, @@ -207,7 +242,7 @@ export const admitDesktopTask = async ( REWRITE, ); } - const end = evaluateTaskChecks(task.checks, viewsAt(target.files, "final")); + const end = evaluateTaskChecks(task.checks, views.end); const failing = end.find((outcome) => !outcome.passed); if (failing !== undefined) throw refuse( @@ -215,10 +250,7 @@ export const admitDesktopTask = async ( "task", "Write checks that the captured end state passes; describe-desktop-application-task shows its views.", ); - const failingOnSeed = evaluateTaskChecks( - task.checks, - viewsAt(target.files, "baseline"), - ) + const failingOnSeed = evaluateTaskChecks(task.checks, views.seed) .filter((outcome) => !outcome.passed) .map((outcome) => outcome.id); if (failingOnSeed.length === 0) diff --git a/src/application/desktop/DesktopZipView.ts b/src/application/desktop/DesktopZipView.ts new file mode 100644 index 0000000000..d83bb70bc1 --- /dev/null +++ b/src/application/desktop/DesktopZipView.ts @@ -0,0 +1,131 @@ +import { + Reader, + Uint8ArrayWriter, + ZipReader, + type FileEntry, +} from "@zip.js/zip.js"; + +import { + decodeZipEntryText, + ZIP_READER_OPTIONS, + zipEntryPath, +} from "../../artifacts/ZipArtifactReader.js"; +import { + DAMAGED_ZIP_ENTRY, + MAX_ZIP_ENTRIES, + UNREADABLE_ZIP, + unreadZipEntry, + zipEntry, + zipEntryRefusal, + zipView, + type ZipEntryHeader, +} from "../../domain/desktop/zipView.js"; +import { + boundedDirectoryTail, + scanZipDirectory, +} from "../../domain/desktop/zipDirectory.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +/** + * An archive as zip.js reads it: its entries in place, then a directory of the + * kept records alone, so zip.js parses no more records than the view lists. + * Reads are copies, as zip.js's own byte array reader returns. + */ +class BoundedArchiveReader extends Reader { + readonly #entries: Uint8Array; + readonly #tail: Uint8Array; + + constructor(entries: Uint8Array, tail: Uint8Array) { + super(entries); + this.#entries = entries; + this.#tail = tail; + this.size = entries.length + tail.length; + } + + override readUint8Array(index: number, length: number): Promise { + const end = Math.min(index + length, this.size); + const split = this.#entries.length; + const read = new Uint8Array(Math.max(0, end - index)); + if (index < split) + read.set(this.#entries.subarray(index, Math.min(end, split))); + if (end > split) + read.set( + this.#tail.subarray(Math.max(0, index - split), end - split), + Math.max(0, split - index), + ); + return Promise.resolve(read); + } +} + +/** An entry's view, its bytes checked against its CRC-32 and declared size, which bounds them. */ +const readEntry = async ( + entry: FileEntry, + header: ZipEntryHeader, + signal: AbortSignal | undefined, +): Promise => { + let bytes: Uint8Array; + try { + bytes = await entry.getData(new Uint8ArrayWriter(), { + checkSignature: true, + ...(signal === undefined ? {} : { signal }), + }); + } catch { + signal?.throwIfAborted(); + return unreadZipEntry(header, DAMAGED_ZIP_ENTRY); + } + return zipEntry(header, bytes); +}; + +/** + * A zip archive's view, read from the digest-checked bytes in memory rather + * than reopened from disk: file entries in archive order up to the listing + * limit, each read by the state view's own readers within the entry and + * archive budgets. Nothing is extracted and no archive inside is opened. + */ +export const readZipView = async ( + bytes: Uint8Array, + signal?: AbortSignal, +): Promise> => { + // The directory is scanned first, so zip.js sees only the records the view lists. + const directory = scanZipDirectory(bytes, MAX_ZIP_ENTRIES); + if (directory === undefined) return err(UNREADABLE_ZIP); + const reader = new ZipReader( + new BoundedArchiveReader( + bytes.subarray(0, directory.start), + boundedDirectoryTail(directory), + ), + ZIP_READER_OPTIONS, + ); + const entries: JsonValue[] = []; + let spent = 0; + try { + for await (const entry of reader.getEntriesGenerator({ + decodeText: decodeZipEntryText, + })) { + signal?.throwIfAborted(); + if (entry.directory) continue; + const name = zipEntryPath(entry); + const header: ZipEntryHeader = { + name, + size: entry.uncompressedSize, + compressedSize: entry.compressedSize, + method: entry.compressionMethod, + encrypted: entry.encrypted, + }; + const refusal = zipEntryRefusal(header, spent); + if (refusal === undefined) spent += header.size; + entries.push( + refusal === undefined + ? await readEntry(entry, header, signal) + : unreadZipEntry(header, refusal), + ); + } + } catch { + signal?.throwIfAborted(); + return err(UNREADABLE_ZIP); + } finally { + await reader.close(); + } + return ok(zipView(entries, directory.count)); +}; diff --git a/src/application/desktop/desktopCapture.fixture.ts b/src/application/desktop/desktopCapture.fixture.ts index 87716917a8..0edada60ac 100644 --- a/src/application/desktop/desktopCapture.fixture.ts +++ b/src/application/desktop/desktopCapture.fixture.ts @@ -15,9 +15,11 @@ export { change, checkpoint, retained }; /** A synthetic capture of a notes app: a settings key file, a document and a SQLite database. */ export const HOME = "/home/ada"; -const sha256 = (text: string) => +/** A retained file's content: text, or bytes such as a zip archive's. */ +type Content = string | Buffer; +const sha256 = (text: Content) => createHash("sha256").update(text).digest("hex"); -const facts = (text: string) => ({ +const facts = (text: Content) => ({ type: "file" as const, size: text.length, mode: 0o644, @@ -26,7 +28,7 @@ const facts = (text: string) => ({ }); const change = ( path: string, - versions: { readonly before?: string; readonly after: string }, + versions: { readonly before?: Content; readonly after: Content }, marks: { readonly state?: boolean } = {}, ) => ({ path, @@ -39,7 +41,7 @@ const change = ( in_state_directory: marks.state === true, opened_by_application: false, }); -const retained = (path: string, text: string) => ({ +const retained = (path: string, text: Content) => ({ path, sha256: sha256(text), size: text.length, @@ -165,10 +167,57 @@ export const capture = (): DesktopApplicationCapture => ({ limitations: [], }); +/** The note as saved before checkpoint `seed`, then as edited and saved before `end`. */ +export const SAVED_NOTE = "milk\n"; +export const EDITED_NOTE = "milk\neggs\n"; + +/** + * The app launches, the user saves a new note and takes `seed`, edits and + * saves it and takes `end`, then stops; the app writes its settings as it + * exits. Write it with `EDITED_NOTE` as extra content. + */ +export const seedEndCapture = (): DesktopApplicationCapture => { + const base = capture(); + const wrap = { before: "[ui]\nwrap=false\n", after: "[ui]\nwrap=true\n" }; + return { + ...base, + state: { + ...base.state, + checkpoints: [ + checkpoint( + "baseline", + "baseline", + [], + [retained(SETTINGS, wrap.before)], + ), + checkpoint( + "seed", + "request", + [change(NOTE, { after: SAVED_NOTE })], + [retained(NOTE, SAVED_NOTE)], + ), + checkpoint( + "end", + "request", + [change(NOTE, { before: SAVED_NOTE, after: EDITED_NOTE })], + [retained(NOTE, EDITED_NOTE)], + ), + { ...checkpoint("stop", "request", [], []), trigger: "stop" }, + checkpoint( + "exit", + "exit", + [change(SETTINGS, wrap, { state: true })], + [retained(SETTINGS, wrap.after)], + ), + ], + }, + }; +}; + export const writeCaptureFolder = async ( result: DesktopApplicationCapture, operation = DESKTOP_OPERATION, - extraContent: readonly string[] = [], + extraContent: readonly Content[] = [], ) => { const folder = await createTestTempDirectory("desktop-state-"); await mkdir(join(folder, "content")); diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index 152c22a116..e1b4e2c2ca 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -28,6 +28,7 @@ import { type DecompressionStreamOptions, type Entry, type FileEntry, + type ZipReaderConstructorOptions, } from "@zip.js/zip.js"; import { @@ -78,6 +79,18 @@ registerCodec({ DecompressionStream: ZipZstandardDecompressionStream, }); +/** zip.js reader options: CRC and overlap checks, and names decoded as REA reads them. */ +export const ZIP_READER_OPTIONS = { + checkSignature: true, + checkOverlappingEntry: true, + // Unflagged names use the ZIP legacy encoding (CP437); the UTF-8 flag + // and validated Unicode Path extra fields remain authoritative. + filenameEncoding: "cp437", + // REA applies its own provider-neutral path validation in scanReader and + // must preserve that typed failure instead of zip.js rejecting first. + filenameValidation: "tolerant", +} as const satisfies ZipReaderConstructorOptions; + class NodeFileReader extends Reader { #handle: FileHandle | undefined; readonly #source: ArtifactFileSource; @@ -169,16 +182,7 @@ export class ZipArtifactReader implements ArtifactReader { ) { this.format = format; this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted); - this.#reader = new ZipReader(this.#source, { - checkSignature: true, - checkOverlappingEntry: true, - // Unflagged names use the ZIP legacy encoding (CP437); the UTF-8 flag - // and validated Unicode Path extra fields remain authoritative. - filenameEncoding: "cp437", - // REA applies its own provider-neutral path validation in scanReader and - // must preserve that typed failure instead of zip.js rejecting first. - filenameValidation: "tolerant", - }); + this.#reader = new ZipReader(this.#source, ZIP_READER_OPTIONS); } async *entries(signal?: AbortSignal): AsyncIterable { @@ -368,7 +372,8 @@ const legacyNameLimitations = (entry: Entry): readonly string[] => { ]; }; -const decodeZipEntryText = ( +/** Decode an entry name as zip.js reports it: strict UTF-8 when flagged, CP437 otherwise. */ +export const decodeZipEntryText = ( bytes: Uint8Array, encoding: string, type: "filename" | "comment", @@ -396,7 +401,8 @@ const decodeCp437 = (bytes: Uint8Array): string => : CP437_HIGH_HALF.charAt(byte - 0x80), ).join(""); -const zipEntryPath = (entry: Entry): string => { +/** An entry's name, from a valid Unicode Path extra field when it has one. */ +export const zipEntryPath = (entry: Entry): string => { const field = entry.extraFieldUnicodePath; if (field?.valid !== true) return entry.filename; return decodeStrictZipUtf8( diff --git a/src/cli/desktopCaptureCommands.ts b/src/cli/desktopCaptureCommands.ts index 9addce69ae..cc84695f9e 100644 --- a/src/cli/desktopCaptureCommands.ts +++ b/src/cli/desktopCaptureCommands.ts @@ -74,6 +74,22 @@ const captureOptions = z.object({ ), }); +/** The checkpoints a bundle's seed and a task's end state are read at. */ +const checkpointOptions = { + seedCheckpoint: z + .string() + .optional() + .describe( + "Checkpoint whose state seeds the bundle; the baseline by default", + ), + endCheckpoint: z + .string() + .optional() + .describe( + "Checkpoint whose state a task must pass on; the capture's last by default", + ), +}; + /** * Run a desktop operation with SQLite and dconf readers, finishing through * the shared command owner so the result, cancellation and the SQLite @@ -147,6 +163,7 @@ const registerDesktopExportCommand = ( .describe( "http or https URL the computer downloads the captured --package-file from when the bundle is put, for an application installed from a .deb file", ), + ...checkpointOptions, }), run: ({ args, options }) => withCommandCancellation((signal) => @@ -170,6 +187,8 @@ const registerDesktopExportCommand = ( absoluteCallerPath(path), ), packageUrl: options.packageUrl, + seedCheckpoint: options.seedCheckpoint, + endCheckpoint: options.endCheckpoint, }, readers, signal, @@ -189,13 +208,14 @@ const registerDesktopTaskCommand = ( ): void => { cli.command(CLI_COMMANDS.describeDesktopApplicationTask, { description: - "Describe how a desktop application capture's files changed from the baseline to the end, with suggested checks and a task draft to complete for export-desktop-application --task", + "Describe how a desktop application capture's files changed from the seed checkpoint to the end, with suggested checks and a task draft to complete for export-desktop-application --task", args: z.object({ capture: z .string() .describe("Folder written by capture-desktop-application"), }), - run: ({ args }) => + options: z.object(checkpointOptions), + run: ({ args, options }) => withCommandCancellation((signal) => logCliCommand( logger, @@ -212,6 +232,7 @@ const registerDesktopTaskCommand = ( absoluteCallerPath(args.capture), readers, signal, + { seed: options.seedCheckpoint, end: options.endCheckpoint }, ), ); }, diff --git a/src/desktop/ApplicationStateDirectories.ts b/src/desktop/ApplicationStateDirectories.ts new file mode 100644 index 0000000000..8c81e887ed --- /dev/null +++ b/src/desktop/ApplicationStateDirectories.ts @@ -0,0 +1,106 @@ +import { lstat, readdir } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import type { DesktopApplicationInventory } from "../domain/desktop/desktopCapture.js"; + +/** The environment whose XDG folders, and the home whose dot-folders, may hold the application's state. */ +interface StateLocations { + readonly environment: Readonly>; + readonly home: string; +} + +/** + * Names the application's state folders may carry, including the entry's + * full reverse-DNS id and its last part. The executable is null for a + * `flatpak run` launch, whose `flatpak` launcher names flatpak's own folders. + */ +export const stateNames = ( + package_: DesktopApplicationInventory["package"], + executable: string | null, + entry: DesktopApplicationInventory["desktop_entry"], +): readonly (string | null)[] => [ + package_.kind === "deb" + ? (package_.name?.split(":")[0] ?? null) + : package_.name, + executable === null ? null : basename(executable), + entry?.id ?? null, + entry?.id?.split(".").at(-1) ?? null, + entry?.startup_wm_class ?? null, + entry?.name ?? null, +]; + +/** Whether a home entry is a dotfile named after the application, such as `.varicad-cfg.ini` or `.notesrc`. */ +const appNamedDotfile = (entry: string, names: ReadonlySet): boolean => + [...names].some((name) => { + // Short names would match unrelated dotfiles. + if (name.length < 3 || !entry.startsWith(`.${name}`)) return false; + const rest = entry.slice(name.length + 1); + return rest === "rc" || /^[-._]./u.test(rest); + }); + +/** A folder's entries that are the prefix and then one of the lowercase names in any letter case, such as `GIMP` for `gimp`. */ +const recasedEntries = async ( + folder: string, + prefix: string, + names: ReadonlySet, +): Promise => + (await readdir(folder).catch(() => [] as string[])) + .filter( + (entry) => + entry.startsWith(prefix) && + names.has(entry.slice(prefix.length).toLowerCase()), + ) + .map((entry) => join(folder, entry)); + +/** + * Folder names under XDG and dot-directories where the application may keep + * state, and the dotfiles in the home named after it, which an application + * may only read during a capture. + */ +export const stateDirectories = async ( + names: readonly (string | null)[], + { environment, home }: StateLocations, + extra: readonly string[], +): Promise => { + const bases = [ + environment.XDG_CONFIG_HOME || join(home, ".config"), + environment.XDG_DATA_HOME || join(home, ".local/share"), + environment.XDG_STATE_HOME || join(home, ".local/state"), + environment.XDG_CACHE_HOME || join(home, ".cache"), + ]; + const variants = new Set( + names + .filter( + (name): name is string => + name !== null && + /^[^/\0]+$/u.test(name) && + name !== "." && + name !== "..", + ) + .flatMap((name) => [name, name.toLowerCase()]), + ); + const lowered = new Set([...variants].map((name) => name.toLowerCase())); + const dotfiles = (await readdir(home).catch(() => [] as string[])) + .filter((entry) => appNamedDotfile(entry, variants)) + .map((entry) => join(home, entry)); + // A folder present in another letter case than any name, such as GIMP's `~/.config/GIMP`, is the application's too. + const recased = await Promise.all([ + ...bases.map((base) => recasedEntries(base, "", lowered)), + recasedEntries(home, ".", lowered), + ]); + const paths = new Set([ + ...[...variants].flatMap((name) => [ + ...bases.map((base) => join(base, name)), + join(home, `.${name}`), + ]), + ...recased.flat(), + ...dotfiles, + ...extra, + ]); + return Promise.all( + [...paths].map(async (path) => ({ + path, + exists_at_start: (await lstat(path).catch(() => undefined)) !== undefined, + })), + ); +}; diff --git a/src/desktop/AptRepositoryInventory.test.ts b/src/desktop/AptRepositoryInventory.test.ts new file mode 100644 index 0000000000..9d11b52837 --- /dev/null +++ b/src/desktop/AptRepositoryInventory.test.ts @@ -0,0 +1,42 @@ +import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { + APT_LOCATIONS, + resolveAptLocations, +} from "./AptRepositoryInventory.js"; +import { runHostQuery, type QueryRunner } from "./HostQuery.js"; + +describe.skipIf( + process.platform !== "linux" || !existsSync("/usr/bin/apt-config"), +)("resolveAptLocations with apt-config", () => { + it("reads the lists folder an APT_CONFIG file names, resolved under Dir::State", async () => { + const root = await createTestTempDirectory("apt-config-"); + const config = join(root, "apt.conf"); + await writeFile( + config, + `Dir::State "${root}/state";\nDir::State::Lists "user lists";\n`, + ); + const configured: QueryRunner = (command, argv, signal) => + runHostQuery("env", [`APT_CONFIG=${config}`, command, ...argv], signal); + + const locations = await resolveAptLocations(configured, undefined); + + expect(locations).toEqual({ + lists: `${root}/state/user lists/`, + trustedParts: "/etc/apt/trusted.gpg.d/", + trusted: "/etc/apt/trusted.gpg", + }); + expect( + await resolveAptLocations( + (_command, argv, signal) => + runHostQuery("/missing/apt-config", argv, signal), + undefined, + ), + ).toEqual(APT_LOCATIONS); + }); +}); diff --git a/src/desktop/AptRepositoryInventory.ts b/src/desktop/AptRepositoryInventory.ts index 7b782f16dd..efbdaf3715 100644 --- a/src/desktop/AptRepositoryInventory.ts +++ b/src/desktop/AptRepositoryInventory.ts @@ -5,6 +5,7 @@ import { aptListFileName, entryForTarget, isUbuntuProRepository, + parseAptConfigShell, parseAptIndexTargets, parseDeb822Sources, parseOneLineSources, @@ -30,6 +31,44 @@ export const APT_LOCATIONS: AptLocations = { lists: "/var/lib/apt/lists", }; +/** + * apt's folders as `apt-config` resolves them under `Dir`, which an + * `APT_CONFIG` file can move, as a user-space `apt-get update` does with its + * lists. Each is the fallback's when apt-config is missing, fails or does not + * name it. + */ +export const resolveAptLocations = async ( + runQuery: QueryRunner, + signal: AbortSignal | undefined, + fallback: AptLocations = APT_LOCATIONS, +): Promise => { + const output = succeeded( + await runQuery( + "apt-config", + [ + "shell", + "LISTS", + "Dir::State::Lists/d", + "PARTS", + "Dir::Etc::TrustedParts/d", + "TRUSTED", + "Dir::Etc::Trusted/f", + ], + signal, + ), + ); + const values = parseAptConfigShell(output ?? ""); + const value = (name: string) => { + const found = values.get(name); + return found?.startsWith("/") === true ? found : undefined; + }; + return { + lists: value("LISTS") ?? fallback.lists, + trustedParts: value("PARTS") ?? fallback.trustedParts, + trusted: value("TRUSTED") ?? fallback.trusted, + }; +}; + /** The host commands and apt folders the repository check reads. */ export interface AptRepositoryHost { readonly runQuery: QueryRunner; diff --git a/src/desktop/DebianPackageInventory.test.ts b/src/desktop/DebianPackageInventory.test.ts index 4f9b0a089a..932ae3d6a7 100644 --- a/src/desktop/DebianPackageInventory.test.ts +++ b/src/desktop/DebianPackageInventory.test.ts @@ -5,6 +5,9 @@ import { mkdir, readFile, realpath, + rename, + rm, + symlink, writeFile, } from "node:fs/promises"; import { join } from "node:path"; @@ -553,6 +556,8 @@ describe.skipIf(process.platform === "win32")("library packages", () => { signing_key: expect.objectContaining({ fingerprint: VENDOR_A.fingerprint, }), + // libnss3 comes from the archive, which does not depend on the repository. + supplies: ["libnspr4"], }, ], library_packages: ["libnspr4", "libnss3"], @@ -610,3 +615,301 @@ describe.skipIf(process.platform === "win32")( }); }, ); + +const WRITER_VERSION = "4:24.2.7-0ubuntu0.24.04.4"; + +/** LibreOffice Writer from the Ubuntu archive: libreoffice-writer owns the entry, libreoffice-common the `libreoffice` it runs. */ +const writerFixture = async () => { + const root = await createTestTempDirectory("desktop-entry-package-"); + for (const folder of [ + "home/.local/share/applications", + "bin", + "share/applications", + "lists", + ]) + await mkdir(join(root, folder), { recursive: true }); + await writeFile(join(root, "bin/libreoffice"), "#!/bin/sh\nexit 0\n"); + await chmod(join(root, "bin/libreoffice"), 0o755); + const entry = + "[Desktop Entry]\nType=Application\nName=LibreOffice Writer\nExec=libreoffice --writer %U\n"; + await writeFile( + join(root, "share/applications/libreoffice-writer.desktop"), + entry, + ); + // A copy the user made, which no package owns. + await writeFile( + join(root, "home/.local/share/applications/my-writer.desktop"), + entry, + ); + await writeFile(join(root, "lists/noble_main_binary-arm64_Packages"), ""); + const runQuery: QueryRunner = async (command, argv) => { + const reply = (stdout: string, exitCode = 0): QueryOutcome => ({ + stdout, + exitCode, + }); + if (command === "apt-cache") + return reply( + argv.length === 1 + ? `Package files:\n 500 ${NOBLE}\n release v=24.04,o=Ubuntu,a=noble,n=noble,l=Ubuntu,c=main,b=arm64\nPinned packages:\n` + : `${argv[1]}:\n Version table:\n *** ${WRITER_VERSION} 500\n 500 ${NOBLE}\n 100 /var/lib/dpkg/status\n`, + ); + if (command === "dpkg-deb") + return reply( + `Package: libreoffice-writer\nVersion: ${WRITER_VERSION}\nArchitecture: arm64\nDepends: libreoffice-core\n`, + ); + if (command !== "dpkg-query") return "missing"; + if (argv[0] === "-W") + return reply(`${argv.at(-1)}\t${WRITER_VERSION}\tarm64\tinstalled\n`); + const path = argv[1] ?? ""; + const owner = path.endsWith("/bin/libreoffice") + ? "libreoffice-common" + : path.endsWith("/share/applications/libreoffice-writer.desktop") + ? "libreoffice-writer" + : undefined; + return owner === undefined ? reply("", 1) : reply(`${owner}: ${path}`); + }; + const inspect = async ( + launch: { desktopEntry: string } | { executable: string }, + options: { packageFile?: string; run?: QueryRunner } = {}, + ) => { + const result = await inspectLinuxApplication( + { + desktopEntry: undefined, + executable: undefined, + ...launch, + arguments: [], + environment: { + PATH: join(root, "bin"), + XDG_DATA_HOME: join(root, "home/.local/share"), + XDG_DATA_DIRS: join(root, "share"), + }, + home: join(root, "home"), + callerDirectory: join(root, "home"), + packageFile: options.packageFile, + }, + undefined, + { + runQuery: options.run ?? runQuery, + apt: { + trustedParts: join(root, "apt/trusted.gpg.d"), + trusted: join(root, "apt/trusted.gpg"), + lists: join(root, "lists"), + }, + }, + ); + if (!result.ok) throw new Error(result.error.reason); + return { + installed: result.value.inventory.package, + limitations: result.value.limitations, + }; + }; + return { root, runQuery, inspect }; +}; + +describe.skipIf(process.platform === "win32")("desktop entry package", () => { + it("records the package that owns the launched entry, and the program's when it differs", async () => { + const { inspect } = await writerFixture(); + + const { installed } = await inspect({ desktopEntry: "libreoffice-writer" }); + + expect(installed).toMatchObject({ + kind: "deb", + name: "libreoffice-writer", + executable_package: "libreoffice-common", + version: WRITER_VERSION, + architecture: "arm64", + sources: [NOBLE], + source_origins: ["Ubuntu"], + package_lists_present: true, + }); + expect(installed).not.toHaveProperty("executable_package_reason"); + }); + + it("finds apt's package lists in the folder apt-config names, or the default when it cannot say", async () => { + const { root, runQuery, inspect } = await writerFixture(); + // A user-space apt-get update under APT_CONFIG keeps its lists elsewhere. + const userLists = join(root, "user's lists"); + await rename(join(root, "lists"), userLists); + await mkdir(join(root, "lists")); + const answering = + (outcome: QueryOutcome): QueryRunner => + async (command, argv, signal) => + command === "apt-config" ? outcome : runQuery(command, argv, signal); + const quoted = `LISTS='${userLists.replaceAll("'", "'\\''")}/'\n`; + + const present = await Promise.all( + [ + { stdout: quoted, exitCode: 0 }, + { stdout: "", exitCode: 100 }, + "missing" as const, + ].map(async (outcome) => { + const { installed } = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: answering(outcome) }, + ); + return installed.package_lists_present; + }), + ); + + expect(present).toEqual([true, false, false]); + }); + + it("records why the computer cannot install the program's package when no source it can use offers it", async () => { + const { runQuery, inspect } = await writerFixture(); + const local: QueryRunner = async (command, argv, signal) => + command === "apt-cache" && argv[1] === "libreoffice-common" + ? { + stdout: `libreoffice-common:\n Version table:\n *** ${WRITER_VERSION} 100\n 100 /var/lib/dpkg/status\n`, + exitCode: 0, + } + : runQuery(command, argv, signal); + const unasked: QueryRunner = async (command, argv, signal) => + command === "apt-cache" && argv[1] === "libreoffice-common" + ? "failed" + : runQuery(command, argv, signal); + + const fromLocal = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: local }, + ); + const fromUnasked = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: unasked }, + ); + + expect(fromLocal.installed).toMatchObject({ + name: "libreoffice-writer", + executable_package: "libreoffice-common", + executable_package_reason: + "neither the Ubuntu 24.04 archive nor a vendor repository the export declares offers libreoffice-common", + }); + expect(fromUnasked.installed).toMatchObject({ + executable_package: "libreoffice-common", + executable_package_reason: + "apt-cache policy did not say where libreoffice-common comes from", + }); + }); + + it.each([ + ["the desktop entry", "libreoffice-writer.desktop"], + ["the executable", "/bin/libreoffice"], + ])( + "leaves the package unknown when dpkg-query cannot say which package owns %s", + async (owned, suffix) => { + const { runQuery, inspect } = await writerFixture(); + const failing = + (outcome: QueryOutcome): QueryRunner => + async (command, argv, signal) => + command === "dpkg-query" && argv[1]?.endsWith(suffix) === true + ? outcome + : runQuery(command, argv, signal); + + for (const outcome of ["failed", { stdout: "", exitCode: 2 }] as const) { + const { installed, limitations } = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: failing(outcome) }, + ); + + expect(installed).toMatchObject({ kind: "unknown", name: null }); + expect(limitations.join("\n")).toMatch( + /^dpkg-query could not say which package owns \//mu, + ); + expect(limitations.join("\n")).toContain( + `${suffix}, so the package that owns ${owned}`, + ); + } + }, + ); + + it("keeps the program's package for an executable launch and an entry no package owns", async () => { + const { inspect } = await writerFixture(); + + for (const launch of [ + { executable: "libreoffice" }, + { desktopEntry: "my-writer" }, + ]) { + const { installed } = await inspect(launch); + expect(installed).toMatchObject({ + kind: "deb", + name: "libreoffice-common", + }); + expect(installed).not.toHaveProperty("executable_package"); + } + }); + + it("matches a package file against the entry's package", async () => { + const { root, inspect } = await writerFixture(); + const deb = join(root, "libreoffice-writer.deb"); + await writeFile(deb, "package bytes"); + + const { installed } = await inspect( + { desktopEntry: "libreoffice-writer" }, + { packageFile: deb }, + ); + + expect(installed).not.toHaveProperty("package_file_reason"); + expect(installed.package_file).toMatchObject({ + package: "libreoffice-writer", + version: WRITER_VERSION, + depends: ["libreoffice-core"], + }); + }); +}); + +describe.skipIf(process.platform === "win32")( + "desktop entry program ownership", + () => { + it("keeps the entry's package and records that no package owns the program it runs", async () => { + const { runQuery, inspect } = await writerFixture(); + // A program of the user's own, such as ~/.local/bin/libreoffice, found first on PATH. + const unpackaged: QueryRunner = async (command, argv, signal) => + command === "dpkg-query" && + argv[1]?.endsWith("/bin/libreoffice") === true + ? { stdout: "", exitCode: 1 } + : runQuery(command, argv, signal); + + const { installed } = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: unpackaged }, + ); + + expect(installed).toMatchObject({ + kind: "deb", + name: "libreoffice-writer", + executable_package_reason: expect.stringMatching( + /^no package owns \/\S+\/bin\/libreoffice$/u, + ), + }); + expect(installed).not.toHaveProperty("executable_package"); + }); + + it("counts the program as owned when dpkg owns the file its path resolves to, as for an alternatives link", async () => { + const { root, runQuery, inspect } = await writerFixture(); + const soffice = join(root, "lib/libreoffice/program/soffice"); + await mkdir(join(root, "lib/libreoffice/program"), { recursive: true }); + await writeFile(soffice, "#!/bin/sh\nexit 0\n"); + await chmod(soffice, 0o755); + await rm(join(root, "bin/libreoffice")); + await symlink(soffice, join(root, "bin/libreoffice")); + const resolvedOnly: QueryRunner = async (command, argv, signal) => { + const path = argv[1] ?? ""; + if (command === "dpkg-query" && path.endsWith("/bin/libreoffice")) + return { stdout: "", exitCode: 1 }; + if (command === "dpkg-query" && path.endsWith("/program/soffice")) + return { stdout: `libreoffice-common: ${path}`, exitCode: 0 }; + return runQuery(command, argv, signal); + }; + + const { installed } = await inspect( + { desktopEntry: "libreoffice-writer" }, + { run: resolvedOnly }, + ); + + expect(installed).toMatchObject({ + name: "libreoffice-writer", + executable_package: "libreoffice-common", + }); + expect(installed).not.toHaveProperty("executable_package_reason"); + }); + }, +); diff --git a/src/desktop/DebianPackageInventory.ts b/src/desktop/DebianPackageInventory.ts index e1f8ddcadf..9fd7b1ca5f 100644 --- a/src/desktop/DebianPackageInventory.ts +++ b/src/desktop/DebianPackageInventory.ts @@ -8,8 +8,14 @@ import { fromUbuntuNobleArchive, repositoryRefusal, } from "../domain/desktop/appExportInstall.js"; -import { declaredPackageFiles } from "../domain/desktop/aptRepositories.js"; -import type { DesktopApplicationInventory } from "../domain/desktop/desktopCapture.js"; +import { + declaredPackageFiles, + repositorySupplies, +} from "../domain/desktop/aptRepositories.js"; +import type { + AptRepository, + DesktopApplicationInventory, +} from "../domain/desktop/desktopCapture.js"; import { DPKG_STATUS_FORMAT, ownershipCandidates, @@ -20,12 +26,14 @@ import { } from "../domain/desktop/linuxPackages.js"; import { inspectAptRepositories, + resolveAptLocations, type AptRepositoryHost, } from "./AptRepositoryInventory.js"; import { succeeded } from "./HostQuery.js"; import { inspectLibraryPackages, installableLibraries, + installablePackages, } from "./LibraryPackageInventory.js"; /** The host commands and apt folders Debian package identification reads. */ @@ -108,7 +116,10 @@ const settled = async ( * the repository check. */ const installFacts = async ( - installed: Pick, + installed: Pick< + Package, + "name" | "sources" | "source_origins" | "executable_package" + >, executable: string, context: InventoryContext, origins: ReadonlyMap | undefined, @@ -132,33 +143,99 @@ const installFacts = async ( context.limitations.push(...vendor.limitations); for (const [sha256, bytes] of vendor.keys) context.signingKeys.set(sha256, bytes); + const usable = (repository: AptRepository) => + repositoryRefusal(repository) === undefined; const declared = declaredPackageFiles( - vendor.repositories.filter( - (repository) => repositoryRefusal(repository) === undefined, - ), + vendor.repositories.filter(usable), vendor.indexTargets, ); + const program = installed.executable_package; + const split = await installablePackages( + [ + ...(libraries?.library_packages ?? []), + ...(program === undefined ? [] : [program]), + ], + { origins, declared }, + runQuery, + context.signal, + ); + // The export checks these against the repositories it still declares. + const supplies = (repository: AptRepository) => + split === undefined || !usable(repository) + ? [] + : repositorySupplies(repository, split.supplied, vendor.indexTargets); return { - apt_repositories: [...vendor.repositories], - ...(libraries === undefined - ? {} - : await installableLibraries( - libraries, - { origins, declared }, - runQuery, - context.signal, - )), + apt_repositories: vendor.repositories.map((repository) => ({ + ...repository, + supplies: [...supplies(repository)], + })), + ...(libraries === undefined ? {} : installableLibraries(libraries, split)), + ...(program === undefined ? {} : executablePackageFacts(program, split)), }; }; +/** Why the computer cannot install the executable's package by name beside the entry's, by the library packages' rule; nothing when it can. */ +const executablePackageFacts = ( + name: string, + split: Awaited>, +): Pick => { + if (split?.installable.includes(name) === true) return {}; + return { + executable_package_reason: + split === undefined + ? `apt-cache policy did not say where ${name} comes from` + : `neither the Ubuntu 24.04 archive nor a vendor repository the export declares offers ${name}`, + }; +}; + +/** Which package dpkg says owns the paths, or why it could not say. */ +type Ownership = + | { readonly kind: "owned"; readonly name: string } + | { readonly kind: "unowned" } + | { readonly kind: "unknown"; readonly reason: string }; + +/** The owner of the first path dpkg owns; unknown once a query fails, since that path's owner would have come first. */ +const dpkgOwner = async ( + paths: readonly string[], + { host, signal }: InventoryContext, +): Promise => { + for (const path of ownershipCandidates(paths)) { + const outcome = await host.runQuery("dpkg-query", ["-S", path], signal); + if (outcome === "missing") + return { kind: "unknown", reason: "dpkg-query is not installed" }; + // dpkg-query exits 1 when no package owns the path and 2 when it could not tell. + if (outcome === "failed" || outcome.exitCode > 1) + return { + kind: "unknown", + reason: `dpkg-query could not say which package owns ${path}`, + }; + const [name] = parseDpkgOwners(outcome.stdout, path); + if (name !== undefined) return { kind: "owned", name }; + } + return { kind: "unowned" }; +}; + +/** A package name without the `:` dpkg may qualify it with. */ +const unqualified = (name: string): string => name.split(":")[0] ?? name; + +/** The files a Debian package is identified from. */ +export interface DebianPackagePaths { + /** The program as launched and as resolved; dpkg may own either. */ + readonly program: readonly string[]; + /** The resolved program, whose libraries are walked. */ + readonly executable: string; + /** The launched desktop entry and its resolved file; empty for an executable launch. */ + readonly entry: readonly string[]; +} + /** - * The Debian package that owns the executable, with the apt sources of its - * installed version, the vendor repositories among them and the packages of - * the libraries the executable loads; `unknown` when dpkg owns no candidate. + * The Debian package that owns the desktop entry, or else the executable, + * with the apt sources of its installed version, the vendor repositories + * among them and the packages of the libraries the executable loads; + * `unknown` when dpkg owns neither. */ export const debianPackage = async ( - paths: readonly string[], - executable: string, + { program, executable, entry }: DebianPackagePaths, context: InventoryContext, ): Promise => { const { host, signal, limitations } = context; @@ -170,57 +247,77 @@ export const debianPackage = async ( sources: [], package_lists_present: null, }; - for (const path of ownershipCandidates(paths)) { - const owners = await host.runQuery("dpkg-query", ["-S", path], signal); - if (owners === "missing") { - limitations.push( - "dpkg-query is not installed, so the package that owns the executable is unknown.", - ); - return unknown; - } - const [name] = parseDpkgOwners(succeeded(owners) ?? "", path); - if (name === undefined) continue; - const status = succeeded( - await host.runQuery( - "dpkg-query", - ["-W", `-f=${DPKG_STATUS_FORMAT}`, name], - signal, - ), + const executableOwner = await dpkgOwner(program, context); + if (executableOwner.kind === "unknown") { + limitations.push( + `${executableOwner.reason}, so the package that owns the executable is unknown.`, ); - const parsed = status === undefined ? undefined : parseDpkgStatus(status); - const policy = succeeded( - await host.runQuery("apt-cache", ["policy", name], signal), - ); - const listsPresent = ( - await readdir(host.apt.lists).catch(() => [] as string[]) - ).some((list) => /_Packages(?:\.[a-z0-9]+)?$/u.test(list)); - if (!listsPresent) - limitations.push( - "No apt package lists are present, so whether the installed version comes from a package archive is unknown.", - ); - const sources = - policy === undefined ? [] : [...parseAptPolicySources(policy)]; - // A local install's library packages are matched to the archive by origin too. - const origins = await policyOrigins(context); - const installed = { - kind: "deb" as const, - name: parsed?.name ?? name, - version: parsed?.version ?? null, - architecture: parsed?.architecture ?? null, - sources, - ...(origins === undefined - ? {} - : { - source_origins: sources.map( - (source) => origins.get(source) ?? null, - ), - }), - package_lists_present: listsPresent, - }; - return { - ...installed, - ...(await installFacts(installed, executable, context, origins)), - }; + return unknown; } - return unknown; + // The entry's package installs the application, whose program can come from another package. + const entryOwner = + entry.length === 0 + ? ({ kind: "unowned" } as const) + : await dpkgOwner(entry, context); + if (entryOwner.kind === "unknown") { + limitations.push( + `${entryOwner.reason}, so the package that owns the desktop entry, which installs the application, is unknown.`, + ); + return unknown; + } + const owner = executableOwner.kind === "owned" ? executableOwner.name : null; + const name = entryOwner.kind === "owned" ? entryOwner.name : owner; + if (name === null) return unknown; + const status = succeeded( + await host.runQuery( + "dpkg-query", + ["-W", `-f=${DPKG_STATUS_FORMAT}`, name], + signal, + ), + ); + const parsed = status === undefined ? undefined : parseDpkgStatus(status); + const policy = succeeded( + await host.runQuery("apt-cache", ["policy", name], signal), + ); + const apt = await resolveAptLocations(host.runQuery, signal, host.apt); + const listsPresent = ( + await readdir(apt.lists).catch(() => [] as string[]) + ).some((list) => /_Packages(?:\.[a-z0-9]+)?$/u.test(list)); + if (!listsPresent) + limitations.push( + "No apt package lists are present, so whether the installed version comes from a package archive is unknown.", + ); + const sources = + policy === undefined ? [] : [...parseAptPolicySources(policy)]; + // A local install's library packages are matched to the archive by origin too. + const origins = await policyOrigins(context); + const installed = { + kind: "deb" as const, + name: parsed?.name ?? name, + ...(owner === null || unqualified(owner) === unqualified(name) + ? {} + : { executable_package: unqualified(owner) }), + // Installing the entry's package does not bring a program no package owns, such as ~/.local/bin/. + ...(owner === null + ? { executable_package_reason: `no package owns ${executable}` } + : {}), + version: parsed?.version ?? null, + architecture: parsed?.architecture ?? null, + sources, + ...(origins === undefined + ? {} + : { + source_origins: sources.map((source) => origins.get(source) ?? null), + }), + package_lists_present: listsPresent, + }; + return { + ...installed, + ...(await installFacts( + installed, + executable, + { ...context, host: { ...host, apt } }, + origins, + )), + }; }; diff --git a/src/desktop/DesktopApplicationSession.test.ts b/src/desktop/DesktopApplicationSession.test.ts index bfefc47361..c330c4b306 100644 --- a/src/desktop/DesktopApplicationSession.test.ts +++ b/src/desktop/DesktopApplicationSession.test.ts @@ -1,4 +1,4 @@ -import { readdir } from "node:fs/promises"; +import { readdir, readFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -64,6 +64,33 @@ describe.skipIf(process.platform !== "linux")("runDesktopSession", () => { expect(checkpoints).toEqual(["stop", "exit"]); expect(vi.mocked(terminateTokenOwnedProcesses)).toHaveBeenCalledTimes(1); }, 30_000); + + it("launches the application without the agent environment's variables", async () => { + const root = await createTestTempDirectory("desktop-session-env-"); + const plan = shellPlan(root, `env > "${join(root, "env")}"`); + + await runDesktopSession({ + ...plan, + environment: { + ...plan.environment, + AGENTENV_CONTROL_TOKEN: "control", + AGENTENV_URL: "http://127.0.0.1:1", + NOT_AGENTENV_KEPT: "kept", + }, + }); + + const names = (await readFile(join(root, "env"), "utf8")) + .split("\n") + .map((line) => line.split("=")[0]); + expect(names.filter((name) => name?.startsWith("AGENTENV_"))).toEqual([]); + expect(names).toEqual( + expect.arrayContaining([ + "NOT_AGENTENV_KEPT", + "PATH", + "REA_PROCESS_RUN_ID", + ]), + ); + }, 30_000); }); describe.skipIf(process.platform !== "linux")( diff --git a/src/desktop/DesktopApplicationSession.ts b/src/desktop/DesktopApplicationSession.ts index 98fc36df42..5908853753 100644 --- a/src/desktop/DesktopApplicationSession.ts +++ b/src/desktop/DesktopApplicationSession.ts @@ -264,6 +264,20 @@ const unreleased = ( ], }; +/** + * The caller's environment without the agent environment's own variables, + * such as its control token: agentenv launches the exported app without + * them, and the app must not hold a credential to the computer it runs on. + */ +const applicationEnvironment = ( + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv => + Object.fromEntries( + Object.entries(environment).filter( + ([name]) => !name.startsWith("AGENTENV_"), + ), + ); + const launch = async ( plan: DesktopSessionPlan, runId: string, @@ -277,7 +291,7 @@ const launch = async ( // Desktop launchers often exec through env, a shell or a wrapper script. expectedCommand: null, cwd: plan.workingDirectory, - hostEnvironment: plan.environment, + hostEnvironment: applicationEnvironment(plan.environment), sweepTokenOwnedProcesses: true, signal, }); diff --git a/src/desktop/LibraryPackageInventory.ts b/src/desktop/LibraryPackageInventory.ts index 9b91d34171..e7f003a89b 100644 --- a/src/desktop/LibraryPackageInventory.ts +++ b/src/desktop/LibraryPackageInventory.ts @@ -205,26 +205,37 @@ const packageSources = async ( return sources; }; -/** - * Keep the library packages the computer can install: those some version of - * which the Ubuntu 24.04 archive or a declared vendor source offers, as - * `apt-cache policy` lists them; the rest are recorded as uninstallable. - */ -export const installableLibraries = async ( - libraries: LibraryPackages, +/** The packages the computer can install by name, and the rest, as `installableLibraryPackages` decides from `apt-cache policy`; undefined when apt could not be asked. */ +export const installablePackages = async ( + names: readonly string[], available: { readonly origins: ReadonlyMap | undefined; readonly declared: ReadonlySet; }, runQuery: QueryRunner, signal: AbortSignal | undefined, -): Promise => { - const names = libraries.library_packages; +): Promise | undefined> => { const sources = names.length === 0 ? new Map() : await packageSources(names, runQuery, signal); - if (sources === undefined) + return sources === undefined + ? undefined + : installableLibraryPackages(names, { ...available, sources }); +}; + +/** + * Keep the library packages the computer can install: those some version of + * which the Ubuntu 24.04 archive or a declared vendor source offers, as + * `apt-cache policy` listed them for `installablePackages`; the rest are + * recorded as uninstallable. + */ +export const installableLibraries = ( + libraries: LibraryPackages, + split: Awaited>, +): LibraryPackages => { + const names = libraries.library_packages; + if (split === undefined && names.length > 0) return { ...libraries, library_packages: [], @@ -236,10 +247,11 @@ export const installableLibraries = async ( .filter((reason) => reason !== undefined) .join("; "), }; - const split = installableLibraryPackages(names, { ...available, sources }); + const installable = (name: string) => + split?.installable.includes(name) === true; return { ...libraries, - library_packages: split.installable, - uninstallable_library_packages: split.uninstallable, + library_packages: names.filter(installable), + uninstallable_library_packages: names.filter((name) => !installable(name)), }; }; diff --git a/src/desktop/LinuxApplicationInventory.test.ts b/src/desktop/LinuxApplicationInventory.test.ts index 6f12851e1d..e530e79eb5 100644 --- a/src/desktop/LinuxApplicationInventory.test.ts +++ b/src/desktop/LinuxApplicationInventory.test.ts @@ -302,6 +302,40 @@ describe.skipIf(process.platform === "win32")( }); }); + it("adds a state folder named after the application in another letter case, such as GIMP's", async () => { + const installed = await fixture(); + const { home, bin } = installed; + await writeFile(join(bin, "gimp-2.10"), "#!/bin/sh\nexit 0\n"); + await chmod(join(bin, "gimp-2.10"), 0o755); + await mkdir(join(home, ".local/share/applications"), { recursive: true }); + await writeFile( + join(home, ".local/share/applications/gimp.desktop"), + "[Desktop Entry]\nType=Application\nName=GNU Image Manipulation Program\nExec=gimp-2.10 %U\n", + ); + await mkdir(join(home, ".config/GIMP/2.10"), { recursive: true }); + await writeFile(join(home, ".config/GIMP/2.10/sessionrc"), "x\n"); + await mkdir(join(home, ".Gimp")); + const unrelated = [ + ".config/GIMPshop", + ".config/MyGIMP", + ".config/Other/GIMP", + ]; + for (const folder of unrelated) + await mkdir(join(home, folder), { recursive: true }); + + const inventory = await inventoryOf("gimp", installed); + + expect(inventory.state_directories).toEqual( + expect.arrayContaining([ + { path: join(home, ".config/GIMP"), exists_at_start: true }, + { path: join(home, ".Gimp"), exists_at_start: true }, + ]), + ); + const paths = inventory.state_directories.map((entry) => entry.path); + for (const folder of unrelated) + expect(paths).not.toContain(join(home, folder)); + }); + it("adds the dotfiles in the home named after the application, which it may only read", async () => { const installed = await fixture(); const { home } = installed; diff --git a/src/desktop/LinuxApplicationInventory.ts b/src/desktop/LinuxApplicationInventory.ts index af12178661..a72261ec41 100644 --- a/src/desktop/LinuxApplicationInventory.ts +++ b/src/desktop/LinuxApplicationInventory.ts @@ -1,4 +1,4 @@ -import { lstat, readdir, realpath, stat } from "node:fs/promises"; +import { lstat, realpath, stat } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import type { DesktopApplicationInventory } from "../domain/desktop/desktopCapture.js"; @@ -21,6 +21,7 @@ import { readStableFileBytes, readStableFilePrefix, } from "../filesystem/readStableFileBytes.js"; +import { stateDirectories, stateNames } from "./ApplicationStateDirectories.js"; import { APT_LOCATIONS } from "./AptRepositoryInventory.js"; import { inspectPackageFile } from "./DebianPackageFile.js"; import { @@ -176,10 +177,12 @@ interface ProgramLocation { readonly program: string; readonly resolved: string | null; readonly flatpak: string | null; + /** The launched desktop entry and its resolved file; empty for an executable launch. */ + readonly entry: readonly string[]; } const identifyPackage = async ( - { program, resolved, flatpak }: ProgramLocation, + { program, resolved, flatpak, entry }: ProgramLocation, context: InventoryContext, ): Promise => { const base = { @@ -201,8 +204,11 @@ const identifyPackage = async ( ) return { kind: "appimage", name: basename(resolved ?? program), ...base }; return debianPackage( - [program, ...(resolved === null ? [] : [resolved])], - resolved ?? program, + { + program: [program, ...(resolved === null ? [] : [resolved])], + executable: resolved ?? program, + entry, + }, context, ); }; @@ -231,61 +237,6 @@ const electronMarkers = async ( return markers; }; -/** Whether a home entry is a dotfile named after the application, such as `.varicad-cfg.ini` or `.notesrc`. */ -const appNamedDotfile = (entry: string, names: ReadonlySet): boolean => - [...names].some((name) => { - // Short names would match unrelated dotfiles. - if (name.length < 3 || !entry.startsWith(`.${name}`)) return false; - const rest = entry.slice(name.length + 1); - return rest === "rc" || /^[-._]./u.test(rest); - }); - -/** - * Folder names under XDG and dot-directories where the application may keep - * state, and the dotfiles in the home named after it, which an application - * may only read during a capture. - */ -const stateDirectories = async ( - names: readonly (string | null)[], - { environment, home }: Pick, - extra: readonly string[], -): Promise => { - const bases = [ - environment.XDG_CONFIG_HOME || join(home, ".config"), - environment.XDG_DATA_HOME || join(home, ".local/share"), - environment.XDG_STATE_HOME || join(home, ".local/state"), - environment.XDG_CACHE_HOME || join(home, ".cache"), - ]; - const variants = new Set( - names - .filter( - (name): name is string => - name !== null && - /^[^/\0]+$/u.test(name) && - name !== "." && - name !== "..", - ) - .flatMap((name) => [name, name.toLowerCase()]), - ); - const dotfiles = (await readdir(home).catch(() => [] as string[])) - .filter((entry) => appNamedDotfile(entry, variants)) - .map((entry) => join(home, entry)); - const paths = new Set([ - ...[...variants].flatMap((name) => [ - ...bases.map((base) => join(base, name)), - join(home, `.${name}`), - ]), - ...dotfiles, - ...extra, - ]); - return Promise.all( - [...paths].map(async (path) => ({ - path, - exists_at_start: (await lstat(path).catch(() => undefined)) !== undefined, - })), - ); -}; - const launchCommand = async ( request: LinuxApplicationRequest, signal: AbortSignal | undefined, @@ -348,26 +299,6 @@ const launchCommand = async ( }); }; -/** - * Names the application's state folders may carry, including the entry's - * full reverse-DNS id and its last part. The executable is null for a - * `flatpak run` launch, whose `flatpak` launcher names flatpak's own folders. - */ -const stateNames = ( - package_: Package, - executable: string | null, - entry: DesktopApplicationInventory["desktop_entry"], -): readonly (string | null)[] => [ - package_.kind === "deb" - ? (package_.name?.split(":")[0] ?? null) - : package_.name, - executable === null ? null : basename(executable), - entry?.id ?? null, - entry?.id?.split(".").at(-1) ?? null, - entry?.startup_wm_class ?? null, - entry?.name ?? null, -]; - /** Resolve what the request launches, where it came from and where it may keep state. */ export const inspectLinuxApplication = async ( request: LinuxApplicationRequest, @@ -404,8 +335,12 @@ export const inspectLinuxApplication = async ( }; const { limitations, signingKeys } = context; const flatpak = wrapper.kind === "flatpak" ? wrapper.application_id : null; + const entryFiles = + entry === null + ? [] + : [entry.path, await realpath(entry.path).catch(() => entry.path)]; const identified = await identifyPackage( - { program: path, resolved, flatpak }, + { program: path, resolved, flatpak, entry: entryFiles }, context, ); const package_ = diff --git a/src/domain/desktop/appExport.test.ts b/src/domain/desktop/appExport.test.ts index 9c5ed33a05..9019a4bcdf 100644 --- a/src/domain/desktop/appExport.test.ts +++ b/src/domain/desktop/appExport.test.ts @@ -29,6 +29,7 @@ const plan: DesktopAppExportPlan = { windowClass: "Mousepad", paths: [".config/Mousepad", "Documents"], capturedHome: "/home/ada", + seedCheckpoint: "baseline", seedFiles: ["Documents/todo.txt"], textSeeds: ["Documents/todo.txt"], unseededFiles: [], @@ -58,7 +59,7 @@ describe("names", () => { it("keeps the class off the module's own names and the module off other modules", () => { expect(className("path")).toBe("PathApp"); expect(className("data-part")).toBe("DataPartApp"); - expect(className("element-tree")).toBe("ElementTreeApp"); + expect(className("run-script")).toBe("RunScriptApp"); expect(className("p-a-t-h-s")).toBe("PATHSApp"); expect(appModuleFile("calendar")).toBe("calendar_app.py"); expect(appModuleFile("gnu-cash")).toBe("gnu_cash_app.py"); @@ -76,6 +77,19 @@ describe("ownedPaths", () => { ).toEqual([".config/app", "Documents"]); }); + it("owns a folder beside a state directory, differing only in letter case, in the file's spelling", () => { + expect( + ownedPaths( + [".config/GIMP/2.10/sessionrc", ".Gimp/x"], + [], + [".config/gimp", ".gimp"], + ), + ).toEqual([".Gimp", ".config/GIMP"]); + expect( + ownedPaths([".config/GIMPshop/2.10/sessionrc"], [], [".config/gimp"]), + ).toEqual([".config/GIMPshop/2.10"]); + }); + it("owns a file directly in the home or a shared folder as itself", () => { expect(ownedPaths(["books.gnucash"], [], [])).toEqual(["books.gnucash"]); expect( diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts index b9d67e7bd0..22d9b66bfb 100644 --- a/src/domain/desktop/appExport.ts +++ b/src/domain/desktop/appExport.ts @@ -13,6 +13,8 @@ import { import type { DesktopApplicationCapture } from "./desktopCapture.js"; import type { DesktopApplicationState } from "./desktopState.js"; import { isWithinAny } from "./desktopStateDiff.js"; +import { SEED_HOME } from "./hostIdentity.js"; +import { stateDirectoryOf } from "./stateRoles.js"; /** agentenv-framework release the exported class was checked against. */ export const DESKTOP_EXPORT_FRAMEWORK_VERSION = "0.9.1431"; @@ -35,6 +37,8 @@ export interface DesktopAppExportPlan { readonly architectures: readonly string[]; /** The package file downloaded and installed in place of the app's apt package; null when apt installs it. */ readonly packageFile: ExportPackageFile | null; + /** The package of the program the desktop entry runs, installed by name after the app's; absent when the app's package owns it. */ + readonly executablePackage?: string; /** Whether the app is Electron, whose launch then runs without Chromium's sandbox. */ readonly electron: boolean; readonly command: readonly string[]; @@ -42,13 +46,15 @@ export interface DesktopAppExportPlan { /** Home-relative paths the app owns for data add, get and reset. */ readonly paths: readonly string[]; readonly capturedHome: string; + /** The checkpoint whose state the seed holds. */ + readonly seedCheckpoint: string; /** Home-relative files copied into the computer's home before the first launch. */ readonly seedFiles: readonly string[]; - /** Seed files read as text, which get the captured home path respelled. */ + /** Seed files read as text, which get the seed's home path respelled. */ readonly textSeeds: readonly string[]; /** Files under the owned paths that existed before the capture but whose bytes were not retained. */ readonly unseededFiles: readonly string[]; - /** `dconf dump` key file of the application's groups at the baseline, or empty. */ + /** `dconf dump` key file of the application's groups at the seed, or empty. */ readonly dconf: string; readonly captureEvidenceId: string; } @@ -101,9 +107,10 @@ const SHARED_FOLDERS = new Set([ ]); /** - * The paths the app owns: the state directory holding each state file, or - * a document's folder; a file directly in the home or a shared folder is - * owned as itself. Nested paths collapse into the outermost one. + * The paths the app owns: the state directory holding each state file, in + * the file's letter case, or a document's folder; a file directly in the home + * or a shared folder is owned as itself. Nested paths collapse into the + * outermost one. */ export const ownedPaths = ( stateFiles: readonly string[], @@ -114,12 +121,10 @@ export const ownedPaths = ( const owner = (file: string, folder: string) => SHARED_FOLDERS.has(folder) ? file : folder; const folderOf = (file: string) => file.split("/").slice(0, -1).join("/"); - for (const file of stateFiles) { - const directory = stateDirectories.find((candidate) => - isWithinAny(file, [candidate]), + for (const file of stateFiles) + folders.add( + owner(file, stateDirectoryOf(file, stateDirectories) ?? folderOf(file)), ); - folders.add(owner(file, directory ?? folderOf(file))); - } for (const file of documents) folders.add(owner(file, folderOf(file))); const sorted = [...folders].filter((folder) => folder.length > 0).sort(); return sorted.filter( @@ -319,10 +324,11 @@ const installName = (plan: DesktopAppExportPlan): string => ? plan.aptPackage : `${plan.aptPackage}/${plan.aptRelease}`; -/** The app's package, or a package file's dependencies, then its library packages and dconf-cli when settings are loaded. */ +/** The app's package, or a package file's dependencies, then its program's package, its library packages and dconf-cli when settings are loaded. */ const aptPackages = (plan: DesktopAppExportPlan): readonly string[] => [ ...new Set([ ...(plan.packageFile?.dependencies ?? [installName(plan)]), + ...(plan.executablePackage === undefined ? [] : [plan.executablePackage]), ...plan.libraryPackages, ...(plan.dconf.length === 0 ? [] : ["dconf-cli"]), ]), @@ -360,12 +366,12 @@ export const renderAppClass = (plan: DesktopAppExportPlan): string => { return `"""${docText(plan.title)} as an agentenv computer app, exported by REA from a desktop application capture. Package: ${plan.aptPackage}${plan.packageVersion === null ? "" : ` (captured at ${plan.packageVersion})`}. Capture Evidence: ${plan.captureEvidenceId}. -The seed is the user's state before the capture: keep this bundle local. +The seed is the user's state at checkpoint ${docText(plan.seedCheckpoint)}: keep this bundle local. """ ${PYTHON_IMPORTS} HERE = Path(__file__).resolve().parent -CAPTURED_HOME = ${py(plan.capturedHome)} +SEED_HOME = ${py(SEED_HOME)} PATHS = ${pyList(plan.paths)} DCONF = ${py(plan.dconf)} TEXT_SEEDS = frozenset(${pyList(plan.textSeeds)}) @@ -430,6 +436,11 @@ const packageFileNotes = ({ packageFile: file }: DesktopAppExportPlan) => const installNotes = (plan: DesktopAppExportPlan): string => [ ...packageFileNotes(plan), + ...(plan.executablePackage === undefined + ? [] + : [ + `- \`${plan.executablePackage}\`, which owns the program the desktop entry runs, is installed with it.`, + ]), ...(plan.libraryPackages.length === 0 ? [] : [ @@ -445,10 +456,36 @@ const installNotes = (plan: DesktopAppExportPlan): string => .map((line) => `\n${line}`) .join(""); +/** Bundle files: seeds the export rewrote to remove the capture host's identity, and files that still hold it. */ +export interface SeedScrubReport { + readonly scrubbed: readonly string[]; + readonly unscrubbed: readonly string[]; +} + +const homeList = (paths: readonly string[]) => + paths.map((path) => `\`~/${path}\``).join(", "); + +const bundleList = (paths: readonly string[]) => + paths.map((path) => `\`${path}\``).join(", "); + +const scrubNotes = ({ scrubbed, unscrubbed }: SeedScrubReport): string => + [ + ...(scrubbed.length === 0 + ? [] + : [`\n Rewritten: ${bundleList(scrubbed)}.`]), + ...(unscrubbed.length === 0 + ? [] + : [ + `\n- Still holding the capture host's name, user@host or home, in their name or in content REA does not rewrite: + ${bundleList(unscrubbed)}. Read them before sharing the bundle.`, + ]), + ].join(""); + /** The bundle README: what was exported, how to run it, and what to keep in mind; a task's section follows. */ export const renderReadme = ( plan: DesktopAppExportPlan, taskSection = "", + scrub: SeedScrubReport = { scrubbed: [], unscrubbed: [] }, ): string => `# ${plan.title} computer app @@ -461,12 +498,13 @@ ${DESKTOP_EXPORT_FRAMEWORK_VERSION}. when a window of class \`${plan.windowClass}\` appears.${installNotes(plan)} - Owned paths: ${plan.paths.length === 0 ? "none" : plan.paths.map((path) => `\`~/${path}\``).join(", ")}. \`data/get\` returns their files and a typed JSON view of each; \`data/reset\` restores the seed. -- Seed: ${plan.seedFiles.length} file(s) from the user's home before the capture, with the captured home path - respelled in text files${plan.dconf.length === 0 ? "" : ", and the application's dconf keys"}.${ +- Seed: ${plan.seedFiles.length} file(s) from the user's home at checkpoint \`${plan.seedCheckpoint}\`${plan.dconf.length === 0 ? "" : ", and the application's dconf keys"}. + Text files and dconf keys spell the capture host's name as \`localhost\`, \`user@host\` as \`user@localhost\` + and the home as \`${SEED_HOME}\`, which the computer respells to its own at boot.${scrubNotes(scrub)}${ plan.unseededFiles.length === 0 ? "" : ` -- Not seeded, because the capture did not retain their starting bytes: ${plan.unseededFiles.map((path) => `\`~/${path}\``).join(", ")}. +- Not seeded, because the capture did not retain their bytes at the seed: ${homeList(plan.unseededFiles)}. Capture again with \`--state-path\` naming them, or a larger \`--max-file-bytes\`, to include them.` } - \`tasks/smoke.json\` deploys the app. diff --git a/src/domain/desktop/appExportInstall.test.ts b/src/domain/desktop/appExportInstall.test.ts index ff4479a0b6..c238aaafb1 100644 --- a/src/domain/desktop/appExportInstall.test.ts +++ b/src/domain/desktop/appExportInstall.test.ts @@ -4,9 +4,9 @@ import { electronLaunchArguments, exportInstall, fromUbuntuNobleArchive, - installLimitations, repositoryRefusal, } from "./appExportInstall.js"; +import { installLimitations } from "./appExportInstallLimitations.js"; import { VARICAD_FILE, VARICAD_URL, @@ -214,6 +214,165 @@ describe("exportInstall", () => { }); }); +describe("exportInstall with the program in another package", () => { + const archive: DesktopPackage = { + ...typora(undefined), + name: "vim-common", + executable_package: "vim", + sources: ["http://archive.ubuntu.com/ubuntu noble/main amd64 Packages"], + source_origins: ["Ubuntu"], + }; + + it("installs the program's package beside the entry's from the archive, a vendor repository or a package file", () => { + const fromVendor = exportInstall( + { ...typora([repository()]), executable_package: "typora-bin" }, + "typora", + "arm64", + ); + const fromFile = exportInstall( + varicadPackage({ executable_package: "varicad-bin" }), + "varicad", + "x64", + VARICAD_URL, + ); + + expect(exportInstall(archive, "vim", "arm64")).toMatchObject({ + ok: true, + value: { aptPackage: "vim-common", executablePackage: "vim" }, + }); + expect(fromVendor).toMatchObject({ + ok: true, + value: { aptPackage: "typora", executablePackage: "typora-bin" }, + }); + expect(fromFile).toMatchObject({ + ok: true, + value: { + packageFile: { url: VARICAD_URL }, + executablePackage: "varicad-bin", + }, + }); + }); + + it("refuses when no package owns the program the entry runs", () => { + const { executable_package: _program, ...entryOnly } = archive; + + expect( + exportInstall( + { + ...entryOnly, + executable_package_reason: "no package owns /home/ada/.local/bin/vim", + }, + "vim", + "arm64", + ), + ).toEqual({ + ok: false, + error: { + field: "capture", + reason: + "The desktop entry belongs to vim-common, but the computer cannot install the program it runs: no package owns /home/ada/.local/bin/vim.", + remediation: + "Capture again with the packaged program first on PATH, or with --executable naming it.", + }, + }); + }); + + it("refuses when the computer cannot install the program's package", () => { + const reason = + "neither the Ubuntu 24.04 archive nor a vendor repository the export declares offers vim"; + + expect( + exportInstall( + { ...archive, executable_package_reason: reason }, + "vim", + "arm64", + ), + ).toEqual({ + ok: false, + error: expect.objectContaining({ + field: "capture", + reason: `The desktop entry belongs to vim-common, but the program it runs belongs to vim, which the computer cannot install: ${reason}.`, + }), + }); + }); +}); + +describe("exportInstall with repositories the export leaves out", () => { + const mirror = (supplies: readonly string[]) => + repository({ + source: "https://mirror.typora.example/linux ./ Packages", + uri: "https://mirror.typora.example/linux", + signing_key: { ...KEY, sha256: "c".repeat(64) }, + supplies: [...supplies], + }); + // The capture could not retain this repository's key, so the export leaves it out. + const dropped = (supplies: readonly string[]) => + repository({ + signing_key: null, + signing_key_reason: + "the verified key /etc/apt/trusted.gpg.d/typora.gpg did not fit the capture's content budget, so it was not retained", + supplies: [...supplies], + }); + const installed = ( + repositories: readonly AptRepository[], + extra: Partial = {}, + ): DesktopPackage => ({ + ...typora(repositories), + library_packages: ["libgbm1", "libvendor1", "libshared1"], + ...extra, + }); + + it("refuses the program's package when only a repository it leaves out supplies it", () => { + const refused = exportInstall( + installed([dropped(["typora-bin"]), mirror([])], { + executable_package: "typora-bin", + }), + "typora", + "arm64", + ); + + expect(refused).toEqual({ + ok: false, + error: expect.objectContaining({ + reason: + "The desktop entry belongs to typora, but the program it runs belongs to typora-bin, which the computer cannot install: only apt repositories the export leaves out offer it.", + }), + }); + }); + + it("installs only the library packages a declared repository or the archive still offers, and says which it left out", () => { + const vendor = exportInstall( + installed([ + dropped(["libshared1", "libvendor1"]), + mirror(["libshared1"]), + ]), + "typora", + "arm64", + ); + const fromFile = exportInstall( + varicadPackage({ apt_repositories: [dropped(["libxt6t64"])] }), + "varicad", + "x64", + VARICAD_URL, + ); + + if (!vendor.ok) throw new Error(vendor.error.reason); + expect(vendor.value).toMatchObject({ + libraryPackages: ["libgbm1", "libshared1"], + strandedPackages: ["libvendor1"], + }); + expect( + installLimitations(vendor.value, installed([]), false).join("\n"), + ).toContain( + "in packages only the apt repositories left out offer: libvendor1", + ); + expect(fromFile).toMatchObject({ + ok: true, + value: { libraryPackages: ["libgl1"], strandedPackages: ["libxt6t64"] }, + }); + }); +}); + describe("exportInstall sources", () => { const amd64 = "https://apt.example.com/repo stable/main amd64 Packages"; const i386 = "https://apt.example.com/repo stable/main i386 Packages"; diff --git a/src/domain/desktop/appExportInstall.ts b/src/domain/desktop/appExportInstall.ts index 849026cd41..ce0902af23 100644 --- a/src/domain/desktop/appExportInstall.ts +++ b/src/domain/desktop/appExportInstall.ts @@ -69,6 +69,10 @@ export interface ExportInstall { readonly architectures: readonly string[]; /** The package file installed in place of an apt package; null when apt installs the package. */ readonly packageFile: ExportPackageFile | null; + /** The package of the program the desktop entry runs, installed by name beside the app's; absent when the app's package owns it. */ + readonly executablePackage?: string; + /** Packages the capture found only in repositories the export leaves out, which the computer cannot install. */ + readonly strandedPackages?: readonly string[]; } /** Why the computer cannot install the captured package: the input to change, and how. */ @@ -146,7 +150,7 @@ const declaredSources = ( const declared = new Map(); const keyFiles = new Map(); const refused: string[] = []; - let first: AptRepository | undefined; + const kept: AptRepository[] = []; for (const repository of repositories) { const { uri, suite, components, signing_key: key } = repository; const reason = @@ -182,12 +186,43 @@ const declaredSources = ( keyFile, keySha256: key.sha256, }); - first ??= repository; + kept.push(repository); } return { sources: [...declared.values()], refused, - release: first?.release ?? null, + release: kept[0]?.release ?? null, + kept, + }; +}; + +/** + * The library packages the computer installs given the repositories the + * export declares, and the packages it cannot: those only repositories it + * leaves out supply, such as one whose key the capture could not retain. + */ +const suppliedLibraries = ( + installed: DesktopPackage, + kept: readonly AptRepository[], +): Pick => { + const repositories = installed.apt_repositories ?? []; + const supplied = new Set( + kept.flatMap((repository) => repository.supplies ?? []), + ); + const stranded = [ + ...new Set( + repositories + .filter((repository) => !kept.includes(repository)) + .flatMap((repository) => repository.supplies ?? []), + ), + ] + .filter((name) => !supplied.has(name)) + .sort(); + return { + libraryPackages: (installed.library_packages ?? []).filter( + (name) => !stranded.includes(name), + ), + ...(stranded.length === 0 ? {} : { strandedPackages: stranded }), }; }; @@ -242,7 +277,7 @@ const vendorInstall = ( ), skipped: [], }); - const { sources, refused, release } = declaredSources( + const { sources, refused, release, kept } = declaredSources( repositories, environment, architecture, @@ -257,7 +292,7 @@ const vendorInstall = ( return ok({ aptPackage, release, - libraryPackages: installed.library_packages ?? [], + ...suppliedLibraries(installed, kept), sources, skipped: refused, architectures: [architecture], @@ -328,7 +363,7 @@ const packageFileInstall = ( return ok({ ...named, release: null, - libraryPackages: installed.library_packages ?? [], + ...suppliedLibraries(installed, []), sources: [], architectures: [architecture], packageFile: { @@ -350,7 +385,7 @@ const packageFileInstall = ( * `--package-url`, is downloaded from it and installed from its captured * package file; anything else is refused with the reason. */ -export const exportInstall = ( +const packageInstall = ( installed: DesktopPackage, environment: string, hostArchitecture: string, @@ -371,6 +406,7 @@ export const exportInstall = ( ? ok({ aptPackage, release: null, + ...suppliedLibraries(installed, []), libraryPackages: [], sources: [], skipped: [], @@ -397,6 +433,38 @@ export const exportInstall = ( ); }; +/** How the computer installs the captured package and, when another package owns the program its desktop entry runs, that package too. */ +export const exportInstall = ( + ...request: Parameters +): Result => { + const install = packageInstall(...request); + const [{ executable_package: program, executable_package_reason: found }] = + request; + if (!install.ok) return install; + if (program === undefined) + return found === undefined + ? install + : err( + refusal( + `The desktop entry belongs to ${install.value.aptPackage}, but the computer cannot install the program it runs: ${found}.`, + "capture", + "Capture again with the packaged program first on PATH, or with --executable naming it.", + ), + ); + const reason = + found ?? + (install.value.strandedPackages?.includes(program) === true + ? "only apt repositories the export leaves out offer it" + : undefined); + return reason === undefined + ? ok({ ...install.value, executablePackage: program }) + : err( + refusal( + `The desktop entry belongs to ${install.value.aptPackage}, but the program it runs belongs to ${program}, which the computer cannot install: ${reason}.`, + ), + ); +}; + /** Chromium's sandbox needs user namespaces a container does not grant, so an Electron launch runs with `--no-sandbox` right after the program. */ export const electronLaunchArguments = ( argv: readonly string[], @@ -407,74 +475,3 @@ export const electronLaunchArguments = ( const at = wrapper.kind === "direct" ? wrapper.index + 1 : 1; return [...argv.slice(0, at), "--no-sandbox", ...argv.slice(at)]; }; - -const unmappedLibraries = (installed: DesktopPackage): readonly string[] => [ - ...(installed.library_packages_reason === undefined - ? [] - : [installed.library_packages_reason]), - ...(installed.missing_libraries?.length - ? [ - `not found on the capture host: ${installed.missing_libraries.join(", ")}`, - ] - : []), - ...(installed.unpackaged_libraries?.length - ? [`owned by no package: ${installed.unpackaged_libraries.join(", ")}`] - : []), - ...(installed.uninstallable_library_packages?.length - ? [ - `in packages neither the Ubuntu 24.04 archive nor a declared repository offers: ${installed.uninstallable_library_packages.join(", ")}`, - ] - : []), -]; - -/** What the export says about a package file the computer downloads and installs. */ -const packageFileLimitations = ( - file: ExportPackageFile, - installed: DesktopPackage, -): readonly string[] => [ - `The computer downloads the package from ${file.url} once, when the bundle is put, and refuses the download unless its SHA-256 is ${file.sha256}, the captured file's; the bundle does not contain the package. It installs exactly that file's version, ${installed.version ?? "unknown"}, with dpkg after the apt packages it depends on, and the bundle supports only its architecture, ${file.architecture}.`, - "Vendor sites may refuse automated downloads, so the URL must point at a copy of the file the operator may use, and the package's licence, such as a trial's terms, is the operator's to check.", - ...(file.unreadDependencies.length === 0 - ? [] - : [ - `REA could not read a package name from these dependencies, so the computer does not install them: ${file.unreadDependencies.join(", ")}.`, - ]), -]; - -/** What the export says about how the computer installs and starts the application. */ -export const installLimitations = ( - install: ExportInstall, - installed: DesktopPackage, - electron: boolean, -): readonly string[] => { - const unmapped = unmappedLibraries(installed); - // Library packages matter for any package the archive does not install by name. - const vendor = install.sources.length > 0 || install.packageFile !== null; - return [ - ...(install.packageFile !== null - ? packageFileLimitations(install.packageFile, installed) - : [ - install.sources.length > 0 - ? `The application is installed as ${install.aptPackage}${install.release === null ? "" : `/${install.release}`} from ${install.sources.map((source) => source.uri).join(", ")}, trusting the key that verified it on the capture host, so the computer installs the version that repository offers then, which may differ from the one captured; the bundle supports only the capture host's architecture, ${install.architectures.join(", ")}.` - : "The application is installed from the distribution archive by name, so the computer may install a different version than the one captured.", - ]), - ...install.skipped.map( - (repository) => `The apt repository ${repository} was left out.`, - ), - ...(vendor && installed.library_packages === undefined - ? [ - "The capture did not record the executable's library packages, so the computer installs only what the package depends on.", - ] - : []), - ...(!vendor || unmapped.length === 0 - ? [] - : [ - `Not every library the executable loads is installed with it (${unmapped.join("; ")}), so the computer may lack some.`, - ]), - ...(electron - ? [ - "Chromium's sandbox is off inside the computer (--no-sandbox): it needs user namespaces the computer's container does not grant, and Electron exits without them.", - ] - : []), - ]; -}; diff --git a/src/domain/desktop/appExportInstallLimitations.ts b/src/domain/desktop/appExportInstallLimitations.ts new file mode 100644 index 0000000000..c3384d3195 --- /dev/null +++ b/src/domain/desktop/appExportInstallLimitations.ts @@ -0,0 +1,81 @@ +import type { ExportInstall, ExportPackageFile } from "./appExportInstall.js"; +import type { DesktopPackage } from "./desktopCapture.js"; + +const unmappedLibraries = ( + installed: DesktopPackage, + stranded: readonly string[], +): readonly string[] => [ + ...(installed.library_packages_reason === undefined + ? [] + : [installed.library_packages_reason]), + ...(installed.missing_libraries?.length + ? [ + `not found on the capture host: ${installed.missing_libraries.join(", ")}`, + ] + : []), + ...(installed.unpackaged_libraries?.length + ? [`owned by no package: ${installed.unpackaged_libraries.join(", ")}`] + : []), + ...(installed.uninstallable_library_packages?.length + ? [ + `in packages neither the Ubuntu 24.04 archive nor a declared repository offers: ${installed.uninstallable_library_packages.join(", ")}`, + ] + : []), + ...(stranded.length > 0 + ? [ + `in packages only the apt repositories left out offer: ${stranded.join(", ")}`, + ] + : []), +]; + +/** What the export says about a package file the computer downloads and installs. */ +const packageFileLimitations = ( + file: ExportPackageFile, + installed: DesktopPackage, +): readonly string[] => [ + `The computer downloads the package from ${file.url} once, when the bundle is put, and refuses the download unless its SHA-256 is ${file.sha256}, the captured file's; the bundle does not contain the package. It installs exactly that file's version, ${installed.version ?? "unknown"}, with dpkg after the apt packages it depends on, and the bundle supports only its architecture, ${file.architecture}.`, + "Vendor sites may refuse automated downloads, so the URL must point at a copy of the file the operator may use, and the package's licence, such as a trial's terms, is the operator's to check.", + ...(file.unreadDependencies.length === 0 + ? [] + : [ + `REA could not read a package name from these dependencies, so the computer does not install them: ${file.unreadDependencies.join(", ")}.`, + ]), +]; + +/** What the export says about how the computer installs and starts the application. */ +export const installLimitations = ( + install: ExportInstall, + installed: DesktopPackage, + electron: boolean, +): readonly string[] => { + const unmapped = unmappedLibraries(installed, install.strandedPackages ?? []); + // Library packages matter for any package the archive does not install by name. + const vendor = install.sources.length > 0 || install.packageFile !== null; + return [ + ...(install.packageFile !== null + ? packageFileLimitations(install.packageFile, installed) + : [ + install.sources.length > 0 + ? `The application is installed as ${install.aptPackage}${install.release === null ? "" : `/${install.release}`} from ${install.sources.map((source) => source.uri).join(", ")}, trusting the key that verified it on the capture host, so the computer installs the version that repository offers then, which may differ from the one captured; the bundle supports only the capture host's architecture, ${install.architectures.join(", ")}.` + : "The application is installed from the distribution archive by name, so the computer may install a different version than the one captured.", + ]), + ...install.skipped.map( + (repository) => `The apt repository ${repository} was left out.`, + ), + ...(vendor && installed.library_packages === undefined + ? [ + "The capture did not record the executable's library packages, so the computer installs only what the package depends on.", + ] + : []), + ...(!vendor || unmapped.length === 0 + ? [] + : [ + `Not every library the executable loads is installed with it (${unmapped.join("; ")}), so the computer may lack some.`, + ]), + ...(electron + ? [ + "Chromium's sandbox is off inside the computer (--no-sandbox): it needs user namespaces the computer's container does not grant, and Electron exits without them.", + ] + : []), + ]; +}; diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts index a44fec8ad3..844df74526 100644 --- a/src/domain/desktop/appExportPython.ts +++ b/src/domain/desktop/appExportPython.ts @@ -1,3 +1,6 @@ +import { PYTHON_XML_READER } from "./appExportPythonXml.js"; +import { PYTHON_ZIP_HELPERS } from "./appExportPythonZip.js"; + /** The generated module's imports. */ /** The class method that holds readiness until the app's own window shows, not a splash screen with its class. */ export const PYTHON_READY_CHECK = ` @@ -10,16 +13,22 @@ export const PYTHON_READY_CHECK = ` export const PYTHON_IMPORTS = `from __future__ import annotations import contextlib +import copy +import hashlib +import io import json import math import os import re import sqlite3 +import struct import subprocess import sys import tempfile -import xml.etree.ElementTree as ElementTree +import zipfile +import zlib from pathlib import Path +from xml.parsers import expat from agentenv_protocol import add_data, get_data, reset_data from agentenv_protocol.computer import AptSource, ComputerAppEnvironment, Download, Launch, Packages, Ready, RunScript, computer_app, linux, ready_check @@ -33,7 +42,9 @@ XML_SUFFIXES = (".xml", ".xbel", ".plist", ".svg", ".ui") GROUP_HEADER = re.compile(r"^\[([^\]]+)\]$") LINE_BREAK = re.compile(r"\r?\n") SQLITE_HEADER = b"SQLite format 3\x00" -XML_SPACE = "{http://www.w3.org/XML/1998/namespace}space" +GVDB_HEADER = b"GVariant" +ZIP_HEADER = b"PK\x03\x04" +XML_SPACE = "xml:space" MAX_VIEW_BYTES = 64 * 1024 * 1024 MAX_SAFE_INTEGER = 2**53 - 1 MAX_TABLES = 64 @@ -42,11 +53,20 @@ MAX_LINES = 5000 MAX_XML_NODES = 5000 MAX_XML_DEPTH = 64 SQLITE_BUSY_S = 0.25 +MAX_PARSED_BYTES = 4 * 1024 * 1024 +PARSED_TOO_LARGE = "larger than the 4 MiB limit for JSON and key file views" +MAX_JSON_VIEW_DEPTH = 500 +JSON_TOO_DEEP = f"nested deeper than the {MAX_JSON_VIEW_DEPTH} levels a JSON view keeps" +#: A JSON token after any whitespace, by group: an opening, a closing, a comma, a colon, a string's quote, or a number +#: or literal. Strings are read by json's scanner, since a pattern repeating per escape holds memory for each. +JSON_TOKEN = re.compile(r'[ \t\n\r]*(?:([\[{])|([\]}])|(,)|(:)|(")|(-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?' + r"|true|false|null))") +JSON_SPACE = re.compile(r"[ \t\n\r]*") def _respelled(text: str, home: Path) -> str: - """The captured home path respelled to this computer's home, only where a whole path segment ends after it.""" - return re.sub(re.escape(CAPTURED_HOME) + r"(?![\w.@+~-])", lambda _: str(home), text) + """The seed's home path respelled to this computer's home, only where a whole path segment ends after it.""" + return re.sub(re.escape(SEED_HOME) + r"(?![\w.@+~-])", lambda _: str(home), text) def _respelled_utf16(data: bytes, home: Path) -> bytes: @@ -57,7 +77,7 @@ def _respelled_utf16(data: bytes, home: Path) -> bytes: text = data[len(mark):].decode(codec) except UnicodeDecodeError: return data - return mark + _respelled(text, home).encode(codec) if CAPTURED_HOME in text else data + return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data return data @@ -81,7 +101,7 @@ def _taskbar_window(window_class: str) -> bool: def _seed_home(home: Path) -> None: - """Copy the captured files into this computer's home; text files, UTF-16 ones too, have the captured home path + """Copy the captured files into this computer's home; text files, UTF-16 ones too, have the seed's home path respelled.""" seed = HERE / "seed" for source in sorted(seed.rglob("*")) if seed.is_dir() else []: @@ -189,10 +209,25 @@ def _keyfile_line(raw: str) -> tuple: return ("entry", key.strip(), value.strip()) if equals and key else ("other",) +def _each_line(text: str): + """Each line of text as REA splits them, one at a time.""" + start = 0 + for line_break in LINE_BREAK.finditer(text): + yield text[start:line_break.start()] + start = line_break.end() + yield text[start:] + + def _is_keyfile(text: str) -> bool: - """REA's rule: at least one group, and nothing a key file view would drop.""" - kinds = {_keyfile_line(line)[0] for line in LINE_BREAK.split(text)} - return "group" in kinds and "other" not in kinds + """REA's rule: at least one group, and nothing a key file view would drop. Lines are read one at a time, stopping + at the first it would drop.""" + group = False + for line in _each_line(text): + kind = _keyfile_line(line)[0] + if kind == "other": + return False + group = group or kind == "group" + return group def _keyfile(text: str) -> dict: @@ -206,56 +241,7 @@ def _keyfile(text: str) -> dict: return groups -class _XmlTree(ElementTree.TreeBuilder): - """Keeps comments and processing instructions, which REA's node limit counts, and refuses DOCTYPEs as REA does.""" - - def __init__(self) -> None: - super().__init__(insert_comments=True, insert_pis=True) - - def doctype(self, name, pubid, system) -> None: - raise ValueError("the document declares a DOCTYPE") - - -def _xml_children(element) -> list: - nodes = [element.text] if element.text else [] - for child in element: - nodes.append(child) - if child.tail: - nodes.append(child.tail) - return nodes - - -def _xml(node, depth: int, budget: dict, scope: tuple): - """A node as REA's XML view keeps it: an element with its attributes and children, or its text; None when it is - formatting whitespace, a comment, or past REA's node and depth limits.""" - if budget["remaining"] <= 0 or depth > MAX_XML_DEPTH: - budget["truncated"] = True - return None - budget["remaining"] -= 1 - preserve, element_content = scope - if isinstance(node, str): - return None if not node.strip() and element_content and not preserve else node - if not isinstance(node.tag, str): - return None - space = node.get(XML_SPACE) - nodes = _xml_children(node) - inner = (space == "preserve" if space else preserve, - any(not isinstance(child, str) and isinstance(child.tag, str) for child in nodes)) - children = [_xml(child, depth + 1, budget, inner) for child in nodes] - return {"name": node.tag, "attributes": dict(node.attrib), "children": [c for c in children if c is not None]} - - -def _xml_view(text: str): - parser = ElementTree.XMLParser(target=_XmlTree()) - try: - parser.feed(text) - root = parser.close() - except (ElementTree.ParseError, ValueError): - return None - budget = {"remaining": MAX_XML_NODES, "truncated": False} - tree = _xml(root, 0, budget, (False, False)) - return None if tree is None else {"root": tree, "truncated": budget["truncated"]} - +${PYTHON_XML_READER} def _not_json(constant: str): raise ValueError(f"{constant} is not JSON") @@ -296,7 +282,7 @@ def _exact_number(text: str): _INEXACT = object() -_INEXACT_END = object() +_LEVEL_END = object() def _number_or_mark(spelling: str): @@ -312,8 +298,8 @@ def _holds_inexact(value) -> bool: iterator per nesting level, so no child list is copied.""" levels = [iter((value,))] while levels: - item = next(levels[-1], _INEXACT_END) - if item is _INEXACT_END: + item = next(levels[-1], _LEVEL_END) + if item is _LEVEL_END: levels.pop() elif item is _INEXACT: return True @@ -324,25 +310,92 @@ def _holds_inexact(value) -> bool: return False -def _text_view(name: str, text: str) -> dict: - """The format REA's state view detects for decoded text, and its view.""" +def _deeper_than(root, limit: int) -> bool: + """Whether a parsed JSON value nests deeper than a limit, as REA checks it: a value's depth is the number of + iterators open above it, one per nesting level, so no child list is copied.""" + levels = [iter((root,))] + while levels: + item = next(levels[-1], _LEVEL_END) + if item is _LEVEL_END: + levels.pop() + elif len(levels) - 1 > limit: + return True + elif isinstance(item, dict): + levels.append(iter(item.values())) + elif isinstance(item, list): + levels.append(iter(item)) + return False + + +def _is_json(text: str) -> bool: + """Whether text is one JSON value as REA's state view reads it, checked a token at a time without building it or + recursing: it holds a byte per open container and one string at a time, so text too deep for Python's parser or + too large to parse into a view is still told apart from other text.""" + containers, expect, empty, at = bytearray(), "value", False, 0 + while True: + token = JSON_TOKEN.match(text, at) + if token is None: + break + kind, at = token.lastindex, token.end() + if kind == 5: + try: + at = json.decoder.scanstring(text, at)[1] + except ValueError: + return False + if kind == 1 and expect == "value": + containers.append(ord(text[at - 1])) + expect, empty = ("value" if text[at - 1] == "[" else "key"), True + # A closing bracket's code is its opening's plus two. + elif kind == 2 and (expect == "after" or empty) and containers and containers[-1] == ord(text[at - 1]) - 2: + containers.pop() + expect, empty = "after", False + elif kind == 3 and expect == "after" and containers: + expect = "value" if containers[-1] == ord("[") else "key" + elif kind == 4 and expect == "colon": + expect = "value" + elif kind >= 5 and (expect == "value" or (kind == 5 and expect == "key")): + expect, empty = ("colon" if expect == "key" else "after"), False + else: + return False + return expect == "after" and not containers and JSON_SPACE.fullmatch(text, at) is not None + + +def _json_view(text: str, size: int): + """A JSON text's format and view, or why it has none, as REA reads it; None when it is not JSON. Text over the + parsed-view limit, or nested past Python's recursion limit, is told apart without building its value.""" + if size > MAX_PARSED_BYTES: + return {"format": "json", "view": None, "error": PARSED_TOO_LARGE} if _is_json(text) else None + # The whole text must parse, and duplicate keys settle, before an inexact number makes it JSON with an error. + try: + view = json.loads(text, parse_constant=_not_json, parse_float=_number_or_mark, parse_int=_number_or_mark) + except RecursionError: + # Nesting too deep for Python to parse is far deeper than REA's view keeps, once the rest of it is JSON too. + return {"format": "json", "view": None, "error": JSON_TOO_DEEP} if _is_json(text) else None + except ValueError: + return None + if _holds_inexact(view): + return {"format": "json", "view": None, + "error": "a number in it cannot be read exactly, such as an integer above 2^53"} + if _deeper_than(view, MAX_JSON_VIEW_DEPTH): + return {"format": "json", "view": None, "error": JSON_TOO_DEEP} + return {"format": "json", "view": view, "truncated": False} + + +def _text_view(name: str, text: str, size: int) -> dict: + """The format REA's state view detects for decoded text of size bytes, and its view; a JSON or key file over the + parsed-view limit is refused before its value is built.""" trimmed = re.sub(r"^[\s]+", "", text) if name.endswith(".json") or trimmed[:1] in ("[", "{"): - # The whole text must parse, and duplicate keys settle, before an inexact number makes it JSON with an error. - try: - view = json.loads(text, parse_constant=_not_json, parse_float=_number_or_mark, parse_int=_number_or_mark) - except ValueError: - pass - else: - if _holds_inexact(view): - return {"format": "json", "view": None, - "error": "a number in it cannot be read exactly, such as an integer above 2^53"} - return {"format": "json", "view": view, "truncated": False} + view = _json_view(text, size) + if view is not None: + return view lowered = name.lower() if lowered.endswith(XML_SUFFIXES) or trimmed.startswith(" MAX_PARSED_BYTES: + return {"format": "keyfile", "view": None, "error": PARSED_TOO_LARGE} return {"format": "keyfile", "view": {"groups": _keyfile(text)}, "truncated": False} return _text(text) @@ -358,13 +411,17 @@ def _view(path: Path) -> dict: data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b"" if data.startswith(SQLITE_HEADER): return _sqlite(path) + if data.startswith(ZIP_HEADER): + return _zip(path) if len(data) > MAX_VIEW_BYTES: return too_large try: text = data.decode("utf-8-sig") if whole else None except UnicodeDecodeError: text = None - return {"format": "binary", "view": None, "truncated": False} if text is None else _text_view(path.name, text) + if text is None: + return {"format": "binary", "view": None, "truncated": False} + return _text_view(path.name, text, len(data)) def _views(home: Path, paths: list) -> dict: @@ -381,12 +438,12 @@ def _views(home: Path, paths: list) -> dict: except (OSError, ValueError, sqlite3.Error, RecursionError) as error: files[key] = {"format": None, "view": None, "error": str(error)} return files -`; +`.concat(PYTHON_ZIP_HELPERS); /** Constants the class module assigns before the helpers. */ const PYTHON_CONSTANTS = [ "HERE", - "CAPTURED_HOME", + "SEED_HOME", "PATHS", "DCONF", "TEXT_SEEDS", diff --git a/src/domain/desktop/appExportPythonXml.ts b/src/domain/desktop/appExportPythonXml.ts new file mode 100644 index 0000000000..fadd5d40d3 --- /dev/null +++ b/src/domain/desktop/appExportPythonXml.ts @@ -0,0 +1,101 @@ +/** The generated module's XML reader: REA's XML view of a document, built with expat. */ +export const PYTHON_XML_READER = String.raw`def _expat(namespaces: bool): + """An expat parser that refuses DOCTYPEs as REA does; with namespaces it also enforces their rules, as REA's + parser does, and without them it reports names as the document spells them.""" + parser = expat.ParserCreate(namespace_separator=" " if namespaces else None) + + def doctype(*_) -> None: + raise ValueError("the document declares a DOCTYPE") + + parser.StartDoctypeDeclHandler = doctype + return parser + + +class _XmlReader: + """REA's XML view built as expat reads: names as spelled, declarations as attributes, text split where REA's DOM + splits it (an empty CDATA section joins the text around it), and each node counted against REA's limits.""" + + def __init__(self) -> None: + self.remaining, self.truncated, self.root = MAX_XML_NODES, False, None + #: Per open element: its view or None when left out, whether xml:space="preserve" applies inside it, and + #: whether it holds an element, which makes whitespace-only text in it formatting. + self.open = [] + self.text, self.cdata = [], None + + def read(self, text: str) -> None: + parser = _expat(False) + parser.buffer_text = True + parser.StartElementHandler, parser.EndElementHandler = self._start, self._end + parser.CharacterDataHandler = self._data + parser.StartCdataSectionHandler, parser.EndCdataSectionHandler = self._cdata_start, self._cdata_end + parser.CommentHandler = parser.ProcessingInstructionHandler = self._other + parser.Parse(text, True) + + def _admit(self) -> bool: + """Whether the next node inside a kept element is kept, within the node and depth limits; one past them + marks the view truncated.""" + if self.open and self.open[-1][0] is None: + return False + if self.remaining <= 0 or len(self.open) > MAX_XML_DEPTH: + self.truncated = True + return False + self.remaining -= 1 + return True + + def _text_node(self, text: str) -> None: + if self._admit(): + self.open[-1][0]["children"].append(text) + + def _flush(self) -> None: + if self.text: + text, self.text = "".join(self.text), [] + self._text_node(text) + + def _data(self, text: str) -> None: + (self.text if self.cdata is None else self.cdata).append(text) + + def _cdata_start(self) -> None: + self.cdata = [] + + def _cdata_end(self) -> None: + text, self.cdata = "".join(self.cdata), None + if text: + self._flush() + self._text_node(text) + + def _other(self, *_) -> None: + """A comment or processing instruction, counted like any node and never kept.""" + self._flush() + if self.open: + self._admit() + + def _start(self, name: str, attributes: dict) -> None: + self._flush() + parent = self.open[-1] if self.open else [None, False, False] + parent[2] = True + view = {"name": name, "attributes": attributes, "children": []} if self._admit() else None + if view is not None and parent[0] is not None: + parent[0]["children"].append(view) + space = attributes.get(XML_SPACE) + self.open.append([view, space == "preserve" if space else parent[1], False]) + + def _end(self, _name: str) -> None: + self._flush() + view, preserve, holds_element = self.open.pop() + if view is not None and holds_element and not preserve: + view["children"] = [child for child in view["children"] if not isinstance(child, str) or child.strip()] + if not self.open: + self.root = view + + +def _xml_view(text: str): + """REA's XML view, or None for a document REA's parser refuses: a pass with namespaces checks the document as REA's + parser does, then a pass without them reads names as spelled, as REA's view keeps them.""" + reader = _XmlReader() + try: + _expat(True).Parse(text, True) + reader.read(text) + except (expat.ExpatError, ValueError): + return None + return {"root": reader.root, "truncated": reader.truncated} +`; diff --git a/src/domain/desktop/appExportPythonZip.ts b/src/domain/desktop/appExportPythonZip.ts new file mode 100644 index 0000000000..c20d197dd1 --- /dev/null +++ b/src/domain/desktop/appExportPythonZip.ts @@ -0,0 +1,237 @@ +/** + * The exported module's zip reader: the central directory read a record at a + * time, entries read in bounded chunks, and each entry's view by the rules a + * file's follow, with REA's limits and wording. Part of the module's helpers. + */ +export const PYTHON_ZIP_HELPERS = String.raw` +MAX_ZIP_ENTRIES = 1000 +MAX_ZIP_ENTRY_BYTES = 16 * 1024 * 1024 +MAX_ZIP_BYTES = 64 * 1024 * 1024 +MAX_ZIP_RATIO = 100 +ZIP_READ_CHUNK = 64 * 1024 +ZIP_END = b"PK\x05\x06" +ZIP_CENTRAL = b"PK\x01\x02" +ZIP64_LOCATOR = b"PK\x06\x07" +ZIP64_END = b"PK\x06\x06" +#: Formats an archive entry may have whose reader needs a tool, a worker or another archive opened. +NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"), + (ZIP_HEADER, "zip", "an archive")) + + +def _entry_view(name: str, data: bytes) -> dict: + """An archive entry's format and view by the rules a file's follow; a database or an archive inside is not + read.""" + for header, form, inside in NOT_READ_INSIDE: + if data.startswith(header): + return {"format": form, "view": None, "error": f"{inside} inside an archive is not read"} + try: + text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None + except UnicodeDecodeError: + text = None + if text is None: + return {"format": "binary", "view": None} + return _text_view(name, text, len(data)) + + +def _zip_refusal(info, spent: int): + """Why an entry is not read, checked in REA's order, or None; spent is the archive's bytes read so far.""" + if info.flag_bits & 0x1: + return "encrypted, so it is not read" + if info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): + return f"compressed with method {info.compress_type}, which the view does not read" + if info.file_size > MAX_ZIP_ENTRY_BYTES: + return "larger than the 16 MiB an entry's view reads" + if info.file_size > MAX_ZIP_RATIO * info.compress_size: + return f"expands more than {MAX_ZIP_RATIO} times, so it is not read" + if spent + info.file_size > MAX_ZIP_BYTES: + return "past the 64 MiB read from one archive" + return None + + +def _entry_bytes(archive, info): + """An entry's uncompressed bytes, or None when they fail its CRC-32 or are not exactly its declared size. Reading + stops one byte past that size, which data past a short declared size fills, and asks for a bounded chunk at a + time, which bounds what the decompressor expands per call, so a forged small size holds no more than that.""" + probe = copy.copy(info) + probe.file_size = info.file_size + 1 + data = bytearray() + try: + with archive.open(probe) as member: + while len(data) <= info.file_size: + chunk = member.read(min(probe.file_size - len(data), ZIP_READ_CHUNK)) + if not chunk: + break + data += chunk + except (zipfile.BadZipFile, zlib.error, EOFError, OSError, ValueError): + return None + return data if len(data) == info.file_size else None + + +def _zip_entry(archive, info, refusal): + """One entry of a zip view: its name, size and digest, and its own format and view, or why it was not read.""" + entry = {"name": info.filename, "size": info.file_size, "sha256": None, "format": None, "view": None, + "view_error": refusal} + if refusal is not None: + return entry + data = _entry_bytes(archive, info) + if data is None: + return {**entry, "view_error": "its compressed data is damaged"} + view = _entry_view(info.filename, data) + return {**entry, "sha256": hashlib.sha256(data).hexdigest(), "format": view["format"], + "view": view.get("view"), "view_error": view.get("error")} + + +def _zip_end(file, size: int): + """Where an archive's central directory ends, how long it is and the offset it declares, from the last end + record whose comment reaches the end of the file and, when a Zip64 locator precedes it whatever that record + holds, the Zip64 one just before the locator; None without an end record, or when sentinels defer to a + Zip64 record that is missing.""" + window = min(size, 22 + 0xFFFF) + file.seek(size - window) + tail = file.read(window) + at = tail.rfind(ZIP_END, 0, len(tail) - 18) + while at >= 0 and at + 22 + int.from_bytes(tail[at + 20:at + 22], "little") != len(tail): + at = tail.rfind(ZIP_END, 0, at + 3) + if at < 0: + return None + total, length, offset = struct.unpack_from("= 5 and len(data) == length and data[0] == 1 and not flags & 0x800 + and int.from_bytes(data[1:5], "little") == zlib.crc32(raw)) + return data[5:] if usable else None + at += 4 + length + return None + + +def _zip_ends_in_slash(raw: bytes, flags: int, extra: bytes): + """Whether an entry's name, as REA decodes it, ends in a slash; None when it must be UTF-8 and is not.""" + utf8 = _zip_unicode_path(raw, flags, extra) + if utf8 is None and flags & 0x800: + utf8 = raw + if utf8 is None: + return raw.endswith(b"/") + try: + return utf8.decode("utf-8").endswith("/") + except UnicodeDecodeError: + return None + + +def _zip_directory(file, size: int, keep: int): + """An archive's central directory read a record at a time, keeping the first file entries' records, as many as keep allows, and + counting the rest, as REA reads it: (start, declared offset, records, count), or None when it cannot be read. + Directories are told apart as zip.js tells them: a final slash, a Unix directory mode, or the MS-DOS attribute.""" + located = _zip_end(file, size) + if located is None: + return None + end, length, declared = located + start = end - length + if start < 0: + return None + file.seek(start) + records, count, at = [], 0, start + while at < end: + fixed = file.read(46) + if at + 46 > end or len(fixed) < 46 or fixed[:4] != ZIP_CENTRAL: + return None + made_by, flags = struct.unpack_from(" end: + return None + slash = _zip_ends_in_slash(rest[:name_length], flags, rest[name_length:name_length + extra_length]) + if slash is None: + return None + if not (slash or (external >> 16) & 0o170000 == 0o040000 or (made_by >> 8 == 0 and external & 0x10)): + count += 1 + if len(records) < keep: + records.append(fixed + rest) + return start, declared, records, count + + +class _BoundedArchive(io.RawIOBase): + """An archive as zipfile reads it: its entries in place, then a directory of the kept records alone and end + records for them, so zipfile parses no more records than the view lists.""" + + def __init__(self, file, start: int, tail: bytes) -> None: + super().__init__() + self._file, self._start, self._tail, self._at = file, start, tail, 0 + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._at + + def seek(self, offset: int, whence: int = 0) -> int: + self._at = offset + (0, self._at, self._start + len(self._tail))[whence] + return self._at + + def readinto(self, buffer) -> int: + view = memoryview(buffer) + if self._at < self._start: + self._file.seek(self._at) + count = self._file.readinto(view[:self._start - self._at]) + else: + chunk = self._tail[self._at - self._start:self._at - self._start + len(view)] + count = len(chunk) + view[:count] = chunk + self._at += count + return count + + +def _zip(path: Path) -> dict: + """A zip archive's file entries in archive order, up to REA's entry, size, ratio and archive limits; nothing is + extracted to disk, no archive inside is opened, and zipfile is shown only the directory records listed.""" + entries, spent = [], 0 + try: + with path.open("rb") as file: + directory = _zip_directory(file, os.fstat(file.fileno()).st_size, MAX_ZIP_ENTRIES) + if directory is None: + raise zipfile.BadZipFile("no readable central directory") + start, declared, records, count = directory + kept = b"".join(records) + offset = declared if declared <= 0xFFFFFFFF else start + # zipfile follows a Zip64 locator signature wherever one sits before the end record, and the kept records + # may end in those bytes, so a real Zip64 end record and locator for them come first. + tail = (kept + + struct.pack("<4sQ2H2I4Q", ZIP64_END, 44, 45, 45, 0, 0, len(records), len(records), len(kept), offset) + + struct.pack("<4sIQI", ZIP64_LOCATOR, 0, offset + len(kept), 1) + + struct.pack("<4s4H2IH", ZIP_END, 0, 0, len(records), len(records), len(kept), offset, 0)) + with zipfile.ZipFile(_BoundedArchive(file, start, tail)) as archive: + for info in archive.infolist(): + refusal = _zip_refusal(info, spent) + spent += info.file_size if refusal is None else 0 + entries.append(_zip_entry(archive, info, refusal)) + except (zipfile.BadZipFile, zlib.error, EOFError, NotImplementedError, OSError, ValueError, struct.error): + return {"format": "zip", "view": None, "error": "not a readable zip archive"} + view = {"entries": entries, "entry_count": count, "truncated": count > len(entries)} + return {"format": "zip", "view": view, "truncated": view["truncated"]} +`; diff --git a/src/domain/desktop/appTask.ts b/src/domain/desktop/appTask.ts index 4272eefa15..8cc0566ace 100644 --- a/src/domain/desktop/appTask.ts +++ b/src/domain/desktop/appTask.ts @@ -48,7 +48,11 @@ const checkSchema = z.discriminatedUnion("kind", [ contains: line.optional(), ...comparison, }), - kind("value_equals", { pointer: z.string(), value: jsonValueSchema }), + kind("value_equals", { + pointer: z.string(), + value: jsonValueSchema, + tolerance: z.number().min(0).optional(), + }), kind("value_absent", { pointer: z.string() }), kind("row_present", { table, where }), kind("row_absent", { table, where }), @@ -143,6 +147,12 @@ const checkProblem = (check: DesktopTaskCheck): string | undefined => { new Set(check.lines).size !== check.lines.length ) return "it lists a line twice"; + if ( + check.kind === "value_equals" && + check.tolerance !== undefined && + typeof check.value !== "number" + ) + return "tolerance compares numbers, so value must be a number"; const stamped = expectedValues(check).find(isTimestamp); return stamped === undefined ? undefined diff --git a/src/domain/desktop/appTaskChecks.fixture.ts b/src/domain/desktop/appTaskChecks.fixture.ts index 5e6c8ffa43..702ed50830 100644 --- a/src/domain/desktop/appTaskChecks.fixture.ts +++ b/src/domain/desktop/appTaskChecks.fixture.ts @@ -1,3 +1,4 @@ +import type { JsonValue } from "../jsonValue.js"; import type { DesktopTaskCheckInput } from "./appTask.js"; import type { TaskFileView } from "./appTaskChecks.js"; @@ -24,6 +25,7 @@ export const CHECK_VIEWS: Readonly> = { zero: 0, tags: ["a", "b"], "a/b": { "~k": null }, + scale: "1.2500", }, }, ".config/notes/app.ini": { @@ -60,6 +62,110 @@ export const CHECK_VIEWS: Readonly> = { format: "sqlite", view: { tables: {}, tables_truncated: true }, }, + "Documents/plate.FCStd": { + format: "zip", + view: { + entries: [ + { + name: "Document.xml", + size: 300, + sha256: "a".repeat(64), + format: "xml", + view: { + root: { + name: "Document", + attributes: {}, + children: [ + element("Object", { name: "Plate", type: "Part::Box" }), + element("Object", { name: "Hole", type: "Part::Cylinder" }, [ + element("Property", { name: "Radius" }, [ + element("Float", { value: "5.0000000000000000" }), + ]), + ]), + element("Object", { name: "Pad", type: "Part::Box" }), + "saved by hand", + ], + }, + truncated: false, + }, + view_error: null, + }, + { + name: "thumbnails/Thumbnail.png", + size: 3, + sha256: "b".repeat(64), + format: "binary", + view: null, + view_error: null, + }, + { + name: "shape.brp", + size: 9000, + sha256: "c".repeat(64), + format: "text", + view: { lines: ["a"], line_count: 6000, truncated: true }, + view_error: null, + }, + { + name: "huge.brp", + size: 17_000_000, + sha256: null, + format: null, + view: null, + view_error: "larger than the 16 MiB an entry's view reads", + }, + { + name: "empty.json", + size: 4, + sha256: "f".repeat(64), + format: "json", + view: null, + view_error: null, + }, + ], + entry_count: 5, + truncated: false, + }, + }, + "Documents/many.zip": { + format: "zip", + view: { entries: [], entry_count: 1200, truncated: true }, + }, + "Documents/cut.xml": { + format: "xml", + view: { + root: element("doc", {}, [element("p", { id: "1" }), "kept"]), + truncated: true, + }, + }, + "Documents/listing.zip": { + format: "zip", + view: { + entries: [ + { + name: "a.json", + size: 7, + sha256: "d".repeat(64), + format: "json", + view: { x: 1 }, + view_error: null, + }, + { + name: "page.xml", + size: 9000, + sha256: "e".repeat(64), + format: "xml", + view: { + root: element("page", {}, [element("p", { id: "1" })]), + truncated: true, + }, + view_error: null, + }, + ], + entry_count: 1500, + truncated: true, + }, + }, "Documents/photo.png": { format: "binary", view: null }, "Documents/huge.txt": { format: null, @@ -68,10 +174,318 @@ export const CHECK_VIEWS: Readonly> = { }, }; +/** An element of REA's XML view. */ +function element( + name: string, + attributes: Record, + children: JsonValue[] = [], +): JsonValue { + return { name, attributes, children }; +} + const todo = "Documents/todo.txt"; const settings = ".config/notes/settings.json"; const db = ".local/share/notes/notes.db"; const absent = "Documents/absent.txt"; +const plate = "Documents/plate.FCStd"; +const objects = "/entries/[name=Document.xml]/view/root/children"; +const radius = `${objects}/[name=Object][@name=Hole]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value`; + +/** Selectors over arrays a view may hold only part of stay undecided; positions stay decided on what it holds. */ +const PARTIAL_SELECTOR_CASES: readonly (readonly [ + DesktopTaskCheckInput, + boolean, +])[] = [ + [ + { + kind: "value_equals", + file: "Documents/cut.xml", + pointer: "/root/children/[name=p]/attributes/id", + value: "1", + }, + false, + ], + [ + { + kind: "value_equals", + file: "Documents/cut.xml", + pointer: "/root/children/0/attributes/id", + value: "1", + }, + true, + ], + [ + { + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/[name=a.json]/view/x", + value: 1, + }, + false, + ], + [ + { + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/0/view/x", + value: 1, + }, + true, + ], + [ + { + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/1/view/root/children/[name=p]/attributes/id", + value: "1", + }, + false, + ], +]; + +/** Selectors: one member matched, none, several, or a malformed one, through a zip view. */ +const SELECTOR_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [ + [ + { + kind: "value_equals", + file: plate, + pointer: `${objects}/[name=Object][@name=Hole]/attributes/type`, + value: "Part::Cylinder", + }, + true, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: `${objects}/[name=Object][@type=Part::Box]/attributes/name`, + value: "Plate", + }, + false, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: `${objects}/[name=Object][@name=Gone]/attributes/name`, + value: "Gone", + }, + false, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: `${objects}/[name=Object][@name=Gone]`, + value: "Gone", + }, + false, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: `${objects}/[name=Object][@name=Gone]`, + }, + true, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: `${objects}/[name=Object][@name=Gone]/attributes/type`, + }, + false, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: `${objects}/[name=Object][@type=Part::Box]`, + }, + false, + ], + [ + { kind: "value_equals", file: settings, pointer: "/tags/[x]", value: "a" }, + false, + ], + [{ kind: "value_absent", file: settings, pointer: "/tags/[name=a]" }, true], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/[name=thumbnails~1Thumbnail.png]/format", + value: "binary", + }, + true, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/0/name", + value: "Document.xml", + }, + true, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: "/entries/[name=Document.xml]/view/root/attributes/x", + }, + true, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: "/entries/[name=shape.brp]/view/lines/5", + }, + false, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: "/entries/[name=huge.brp]/view/lines/0", + }, + false, + ], + [ + { kind: "value_absent", file: plate, pointer: "/entries/[name=gone.txt]" }, + true, + ], + [ + { + kind: "value_absent", + file: "Documents/many.zip", + pointer: "/entries/[name=gone.txt]", + }, + false, + ], + [{ kind: "lines_present", file: plate, lines: ["a"] }, false], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/[name=huge.brp]/view", + value: null, + }, + false, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/[name=empty.json]/view", + value: null, + }, + true, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/[name=thumbnails~1Thumbnail.png]/view", + value: null, + }, + false, + ], + [ + { + kind: "value_absent", + file: plate, + pointer: "/entries/[name=thumbnails~1Thumbnail.png]/view/x", + }, + false, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: "/entries/[name=thumbnails~1Thumbnail.png]/size", + value: 3, + }, + true, + ], + ...PARTIAL_SELECTOR_CASES, +]; + +/** Numbers within a tolerance, numeral strings read as numbers, anything else refused. */ +const TOLERANCE_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = + [ + [ + { + kind: "value_equals", + file: plate, + pointer: radius, + value: 5, + tolerance: 0.01, + }, + true, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: radius, + value: 5.1, + tolerance: 0.01, + }, + false, + ], + [ + { + kind: "value_equals", + file: plate, + pointer: `${objects}/[name=Object][@name=Hole]/attributes/type`, + value: 5, + tolerance: 1, + }, + false, + ], + [ + { + kind: "value_equals", + file: settings, + pointer: "/ratio", + value: 0.49, + tolerance: 0.02, + }, + true, + ], + [ + { + kind: "value_equals", + file: settings, + pointer: "/wrap", + value: 1, + tolerance: 0, + }, + false, + ], + [ + { + kind: "value_equals", + file: settings, + pointer: "/scale", + value: 1.25, + tolerance: 0, + }, + true, + ], + [ + { + kind: "value_equals", + file: settings, + pointer: "/missing", + value: 1, + tolerance: 0, + }, + false, + ], + ]; /** Every kind of check against those views, each with the outcome both evaluators must give. */ export const CHECK_CASES: readonly (readonly [ @@ -319,4 +733,6 @@ export const CHECK_CASES: readonly (readonly [ [{ kind: "row_count", file: db, table: "notes", at_least: 3 }, false], [{ kind: "row_count", file: db, table: "big", at_least: 1 }, false], [{ kind: "row_count", file: absent, table: "notes", equals: 0 }, true], + ...SELECTOR_CASES, + ...TOLERANCE_CASES, ]; diff --git a/src/domain/desktop/appTaskChecks.test.ts b/src/domain/desktop/appTaskChecks.test.ts index be86236d4c..a050a5148a 100644 --- a/src/domain/desktop/appTaskChecks.test.ts +++ b/src/domain/desktop/appTaskChecks.test.ts @@ -118,6 +118,34 @@ const REFUSALS: readonly (readonly [Record, string])[] = [ { checks: [{ kind: "row_present", file: "a", table: "t", where: {} }] }, "name at least one column", ], + [ + { + checks: [ + { + kind: "value_equals", + file: "a", + pointer: "/x", + value: "5", + tolerance: 0.1, + }, + ], + }, + "tolerance compares numbers, so value must be a number", + ], + [ + { + checks: [ + { + kind: "value_equals", + file: "a", + pointer: "/x", + value: 5, + tolerance: -1, + }, + ], + }, + "at checks.0.tolerance", + ], ]; describe("evaluateTaskChecks", () => { @@ -152,6 +180,136 @@ describe("evaluateTaskChecks", () => { }); }); +describe("evaluateTaskChecks selectors and tolerance", () => { + const plate = "Documents/plate.FCStd"; + const objects = "/entries/[name=Document.xml]/view/root/children"; + const outcome = (check: Record) => + evaluateTaskChecks( + [{ id: "c", file: plate, ...check } as never], + new Map(Object.entries(CHECK_VIEWS)), + )[0]; + + it("leaves a selector undecided where the array it searches may be cut short, and a position decided", () => { + const cut = + "the view of Documents/listing.zip is truncated, so what it leaves out is unknown"; + + expect( + outcome({ + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/[name=a.json]/view/x", + value: 1, + }), + ).toEqual({ id: "c", passed: false, description: cut }); + expect( + outcome({ + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/1/view/root/children/[name=p]/attributes/id", + value: "1", + }), + ).toEqual({ id: "c", passed: false, description: cut }); + expect( + outcome({ + kind: "value_equals", + file: "Documents/listing.zip", + pointer: "/entries/0/view/x", + value: 1, + })?.passed, + ).toBe(true); + }); + + it("fails a check into an entry's view that could not be read, where JSON null passes", () => { + expect( + outcome({ + kind: "value_equals", + pointer: "/entries/[name=huge.brp]/view", + value: null, + }), + ).toEqual({ + id: "c", + passed: false, + description: + "Documents/plate.FCStd entry huge.brp has no view: larger than the 16 MiB an entry's view reads", + }); + expect( + outcome({ + kind: "value_equals", + pointer: "/entries/[name=empty.json]/view", + value: null, + })?.passed, + ).toBe(true); + expect( + outcome({ + kind: "value_equals", + pointer: "/entries/[name=thumbnails~1Thumbnail.png]/view", + value: null, + }), + ).toEqual({ + id: "c", + passed: false, + description: + "Documents/plate.FCStd entry thumbnails/Thumbnail.png is binary and has no view", + }); + }); + + it("fails a selector that matches several members or, on the way, none, and says which", () => { + expect( + outcome({ + kind: "value_equals", + pointer: `${objects}/[name=Object][@type=Part::Box]/attributes/name`, + value: "Plate", + }), + ).toMatchObject({ + passed: false, + description: `${plate} at ${objects}/[name=Object][@type=Part::Box]/attributes/name: [name=Object][@type=Part::Box] matches 2 members, not one`, + }); + expect( + outcome({ + kind: "value_absent", + pointer: `${objects}/[name=Object][@name=Gone]/attributes/type`, + })?.description, + ).toMatch(/: \[name=Object\]\[@name=Gone\] matches no member$/u); + expect( + outcome({ + kind: "value_equals", + pointer: `${objects}/[name=Object][@name=Gone]`, + value: 1, + })?.description, + ).toMatch(/matches no member$/u); + expect( + outcome({ + kind: "value_equals", + pointer: `${objects}/[Object]`, + value: 1, + })?.description, + ).toMatch(/: \[Object\] is not a selector of the form \[KEY=VALUE\]$/u); + }); + + it("compares numbers and numeral strings within a tolerance, and nothing else", () => { + const radius = `${objects}/[name=Object][@name=Hole]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value`; + + expect( + outcome({ + kind: "value_equals", + pointer: radius, + value: 5.1, + tolerance: 0.01, + })?.description, + ).toBe( + `${plate} at ${radius} is "5.0000000000000000", not within 0.01 of 5.1`, + ); + expect( + outcome({ + kind: "value_equals", + pointer: `${objects}/[name=Object][@name=Hole]/attributes/type`, + value: 5, + tolerance: 1, + })?.description, + ).toMatch(/is "Part::Cylinder", not a number$/u); + }); +}); + describe("parseDesktopTask", () => { it("settles ids and keeps the prompt trimmed", () => { const parsed = parseDesktopTask( diff --git a/src/domain/desktop/appTaskChecks.ts b/src/domain/desktop/appTaskChecks.ts index 96fcee4794..bed05c4dc1 100644 --- a/src/domain/desktop/appTaskChecks.ts +++ b/src/domain/desktop/appTaskChecks.ts @@ -1,6 +1,12 @@ import type { JsonValue } from "../jsonValue.js"; import { countComparison, type DesktopTaskCheck } from "./appTask.js"; import type { DesktopStatePath } from "./desktopState.js"; +import { locatePointer, type Located } from "./viewPointer.js"; +import { + viewlessEntryAt, + zipArrayLeavesOut, + zipLeavesOutAt, +} from "./zipView.js"; /** One file as the app's data/get reports it: its view, or why it has none. An absent file has no entry. */ export interface TaskFileView { @@ -62,24 +68,15 @@ export const sameJson = ( return actual === expected; }; -/** The value a JSON pointer names, or undefined when it names none. */ -export const resolvePointer = ( - view: JsonValue | null, - pointer: string, -): JsonValue | undefined => { - let current: JsonValue | null = view; - for (const raw of pointer.slice(1).split("/")) { - const token = raw.replaceAll("~1", "/").replaceAll("~0", "~"); - if (Array.isArray(current)) { - if (!/^(?:0|[1-9][0-9]*)$/u.test(token)) return undefined; - const index = Number(token); - if (index >= current.length) return undefined; - current = current[index] ?? null; - } else if (isRecord(current) && Object.hasOwn(current, token)) - current = current[token] ?? null; - else return undefined; - } - return current ?? null; +/** A decimal numeral as FreeCAD and other apps write numbers in text, such as `5.0000000000000000`. */ +const NUMERAL = /^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$/u; + +/** A number, or a numeral string read as one; undefined for anything else or a numeral past the doubles. */ +const numberOf = (value: JsonValue): number | undefined => { + if (typeof value === "number") return value; + if (typeof value !== "string" || !NUMERAL.test(value)) return undefined; + const number = Number(value); + return Number.isFinite(number) ? number : undefined; }; /** A file's version as the app's data/get serves it, or undefined when it is absent. */ @@ -92,19 +89,6 @@ export const versionView = ( ? { format: version.format, view: version.view } : { format: version.format, view: null, error: version.view_error }; -/** Whether an XML view names a namespace, whose names the verifier's reader spells differently. */ -export const namesXmlNamespace = (node: JsonValue | null): boolean => { - if (Array.isArray(node)) return node.some(namesXmlNamespace); - if (!isRecord(node)) return false; - const named = (name: JsonValue | undefined) => - typeof name === "string" && (name.includes(":") || name === "xmlns"); - return ( - named(node.name) || - (isRecord(node.attributes) && Object.keys(node.attributes).some(named)) || - Object.values(node).some(namesXmlNamespace) - ); -}; - /** Whether a view says it holds only part of its file, as REA's own readers flag it. */ const truncatedView = (format: string | null, view: JsonValue | null) => { if (!isRecord(view)) return false; @@ -196,33 +180,85 @@ const lineVerdict = ( ); }; +/** A value compared within a tolerance: numbers, and numeral strings, by their difference. */ +const nearVerdict = ( + at: string, + found: JsonValue, + expected: number, + tolerance: number, +): Verdict => { + const number = numberOf(found); + const within = `within ${shown(tolerance)} of ${shown(expected)}`; + if (number === undefined) + return fail(`${at} is ${shown(found)}, not a number`); + return Math.abs(number - expected) <= tolerance + ? pass(`${at} is ${shown(found)}, ${within}`) + : fail(`${at} is ${shown(found)}, not ${within}`); +}; + +/** + * Why a value check cannot be decided on what the view holds: the entry whose + * view its pointer goes into has none, or a selector searched an array the view + * may hold only part of. Undefined when it can be. + */ +const unreadOrPartial = ( + check: Check<"value_equals" | "value_absent">, + format: string | null | undefined, + read: { readonly view: JsonValue | null; readonly truncated: boolean }, + located: Located, +): Verdict | undefined => { + const zip = format === "zip"; + const viewless = zip ? viewlessEntryAt(read.view, check.pointer) : undefined; + if (viewless !== undefined) + return fail(`${check.file} entry ${viewless.name} ${viewless.reason}`); + // A selector tells one member from several only in an array the view holds whole. + const partial = (array: string) => + zip ? zipArrayLeavesOut(read.view, array) : read.truncated; + return located.searched.some(partial) ? undecided(check.file) : undefined; +}; + const valueVerdict = ( check: Check<"value_equals" | "value_absent">, entry: TaskFileView | undefined, ): Verdict => { const read = viewOf(check, entry, undefined); if ("passed" in read) return read; - const found = - entry === undefined ? undefined : resolvePointer(read.view, check.pointer); + const located: Located = + entry === undefined + ? { kind: "none", searched: [], byPosition: [] } + : locatePointer(read.view, check.pointer); const at = `${check.file} at ${check.pointer}`; + const unknown = unreadOrPartial(check, entry?.format, read, located); + if (unknown !== undefined) return unknown; + if (located.kind === "unresolved") return fail(`${at}: ${located.problem}`); + // A zip view's truncation that matters is the entry the pointer goes into. + const truncated = + entry?.format === "zip" + ? zipLeavesOutAt(read.view, check.pointer) + : read.truncated; // A whole object or array equals the expected one only if nothing of it was left out. const whole = check.kind === "value_equals" && typeof check.value === "object" && check.value !== null; - if (whole && read.truncated) return undecided(check.file); - if (check.kind === "value_equals") - return found !== undefined && sameJson(found, check.value) + if (whole && truncated) return undecided(check.file); + if (check.kind === "value_equals") { + if (located.kind === "none") + return fail( + located.reason === undefined + ? `${at} has no value` + : `${at}: ${located.reason}`, + ); + if (check.tolerance !== undefined && typeof check.value === "number") + return nearVerdict(at, located.value, check.value, check.tolerance); + return sameJson(located.value, check.value) ? pass(`${at} is ${shown(check.value)}`) - : fail( - found === undefined - ? `${at} has no value` - : `${at} is ${shown(found)}, not ${shown(check.value)}`, - ); - if (read.truncated) return undecided(check.file); - return found === undefined + : fail(`${at} is ${shown(located.value)}, not ${shown(check.value)}`); + } + if (truncated) return undecided(check.file); + return located.kind === "none" ? pass(`${at} has no value`) - : fail(`${at} is ${shown(found)}`); + : fail(`${at} is ${shown(located.value)}`); }; /** A table's columns and rows; a table the view lacks has none, unless the view left tables out. */ diff --git a/src/domain/desktop/appTaskDraft.test.ts b/src/domain/desktop/appTaskDraft.test.ts index 3f3fe6b246..c44d5b4918 100644 --- a/src/domain/desktop/appTaskDraft.test.ts +++ b/src/domain/desktop/appTaskDraft.test.ts @@ -46,6 +46,7 @@ const draft = (files: Record) => draftTaskFiles( new Map(Object.entries(files)), (home) => home !== "elsewhere", + { seed: 0, end: 1 }, ); describe("draftTaskFiles", () => { @@ -269,6 +270,46 @@ describe("draftTaskFiles for structured views", () => { }); }); +describe("draftTaskFiles for XML that uses namespaces", () => { + it("suggests values inside an XML file by pointer, as for any XML", () => { + const poster = (headline: string): JsonValue => ({ + root: { + name: "svg", + attributes: { + xmlns: "http://www.w3.org/2000/svg", + "sodipodi:docname": "poster.svg", + }, + children: [ + { + name: "text", + attributes: { id: "headline" }, + children: [headline], + }, + ], + }, + truncated: false, + }); + + const [document] = draft({ + "Pictures/poster.svg": path( + "document", + "xml", + poster("Spring Bake Sale"), + poster("Summer Bake Sale"), + ), + }); + + expect(document?.suggested).toEqual([ + { + kind: "value_equals", + file: "Pictures/poster.svg", + pointer: "/root/children/0/children/0", + value: "Summer Bake Sale", + }, + ]); + }); +}); + describe("draftTaskFiles with partial evidence", () => { const notes = (...rows: JsonValue[][]) => ({ tables: { @@ -349,3 +390,465 @@ describe("draftTaskFiles with partial evidence", () => { }); }); }); + +/** An element of REA's XML view. */ +const xmlElement = ( + name: string, + attributes: Record = {}, + children: JsonValue[] = [], +): JsonValue => ({ name, attributes, children }); + +/** A zip view's entry, read, under a name. */ +const zipEntry = ( + name: string, + format: string, + view: JsonValue, +): JsonValue => ({ + name, + size: 1, + sha256: name, + format, + view, + view_error: null, +}); + +const listing = (entries: JsonValue[]): JsonValue => ({ + entries, + entry_count: entries.length, + truncated: false, +}); + +describe("draftTaskFiles archives added to whole", () => { + it("suggests checks inside a JSON value or a readable entry added whole", () => { + const [file] = draft({ + "Documents/a.zip": path( + "document", + "zip", + listing([zipEntry("a.json", "json", { a: 1 })]), + listing([ + zipEntry("a.json", "json", { a: 1, added: { x: 1 } }), + zipEntry("new.json", "json", { y: [2] }), + zipEntry("new.xml", "xml", { + root: xmlElement("doc"), + truncated: false, + }), + zipEntry("new.png", "binary", null), + ]), + ), + }); + + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=new.xml]/view/root/name", + value: "doc", + }, + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=a.json]/view/added/x", + value: 1, + }, + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=new.json]/view/y/0", + value: 2, + }, + ]); + }); + + it("checks an empty array or object added as a value, and nothing absent that a type change keeps", () => { + const [file] = draft({ + "Documents/a.zip": path( + "document", + "zip", + listing([zipEntry("a.json", "json", { a: 1, kept: [1] })]), + listing([ + zipEntry("a.json", "json", { + a: 1, + kept: { "0": 1 }, + items: [], + meta: {}, + }), + ]), + ), + }); + + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=a.json]/view/items", + value: [], + }, + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=a.json]/view/meta", + value: {}, + }, + ]); + }); + + it("suggests no element by its position, even at the last step", () => { + const page = (paragraphs: string[]) => + listing([ + zipEntry("page.xml", "xml", { + root: xmlElement( + "page", + {}, + paragraphs.map((text) => xmlElement("p", {}, [text])), + ), + truncated: false, + }), + ]); + + const [file] = draft({ + "Documents/page.zip": path( + "document", + "zip", + page(["a"]), + page(["a", "b"]), + ), + }); + + expect(file).toMatchObject({ change: { kind: "values" }, suggested: [] }); + }); +}); + +describe("draftTaskFiles archive elements by identity", () => { + it("checks an element replaced by another of its type as one added and one gone, not renamed", () => { + const document = (name: string, id: string) => + listing([ + zipEntry("Document.xml", "xml", { + root: xmlElement("Objects", {}, [ + xmlElement("Object", { type: "Part::Box", name: "Plate" }), + xmlElement("Object", { type: "Part::Cylinder", name, id }), + ]), + truncated: false, + }), + ]); + + const [file] = draft({ + "Documents/plate.zip": path( + "document", + "zip", + document("Hole", "2"), + document("Hole001", "3"), + ), + }); + + const objects = "/entries/[name=Document.xml]/view/root/children"; + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/plate.zip", + pointer: `${objects}/[name=Object][@name=Hole001]/name`, + value: "Object", + }, + { + kind: "value_absent", + file: "Documents/plate.zip", + pointer: `${objects}/[name=Object][@name=Hole]`, + }, + ]); + }); + + it("checks an edit to an element its identifier still names, not a replacement", () => { + const list = (color: string) => + listing([ + zipEntry("list.xml", "xml", { + root: xmlElement("list", {}, [ + xmlElement("item", { color, id: "1" }), + xmlElement("item", { id: "2" }), + ]), + truncated: false, + }), + ]); + + const [file] = draft({ + "Documents/list.zip": path("document", "zip", list("red"), list("green")), + }); + + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/list.zip", + pointer: + "/entries/[name=list.xml]/view/root/children/[name=item][@id=1]/attributes/color", + value: "green", + }, + ]); + }); +}); + +/** The checks drafted for `list.xml` in an archive whose `item` elements change. */ +const itemChecks = (before: JsonValue[], after: JsonValue[]) => { + const list = (items: JsonValue[]) => + listing([ + zipEntry("list.xml", "xml", { + root: xmlElement("list", {}, items), + truncated: false, + }), + ]); + const [file] = draft({ + "Documents/list.zip": path("document", "zip", list(before), list(after)), + }); + return file?.suggested; +}; + +const ITEMS = "/entries/[name=list.xml]/view/root/children"; + +describe("draftTaskFiles archive elements beside identified ones", () => { + it("counts an identified element's other attributes, so a selector it shares is not taken for absent", () => { + expect( + itemChecks( + [ + xmlElement("item", { id: "1", color: "red" }), + xmlElement("item", { color: "blue" }), + ], + [ + xmlElement("item", { id: "1", color: "blue" }), + xmlElement("item", { color: "red" }), + ], + ), + ).toEqual([ + { + kind: "value_equals", + file: "Documents/list.zip", + pointer: `${ITEMS}/[name=item][@id=1]/attributes/color`, + value: "blue", + }, + ]); + }); + + it("keys an element by a selector naming it alone, not one an identified sibling also matches", () => { + const body = (text: string) => + xmlElement("item", { color: "red", role: "body" }, [text]); + const id = xmlElement("item", { id: "1", color: "red" }); + + expect(itemChecks([id, body("old")], [id, body("new")])).toEqual([ + { + kind: "value_equals", + file: "Documents/list.zip", + pointer: `${ITEMS}/[name=item][@role=body]/children/0`, + value: "new", + }, + ]); + }); + + it("passes over a selector an identified sibling shares for one naming the element alone", () => { + const second = (color: string) => + xmlElement("item", { class: "common", role: "second", color }); + const id = xmlElement("item", { id: "1", class: "common" }); + + expect(itemChecks([id, second("red")], [id, second("blue")])).toEqual([ + { + kind: "value_equals", + file: "Documents/list.zip", + pointer: `${ITEMS}/[name=item][@role=second]/attributes/color`, + value: "blue", + }, + ]); + }); +}); + +describe("draftTaskFiles archive entries without a view, and type changes", () => { + it("checks no entry whose view could not be read, though JSON null is a value", () => { + const [file] = draft({ + "Documents/a.zip": path( + "document", + "zip", + listing([]), + listing([ + { + name: "big.json", + size: 1, + sha256: "big.json", + format: "json", + view: null, + view_error: + "a number in it cannot be read exactly, such as an integer above 2^53", + }, + zipEntry("null.json", "json", null), + ]), + ), + }); + + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=null.json]/view", + value: null, + }, + ]); + }); + + it("suggests no absence inside an entry that ends with no view, such as one now binary", () => { + const [file] = draft({ + "Documents/a.zip": path( + "document", + "zip", + listing([zipEntry("a.json", "json", { a: 1 })]), + listing([zipEntry("a.json", "binary", null)]), + ), + }); + + expect(file).toMatchObject({ change: { kind: "values" }, suggested: [] }); + }); + + it("suggests no absence of a value a selector still names after a type change", () => { + const [file] = draft({ + "Documents/a.zip": path( + "document", + "zip", + listing([zipEntry("a.json", "json", { items: { "[name=x]": 1 } })]), + listing([zipEntry("a.json", "json", { items: [{ name: "x" }] })]), + ), + }); + + expect(file?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/a.zip", + pointer: "/entries/[name=a.json]/view/items/[name=x]/name", + value: "x", + }, + ]); + }); +}); + +describe("draftTaskFiles archives", () => { + it("suggests checks inside an archive's XML by name: elements added or removed whole, then values", () => { + const element = ( + name: string, + attributes: Record, + children: JsonValue[] = [], + ): JsonValue => ({ name, attributes, children }); + const archive = (objects: JsonValue[], saved: string): JsonValue => ({ + entries: [ + { + name: "Document.xml", + size: 1, + sha256: saved, + format: "xml", + view: { + root: element("Document", { saved }, [ + element("Objects", {}, objects), + ]), + truncated: false, + }, + view_error: null, + }, + { + name: "thumb.png", + size: 1, + sha256: saved, + format: "binary", + view: null, + view_error: null, + }, + ], + entry_count: 2, + truncated: false, + }); + const plate = { name: "Plate", type: "Part::Box" }; + const box = element("Object", { ...plate, count: "1" }); + const sketch = element("Object", { name: "Sketch", type: "Sketcher" }); + const hole = element("Object", { name: "Hole", type: "Part::Cylinder" }, [ + element("Property", { name: "Radius" }, [ + element("Float", { value: "5.0" }), + ]), + ]); + + const [document] = draft({ + "Documents/plate.FCStd": path( + "document", + "zip", + archive([box, sketch], "2026-11-04T10:12:00Z"), + archive( + [element("Object", { ...plate, count: "2" }), hole], + "2026-11-04T10:13:00Z", + ), + ), + }); + + const objects = + "/entries/[name=Document.xml]/view/root/children/[name=Objects]/children"; + expect(document?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/plate.FCStd", + pointer: `${objects}/[name=Object][@name=Hole]/name`, + value: "Object", + }, + { + kind: "value_absent", + file: "Documents/plate.FCStd", + pointer: `${objects}/[name=Object][@name=Sketch]`, + }, + { + kind: "value_equals", + file: "Documents/plate.FCStd", + pointer: `${objects}/[name=Object][@name=Plate]/attributes/count`, + value: "2", + }, + ]); + }); + + it("suggests checks inside XML that uses namespaces, by its prefixed names", () => { + const paragraph = (style: string, text: string): JsonValue => ({ + name: "text:p", + attributes: { "text:style-name": style }, + children: [text], + }); + const odt = (date: string): JsonValue => ({ + entries: [ + { + name: "content.xml", + size: 1, + sha256: date, + format: "xml", + view: { + root: { + name: "office:text", + attributes: { + "xmlns:text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0", + }, + children: [ + paragraph("Title", "Summer Bake Sale"), + paragraph("Subtitle", date), + ], + }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, + }); + + const [flyer] = draft({ + "Documents/flyer.odt": path( + "document", + "zip", + odt("Monday 17 November"), + odt("Monday 24 November"), + ), + }); + + expect(flyer?.suggested).toEqual([ + { + kind: "value_equals", + file: "Documents/flyer.odt", + pointer: + "/entries/[name=content.xml]/view/root/children/[name=text:p][@text:style-name=Subtitle]/children/0", + value: "Monday 24 November", + }, + ]); + }); +}); diff --git a/src/domain/desktop/appTaskDraft.ts b/src/domain/desktop/appTaskDraft.ts index 1bab5cf508..d0c3b1c9b9 100644 --- a/src/domain/desktop/appTaskDraft.ts +++ b/src/domain/desktop/appTaskDraft.ts @@ -1,8 +1,13 @@ import type { JsonValue } from "../jsonValue.js"; import { isTimestamp, type DesktopTaskCheckInput } from "./appTask.js"; -import { namesXmlNamespace, resolvePointer } from "./appTaskChecks.js"; -import type { DesktopStatePath } from "./desktopState.js"; -import { diffStateViews } from "./stateChanges.js"; +import { + versionAt, + type DesktopStatePath, + type StateEnds, +} from "./desktopState.js"; +import { diffStateViews, diffZipMembers } from "./stateChanges.js"; +import { locatePointer, pairMembers, resolvePointer } from "./viewPointer.js"; +import { entryWithoutView, zipArrayLeavesOut, zipEntryAt } from "./zipView.js"; type Version = DesktopStatePath["versions"][number]; type Suggestion = DesktopTaskCheckInput; @@ -124,15 +129,142 @@ const innermost = (pointers: readonly string[]): readonly string[] => (pointer) => !pointers.some((other) => other.startsWith(`${pointer}/`)), ); +/** + * The entry a pointer into a zip view goes into, when it has a view and both + * evaluators read it alike: JSON, a key file or XML. A binary or unread + * entry also holds a `null` view, which is no JSON `null`. + */ +const readableEntry = (view: JsonValue | null, pointer: string) => { + const entry = zipEntryAt(view, pointer); + return typeof entry === "object" && + entryWithoutView(entry) === undefined && + (entry.format === "json" || + entry.format === "keyfile" || + entry.format === "xml") + ? entry + : undefined; +}; + +/** + * Where checks for a pointer the diff lists start: the view of an entry it + * names whole, or its root element for XML; the pointer itself inside that; + * undefined for an entry neither evaluator reads alike, or the entry's fields. + */ +const checkRoot = ( + view: JsonValue | null, + pointer: string, +): string | undefined => { + const entry = readableEntry(view, pointer); + const [entryPointer] = /^\/entries\/[^/]+/u.exec(pointer) ?? []; + if (entry === undefined || entryPointer === undefined) return undefined; + const inner = pointer.slice(entryPointer.length).split("/").slice(1); + const root = entry.format === "xml" ? ["view", "root"] : ["view"]; + const prefix = (short: readonly string[], long: readonly string[]) => + short.length <= long.length && + short.every((part, index) => long[index] === part); + if (prefix(inner, root)) return `${entryPointer}/${root.join("/")}`; + return prefix(root, inner) ? pointer : undefined; +}; + +/** Whether a check at a pointer is decided by name: no object or array reached by position, no array searched that the view may cut. */ +const byName = (view: JsonValue | null, pointer: string) => { + const located = locatePointer(view, pointer); + return ( + located.kind !== "unresolved" && + located.byPosition.length === 0 && + !located.searched.some((array) => zipArrayLeavesOut(view, array)) + ); +}; + +/** An element of REA's XML view, by its tag name, or undefined for any other value. */ +const elementName = (value: JsonValue | undefined): string | undefined => + isRecord(value) && + typeof value.name === "string" && + isRecord(value.attributes) && + Array.isArray(value.children) + ? value.name + : undefined; + +/** + * Checks on the entries of an archive, by the selectors its diff names + * members with: elements added or removed whole first, as the likeliest + * record of what the user did, then values changed, added and removed. A + * JSON value or an entry added whole is checked through its members. + */ +const zipSuggestions = ( + file: string, + baseline: Version, + final: Version, +): Suggestion[] => { + const change = diffZipMembers(baseline.view, final.view); + const elements: Suggestion[] = []; + const values: Suggestion[] = []; + const equals = (pointer: string, value: JsonValue | undefined) => { + if ( + values.length < MAX_SUGGESTIONS && + value !== undefined && + !(isRecord(value) && Object.keys(value).length > 0) && + !isTimestamp(value) && + byName(final.view, pointer) + ) + values.push({ kind: "value_equals", file, pointer, value }); + }; + const added = (pointer: string, value: JsonValue | undefined): void => { + if (elements.length >= MAX_SUGGESTIONS && values.length >= MAX_SUGGESTIONS) + return; + const name = elementName(value); + if (name !== undefined) { + if ( + elements.length < MAX_SUGGESTIONS && + byName(final.view, `${pointer}/name`) + ) + elements.push({ + kind: "value_equals", + file, + pointer: `${pointer}/name`, + value: name, + }); + } else if (Array.isArray(value) && value.length > 0) { + const { keys } = pairMembers(value, []).newer; + value.forEach((item, index) => + added(`${pointer}/${escapeToken(keys[index] ?? String(index))}`, item), + ); + } else if (isRecord(value) && Object.keys(value).length > 0) + for (const [key, item] of Object.entries(value)) + added(`${pointer}/${escapeToken(key)}`, item); + // A scalar, or an empty array or object, is checked as it is. + else equals(pointer, value); + }; + for (const pointer of change.added) { + const root = checkRoot(final.view, pointer); + if (root !== undefined) added(root, resolvePointer(final.view, root)); + } + for (const pointer of innermost(change.changed)) + if (checkRoot(final.view, pointer) === pointer) + equals(pointer, resolvePointer(final.view, pointer)); + // An absence is checked only where the entry still has a view to lack it in. + for (const pointer of change.removed) + if ( + checkRoot(baseline.view, pointer) === pointer && + checkRoot(final.view, pointer) === pointer && + byName(baseline.view, pointer) && + byName(final.view, pointer) + ) + (elementName(resolvePointer(baseline.view, pointer)) === undefined + ? values + : elements + ).push({ kind: "value_absent", file, pointer }); + return [...elements, ...values]; +}; + const valueSuggestions = ( file: string, baseline: Version, final: Version, ): Suggestion[] => { const format = final.format; + if (format === "zip") return zipSuggestions(file, baseline, final); if (format !== "json" && format !== "xml" && format !== "keyfile") return []; - if (format === "xml" && [baseline.view, final.view].some(namesXmlNamespace)) - return []; const change = diffStateViews(format, baseline.view, final.view); const pointers = keyPointers([baseline.view, final.view]); const pointer = (entry: string) => @@ -350,19 +482,20 @@ export interface TaskDraftFile { } /** - * Each file under the owned paths whose state differs between the baseline and - * the final version, with suggested checks for the files the user worked on: - * documents, or the app's own state when no document changed. + * Each file under the owned paths whose state differs between its versions at + * the seed and end checkpoints, with suggested checks for the files the user + * worked on: documents, or the app's own state when no document changed. */ export const draftTaskFiles = ( files: ReadonlyMap, served: (home: string) => boolean, + ends: StateEnds, ): readonly TaskDraftFile[] => { const changed = [...files] .filter(([home]) => served(home)) .flatMap(([home, path]) => { - const baseline = path.versions[0]; - const final = path.versions.at(-1); + const baseline = versionAt(path, ends.seed); + const final = versionAt(path, ends.end); const change = baseline === undefined || final === undefined ? undefined diff --git a/src/domain/desktop/appTaskPython.ts b/src/domain/desktop/appTaskPython.ts index 73a8228655..50d0fbd68e 100644 --- a/src/domain/desktop/appTaskPython.ts +++ b/src/domain/desktop/appTaskPython.ts @@ -4,6 +4,9 @@ import type { DesktopTask } from "./appTask.js"; /** The verifier's evaluator: the checks' semantics, kept equal to `evaluateTaskChecks`. */ export const PYTHON_TASK_EVALUATOR = String.raw` MISSING = object() +INDEX = re.compile(r"0|[1-9][0-9]*") +VIEW_FORMATS = ("text", "json", "keyfile", "xml") +NUMERAL = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?") def _shown(value) -> str: @@ -26,19 +29,130 @@ def _same(actual, expected) -> bool: return type(actual) is type(expected) and actual == expected -def _resolve(view, pointer: str): - current = view - for raw in pointer[1:].split("/"): - token = raw.replace("~1", "/").replace("~0", "~") +def _bracketed(token: str) -> bool: + return len(token) >= 2 and token.startswith("[") and token.endswith("]") + + +def _selector(token: str): + """A selector's conditions, [KEY=VALUE] or [@KEY=VALUE] one or more times, or None when the token is not one.""" + if not _bracketed(token): + return None + conditions = [] + for part in token[1:-1].split("]["): + key, equals, value = part.partition("=") + name = key[1:] if key.startswith("@") else key + if not equals or not name: + return None + conditions.append((key.startswith("@"), name, value)) + return conditions + + +def _selects(member, conditions) -> bool: + """Whether an array member meets every condition; selectors compare strings only.""" + if not isinstance(member, dict): + return False + for attribute, key, value in conditions: + fields = member.get("attributes") if attribute else member + if not isinstance(fields, dict) or not isinstance(fields.get(key), str) or fields[key] != value: + return False + return True + + +def _step(items: list, token: str, last: bool): + """One step into an array, by index or by a selector that must match one member, or none when it is the last.""" + if INDEX.fullmatch(token): + return ("member", items[int(token)]) if int(token) < len(items) else ("none", None) + conditions = _selector(token) + if conditions is None: + return ("unresolved", f"{token} is not a selector of the form [KEY=VALUE]") if _bracketed(token) else ( + "none", None) + matched = [item for item in items if _selects(item, conditions)] + if len(matched) == 1: + return "member", matched[0] + if len(matched) > 1: + return "unresolved", f"{token} matches {len(matched)} members, not one" + return ("none" if last else "unresolved"), f"{token} matches no member" + + +def _locate(view, pointer: str): + """Where a pointer leads: ("value", value), ("none", why or None) or ("unresolved", problem), with the pointers + to the arrays a selector searched on the way.""" + current, searched = view, [] + segments = pointer[1:].split("/") + for position, segment in enumerate(segments): + token = segment.replace("~1", "/").replace("~0", "~") if isinstance(current, list): - if not re.fullmatch(r"0|[1-9][0-9]*", token) or int(token) >= len(current): - return MISSING - current = current[int(token)] + if not INDEX.fullmatch(token) and _selector(token) is not None: + searched.append("".join("/" + part for part in segments[:position])) + kind, found = _step(current, token, position == len(segments) - 1) + if kind != "member": + return kind, found, searched + current = found elif isinstance(current, dict) and token in current: current = current[token] else: - return MISSING - return current + return "none", None, searched + return "value", current, searched + + +def _number(value): + """A number, or a numeral string read as one; None for anything else, a boolean, or a numeral past the doubles.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + if not isinstance(value, str) or not NUMERAL.fullmatch(value): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def _entry_leaves_out(entry) -> bool: + return isinstance(entry, dict) and (isinstance(entry.get("view_error"), str) or ( + entry.get("format") in ("text", "xml") and isinstance(entry.get("view"), dict) + and entry["view"].get("truncated") is True)) + + +def _viewless_entry(view, pointer: str): + """The name of the entry whose view a pointer goes into when it has none, so its null is no JSON value, and + why: binary, unread, or of a format without a view; None when it has one.""" + parts = pointer[1:].split("/") + if len(parts) < 3 or parts[0] != "entries" or parts[2] != "view": + return None + kind, item, _ = _locate(view, "/entries/" + parts[1]) + if kind != "value" or not isinstance(item, dict): + return None + if isinstance(item.get("view_error"), str): + reason = f"has no view: {item['view_error']}" + elif item.get("format") == "binary": + reason = "is binary and has no view" + elif item.get("format") not in VIEW_FORMATS: + reason = "has no view" + else: + return None + return (item["name"] if isinstance(item.get("name"), str) else ""), reason + + +def _array_leaves_out(form, view, array: str) -> bool: + """Whether an array a selector searched may lack members: in a zip the listing or its entry's view, else the + view's own truncation.""" + if form != "zip": + return _truncated(form, view) + if array == "/entries": + return isinstance(view, dict) and view.get("truncated") is True + return _zip_leaves_out(view, array) + + +def _zip_leaves_out(view, pointer: str) -> bool: + """Whether a zip view leaves out what a pointer reaches: for a pointer into one listed entry, only that entry.""" + if not isinstance(view, dict): + return False + parts = pointer[1:].split("/") + if parts[0] == "entries" and len(parts) > 1: + kind, entry, _ = _locate(view, "/entries/" + parts[1]) + return _entry_leaves_out(entry) if kind == "value" else view.get("truncated") is True + entries = view.get("entries") if isinstance(view.get("entries"), list) else [] + return view.get("truncated") is True or any(_entry_leaves_out(entry) for entry in entries) def _truncated(form, view) -> bool: @@ -102,21 +216,47 @@ def _lines(check, entry): return _compared(check, n, f"line(s) of {file} match {_shown(check['exact'])}") +def _near(at: str, found, expected, tolerance): + """A value compared within a tolerance: numbers, and numeral strings, by their difference.""" + number = _number(found) + within = f"within {_shown(tolerance)} of {_shown(expected)}" + if number is None: + return False, f"{at} is {_shown(found)}, not a number" + if abs(number - expected) <= tolerance: + return True, f"{at} is {_shown(found)}, {within}" + return False, f"{at} is {_shown(found)}, not {within}" + + def _value(check, entry): view, truncated, problem = _view_of(check, entry, None) if problem is not None: return False, problem - found = MISSING if entry is None else _resolve(view, check["pointer"]) + kind, found, searched = ("none", None, []) if entry is None else _locate(view, check["pointer"]) at = f"{check['file']} at {check['pointer']}" + viewless = _viewless_entry(view, check["pointer"]) if entry is not None and entry.get("format") == "zip" else None + if viewless is not None: + return False, f"{check['file']} entry {viewless[0]} {viewless[1]}" + # A selector tells one member from several only in an array the view holds whole. + if any(_array_leaves_out(entry.get("format"), view, array) for array in searched): + return _undecided(check) + if kind == "unresolved": + return False, f"{at}: {found}" + if entry is not None and entry.get("format") == "zip": + truncated = _zip_leaves_out(view, check["pointer"]) if check["kind"] == "value_equals" and isinstance(check["value"], (dict, list)) and truncated: return _undecided(check) if check["kind"] == "value_equals": - if found is not MISSING and _same(found, check["value"]): + if kind == "none": + return False, f"{at} has no value" if found is None else f"{at}: {found}" + expected = check["value"] + if check.get("tolerance") is not None and isinstance(expected, (int, float)) and not isinstance(expected, bool): + return _near(at, found, expected, check["tolerance"]) + if _same(found, check["value"]): return True, f"{at} is {_shown(check['value'])}" - return False, f"{at} has no value" if found is MISSING else f"{at} is {_shown(found)}, not {_shown(check['value'])}" + return False, f"{at} is {_shown(found)}, not {_shown(check['value'])}" if truncated: return _undecided(check) - return (True, f"{at} has no value") if found is MISSING else (False, f"{at} is {_shown(found)}") + return (True, f"{at} has no value") if kind == "none" else (False, f"{at} is {_shown(found)}") def _rows(check, entry): @@ -194,6 +334,7 @@ It reads the app's data/get through the computer, takes the typed views of its f model, no code from the task. A failure to reach the app raises, so the run fails rather than scoring the agent. """ import json +import math import re from agentenv_protocol import client as protocol diff --git a/src/domain/desktop/aptRepositories.ts b/src/domain/desktop/aptRepositories.ts index 7a1164be80..9ebfe7c50d 100644 --- a/src/domain/desktop/aptRepositories.ts +++ b/src/domain/desktop/aptRepositories.ts @@ -291,6 +291,33 @@ export const declaredPackageFiles = ( ); }; +/** The packages of `supplied` whose declared sources include one of the repository's package files, sorted. */ +export const repositorySupplies = ( + repository: Pick, + supplied: ReadonlyMap, + targets: readonly AptIndexTarget[], +): readonly string[] => { + const files = declaredPackageFiles([repository], targets); + return [...supplied] + .filter(([, sources]) => sources.some((source) => files.has(source))) + .map(([name]) => name) + .sort(); +}; + +/** The `NAME='value'` lines `apt-config shell` prints, by name; apt writes a quote in a value as `'\''`. */ +export const parseAptConfigShell = ( + output: string, +): ReadonlyMap => + new Map( + output.split("\n").flatMap((line) => { + const match = /^(?[A-Za-z_]\w*)='(?.*)'$/u.exec(line); + const { name, value } = match?.groups ?? {}; + return name === undefined || value === undefined + ? [] + : [[name, value.replaceAll(`'\\''`, "'")] as const]; + }), + ); + const QUOTED = new Set('\\|{}[]<>"^~_=!@#$%^&*'); /** apt's file name for a URI in its lists directory: no scheme or credentials, special characters quoted, `/` as `_`. */ diff --git a/src/domain/desktop/desktopCapture.ts b/src/domain/desktop/desktopCapture.ts index 484e8c089e..07f251b41e 100644 --- a/src/domain/desktop/desktopCapture.ts +++ b/src/domain/desktop/desktopCapture.ts @@ -90,6 +90,8 @@ const aptRepositorySchema = z.strictObject({ signing_key: aptSigningKeySchema.nullable(), /** Why no signing key was verified, or why the repository was not checked. */ signing_key_reason: z.string().optional(), + /** Installable library packages, and the executable's package, that the Ubuntu 24.04 archive does not offer and this repository's package files list; absent from captures written before it was recorded. */ + supplies: z.array(z.string()).optional(), }); export type AptRepository = z.infer; @@ -110,7 +112,12 @@ export type DesktopPackageFile = z.infer; const packageSchema = z.strictObject({ kind: z.enum(["deb", "snap", "flatpak", "appimage", "unknown"]), + /** A Debian package is the launched desktop entry's owner, or else the executable's. */ name: z.string().nullable(), + /** The executable's Debian package when another package owns the desktop entry; absent otherwise. */ + executable_package: z.string().optional(), + /** Why the computer cannot install the program the desktop entry runs: `executable_package` is not offered, or, without it, no package owns the program; absent when it can. */ + executable_package_reason: z.string().optional(), version: z.string().nullable(), architecture: z.string().nullable(), /** Package sources listing the installed version; empty for local installs. */ @@ -224,6 +231,9 @@ export const desktopApplicationCaptureSchema = z.strictObject({ architecture: z.string().min(1), display: z.string().min(1), home: z.string().min(1), + /** The host's name and the user's, which captured state can hold; absent from captures written before they were recorded, or when the system could not say. */ + hostname: z.string().min(1).optional(), + user: z.string().min(1).optional(), }), application: desktopApplicationInventorySchema, session: desktopSessionSchema, diff --git a/src/domain/desktop/desktopState.ts b/src/domain/desktop/desktopState.ts index d43a2070d4..11b7ebd97a 100644 --- a/src/domain/desktop/desktopState.ts +++ b/src/domain/desktop/desktopState.ts @@ -11,6 +11,7 @@ const format = z.enum([ "xml", "sqlite", "dconf", + "zip", "binary", ]); const index = z.int().min(0); @@ -86,6 +87,19 @@ export const desktopStatePathSchema = z.strictObject({ }); export type DesktopStatePath = z.infer; +/** The checkpoints, by position, a bundle's seed is taken at and a task's end state is read at. */ +export interface StateEnds { + readonly seed: number; + readonly end: number; +} + +/** A path's version as it stood at a checkpoint: the last one recorded at or before that position. */ +export const versionAt = ( + path: DesktopStatePath, + checkpoint: number, +): DesktopStatePath["versions"][number] | undefined => + path.versions.findLast((version) => version.checkpoint_index <= checkpoint); + /** The normalized result of `inspect_desktop_application_state`. */ export const desktopApplicationStateSchema = z.strictObject({ capture: z.strictObject({ diff --git a/src/domain/desktop/hostIdentity.test.ts b/src/domain/desktop/hostIdentity.test.ts new file mode 100644 index 0000000000..43d7ea458b --- /dev/null +++ b/src/domain/desktop/hostIdentity.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; + +import { + SEED_HOME, + identityLeftIn, + scrubHostIdentity, + type HostIdentity, +} from "./hostIdentity.js"; + +const GEANY: HostIdentity = { + home: "/home/ada", + hostname: "e81232be457d", + user: "ada", +}; + +/** A capture whose host is named localhost, so only user@host gives it away. */ +const LOCAL: HostIdentity = { ...GEANY, hostname: "localhost" }; + +describe("scrubHostIdentity", () => { + it("replaces user@host, keeping a short user name standing alone", () => { + expect( + scrubHostIdentity( + "pref_template_mail=ada@e81232be457d\npref_template_developer=ada\nby ada@example.com\n", + GEANY, + ), + ).toBe( + "pref_template_mail=user@localhost\npref_template_developer=ada\nby ada@example.com\n", + ); + }); + + it("replaces the host name only as a whole name, not inside a word or a domain", () => { + expect( + scrubHostIdentity( + "host=e81232be457d\non e81232be457d.\nhttp://e81232be457d.local/\nxe81232be457d e81232be457d-2\n", + GEANY, + ), + ).toBe( + "host=localhost\non localhost.\nhttp://e81232be457d.local/\nxe81232be457d e81232be457d-2\n", + ); + const ubuntu = { home: "/home/ubuntu", hostname: "ubuntu", user: "user" }; + expect( + scrubHostIdentity( + "deb http://ports.ubuntu.com/ubuntu-ports noble\nuser settings\n", + ubuntu, + ), + ).toBe("deb http://ports.ubuntu.com/ubuntu-ports noble\nuser settings\n"); + }); + + it("replaces the home path where a whole path segment ends after it", () => { + expect( + scrubHostIdentity( + "a=/home/ada/Documents/x\nb=file:///home/ada\nc=/home/adam/x\nd=/home/ada.bak\n", + GEANY, + ), + ).toBe( + `a=${SEED_HOME}/Documents/x\nb=file://${SEED_HOME}\nc=/home/adam/x\nd=/home/ada.bak\n`, + ); + }); + + it("replaces only the home when the capture recorded no host name or user", () => { + expect( + scrubHostIdentity("/home/ada/x ada@e81232be457d e81232be457d", { + home: "/home/ada", + hostname: null, + user: null, + }), + ).toBe(`${SEED_HOME}/x ada@e81232be457d e81232be457d`); + }); +}); + +describe("scrubHostIdentity on a host named localhost", () => { + it("replaces the captured user@localhost and leaves localhost alone", () => { + expect( + scrubHostIdentity("mail=ada@localhost\nhost=localhost\n", LOCAL), + ).toBe("mail=user@localhost\nhost=localhost\n"); + }); +}); + +describe("identityLeftIn", () => { + it("finds the host name and user@host anywhere, the home as a whole path, and not the user name alone", () => { + const texts = [ + '["mail=ada@e81232be457d"]', + "e81232be457d-notes.txt", + "http://e81232be457d.local/", + "x/home/ada/y", + "/home/adam/x", + '["author=ada"]', + ]; + + expect(texts.map((text) => identityLeftIn(text, GEANY))).toEqual([ + "ada@e81232be457d", + "e81232be457d", + "e81232be457d", + "/home/ada", + undefined, + undefined, + ]); + }); + + it("does not take the replacements for what they replaced", () => { + const host = { home: "/home", hostname: "host", user: "user" }; + + expect( + identityLeftIn(`mail=user@localhost on localhost in ${SEED_HOME}`, host), + ).toBeUndefined(); + expect(identityLeftIn("ghost /home/ada", host)).toBe("/home"); + expect(identityLeftIn("mail=ada@localhost", LOCAL)).toBe("ada@localhost"); + expect(identityLeftIn("mail=user@localhost", LOCAL)).toBeUndefined(); + }); +}); diff --git a/src/domain/desktop/hostIdentity.ts b/src/domain/desktop/hostIdentity.ts new file mode 100644 index 0000000000..c51971e26b --- /dev/null +++ b/src/domain/desktop/hostIdentity.ts @@ -0,0 +1,106 @@ +/** The capture host's identity, which captured state can hold and a shared bundle must not. */ +export interface HostIdentity { + readonly home: string; + /** Null when the capture did not record it. */ + readonly hostname: string | null; + readonly user: string | null; +} + +/** The home the seed spells, which the computer respells to its own at boot; the computer's user is `user`. */ +export const SEED_HOME = "/home/user"; +const SEED_HOST = "localhost"; +const SEED_LOGIN = `user@${SEED_HOST}`; +/** What the scrub writes, which a search for what is left must not take for what it replaced. */ +const REPLACEMENTS = [SEED_LOGIN, SEED_HOME, SEED_HOST]; + +const WORD = String.raw`\p{L}\p{N}_`; +/** A home path ends where no file name character follows, as the computer's boot respelling reads it. */ +const HOME_END = String.raw`(?![${WORD}.@+~-])`; +/** A host name ends where no name character or `.name` follows, so `host.example.com` is left whole. */ +const HOST_END = String.raw`(?![${WORD}-]|\.[${WORD}])`; + +const escaped = (text: string): string => + text.replace(/[\\^$.*+?()[\]{}|/]/gu, "\\$&"); + +/** The captured home, unless it is the root or the seed's own home. */ +const homeOf = ({ home }: HostIdentity): string | undefined => + home.length > 1 && home !== SEED_HOME ? home : undefined; + +/** The captured `user@host`, unless it is already what the scrub writes. */ +const loginOf = ({ hostname, user }: HostIdentity): string | undefined => + hostname === null || user === null || `${user}@${hostname}` === SEED_LOGIN + ? undefined + : `${user}@${hostname}`; + +/** The host name, unless it is `localhost`, which every host calls itself. */ +const hostOf = ({ hostname }: HostIdentity): string | undefined => + hostname === null || hostname === SEED_HOST ? undefined : hostname; + +/** The spellings a bundle must not hold: the captured home, `user@host` and the host name. */ +export const identitySpellings = (identity: HostIdentity): readonly string[] => + [homeOf(identity), loginOf(identity), hostOf(identity)].filter( + (spelling): spelling is string => spelling !== undefined, + ); + +/** + * The identity's spellings the scrub replaces: the home path where a whole + * path segment ends after it, as the computer respells it at boot; + * `user@host`; and the host name as a whole name. The user name alone is + * never matched, since a short one such as `ada` is also ordinary text. + */ +const identityPattern = (identity: HostIdentity): RegExp | undefined => { + const home = homeOf(identity); + const login = loginOf(identity); + const host = hostOf(identity); + const alternatives = [ + ...(home === undefined ? [] : [`(?${escaped(home)})${HOME_END}`]), + ...(login === undefined + ? [] + : [`(?${escaped(login)})${HOST_END}`]), + ...(host === undefined + ? [] + : [`(?${escaped(host)})${HOST_END}`]), + ]; + return alternatives.length === 0 + ? undefined + : new RegExp(alternatives.join("|"), "gu"); +}; + +/** The text with the identity replaced: `/home/user` for the home, `user@localhost` and `localhost` for the host. */ +export const scrubHostIdentity = ( + text: string, + identity: HostIdentity, +): string => { + const pattern = identityPattern(identity); + return pattern === undefined + ? text + : text.replace(pattern, (...match: unknown[]) => { + const groups = match.at(-1) as Record; + if (groups.home !== undefined) return SEED_HOME; + return groups.login === undefined ? SEED_HOST : SEED_LOGIN; + }); +}; + +/** + * The first of the identity's spellings a text still holds, by the rule for + * what a bundle may not ship and a check may not name: the captured home where + * a whole path segment ends after it, and `user@host` and the host name + * anywhere, as in `host.local` or `host-notes.txt`. The scrub's own + * replacements are set aside first, so a host named `host` is not found in + * `localhost`. + */ +export const identityLeftIn = ( + text: string, + identity: HostIdentity, +): string | undefined => { + const home = homeOf(identity); + return identitySpellings(identity).find((spelling) => { + const rest = REPLACEMENTS.filter((own) => own.includes(spelling)).reduce( + (remaining, own) => remaining.replaceAll(own, "\u0000"), + text, + ); + return spelling === home + ? new RegExp(`${escaped(home)}${HOME_END}`, "u").test(rest) + : rest.includes(spelling); + }); +}; diff --git a/src/domain/desktop/libraryPackages.test.ts b/src/domain/desktop/libraryPackages.test.ts index 2b410c1c19..d185899a8b 100644 --- a/src/domain/desktop/libraryPackages.test.ts +++ b/src/domain/desktop/libraryPackages.test.ts @@ -209,6 +209,7 @@ describe("installableLibraryPackages", () => { ).toEqual({ installable: ["libnss3", "typora-helper"], uninstallable: ["libfoo", "libgone"], + supplied: new Map([["typora-helper", [TYPORA]]]), }); }); }); diff --git a/src/domain/desktop/libraryPackages.ts b/src/domain/desktop/libraryPackages.ts index 548d50c9d0..17d0858dd0 100644 --- a/src/domain/desktop/libraryPackages.ts +++ b/src/domain/desktop/libraryPackages.ts @@ -182,7 +182,8 @@ export const libraryPackageNames = ( /** * Library packages the computer can install, and the rest: a package is * installable when some version of it comes from the Ubuntu 24.04 archive - * or from one of the vendor sources the export declares. + * or from one of the vendor sources the export declares. `supplied` names the + * declared sources of each installable package the archive does not offer. */ export const installableLibraryPackages = ( names: readonly string[], @@ -193,21 +194,26 @@ export const installableLibraryPackages = ( readonly origins: ReadonlyMap | undefined; readonly declared: ReadonlySet; }, -): { readonly installable: string[]; readonly uninstallable: string[] } => { +): { + readonly installable: string[]; + readonly uninstallable: string[]; + readonly supplied: ReadonlyMap; +} => { const installable: string[] = []; const uninstallable: string[] = []; + const supplied = new Map(); for (const name of names) { const sources = available.sources.get(name) ?? []; - const offered = - sources.some((source) => available.declared.has(source)) || - fromUbuntuNobleArchive( - sources, - available.origins === undefined - ? undefined - : sources.map((source) => available.origins?.get(source) ?? null), - ); - if (offered) installable.push(name); + const declared = sources.filter((source) => available.declared.has(source)); + const archive = fromUbuntuNobleArchive( + sources, + available.origins === undefined + ? undefined + : sources.map((source) => available.origins?.get(source) ?? null), + ); + if (archive || declared.length > 0) installable.push(name); else uninstallable.push(name); + if (!archive && declared.length > 0) supplied.set(name, declared); } - return { installable, uninstallable }; + return { installable, uninstallable, supplied }; }; diff --git a/src/domain/desktop/seedScrub.test.ts b/src/domain/desktop/seedScrub.test.ts new file mode 100644 index 0000000000..a283e715ba --- /dev/null +++ b/src/domain/desktop/seedScrub.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; + +import { SEED_HOME, type HostIdentity } from "./hostIdentity.js"; +import { scrubDconf, scrubSeed } from "./seedScrub.js"; +import { detectStateFormat } from "./stateFormats.js"; + +const GEANY: HostIdentity = { + home: "/home/ada", + hostname: "e81232be457d", + user: "ada", +}; + +/** Text as UTF-16 after a byte-order mark, in either byte order. */ +const utf16 = (text: string, order: "le" | "be") => { + const bytes = Buffer.from(`\uFEFF${text}`, "utf16le"); + return order === "be" ? bytes.swap16() : bytes; +}; + +/** A seed at a home path that says nothing of its format. */ +const seed = (format: string) => ({ home: "seed", format }); + +const scrubbed = (text: string, format: string, identity = GEANY) => { + const outcome = scrubSeed(Buffer.from(text), seed(format), identity); + return { + text: outcome.bytes?.toString("utf8"), + holdsIdentity: outcome.holdsIdentity, + }; +}; + +describe("scrubSeed on text", () => { + it("rewrites a UTF-8 text seed, keeping its byte-order mark, and leaves a clean one as it is", () => { + expect(scrubbed("\uFEFFmail=ada@e81232be457d\n", "keyfile")).toEqual({ + text: "\uFEFFmail=user@localhost\n", + holdsIdentity: false, + }); + expect(scrubbed("wrap=true\n", "text")).toEqual({ + text: undefined, + holdsIdentity: false, + }); + }); + + it.each(["le", "be"] as const)( + "rewrites a UTF-16 %s seed with a byte-order mark in its own order", + (order) => { + const outcome = scrubSeed( + utf16("20260216;/home/ada/.varicad;ada@e81232be457d\n", order), + seed("binary"), + GEANY, + ); + + expect(outcome.bytes?.toString("hex")).toBe( + utf16( + `20260216;${SEED_HOME}/.varicad;user@localhost\n`, + order, + ).toString("hex"), + ); + }, + ); + + it("replaces the captured user@localhost of a host named localhost and finds it in bytes it cannot rewrite", () => { + const local = { ...GEANY, hostname: "localhost" }; + + expect(scrubbed("mail=ada@localhost\n", "keyfile", local)).toEqual({ + text: "mail=user@localhost\n", + holdsIdentity: false, + }); + expect(scrubbed("SQLite format 3\0ada@localhost", "sqlite", local)).toEqual( + { text: undefined, holdsIdentity: true }, + ); + }); +}); + +describe("scrubSeed on JSON and XML", () => { + it("rewrites JSON strings only, so a host named like a literal leaves the syntax valid", () => { + const named = { ...GEANY, hostname: "true" }; + const outcome = scrubbed( + '{"enabled": true, "host": "true", "list": [true, "a true"]}', + "json", + named, + ); + + expect(outcome.text).toBe( + '{"enabled": true, "host": "localhost", "list": [true, "a localhost"]}', + ); + expect(JSON.parse(outcome.text ?? "")).toEqual({ + enabled: true, + host: "localhost", + list: [true, "a localhost"], + }); + }); + + it("keeps member names, reporting one that holds the host rather than rewriting it into another", () => { + const named = { ...GEANY, hostname: "ubuntu" }; + const text = String.raw`{"ubuntu": 1, "localhost": 2, "\u0075buntu-dock" + : "ubuntu", "list": ["ubuntu"]}`; + + const outcome = scrubbed(text, "json", named); + + expect(outcome).toEqual({ + text: String.raw`{"ubuntu": 1, "localhost": 2, "\u0075buntu-dock" + : "localhost", "list": ["localhost"]}`, + holdsIdentity: true, + }); + expect(Object.keys(JSON.parse(outcome.text ?? "") as object)).toEqual([ + "ubuntu", + "localhost", + "ubuntu-dock", + "list", + ]); + }); + + it("walks a JSON string of 16,000,000 characters, which the capture can retain, without running out of stack", () => { + const note = "x".repeat(16_000_000); + const text = `{"note": "${note}", "mail": "ada@e81232be457d"}`; + + const outcome = scrubSeed(Buffer.from(text), seed("json"), GEANY); + + expect(outcome.holdsIdentity).toBe(false); + expect(outcome.bytes?.length).toBe(text.length - 2); + expect(outcome.bytes?.subarray(-29).toString("utf8")).toBe( + `x", "mail": "user@localhost"}`, + ); + }); + + it("decodes JSON escapes, rewriting an escaped identity and finding one left inside a domain", () => { + expect( + scrubbed( + String.raw`{"mail": "ada@\u006581232be457d", "kept": "\u00e9"}`, + "json", + ), + ).toEqual({ + text: String.raw`{"mail": "user@localhost", "kept": "\u00e9"}`, + holdsIdentity: false, + }); + expect( + scrubbed(String.raw`{"url": "http://\u006581232be457d.local/"}`, "json"), + ).toEqual({ text: undefined, holdsIdentity: true }); + }); + + it("rewrites XML text, attribute values, comments and CDATA but not names, decoding references", () => { + const xml = `\nada@e81232be457d`; + const named = { ...GEANY, hostname: "item" }; + + const outcome = scrubbed(xml, "xml"); + const items = scrubbed('item', "xml", named); + + expect(outcome).toEqual({ + text: `\nuser@localhost`, + holdsIdentity: false, + }); + expect(detectStateFormat("m.xml", Buffer.from(outcome.text ?? ""))).toBe( + "xml", + ); + expect(items.text).toBe('localhost'); + }); + + it.each(["le", "be"] as const)( + "scrubs UTF-16 %s JSON and XML by value, as their UTF-8 forms are, in their own byte order", + (order) => { + const named = { ...GEANY, hostname: "ubuntu" }; + const json = '{"ubuntu": 1, "localhost": 2, "dock": "ubuntu"}'; + const xml = `\nubuntu`; + + const outcomes = [ + { text: json, home: ".config/dock.json" }, + { text: xml, home: ".config/dock.xml" }, + ].map(({ text, home }) => + scrubSeed(utf16(text, order), { home, format: "binary" }, named), + ); + + expect( + outcomes.map(({ bytes, holdsIdentity }) => [ + bytes?.toString("hex"), + holdsIdentity, + ]), + ).toEqual([ + [ + utf16( + '{"ubuntu": 1, "localhost": 2, "dock": "localhost"}', + order, + ).toString("hex"), + true, + ], + [ + utf16( + `\nlocalhost`, + order, + ).toString("hex"), + true, + ], + ]); + }, + ); + + it("finds an identity XML references hide inside a domain, which it does not rewrite", () => { + expect(scrubbed("http://e81232be457d.local/", "xml")).toEqual({ + text: undefined, + holdsIdentity: true, + }); + }); + + it("scrubs around a reference past the last code point instead of throwing", () => { + const outcome = scrubbed( + '� ada@e81232be457d', + "xml", + ); + expect(outcome.holdsIdentity).toBe(false); + expect(outcome.text).toContain("user@localhost"); + expect(outcome.text).not.toContain("e81232be457d"); + }); +}); + +describe("scrubSeed on bytes it does not rewrite", () => { + it("reports a seed whose shipped bytes still hold the host name or the captured home", () => { + const database = Buffer.concat([ + Buffer.from("SQLite format 3\0"), + Buffer.from([0, 1, 2]), + Buffer.from("e81232be457d"), + ]); + const outcomes = [ + scrubSeed(database, seed("sqlite"), GEANY), + scrubSeed( + Buffer.from("url=http://e81232be457d.local/\n"), + seed("text"), + GEANY, + ), + scrubSeed(Buffer.from("SQLite format 3\0ada"), seed("sqlite"), GEANY), + scrubSeed(Buffer.from([0xff, 0xfe, 0x41]), seed("binary"), GEANY), + ]; + + expect( + outcomes.map(({ bytes, holdsIdentity }) => [bytes, holdsIdentity]), + ).toEqual([ + [undefined, true], + [undefined, true], + [undefined, false], + [undefined, false], + ]); + }); + + it("leaves the seed's own home alone, as a capture inside the computer spells it", () => { + const inside = { home: SEED_HOME, hostname: "4f0c1d2e3a4b", user: "user" }; + + expect( + scrubSeed( + Buffer.from(`SQLite format 3\0${SEED_HOME}/x`), + seed("sqlite"), + inside, + ), + ).toEqual({ bytes: undefined, holdsIdentity: false }); + expect( + scrubbed(`${SEED_HOME}/x user@4f0c1d2e3a4b`, "text", inside), + ).toEqual({ text: `${SEED_HOME}/x user@localhost`, holdsIdentity: false }); + }); +}); + +describe("scrubDconf", () => { + const named = { ...GEANY, hostname: "true" }; + + it("rewrites GVariant strings by their decoded values, keeping groups, keys, dictionary keys, literals and bytestrings", () => { + const outcome = scrubDconf( + { + "org/true": { + true: "true", + mail: String.raw`'ada@\u0074rue'`, + quoted: `"it's true"`, + list: "['true', \"a 'true'\", 'x']", + hosts: "{'true': <'true'>}", + blob: "b'true'", + count: "uint32 7", + }, + }, + named, + ); + + expect(outcome.text).toBe( + [ + "[org/true]", + "true=true", + "mail='user@localhost'", + `quoted="it's localhost"`, + "list=['localhost', \"a 'localhost'\", 'x']", + "hosts={'true': <'localhost'>}", + "blob=b'true'", + "count=uint32 7", + "", + ].join("\n"), + ); + expect(outcome.holdsIdentity).toBe(true); + }); + + it("escapes what a rewritten string holds that would end it or its line", () => { + const outcome = scrubDconf( + { "org/notes": { note: String.raw`'true\nit\'s \\ true'` } }, + named, + ); + + expect(outcome.text).toBe( + String.raw`[org/notes] +note='localhost\u000ait\'s \\ localhost' +`, + ); + }); + + it.each([ + ["an unterminated string", "'ada@e81232be457d"], + ["an escape naming no character", String.raw`'ada@e81232be457d\u0000'`], + ["an object path", "[objectpath '/e81232be457d', '/x']"], + ["a signature", "@mg 'e81232be457d'"], + ])( + "keeps a value holding %s as captured and finds the host in it", + (_name, value) => { + expect(scrubDconf({ "org/notes": { key: value } }, GEANY)).toEqual({ + text: `[org/notes]\nkey=${value}\n`, + holdsIdentity: true, + }); + }, + ); + + it("finds a host its escapes spell where the scrub does not replace it", () => { + expect( + scrubDconf( + { "org/notes": { url: String.raw`'http://\u006581232be457d.local/'` } }, + GEANY, + ), + ).toEqual({ + text: String.raw`[org/notes] +url='http://\u006581232be457d.local/' +`, + holdsIdentity: true, + }); + }); +}); diff --git a/src/domain/desktop/seedScrub.ts b/src/domain/desktop/seedScrub.ts new file mode 100644 index 0000000000..1860ceb4e9 --- /dev/null +++ b/src/domain/desktop/seedScrub.ts @@ -0,0 +1,418 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + identityLeftIn, + identitySpellings, + scrubHostIdentity, + type HostIdentity, +} from "./hostIdentity.js"; +import { isTextualFormat, textSyntaxFormat } from "./stateFormats.js"; + +/** What the export ships for one seed. */ +export interface SeedScrub { + /** The rewritten bytes, or undefined when the seed ships as captured. */ + readonly bytes: Buffer | undefined; + /** Whether what it ships still holds the captured home, `user@host` or the host name, its escapes decoded. */ + readonly holdsIdentity: boolean; +} + +/** Seeds up to this size are decoded, well inside a JavaScript string; a larger one is only searched byte for byte. */ +const MAX_SCRUBBED_BYTES = 256 * 1024 * 1024; + +/** A text with the identity scrubbed, and the text a search for what is left reads, its escapes decoded. */ +interface TextScrub { + readonly text: string; + readonly decoded: string; +} + +/** Whitespace JSON allows between tokens. */ +const JSON_SPACE = new Set([" ", "\t", "\n", "\r"]); + +/** Where the string literal opening at `start` ends, just past its closing quote: the first of its quotes no odd run of backslashes escapes, or -1. */ +const stringEnd = (text: string, start: number): number => { + const mark = text[start] ?? ""; + for (let quote = text.indexOf(mark, start + 1); quote !== -1;) { + let slashes = 0; + while (text[quote - 1 - slashes] === "\\") slashes += 1; + if (slashes % 2 === 0) return quote + 1; + quote = text.indexOf(mark, quote + 1); + } + return -1; +}; + +/** Whether the token after a string literal is a colon, which makes it a member name or dictionary key. */ +const isMemberName = (text: string, end: number): boolean => { + let next = end; + while (JSON_SPACE.has(text[next] ?? "")) next += 1; + return text[next] === ":"; +}; + +/** + * JSON with each string value scrubbed as its decoded value and rewritten + * only when that changed, so names, literals, numbers and other strings keep + * their spelling. A name is never rewritten, since two could then collide; + * one holding the identity is left for the search for what is left. The + * document, already read as JSON, is walked one string at a time, so a long + * string costs no more stack than a short one. + */ +const scrubJson = (text: string, identity: HostIdentity): TextScrub => { + const parts: string[] = []; + const strings: string[] = []; + let copied = 0; + for (let start = text.indexOf('"'); start !== -1;) { + const end = stringEnd(text, start); + if (end === -1) break; + const value = JSON.parse(text.slice(start, end)) as string; + const clean = isMemberName(text, end) + ? value + : scrubHostIdentity(value, identity); + strings.push(clean); + if (clean !== value) { + parts.push(text.slice(copied, start), JSON.stringify(clean)); + copied = end; + } + start = text.indexOf('"', end); + } + parts.push(text.slice(copied)); + const scrubbed = parts.join(""); + return { text: scrubbed, decoded: [scrubbed, ...strings].join("\n") }; +}; + +/** Comments, CDATA sections, processing instructions and tags of a document already read as XML; the rest is character data. */ +const XML_MARKUP = + /||<\?[\s\S]*?\?>|<[^>"']*(?:(?:"[^"]*"|'[^']*')[^>"']*)*>/gu; +const XML_VALUE = /"([^"]*)"|'([^']*)'/gu; +const XML_REFERENCE = + /&(?:#x([0-9a-fA-F]+)|#([0-9]+)|(lt|gt|amp|quot|apos));/gu; +const XML_ENTITIES: Readonly> = { + lt: "<", + gt: ">", + amp: "&", + quot: '"', + apos: "'", +}; + +/** A character reference's character, or the reference as written when it names no Unicode code point. */ +const referenced = (whole: string, point: number): string => + point <= 0x10ffff ? String.fromCodePoint(point) : whole; + +const decodeXml = (text: string): string => + text.replace( + XML_REFERENCE, + (whole, hex?: string, decimal?: string, name?: string) => + hex !== undefined + ? referenced(whole, Number.parseInt(hex, 16)) + : decimal !== undefined + ? referenced(whole, Number(decimal)) + : (XML_ENTITIES[name ?? ""] ?? whole), + ); + +/** Character data, or an attribute value in the given quote, escaped so XML reads it back as written. */ +const encodeXml = (text: string, quote: string | undefined): string => + text.replace(/[&<>\r\t\n"']/gu, (character) => { + if (character === "&") return "&"; + if (character === "<") return "<"; + if (character === ">") return ">"; + if (character === "\r") return " "; + if (quote === undefined || (character !== quote && character.trim() !== "")) + return character; + return `&#${String(character.codePointAt(0))};`; + }); + +/** + * XML with character data, attribute values, comments and CDATA scrubbed, + * character data and attribute values as their decoded text and rewritten + * only when that changed; names and processing instructions are kept. Only + * changed spans are copied out, so however many elements the document has, + * the scrub holds no more than its changes. + */ +const scrubXml = (text: string, identity: HostIdentity): TextScrub => { + const data = (raw: string, quote?: string) => { + const value = decodeXml(raw); + const clean = scrubHostIdentity(value, identity); + return clean === value ? raw : encodeXml(clean, quote); + }; + const parts: string[] = []; + let copied = 0; + /** Replace the span at `start` written `raw` with `clean` when they differ. */ + const replace = (start: number, raw: string, clean: string) => { + if (clean === raw) return; + parts.push(text.slice(copied, start), clean); + copied = start + raw.length; + }; + let end = 0; + for (const match of text.matchAll(XML_MARKUP)) { + const [markup, comment, cdata] = match; + const characters = text.slice(end, match.index); + replace(end, characters, data(characters)); + end = match.index + markup.length; + if (comment !== undefined) + replace( + match.index, + markup, + ``, + ); + else if (cdata !== undefined) + replace( + match.index, + markup, + ``, + ); + else if (!markup.startsWith(" + double === undefined + ? `'${data(single ?? "", "'")}'` + : `"${data(double, '"')}"`, + ), + ); + } + const rest = text.slice(end); + replace(end, rest, data(rest)); + parts.push(text.slice(copied)); + const scrubbed = parts.join(""); + return { text: scrubbed, decoded: decodeXml(scrubbed) }; +}; + +/** GVariant escapes that stand for one character, as `g_variant_parse` reads them; a backslash before a line break joins lines. */ +const GVARIANT_ESCAPES: Readonly> = { + a: "\u0007", + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + v: "\v", + "\n": "", +}; +const GVARIANT_ESCAPE = /\\(?:u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8})|([\s\S]))/gu; +/** What, outside quotes, types a GVariant literal as an object path or signature rather than a string. */ +const GVARIANT_PATH_OR_SIGNATURE = + /\b(?:objectpath|signature)\b|@[a-z(){}]*[og]/u; + +/** Whether a code point is one a GVariant string can hold: not NUL, a surrogate or past Unicode. */ +const isCharacter = (point: number): boolean => + point > 0 && point <= 0x10ffff && (point < 0xd800 || point >= 0xe000); + +/** A GVariant string literal's body decoded, or undefined where `g_variant_parse` refuses it. */ +const decodeGVariant = (body: string): string | undefined => { + let refused = body.includes("\0"); + const value = body.replace( + GVARIANT_ESCAPE, + (_escape, short?: string, long?: string, other?: string) => { + if (other !== undefined) { + refused ||= other === "u" || other === "U"; + return GVARIANT_ESCAPES[other] ?? other; + } + const point = Number.parseInt(short ?? long ?? "0", 16); + if (isCharacter(point)) return String.fromCodePoint(point); + refused = true; + return ""; + }, + ); + return refused ? undefined : value; +}; + +/** A GVariant string literal in the given quote, escaping what would end it or break its key file line. */ +const encodeGVariant = (value: string, quote: string): string => + `${quote}${value.replace(/[\\'"\p{Cc}]/gu, (character) => { + if (character === "\\" || character === quote) return `\\${character}`; + if (character === "'" || character === '"') return character; + return `\\u${(character.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`; + })}${quote}`; + +/** Whether the literal opening at `start` is a bytestring, its `b` starting a token. */ +const isBytestring = (text: string, start: number): boolean => + text[start - 1] === "b" && !/[A-Za-z0-9]/u.test(text[start - 2] ?? ""); + +/** + * A dconf value, GVariant text as `dconf dump` prints it, with each string + * scrubbed as its decoded value and rewritten only when that changed, as JSON + * strings are: dictionary keys, bytestrings and everything outside quotes + * keep their spelling. Undefined, so the value ships as captured, when a + * literal does not parse or the value holds an object path or signature, + * whose quoted literals are not strings. + */ +const scrubGVariant = ( + text: string, + identity: HostIdentity, +): TextScrub | undefined => { + const parts: string[] = []; + const strings: string[] = []; + const quotes = /['"]/gu; + let outside = ""; + let last = 0; + let copied = 0; + for ( + let match = quotes.exec(text); + match !== null; + match = quotes.exec(text) + ) { + const start = match.index; + const end = stringEnd(text, start); + if (end === -1) return undefined; + outside += ` ${text.slice(last, start)}`; + last = end; + quotes.lastIndex = end; + if (isBytestring(text, start)) continue; + const value = decodeGVariant(text.slice(start + 1, end - 1)); + if (value === undefined) return undefined; + const clean = isMemberName(text, end) + ? value + : scrubHostIdentity(value, identity); + strings.push(clean); + if (clean !== value) { + parts.push(text.slice(copied, start), encodeGVariant(clean, match[0])); + copied = end; + } + } + if (GVARIANT_PATH_OR_SIGNATURE.test(`${outside} ${text.slice(last)}`)) + return undefined; + parts.push(text.slice(copied)); + const scrubbed = parts.join(""); + return { text: scrubbed, decoded: [scrubbed, ...strings].join("\n") }; +}; + +const isRecord = ( + value: JsonValue | undefined, +): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** + * A dconf view's groups as the key file the computer compiles, each value's + * strings scrubbed, and whether it still holds the identity, strings decoded. + * Group names and keys are kept, so no two can collide; a value that does + * not parse ships as captured. + */ +export const scrubDconf = ( + groups: Readonly>, + identity: HostIdentity, +): { readonly text: string; readonly holdsIdentity: boolean } => { + const lines: string[] = []; + const decoded: string[] = []; + for (const [group, keys] of Object.entries(groups)) { + if (!isRecord(keys)) continue; + lines.push(`[${group}]`); + for (const [key, value] of Object.entries(keys)) { + if (typeof value !== "string") continue; + const scrub = scrubGVariant(value, identity); + lines.push(`${key}=${scrub?.text ?? value}`); + if (scrub !== undefined) decoded.push(scrub.decoded); + } + lines.push(""); + } + const text = lines.join("\n"); + return { + text, + holdsIdentity: + identityLeftIn([text, ...decoded].join("\n"), identity) !== undefined, + }; +}; + +/** Text and key files, which hold the identity as written. */ +const plainScrub = (text: string, identity: HostIdentity): TextScrub => { + const scrubbed = scrubHostIdentity(text, identity); + return { text: scrubbed, decoded: scrubbed }; +}; + +/** UTF-16 text after a byte-order mark, decoded and encoded as the computer's boot respelling does; undefined otherwise. */ +const utf16Text = (bytes: Buffer) => { + const order = [ + { mark: [0xff, 0xfe], swap: false }, + { mark: [0xfe, 0xff], swap: true }, + ].find(({ mark }) => mark.every((byte, index) => bytes[index] === byte)); + if (order === undefined) return undefined; + const little = Buffer.from(bytes.subarray(2)); + if (order.swap && little.length % 2 === 0) little.swap16(); + try { + return { + text: new TextDecoder("utf-16le", { fatal: true }).decode(little), + encode: (scrubbed: string) => { + const encoded = Buffer.from(scrubbed, "utf16le"); + if (order.swap) encoded.swap16(); + return Buffer.concat([Buffer.from(order.mark), encoded]); + }, + }; + } catch { + return undefined; + } +}; + +/** A seed by its home path and the format the state view read it as. */ +interface Seed { + readonly home: string; + readonly format: string | null; +} + +/** + * The seed's text, the format it is scrubbed as, and how to encode it again: + * UTF-8 for a text format, or UTF-16 after a byte-order mark, JSON or XML as + * its text would be in UTF-8, told by syntax alone so no view is built. + */ +const seedText = (bytes: Buffer, { home, format }: Seed) => { + if (bytes.length > MAX_SCRUBBED_BYTES) return undefined; + if (!isTextualFormat(format)) { + const utf16 = utf16Text(bytes); + return utf16 === undefined + ? undefined + : { ...utf16, format: textSyntaxFormat(home, utf16.text) }; + } + try { + return { + // The byte-order mark is kept, as Python's utf-8 codec keeps it. + text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes, + ), + encode: (scrubbed: string) => Buffer.from(scrubbed, "utf8"), + format, + }; + } catch { + return undefined; + } +}; + +/** Whether bytes it does not rewrite hold the identity: read as Latin-1, so each byte is one character, or searched byte for byte past the size it decodes. */ +const holdsIdentity = (bytes: Buffer, identity: HostIdentity): boolean => { + if (bytes.length > MAX_SCRUBBED_BYTES) + return identitySpellings(identity).some((spelling) => + bytes.includes(spelling), + ); + const latin1 = (value: string | null) => + value === null ? null : Buffer.from(value).toString("latin1"); + return ( + identityLeftIn(bytes.toString("latin1"), { + home: latin1(identity.home) ?? "", + hostname: latin1(identity.hostname), + user: latin1(identity.user), + }) !== undefined + ); +}; + +/** + * Rewrite a seed as the computer reads it at boot: a text format as UTF-8, + * and another file as UTF-16 when it starts with a byte-order mark, keeping + * the mark and byte order. JSON is scrubbed by its strings and XML by its + * character data and attribute values, in either encoding, so neither's + * syntax changes. Other bytes ship as captured. Either way what ships is + * searched, its escapes decoded, for what is left. + */ +export const scrubSeed = ( + bytes: Buffer, + seed: Seed, + identity: HostIdentity, +): SeedScrub => { + const decoded = seedText(bytes, seed); + if (decoded === undefined) + return { bytes: undefined, holdsIdentity: holdsIdentity(bytes, identity) }; + const scrub = + decoded.format === "json" + ? scrubJson(decoded.text, identity) + : decoded.format === "xml" + ? scrubXml(decoded.text, identity) + : plainScrub(decoded.text, identity); + return { + bytes: scrub.text === decoded.text ? undefined : decoded.encode(scrub.text), + holdsIdentity: identityLeftIn(scrub.decoded, identity) !== undefined, + }; +}; diff --git a/src/domain/desktop/stateChanges.test.ts b/src/domain/desktop/stateChanges.test.ts index f2d4e76d15..8dbc97343c 100644 --- a/src/domain/desktop/stateChanges.test.ts +++ b/src/domain/desktop/stateChanges.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; import { diffStateViews } from "./stateChanges.js"; +import { resolvePointer } from "./viewPointer.js"; describe("diffStateViews", () => { it("reports added and removed lines as a multiset", () => { @@ -193,3 +194,430 @@ describe("diffStateViews fidelity", () => { ).toBe(true); }); }); + +/** An archive holding one JSON entry, `a.json`, whose view is `{items}`. */ +const jsonArchive = (items: JsonValue): JsonValue => ({ + entries: [ + { + name: "a.json", + size: 1, + sha256: "a", + format: "json", + view: { items }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, +}); + +/** + * Diff an object of selector keys, each naming one of `size` members named + * n0, n1 and on, against them, and expect a few reads of each member, not a + * pass over the array per key. + */ +const expectFewReads = (keyed: Record, size: number) => { + let reads = 0; + const listed = new Proxy( + Array.from({ length: size }, (_, index) => ({ + name: `n${String(index)}`, + })), + { + get: (target, key, receiver) => { + if (typeof key === "string" && /^[0-9]+$/u.test(key)) reads += 1; + return Reflect.get(target, key, receiver) as unknown; + }, + }, + ); + + const difference = diffStateViews( + "json", + { items: keyed }, + { items: listed as JsonValue }, + ); + + expect(difference).toMatchObject({ removed: [], removed_count: 0 }); + expect(reads).toBeLessThan(10 * size); +}; + +describe("diffStateViews selector lookups", () => { + it("reads every spelling of a selector as the evaluator does", () => { + const listed = [ + { name: "n0", kind: "a" }, + { name: "n1", kind: "a" }, + ]; + const keyed = { + "[kind=a][name=n0]": 1, + "[name=n1][kind=a][name=n1]": 2, + "[name=n0][name=n1]": 3, + "[kind=a]": 4, + }; + + const difference = diffStateViews( + "json", + { items: keyed }, + { items: listed }, + ); + + expect(difference.removed).toEqual([ + "/items/[name=n0][name=n1]", + "/items/[kind=a]", + ]); + for (const key of Object.keys(keyed)) + expect( + resolvePointer({ items: listed }, `/items/${key}`) === undefined, + ).toBe(difference.removed.includes(`/items/${key}`)); + }); + + it("resolves selectors that repeat a condition as one, in one pass over the array", () => { + // An index per spelling of the same selector read the 800 members once for each of the 160. + expectFewReads( + Object.fromEntries( + Array.from({ length: 160 }, (_, index) => [ + "[name=n0]".repeat(index + 1), + index, + ]), + ), + 800, + ); + }); + + it("compares a wide object of selector keys with an array in one pass over the array, not one per key", () => { + const size = 2000; + + // A scan of the array per key read its members about size * size times. + expectFewReads( + Object.fromEntries( + Array.from({ length: size }, (_, index) => [ + `[name=n${String(index)}]`, + index, + ]), + ), + size, + ); + }); +}); + +describe("diffStateViews zip member pairing", () => { + it("compares counterparts by the evaluator's pointer rules, a bracketed key against an array as a selector", () => { + const keyed = { "[name=x]": 1 }; + const listed = [{ name: "x" }]; + const items = "/entries/[name=a.json]/view/items"; + + expect( + diffStateViews("zip", jsonArchive(keyed), jsonArchive(listed)), + ).toMatchObject({ + added: [`${items}/[name=x]/name`], + removed: [], + changed: [items, `${items}/[name=x]`], + }); + expect( + diffStateViews("zip", jsonArchive(listed), jsonArchive(keyed)), + ).toMatchObject({ + added: [], + removed: [`${items}/[name=x]/name`], + changed: [items, `${items}/[name=x]`], + }); + expect( + diffStateViews("json", { items: keyed }, { items: listed }), + ).toMatchObject({ + added: ["/items/0", "/items/0/name"], + removed: [], + changed: ["/items"], + }); + }); + + it("compares an array's members with what their pointers name once it becomes an object", () => { + const unchanged = { + added: [], + removed: [], + changed: ["/entries/[name=a.json]/view/items"], + complete: true, + }; + + expect( + diffStateViews("zip", jsonArchive([1]), jsonArchive({ "0": 1 })), + ).toMatchObject(unchanged); + expect( + diffStateViews("zip", jsonArchive({ "0": 1 }), jsonArchive([1])), + ).toMatchObject(unchanged); + }); + + it("pairs each element with at most one in the other version, so one split in two is an addition", () => { + const item = (attributes: Record) => ({ + name: "item", + attributes, + children: [], + }); + const archive = (items: JsonValue[]): JsonValue => ({ + entries: [ + { + name: "list.xml", + size: 1, + sha256: String(items.length), + format: "xml", + view: { + root: { name: "list", attributes: {}, children: items }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, + }); + + const difference = diffStateViews( + "zip", + archive([item({ a: "x", b: "y" })]), + archive([item({ b: "y" }), item({ a: "x" })]), + ); + + const items = "/entries/[name=list.xml]/view/root/children"; + expect(difference).toMatchObject({ + added: [ + `${items}/1`, + `${items}/1/name`, + `${items}/1/attributes`, + `${items}/1/attributes/a`, + `${items}/1/children`, + ], + removed: [`${items}/[name=item][@b=y]/attributes/a`], + changed: ["/entries/[name=list.xml]/sha256"], + complete: true, + }); + }); +}); + +describe("diffStateViews zip member pairing by identity", () => { + it("pairs an element by an attribute both versions keep, so another attribute's new value is a change", () => { + const item = (attributes: Record) => ({ + name: "item", + attributes, + children: [], + }); + const archive = (color: string): JsonValue => ({ + entries: [ + { + name: "list.xml", + size: 1, + sha256: color, + format: "xml", + view: { + root: { + name: "list", + attributes: {}, + children: [item({ color, id: "1" }), item({ id: "2" })], + }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, + }); + + expect( + diffStateViews("zip", archive("red"), archive("green")), + ).toMatchObject({ + added: [], + removed: [], + changed: [ + "/entries/[name=list.xml]/sha256", + "/entries/[name=list.xml]/view/root/children/[name=item][@id=1]/attributes/color", + ], + complete: true, + }); + }); + + it("counts every attribute of an identified element, so a selector it shares names nothing removed", () => { + const archive = (first: string, second: string): JsonValue => ({ + entries: [ + { + name: "list.xml", + size: 1, + sha256: first, + format: "xml", + view: { + root: { + name: "list", + attributes: {}, + children: [ + { name: "item", attributes: { id: "1", color: first } }, + { name: "item", attributes: { color: second } }, + ], + }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, + }); + + expect( + diffStateViews("zip", archive("red", "blue"), archive("blue", "red")), + ).toMatchObject({ + added: [], + removed: [], + changed: [ + "/entries/[name=list.xml]/sha256", + "/entries/[name=list.xml]/view/root/children/[name=item][@id=1]/attributes/color", + "/entries/[name=list.xml]/view/root/children/1/attributes/color", + ], + complete: true, + }); + }); + + it("keys an Inkscape group by its label when an identified group shares its fill", () => { + const archive = (transform: string): JsonValue => ({ + entries: [ + { + name: "poster.svg", + size: 1, + sha256: transform, + format: "xml", + view: { + root: { + name: "svg", + attributes: {}, + children: [ + { + name: "g", + attributes: { + fill: "red", + "inkscape:label": "note", + transform, + }, + }, + { name: "g", attributes: { fill: "red", id: "layer" } }, + ], + }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 1, + truncated: false, + }); + + expect( + diffStateViews("zip", archive("scale(1)"), archive("scale(2)")), + ).toMatchObject({ + added: [], + removed: [], + changed: [ + "/entries/[name=poster.svg]/sha256", + "/entries/[name=poster.svg]/view/root/children/[name=g][@inkscape:label=note]/attributes/transform", + ], + complete: true, + }); + }); +}); + +describe("diffStateViews zip archives", () => { + it("matches zip entries and XML elements by name, so an inserted one shifts nothing", () => { + const element = (name: string, attributes: Record) => ({ + name, + attributes, + children: [], + }); + const archive = (objects: JsonValue[], radius: string): JsonValue => ({ + entries: [ + { + name: "thumb.png", + size: 3, + sha256: "a", + format: "binary", + view: null, + }, + { + name: "Document.xml", + size: 9, + sha256: radius, + format: "xml", + view: { + root: { name: "Document", attributes: {}, children: objects }, + truncated: false, + }, + view_error: null, + }, + ], + entry_count: 2, + truncated: false, + }); + const plate = element("Object", { name: "Plate", type: "Part::Box" }); + const hole = element("Object", { name: "Hole", type: "Part::Cylinder" }); + + const difference = diffStateViews( + "zip", + archive([plate], "1"), + archive( + [hole, { ...plate, attributes: { name: "Plate", type: "Part::Cut" } }], + "2", + ), + ); + + const objects = "/entries/[name=Document.xml]/view/root/children"; + expect(difference).toMatchObject({ + kind: "values", + added: [ + `${objects}/[name=Object][@name=Hole]`, + `${objects}/[name=Object][@name=Hole]/name`, + `${objects}/[name=Object][@name=Hole]/attributes`, + `${objects}/[name=Object][@name=Hole]/attributes/name`, + `${objects}/[name=Object][@name=Hole]/attributes/type`, + `${objects}/[name=Object][@name=Hole]/children`, + ], + removed: [], + changed: [ + "/entries/[name=Document.xml]/sha256", + `${objects}/[name=Object][@name=Plate]/attributes/type`, + ], + complete: true, + }); + }); + + it("marks a zip comparison incomplete when an entry was left unread or its view cut", () => { + const listing = (entry: JsonValue): JsonValue => ({ + entries: [entry], + entry_count: 1, + truncated: false, + }); + const read = { + name: "a.txt", + size: 1, + format: "text", + view: { lines: [] }, + view_error: null, + }; + + expect( + diffStateViews( + "zip", + listing(read), + listing({ ...read, view_error: "encrypted, so it is not read" }), + ).complete, + ).toBe(false); + expect( + diffStateViews( + "zip", + listing(read), + listing({ ...read, view: { lines: [], truncated: true } }), + ).complete, + ).toBe(false); + expect( + diffStateViews("zip", listing(read), { + entries: [], + entry_count: 1001, + truncated: true, + }).complete, + ).toBe(false); + expect(diffStateViews("zip", listing(read), listing(read)).complete).toBe( + true, + ); + }); +}); diff --git a/src/domain/desktop/stateChanges.ts b/src/domain/desktop/stateChanges.ts index 7912d37295..c7e78a7e27 100644 --- a/src/domain/desktop/stateChanges.ts +++ b/src/domain/desktop/stateChanges.ts @@ -1,5 +1,7 @@ import type { JsonValue } from "../jsonValue.js"; import type { DesktopStateFormat } from "./stateFormats.js"; +import { memberLookup, pairMembers } from "./viewPointer.js"; +import { zipLeavesOut } from "./zipView.js"; const MAX_LISTED = 100; @@ -57,54 +59,98 @@ const ownEntry = (value: JsonValue): string => { return isRecord(value) ? "{}" : JSON.stringify(value); }; -/** The member a pointer segment names, the way JSON pointers resolve it. */ -const memberOf = (value: JsonValue, key: string): JsonValue | undefined => { - if (Array.isArray(value)) - return String(Number(key)) === key ? value[Number(key)] : undefined; - return isRecord(value) && Object.hasOwn(value, key) ? value[key] : undefined; +/** How a walk matches array members: by selector rather than position, and whether it notes a missing value's members too. */ +interface WalkOptions { + readonly keyed?: boolean; + readonly outermost?: boolean; +} + +/** What a walk notes, and how; `older` when it walks the earlier version against the later one. */ +interface Noting extends WalkOptions { + readonly missing: Found; + readonly changed?: Found; + readonly older?: boolean; +} + +/** Each member's key and the other version's member it pairs with, one to one, both walks pairing alike. */ +const keyedMembers = ( + from: readonly JsonValue[], + other: JsonValue | undefined, + older: boolean, +) => { + const others = Array.isArray(other) ? other : []; + const pairing = older ? pairMembers(others, from) : pairMembers(from, others); + const side = older ? pairing.older : pairing.newer; + const lookup = memberLookup(other); + return side.keys.map((key, index) => { + const partner = side.partners[index]; + // Once the array became another type, a member's counterpart is what its pointer names there. + if (!Array.isArray(other)) return { key, counterpart: lookup(key) }; + return { + key, + counterpart: partner === undefined ? undefined : others[partner], + }; + }); }; /** * Walk `from` in document order against `other`, noting each value `other` * lacks in `missing` and, when given, each whose own entry differs in - * `changed`. One path is shared, so memory follows depth, not the document. + * `changed`. One path is shared, so memory follows depth, not the document; + * a keyed walk also holds the member keys of the arrays along it. */ const walk = ( from: JsonValue, other: JsonValue | undefined, path: PointerPath, - into: { readonly missing: Found; readonly changed?: Found }, + into: Noting, ): void => { - if (other === undefined) note(into.missing, path); - else if (into.changed !== undefined && ownEntry(from) !== ownEntry(other)) + if (other === undefined) { + note(into.missing, path); + if (into.outermost === true) return; + } else if (into.changed !== undefined && ownEntry(from) !== ownEntry(other)) note(into.changed, path); - const visit = (key: string, item: JsonValue) => { + const visit = (key: string, item: JsonValue, counterpart?: JsonValue) => { const segment = escapePointer(key); path.segments.push(segment); path.length += segment.length + 1; - walk( - item, - other === undefined ? undefined : memberOf(other, key), - path, - into, - ); + walk(item, counterpart, path, into); path.segments.pop(); path.length -= segment.length + 1; }; - if (Array.isArray(from)) - from.forEach((item, index) => visit(String(index), item)); - else if (isRecord(from)) - for (const [key, item] of Object.entries(from)) visit(key, item); + if (Array.isArray(from) && into.keyed === true) { + const members = keyedMembers(from, other, into.older === true); + from.forEach((item, index) => { + const member = members[index]; + if (member !== undefined) visit(member.key, item, member.counterpart); + }); + } else if (Array.isArray(from)) { + const lookup = memberLookup(other); + from.forEach((item, index) => + visit(String(index), item, lookup(String(index))), + ); + } else if (isRecord(from)) { + // The keys go to the lookup together, so an array compared with an object is searched once per field set. + const lookup = memberLookup(other, Object.keys(from)); + for (const [key, item] of Object.entries(from)) + visit(key, item, lookup(key)); + } }; -/** Values added, removed and changed between two JSON documents, by JSON pointer. */ +/** + * Values added, removed and changed between two JSON documents, by JSON + * pointer; keyed, array members are paired one to one by the selectors naming + * them, so an inserted member does not shift the ones after it. + */ const compareTrees = ( before: JsonValue, after: JsonValue, + options: WalkOptions = {}, ): DesktopStateDifference => { const [added, removed, changed] = [found(), found(), found()]; - walk(after, before, { segments: [], length: 0 }, { missing: added, changed }); - walk(before, after, { segments: [], length: 0 }, { missing: removed }); + const start = () => ({ segments: [], length: 0 }); + walk(after, before, start(), { ...options, missing: added, changed }); + walk(before, after, start(), { ...options, missing: removed, older: true }); return { kind: "values", added: added.listed, @@ -257,9 +303,11 @@ export const diffStateViews = ( const difference = compareViews(format, before, after); // Only REA's own readers write coverage flags; JSON and key files are data. const flagged = format === "text" || format === "xml" || format === "sqlite"; - return flagged && (truncatedView(before) || truncatedView(after)) - ? { ...difference, complete: false } - : difference; + const partial = + format === "zip" + ? zipLeavesOut(before) || zipLeavesOut(after) + : flagged && (truncatedView(before) || truncatedView(after)); + return partial ? { ...difference, complete: false } : difference; }; const compareViews = ( @@ -273,6 +321,8 @@ const compareViews = ( case "json": case "xml": return compareTrees(before, after); + case "zip": + return compareTrees(before, after, { keyed: true }); case "keyfile": case "dconf": return compareMaps("keys", keyEntries(before), keyEntries(after)); @@ -288,3 +338,10 @@ const compareViews = ( return compareMaps("content", new Map(), new Map()); } }; + +/** How two zip views differ by name-addressed pointer, a member added or removed whole noted once, without its contents. */ +export const diffZipMembers = ( + before: JsonValue, + after: JsonValue, +): DesktopStateDifference => + compareTrees(before, after, { keyed: true, outermost: true }); diff --git a/src/domain/desktop/stateFormats.test.ts b/src/domain/desktop/stateFormats.test.ts index 9ca5a81252..c2558f6119 100644 --- a/src/domain/desktop/stateFormats.test.ts +++ b/src/domain/desktop/stateFormats.test.ts @@ -21,6 +21,15 @@ describe("detectStateFormat", () => { expect(detectStateFormat("/h/blob", Uint8Array.of(1, 0, 2))).toBe("binary"); }); + it("recognises a zip archive by its local file header, whatever its name", () => { + const header = Uint8Array.of(0x50, 0x4b, 3, 4, 20, 0, 0, 0); + expect(detectStateFormat("/h/plate.FCStd", header)).toBe("zip"); + expect(detectStateFormat("/h/plate.20261105.FCBak", header)).toBe("zip"); + expect( + detectStateFormat("/h/empty.odt", Uint8Array.of(0x50, 0x4b, 5, 6, 0, 0)), + ).toBe("binary"); + }); + it("tells JSON, XML, key files and plain text apart", () => { expect(detectStateFormat("/h/a.json", bytes('{"a":1}'))).toBe("json"); expect(detectStateFormat("/h/notes", bytes("{not json"))).toBe("text"); diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index 11e2f79f11..e4eed2e211 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -4,7 +4,7 @@ import { isExactJsonNumber } from "../exactJsonNumber.js"; import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; import { err, ok, type Result } from "../result.js"; import { safeParseJson } from "../safeJson.js"; -import { parseXmlDocument } from "../xmlDocument.js"; +import { isWellFormedXml, parseXmlDocument } from "../xmlDocument.js"; /** Formats the state view reads into JSON. */ export type DesktopStateFormat = @@ -14,6 +14,7 @@ export type DesktopStateFormat = | "xml" | "sqlite" | "dconf" + | "zip" | "binary"; const SQLITE_HEADER = "SQLite format 3\0"; @@ -23,12 +24,17 @@ export const SQLITE_SIDECARS = [ { suffix: "-journal", label: "rollback journal" }, ] as const; const GVDB_HEADER = "GVariant"; +/** A zip archive's local file header, which FreeCAD, ODF, OOXML, EPUB and JAR files begin with. */ +const ZIP_HEADER = "PK\u0003\u0004"; const XML_EXTENSIONS = /\.(?:xml|xbel|plist|svg|ui)$/iu; const MAX_TEXT_LINES = 5000; const MAX_XML_NODES = 5000; const MAX_XML_DEPTH = 64; /** Half of Evidence's nesting limit, leaving room for the result around a view. */ const MAX_JSON_VIEW_DEPTH = MAX_JSON_DEPTH / 2; +const MIB = 1024 * 1024; +/** Largest JSON or key file read into a view, whose whole value enters Evidence. */ +const MAX_PARSED_VIEW_BYTES = 4 * MIB; type KeyfileLine = | { readonly kind: "skip" } @@ -80,6 +86,149 @@ export const decodeStateText = (bytes: Uint8Array): string | undefined => { } }; +const JSON_NUMBER = /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/uy; +const JSON_HEX4 = /[0-9a-fA-F]{4}/uy; +const JSON_ESCAPES = new Set(['"', "\\", "/", "b", "f", "n", "r", "t"]); + +/** Where the JSON string opening at `start` ends, past its closing quote, or -1 when it is malformed. */ +const jsonStringEnd = (text: string, start: number): number => { + for (let at = start + 1; at < text.length; at += 1) { + const code = text.charCodeAt(at); + if (code === 0x22) return at + 1; + if (code < 0x20) return -1; + if (code === 0x5c) { + const escape = text.charAt(at + 1); + JSON_HEX4.lastIndex = at + 2; + if (escape === "u" && JSON_HEX4.test(text)) at += 5; + else if (JSON_ESCAPES.has(escape)) at += 1; + else return -1; + } + } + return -1; +}; + +/** Where the JSON string, number or literal starting at `start` ends, or -1 when none does. */ +const jsonScalarEnd = (text: string, start: number): number => { + if (text.charCodeAt(start) === 0x22) return jsonStringEnd(text, start); + for (const literal of ["true", "false", "null"]) + if (text.startsWith(literal, start)) return start + literal.length; + JSON_NUMBER.lastIndex = start; + return JSON_NUMBER.test(text) ? JSON_NUMBER.lastIndex : -1; +}; + +/** How far a JSON text has been read: what may come next, and the containers still open. */ +interface JsonScan { + expect: "value" | "key" | "colon" | "after"; + /** The innermost container was just opened, so it may close at once. */ + empty: boolean; + /** Each open container's opening bracket, innermost at `depth - 1`. */ + open: Uint8Array; + depth: number; +} + +/** Open a container, growing the brackets held as it nests deeper. */ +const openJson = (scan: JsonScan, bracket: number): void => { + if (scan.depth === scan.open.length) { + const grown = new Uint8Array(scan.open.length * 2); + grown.set(scan.open); + scan.open = grown; + } + scan.open[scan.depth] = bracket; + scan.depth += 1; + scan.expect = bracket === 0x5b ? "value" : "key"; + scan.empty = true; +}; + +/** Read one bracket, comma or colon into the scan; false when the text cannot hold it there. */ +const jsonPunctuation = (scan: JsonScan, char: string): boolean => { + const code = char.charCodeAt(0); + const inner = scan.depth > 0 ? scan.open[scan.depth - 1] : undefined; + switch (char) { + case "[": + case "{": + if (scan.expect !== "value") return false; + openJson(scan, code); + return true; + case "]": + case "}": + // A closing bracket's code is its opening's plus two. + if ((scan.expect !== "after" && !scan.empty) || inner !== code - 2) + return false; + scan.depth -= 1; + break; + case ",": + if (scan.expect !== "after" || inner === undefined) return false; + scan.expect = inner === 0x5b ? "value" : "key"; + return true; + default: + if (scan.expect !== "colon") return false; + scan.expect = "value"; + return true; + } + scan.expect = "after"; + scan.empty = false; + return true; +}; + +/** Read a string, number or literal into the scan; false when the text cannot hold it there. */ +const jsonScalar = (scan: JsonScan, string: boolean): boolean => { + if (scan.expect === "value") scan.expect = "after"; + else if (string && scan.expect === "key") scan.expect = "colon"; + else return false; + scan.empty = false; + return true; +}; + +/** + * Whether text is one JSON value as `JSON.parse` reads it, checked a token at + * a time without building the value: it holds a byte per open container, so + * text too large to parse into a view is still told apart from other text. + */ +export const isJsonText = (text: string): boolean => { + const scan: JsonScan = { + expect: "value", + empty: false, + open: new Uint8Array(64), + depth: 0, + }; + for (let at = 0; at < text.length;) { + const char = text.charAt(at); + if (" \t\n\r".includes(char)) at += 1; + else if ("[]{},:".includes(char)) { + if (!jsonPunctuation(scan, char)) return false; + at += 1; + } else { + at = jsonScalarEnd(text, at); + if (at < 0 || !jsonScalar(scan, char === '"')) return false; + } + } + return scan.expect === "after" && scan.depth === 0; +}; + +/** Whether a file's name or leading text marks it as JSON, which it is if it parses. */ +const namedJson = (path: string, trimmed: string): boolean => + path.endsWith(".json") || /^[[{]/u.test(trimmed); + +/** Whether a file's name or XML declaration marks it as XML, which it is if it parses. */ +const namedXml = (path: string, trimmed: string): boolean => + XML_EXTENSIONS.test(path) || trimmed.startsWith(" { + const trimmed = text.trimStart(); + if (namedJson(path, trimmed) && isJsonText(text)) return "json"; + if (namedXml(path, trimmed) && isWellFormedXml(text, { doctype: "reject" })) + return "xml"; + return "text"; +}; + /** Pick the reader for one retained file from its leading bytes and name. */ export const detectStateFormat = ( path: string, @@ -88,15 +237,19 @@ export const detectStateFormat = ( const header = new TextDecoder("latin1").decode(bytes.subarray(0, 16)); if (header === SQLITE_HEADER) return "sqlite"; if (header.startsWith(GVDB_HEADER)) return "dconf"; + if (header.startsWith(ZIP_HEADER)) return "zip"; const text = decodeStateText(bytes); if (text === undefined) return "binary"; const trimmed = text.trimStart(); if ( - (path.endsWith(".json") || /^[[{]/u.test(trimmed)) && - safeParseJson(text).ok + namedJson(path, trimmed) && + // JSON too large for a view is told apart without building its value. + (bytes.length > MAX_PARSED_VIEW_BYTES + ? isJsonText(text) + : safeParseJson(text).ok) ) return "json"; - if (XML_EXTENSIONS.test(path) || trimmed.startsWith(" { - const pending: [JsonValue, number][] = [[root, 0]]; - for (let next = pending.pop(); next !== undefined; next = pending.pop()) { - const [value, depth] = next; - if (depth > limit) return true; - if (typeof value === "object" && value !== null) - for (const item of Object.values(value)) pending.push([item, depth + 1]); +/** An object's own values one at a time, without copying them into an array. */ +function* ownValues(record: Record): Generator { + for (const key in record) + if (Object.hasOwn(record, key)) yield record[key] ?? null; +} + +/** + * Whether a parsed JSON value nests deeper than a limit, checked without + * recursion: a value's depth is the number of levels open above it, each + * walked by one iterator, so no container's members are copied or queued. + */ +export const deeperThan = (root: JsonValue, limit: number): boolean => { + const levels: Iterator[] = [[root].values()]; + for (let level = levels.at(-1); level !== undefined; level = levels.at(-1)) { + const next = level.next(); + if (next.done === true) levels.pop(); + else if (levels.length - 1 > limit) return true; + else if (Array.isArray(next.value)) levels.push(next.value.values()); + else if (typeof next.value === "object" && next.value !== null) + levels.push(ownValues(next.value)); } return false; }; @@ -256,3 +421,44 @@ export const xmlView = (text: string): JsonValue | undefined => { ? undefined : { root: tree, truncated: budget.truncated }; }; + +/** Formats read from a file's bytes alone, without a tool or worker. */ +export type TextualStateFormat = "text" | "json" | "keyfile" | "xml"; + +/** Whether a format is one of the text formats, read as UTF-8. */ +export const isTextualFormat = ( + format: string | null, +): format is TextualStateFormat => + format === "text" || + format === "json" || + format === "keyfile" || + format === "xml"; + +/** A text, JSON, key file or XML file's view, or why it has none. */ +export const textualView = ( + format: TextualStateFormat, + bytes: Uint8Array, +): Result => { + if ( + (format === "json" || format === "keyfile") && + bytes.length > MAX_PARSED_VIEW_BYTES + ) + return err( + `larger than the ${String(MAX_PARSED_VIEW_BYTES / MIB)} MiB limit for JSON and key file views`, + ); + const text = decodeStateText(bytes) ?? ""; + switch (format) { + case "text": + return ok(textView(text)); + case "json": + return jsonView(text); + case "keyfile": + return ok({ groups: keyfileView(text) }); + case "xml": { + const tree = xmlView(text); + return tree === undefined + ? err("not well-formed XML, or it declares a DOCTYPE") + : ok(tree); + } + } +}; diff --git a/src/domain/desktop/stateRoles.test.ts b/src/domain/desktop/stateRoles.test.ts index 466219dd53..a76c1424ee 100644 --- a/src/domain/desktop/stateRoles.test.ts +++ b/src/domain/desktop/stateRoles.test.ts @@ -58,6 +58,31 @@ describe("stateReasons", () => { }); }); +describe("stateReasons for state directories", () => { + it("counts a folder beside a state directory whose name differs only in letter case, as GIMP's ~/.config/GIMP does", () => { + const facts: DesktopRoleFacts = { + home: "/home/ada", + stateDirectories: ["/data/ada/.config/gimp"], + stateTails: [".config/gimp"], + selectedRoots: [], + titles: [], + openedFiles: new Set(), + }; + const reasons = (path: string) => stateReasons(facts, path, [], false); + + expect(reasons("/data/ada/.config/GIMP/2.10/sessionrc")).toEqual([ + "state_directory", + ]); + expect(reasons("/home/ada/.config/GIMP/2.10/sessionrc")).toEqual([ + "state_directory", + ]); + expect(reasons("/home/ada/.config/gimp")).toEqual(["state_directory"]); + expect(reasons("/home/ada/.config/GIMPshop/sessionrc")).toEqual([]); + expect(reasons("/home/ada/.config/other/GIMP/sessionrc")).toEqual([]); + expect(reasons("/home/ada/.Config/gimp/sessionrc")).toEqual([]); + }); +}); + describe("isApplicationDconfGroup", () => { it("matches a whole path segment, not a substring", () => { const names = new Set(["mousepad"]); diff --git a/src/domain/desktop/stateRoles.ts b/src/domain/desktop/stateRoles.ts index f25084a087..6b92aded99 100644 --- a/src/domain/desktop/stateRoles.ts +++ b/src/domain/desktop/stateRoles.ts @@ -85,6 +85,24 @@ const hiddenTail = (path: string): string | null => { return index < 0 ? null : path.slice(index + 1); }; +/** The state directory a path lies in, or the folder beside one whose name differs only in letter case that the application may create during the capture, as the path spells it. */ +export const stateDirectoryOf = ( + path: string, + directories: readonly string[], +): string | undefined => { + for (const directory of directories) { + const parent = directory.slice(0, directory.lastIndexOf("/") + 1); + if (!path.startsWith(parent)) continue; + const folder = path.slice(parent.length).split("/")[0] ?? ""; + if ( + folder.length > 0 && + folder.toLowerCase() === directory.slice(parent.length).toLowerCase() + ) + return `${parent}${folder}`; + } + return undefined; +}; + /** Facts about the whole capture that every path's reasons draw on. */ export interface DesktopRoleFacts { readonly home: string; @@ -170,8 +188,9 @@ export const stateReasons = ( const reasons: DesktopStateReason[] = []; if ( changes.some((change) => change.in_state_directory) || - isWithinAny(path, facts.stateDirectories) || - (relative !== null && isWithinAny(relative, facts.stateTails)) + stateDirectoryOf(path, facts.stateDirectories) !== undefined || + (relative !== null && + stateDirectoryOf(relative, facts.stateTails) !== undefined) ) reasons.push("state_directory"); if ( diff --git a/src/domain/desktop/viewPointer.test.ts b/src/domain/desktop/viewPointer.test.ts new file mode 100644 index 0000000000..e0e390083d --- /dev/null +++ b/src/domain/desktop/viewPointer.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from "vitest"; + +import type { JsonValue } from "../jsonValue.js"; +import { locatePointer, pairMembers } from "./viewPointer.js"; + +const element = ( + name: string, + attributes: Record = {}, + children: JsonValue[] = [], +): JsonValue => ({ name, attributes, children }); + +const VIEW: JsonValue = { + root: element("Objects", {}, [ + "text between", + element("Object", { name: "Plate", type: "Part::Box" }), + element("Object", { name: "Hole", type: "Part::Cylinder" }), + element("Object", { name: "Pad", type: "Part::Box" }), + element("Note", { "odd]key": "x" }), + ]), + entries: [ + { name: "a/b.xml", size: 1 }, + { name: "x][y", size: 2 }, + { name: 3, size: 3 }, + ], +}; + +describe("locatePointer", () => { + it("selects the one member a selector names, by field, by attribute, or both", () => { + expect( + locatePointer( + VIEW, + "/root/children/[name=Object][@name=Hole]/attributes/type", + ), + ).toMatchObject({ kind: "value", value: "Part::Cylinder" }); + expect( + locatePointer(VIEW, "/root/children/[@name=Pad]/name"), + ).toMatchObject({ + kind: "value", + value: "Object", + }); + expect(locatePointer(VIEW, "/entries/[name=a~1b.xml]/size")).toMatchObject({ + kind: "value", + value: 1, + }); + expect(locatePointer(VIEW, "/root/children/0")).toMatchObject({ + kind: "value", + value: "text between", + }); + }); + + it("reports the arrays a selector searched, and those it entered by position at an object", () => { + expect( + locatePointer( + VIEW, + "/root/children/[name=Object][@name=Hole]/attributes/type", + ), + ).toMatchObject({ searched: ["/root/children"], byPosition: [] }); + expect(locatePointer(VIEW, "/root/children/1/name")).toMatchObject({ + searched: [], + byPosition: ["/root/children"], + }); + expect(locatePointer(VIEW, "/root/children/0")).toMatchObject({ + byPosition: [], + }); + expect( + locatePointer(VIEW, "/root/children/[@type=Part::Box]"), + ).toMatchObject({ kind: "unresolved", searched: ["/root/children"] }); + }); + + it("names no value when the last selector matches nothing, and is unresolved otherwise", () => { + expect( + locatePointer(VIEW, "/root/children/[name=Object][@name=Gone]"), + ).toMatchObject({ + kind: "none", + reason: "[name=Object][@name=Gone] matches no member", + }); + expect( + locatePointer(VIEW, "/root/children/[name=Object][@name=Gone]/name"), + ).toMatchObject({ + kind: "unresolved", + problem: "[name=Object][@name=Gone] matches no member", + }); + expect( + locatePointer(VIEW, "/root/children/[@type=Part::Box]"), + ).toMatchObject({ + kind: "unresolved", + problem: "[@type=Part::Box] matches 2 members, not one", + }); + expect(locatePointer(VIEW, "/root/children/[Object]")).toMatchObject({ + kind: "unresolved", + problem: "[Object] is not a selector of the form [KEY=VALUE]", + }); + expect(locatePointer(VIEW, "/root/children/[=x]")).toMatchObject({ + kind: "unresolved", + }); + }); + + it("compares strings only and keeps an ordinary key on an object", () => { + expect(locatePointer(VIEW, "/entries/[name=3]")).toMatchObject({ + kind: "none", + reason: "[name=3] matches no member", + }); + expect(locatePointer({ "[a=b]": 1 }, "/[a=b]")).toMatchObject({ + kind: "value", + value: 1, + }); + expect(locatePointer(VIEW, "/root/children/01")).toMatchObject({ + kind: "none", + }); + expect(locatePointer(null, "/x")).toMatchObject({ kind: "none" }); + }); +}); + +describe("pairMembers", () => { + it("keys each member by the most general selector naming it alone, else by position", () => { + const children = (VIEW as { root: { children: JsonValue[] } }).root + .children; + const entries = (VIEW as { entries: JsonValue[] }).entries; + + expect(pairMembers(children, []).newer.keys).toEqual([ + "0", + "[name=Object][@name=Plate]", + "[name=Object][@name=Hole]", + "[name=Object][@name=Pad]", + "[name=Note]", + ]); + expect(pairMembers(entries, []).newer.keys).toEqual([ + "[name=a/b.xml]", + "1", + "2", + ]); + }); + + it("pairs one to one, so a member another selector also names is not paired twice", () => { + const item = (attributes: Record) => + element("item", attributes); + + expect( + pairMembers( + [item({ b: "y" }), item({ a: "x" })], + [item({ a: "x", b: "y" })], + ), + ).toEqual({ + newer: { keys: ["[name=item][@b=y]", "1"], partners: [0, undefined] }, + older: { keys: ["[name=item][@b=y]"], partners: [0] }, + }); + }); + + it("passes over a selector that names several members of the other version", () => { + const pairing = pairMembers( + [ + element("Object", { name: "Plate" }), + element("Object", { name: "Hole" }), + ], + [element("Object", { name: "Plate" })], + ); + + expect(pairing).toEqual({ + newer: { + keys: ["[name=Object][@name=Plate]", "[name=Object][@name=Hole]"], + partners: [0, undefined], + }, + older: { keys: ["[name=Object][@name=Plate]"], partners: [0] }, + }); + }); + + it("pairs an element by its identifier, so another attribute that changed is an edit", () => { + const pairing = pairMembers( + [ + element("item", { color: "green", id: "1" }), + element("item", { id: "2" }), + ], + [ + element("item", { color: "red", id: "1" }), + element("item", { id: "2" }), + ], + ); + + const keys = ["[name=item][@id=1]", "[name=item][@id=2]"]; + expect(pairing).toEqual({ + newer: { keys, partners: [0, 1] }, + older: { keys, partners: [0, 1] }, + }); + }); + + it("counts an identified sibling's other attributes, keying a group by its label rather than a fill both share", () => { + const note = (transform: string) => + element("g", { fill: "red", "inkscape:label": "note", transform }); + const layer = element("g", { fill: "red", id: "layer" }); + + const keys = ["[name=g][@inkscape:label=note]", "[name=g][@id=layer]"]; + expect( + pairMembers([note("scale(2)"), layer], [note("scale(1)"), layer]), + ).toEqual({ + newer: { keys, partners: [0, 1] }, + older: { keys, partners: [0, 1] }, + }); + }); + + it("pairs an element with identifiers by those alone, so one replaced by another of its type is removed and added", () => { + const object = (name: string, id: string) => + element("Object", { type: "Part::Cylinder", name, id }); + const plate = element("Object", { type: "Part::Box", name: "Plate" }); + + expect( + pairMembers( + [plate, object("Hole001", "3")], + [plate, object("Hole", "2")], + ), + ).toEqual({ + newer: { + keys: ["[name=Object][@name=Plate]", "[name=Object][@name=Hole001]"], + partners: [0, undefined], + }, + older: { + keys: ["[name=Object][@name=Plate]", "[name=Object][@name=Hole]"], + partners: [0, undefined], + }, + }); + }); + + it("pairs members no selector names by position", () => { + expect(pairMembers(["a", "b", "c"], ["a", "x"])).toEqual({ + newer: { keys: ["0", "1", "2"], partners: [0, 1, undefined] }, + older: { keys: ["0", "1"], partners: [0, 1] }, + }); + }); + + it("leaves out values a selector cannot carry and times an app stamps", () => { + expect( + pairMembers( + [ + element("Save", { at: "2026-11-04T10:12:00Z", id: "1" }), + element("Save", { at: "2026-11-04T10:13:00Z", id: "2" }), + ], + [], + ).newer.keys, + ).toEqual(["[name=Save][@id=1]", "[name=Save][@id=2]"]); + }); +}); diff --git a/src/domain/desktop/viewPointer.ts b/src/domain/desktop/viewPointer.ts new file mode 100644 index 0000000000..2f5b5b735d --- /dev/null +++ b/src/domain/desktop/viewPointer.ts @@ -0,0 +1,400 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isTimestamp } from "./appTask.js"; + +/** One condition of a selector: a member's string field, or with `@` one of its attributes, equal to a value. */ +interface Condition { + readonly attribute: boolean; + readonly key: string; + readonly value: string; +} + +/** The arrays a pointer stepped into, each as the pointer to it. */ +interface ArraySteps { + /** Arrays a selector searched, whose every member it had to see. */ + readonly searched: readonly string[]; + /** Arrays it entered by position at a member that is an object or array. */ + readonly byPosition: readonly string[]; +} + +/** Where a pointer leads in a view, and the arrays it stepped into on the way. */ +export type Located = ArraySteps & + ( + | { readonly kind: "value"; readonly value: JsonValue } + /** It names no value; `reason` says which last selector matched nothing, when one did. */ + | { readonly kind: "none"; readonly reason?: string } + /** A selector before the last segment matched no member, or any matched several, or one is malformed. */ + | { readonly kind: "unresolved"; readonly problem: string } + ); + +/** Where a step into an array leads, before the steps are attached. */ +type Stepped = + | { readonly kind: "none"; readonly reason?: string } + | { readonly kind: "unresolved"; readonly problem: string }; + +const INDEX = /^(?:0|[1-9][0-9]*)$/u; + +const isRecord = ( + value: JsonValue | null | undefined, +): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const own = (record: Record, key: string) => + Object.hasOwn(record, key) ? record[key] : undefined; + +const bracketed = (segment: string) => + segment.length >= 2 && segment.startsWith("[") && segment.endsWith("]"); + +/** A segment's conditions when it is a selector; otherwise undefined. A value cannot hold `][`. */ +const selectorOf = (segment: string): readonly Condition[] | undefined => { + if (!bracketed(segment)) return undefined; + const conditions: Condition[] = []; + for (const part of segment.slice(1, -1).split("][")) { + const equals = part.indexOf("="); + const key = equals < 0 ? "" : part.slice(0, equals); + const attribute = key.startsWith("@"); + const name = attribute ? key.slice(1) : key; + if (name === "") return undefined; + conditions.push({ attribute, key: name, value: part.slice(equals + 1) }); + } + return conditions; +}; + +/** Whether an array member meets every condition; selectors compare strings only. */ +const selects = (member: JsonValue, conditions: readonly Condition[]) => + isRecord(member) && + conditions.every(({ attribute, key, value }) => { + const fields = attribute ? own(member, "attributes") : member; + return isRecord(fields) && own(fields, key) === value; + }); + +/** One step into an array: by index, or by a selector that must match one member, or none when it is the last. */ +const arrayStep = ( + items: readonly JsonValue[], + token: string, + last: boolean, +): { readonly member: JsonValue } | Stepped => { + if (INDEX.test(token)) { + const index = Number(token); + return index < items.length + ? { member: items[index] ?? null } + : { kind: "none" }; + } + const conditions = selectorOf(token); + if (conditions === undefined) + return bracketed(token) + ? { + kind: "unresolved", + problem: `${token} is not a selector of the form [KEY=VALUE]`, + } + : { kind: "none" }; + const matched = items.filter((item) => selects(item, conditions)); + if (matched.length === 1) return { member: matched[0] ?? null }; + if (matched.length > 1) + return { + kind: "unresolved", + problem: `${token} matches ${String(matched.length)} members, not one`, + }; + const reason = `${token} matches no member`; + return last + ? { kind: "none", reason } + : { kind: "unresolved", problem: reason }; +}; + +/** + * Follow an RFC 6901 pointer through a view, where a segment applied to an + * array may also be a selector: `[KEY=VALUE]` one or more times, naming the one + * member whose string field KEY equals VALUE, or with `@KEY` whose `attributes` + * hold it, as REA's XML view keeps them. So `/root/children/[name=Object][@name=Hole]` + * is the `Object` element named `Hole`, wherever it sits among its siblings. + */ +export const locatePointer = ( + view: JsonValue | null, + pointer: string, +): Located => { + let current: JsonValue = view; + const segments = pointer.slice(1).split("/"); + const steps = { searched: [] as string[], byPosition: [] as string[] }; + for (const [position, segment] of segments.entries()) { + const token = segment.replaceAll("~1", "/").replaceAll("~0", "~"); + if (Array.isArray(current)) { + const array = segments + .slice(0, position) + .map((part) => `/${part}`) + .join(""); + const indexed = INDEX.test(token); + if (!indexed && selectorOf(token) !== undefined) + steps.searched.push(array); + const step = arrayStep(current, token, position === segments.length - 1); + if ("kind" in step) return { ...step, ...steps }; + if (indexed && typeof step.member === "object" && step.member !== null) + steps.byPosition.push(array); + current = step.member; + } else if (isRecord(current) && Object.hasOwn(current, token)) + current = current[token] ?? null; + else return { kind: "none", ...steps }; + } + return { kind: "value", value: current, ...steps }; +}; + +/** The field a condition reads from a member, when it is a string. */ +const fieldOf = (member: JsonValue, { attribute, key }: Condition) => { + if (!isRecord(member)) return undefined; + const fields = attribute ? own(member, "attributes") : member; + const value = isRecord(fields) ? own(fields, key) : undefined; + return typeof value === "string" ? value : undefined; +}; + +/** Marks selector values several members share, which therefore name none. */ +const SHARED = Symbol("shared"); + +/** + * A selector's conditions once per field, in field order, so every spelling of + * one selector reads alike; undefined when it is malformed or two conditions on + * one field differ, which no member meets. + */ +const normalSelector = (token: string): readonly Condition[] | undefined => { + const conditions = selectorOf(token); + if (conditions === undefined) return undefined; + const byField = new Map(); + for (const condition of conditions) { + const field = `${condition.attribute ? "@" : ""}${condition.key}`; + const seen = byField.get(field); + if (seen !== undefined && seen.value !== condition.value) return undefined; + byField.set(field, condition); + } + return [...byField.keys()] + .sort() + .flatMap((field) => byField.get(field) ?? []); +}; + +/** Selectors grouped by the fields they read, each wanted value list with the tokens spelling it. */ +const selectorGroups = (tokens: Iterable) => { + const groups = new Map< + string, + { conditions: readonly Condition[]; wanted: Map } + >(); + for (const token of tokens) { + if (INDEX.test(token)) continue; + const conditions = normalSelector(token); + if (conditions === undefined) continue; + const fields = JSON.stringify( + conditions.map(({ attribute, key }) => [attribute, key]), + ); + const values = JSON.stringify(conditions.map(({ value }) => value)); + const group = groups.get(fields) ?? { conditions, wanted: new Map() }; + groups.set(fields, group); + const spellings = group.wanted.get(values); + if (spellings === undefined) group.wanted.set(values, [token]); + else spellings.push(token); + } + return groups.values(); +}; + +/** + * The member each selector among `tokens` names alone, found in one pass over + * the array per set of fields they read, holding only the values they ask for. + */ +const selectedMembers = ( + items: readonly JsonValue[], + tokens: Iterable, +): ReadonlyMap => { + const selected = new Map(); + for (const { conditions, wanted } of selectorGroups(tokens)) { + const found = new Map(); + for (const item of items) { + const values = conditions.map((condition) => fieldOf(item, condition)); + if (values.some((value) => value === undefined)) continue; + const key = JSON.stringify(values); + if (wanted.has(key)) found.set(key, found.has(key) ? SHARED : item); + } + for (const [key, member] of found) + if (member !== SHARED) + for (const token of wanted.get(key) ?? []) selected.set(token, member); + } + return selected; +}; + +/** + * What pointer tokens name in one value by the evaluator's rules: an own key, + * an index, or the one member a selector among `tokens` matches. The selectors + * are resolved together up front, so many cost one pass per set of fields they + * read, and what is kept grows with the tokens, not with the array. + */ +export const memberLookup = ( + value: JsonValue | undefined, + tokens: Iterable = [], +): ((token: string) => JsonValue | undefined) => { + if (!Array.isArray(value)) + return (token) => + isRecord(value) && Object.hasOwn(value, token) ? value[token] : undefined; + const selected = selectedMembers(value, tokens); + return (token) => + INDEX.test(token) ? value[Number(token)] : selected.get(token); +}; + +/** The value a pointer names, or undefined when it names none or a selector on its way is unresolved. */ +export const resolvePointer = ( + view: JsonValue | null, + pointer: string, +): JsonValue | undefined => { + const located = locatePointer(view, pointer); + return located.kind === "value" ? located.value : undefined; +}; + +/** A value a selector can carry and that stays the same between saves. */ +const selectable = (value: string) => + !value.includes("][") && !isTimestamp(value); + +/** An attribute name a selector can carry. */ +const attributeName = (key: string) => key !== "" && !/[=[\]]/u.test(key); + +/** + * A member's selectors as its name and the conditions after it, the ids they + * are counted by, and how many of them, from the first, may key it. + */ +interface Candidates { + readonly name: string; + readonly suffixes: readonly string[]; + readonly ids: readonly string[]; + readonly choosable: number; +} + +/** An attribute that identifies its element: `name`, `id`, `xid`, or a prefixed `id` such as `xml:id`. */ +const identifying = (key: string) => /^(?:name|id|xid|.+:id)$/iu.test(key); + +/** + * The selectors that could name a member, most general first: `[name=N]`, then + * `[name=N][@A=V]` for each attribute, a `name` attribute before the others. + * All are counted, so a selector several siblings match is never taken for one + * naming a member alone; but an element with identifying attributes is keyed + * by those alone, so one whose identifiers are new is another element, + * whatever else it shares. The ids number the name among those seen, so a + * name is spelled into a selector only once one is chosen, not per attribute. + */ +const candidatesOf = ( + member: JsonValue, + names: Map, +): Candidates | undefined => { + if (!isRecord(member)) return undefined; + const name = own(member, "name"); + if (typeof name !== "string" || !selectable(name)) return undefined; + const attributes = own(member, "attributes"); + const usable = ( + isRecord(attributes) ? Object.entries(attributes) : [] + ).flatMap(([key, value]) => + typeof value === "string" && attributeName(key) && selectable(value) + ? [{ key, value }] + : [], + ); + const identities = usable.filter(({ key }) => identifying(key)); + const named = (key: string) => key.toLowerCase() === "name"; + const suffixes = [ + "", + ...[ + ...identities.filter(({ key }) => named(key)), + ...identities.filter(({ key }) => !named(key)), + ...usable.filter(({ key }) => !identifying(key)), + ].map(({ key, value }) => `[@${key}=${value}]`), + ]; + const number = names.get(name) ?? names.size; + names.set(name, number); + // A suffix is empty or opens with `[`, so the number before it cannot run into it. + const ids = suffixes.map((suffix) => `${String(number)}${suffix}`); + const choosable = + identities.length > 0 ? identities.length + 1 : suffixes.length; + return { name, suffixes, ids, choosable }; +}; + +/** The selector a member's candidate at `index` spells. */ +const selectorAt = ({ name, suffixes }: Candidates, index: number) => + `[name=${name}]${suffixes[index] ?? ""}`; + +const countIds = (members: readonly (Candidates | undefined)[]) => { + const counts = new Map(); + for (const candidates of members) + for (const id of candidates?.ids ?? []) + counts.set(id, (counts.get(id) ?? 0) + 1); + return counts; +}; + +/** One version's side of a pairing: each member's key, and the other version's member it pairs with. */ +interface PairedSide { + readonly keys: readonly string[]; + readonly partners: readonly (number | undefined)[]; +} + +/** + * Pair the members of two versions of an array one to one. A member pairs by + * its first candidate selector naming it alone in its version and one unpaired + * member of the other, whichever before it name nothing there; one with no + * partner keeps its first selector naming it alone and nothing in the other + * version. Members left without a selector are keyed and paired by position. + */ +export const pairMembers = ( + newer: readonly JsonValue[], + older: readonly JsonValue[], +): { readonly newer: PairedSide; readonly older: PairedSide } => { + const names = new Map(); + const newCandidates = newer.map((member) => candidatesOf(member, names)); + const oldCandidates = older.map((member) => candidatesOf(member, names)); + const newCounts = countIds(newCandidates); + const oldCounts = countIds(oldCandidates); + const count = (counts: ReadonlyMap, id: string) => + counts.get(id) ?? 0; + const oldById = new Map(); + oldCandidates.forEach((candidates, index) => { + for (const id of candidates?.ids.slice(0, candidates.choosable) ?? []) + if (count(oldCounts, id) === 1) oldById.set(id, index); + }); + const newKeys: (string | undefined)[] = newer.map(() => undefined); + const oldKeys: (string | undefined)[] = older.map(() => undefined); + const newPartners: (number | undefined)[] = newer.map(() => undefined); + const oldPartners: (number | undefined)[] = older.map(() => undefined); + newCandidates.forEach((candidates, index) => { + if (candidates === undefined) return; + for (const [at, id] of candidates.ids + .slice(0, candidates.choosable) + .entries()) { + const match = count(newCounts, id) === 1 ? oldById.get(id) : undefined; + if (match === undefined || oldPartners[match] !== undefined) continue; + newKeys[index] = oldKeys[match] = selectorAt(candidates, at); + newPartners[index] = match; + oldPartners[match] = index; + return; + } + }); + /** The first selector naming a member alone in its version and nothing in the other. */ + const unmatched = ( + candidates: Candidates | undefined, + here: ReadonlyMap, + there: ReadonlyMap, + ) => { + if (candidates === undefined) return undefined; + const at = candidates.ids + .slice(0, candidates.choosable) + .findIndex((id) => count(here, id) === 1 && count(there, id) === 0); + return at < 0 ? undefined : selectorAt(candidates, at); + }; + newCandidates.forEach((candidates, index) => { + newKeys[index] ??= unmatched(candidates, newCounts, oldCounts); + }); + oldCandidates.forEach((candidates, index) => { + oldKeys[index] ??= unmatched(candidates, oldCounts, newCounts); + }); + for (let index = 0; index < Math.min(newer.length, older.length); index += 1) + if (newKeys[index] === undefined && oldKeys[index] === undefined) { + newPartners[index] = index; + oldPartners[index] = index; + } + const side = ( + keys: readonly (string | undefined)[], + partners: readonly (number | undefined)[], + ): PairedSide => ({ + keys: keys.map((key, index) => key ?? String(index)), + partners, + }); + return { + newer: side(newKeys, newPartners), + older: side(oldKeys, oldPartners), + }; +}; diff --git a/src/domain/desktop/zipArchive.fixture.ts b/src/domain/desktop/zipArchive.fixture.ts new file mode 100644 index 0000000000..6659c3c46a --- /dev/null +++ b/src/domain/desktop/zipArchive.fixture.ts @@ -0,0 +1,172 @@ +import { crc32, deflateRawSync } from "node:zlib"; + +/** One archive member; the overrides let a test frame what a real writer would not. */ +export interface ZipMember { + readonly name: string; + readonly data?: string | Uint8Array; + /** 0 stores, 8 deflates; another method stores the data under that number. */ + readonly method?: number; + readonly declaredSize?: number; + readonly crc?: number; + /** General purpose flags; bit 0 marks the member encrypted. */ + readonly flags?: number; + /** "Version made by": its high byte names the host, 0 for MS-DOS and 3 for Unix. */ + readonly madeBy?: number; + /** External attributes; a slash-ended name gets the MS-DOS directory bit. */ + readonly external?: number; +} + +const UTF8_NAME = 0x800; + +/** A zip archive of the members in order, framed from the PKWARE APPNOTE records. */ +export const zipArchive = (members: readonly ZipMember[]): Buffer => { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + for (const member of members) { + const content = Buffer.from(member.data ?? ""); + const method = member.method ?? 8; + const stored = method === 8 ? deflateRawSync(content) : content; + const name = Buffer.from(member.name); + const flags = + (member.flags ?? 0) | + (/^[\x20-\x7e]*$/u.test(member.name) ? 0 : UTF8_NAME); + const fields = { + method, + flags, + crc: member.crc ?? crc32(content), + compressed: stored.length, + size: member.declaredSize ?? content.length, + }; + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(fields.flags, 6); + local.writeUInt16LE(fields.method, 8); + local.writeUInt16LE(0x5021, 12); // 2020-01-01, DOS date. + local.writeUInt32LE(fields.crc, 14); + local.writeUInt32LE(fields.compressed, 18); + local.writeUInt32LE(fields.size, 22); + local.writeUInt16LE(name.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(member.madeBy ?? 20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(fields.flags, 8); + central.writeUInt16LE(fields.method, 10); + central.writeUInt16LE(0x5021, 14); + central.writeUInt32LE(fields.crc, 16); + central.writeUInt32LE(fields.compressed, 20); + central.writeUInt32LE(fields.size, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE( + member.external ?? (member.name.endsWith("/") ? 0x10 : 0), + 38, + ); + central.writeUInt32LE(offset, 42); + locals.push(local, name, stored); + centrals.push(central, name); + offset += local.length + name.length + stored.length; + } + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + // Past 65,535 members the 16-bit counts wrap, as writers without Zip64 leave them. + end.writeUInt16LE(members.length & 0xffff, 8); + end.writeUInt16LE(members.length & 0xffff, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, directory, end]); +}; + +/** A minimal OpenDocument text: the stored `mimetype` first, then its content and manifest, then any other members. */ +export const odtArchive = ( + paragraph: string, + others: readonly ZipMember[] = [], +): Buffer => + zipArchive([ + { + name: "mimetype", + data: "application/vnd.oasis.opendocument.text", + method: 0, + }, + { + name: "content.xml", + data: ` +${paragraph} +`, + }, + { + name: "META-INF/manifest.xml", + data: ` + +`, + }, + ...others, + ]); + +/** An archive of `count` empty stored entries named `e/0`, `e/1` and so on, written straight into one buffer. */ +export const emptyEntriesArchive = (count: number): Buffer => { + const names = Array.from({ length: count }, (_, index) => + Buffer.from(`e/${String(index)}`), + ); + const named = names.reduce((total, name) => total + name.length, 0); + const directory = 30 * count + named; + const archive = Buffer.alloc(directory + 46 * count + named + 22); + let local = 0; + let central = directory; + for (const name of names) { + archive.writeUInt32LE(0x04034b50, local); + archive.writeUInt16LE(20, local + 4); + archive.writeUInt16LE(name.length, local + 26); + name.copy(archive, local + 30); + archive.writeUInt32LE(0x02014b50, central); + archive.writeUInt16LE(20, central + 4); + archive.writeUInt16LE(20, central + 6); + archive.writeUInt16LE(name.length, central + 28); + archive.writeUInt32LE(local, central + 42); + name.copy(archive, central + 46); + local += 30 + name.length; + central += 46 + name.length; + } + archive.writeUInt32LE(0x06054b50, central); + // Past 65,535 entries the 16-bit counts wrap, as writers without Zip64 leave them. + archive.writeUInt16LE(count & 0xffff, central + 8); + archive.writeUInt16LE(count & 0xffff, central + 10); + archive.writeUInt32LE(central - directory, central + 12); + archive.writeUInt32LE(directory, central + 16); + return archive; +}; + +/** + * The same archive ended by Zip64 records: a Zip64 end record and its locator, + * and a classic one that defers to them, or with `sentinels` false keeps its + * own counts, size and offset, as a writer may when they fit. + */ +export const withZip64End = (archive: Buffer, sentinels = true): Buffer => { + const end = archive.length - 22; + const entries = archive.readUInt16LE(end + 10); + const size = archive.readUInt32LE(end + 12); + const offset = archive.readUInt32LE(end + 16); + const record = Buffer.alloc(56); + record.writeUInt32LE(0x06064b50, 0); + record.writeBigUInt64LE(44n, 4); + record.writeUInt16LE(45, 12); + record.writeUInt16LE(45, 14); + record.writeBigUInt64LE(BigInt(entries), 24); + record.writeBigUInt64LE(BigInt(entries), 32); + record.writeBigUInt64LE(BigInt(size), 40); + record.writeBigUInt64LE(BigInt(offset), 48); + const locator = Buffer.alloc(20); + locator.writeUInt32LE(0x07064b50, 0); + locator.writeBigUInt64LE(BigInt(end), 8); + locator.writeUInt32LE(1, 16); + const classic = Buffer.from(archive.subarray(end)); + if (sentinels) { + classic.writeUInt16LE(0xffff, 8); + classic.writeUInt16LE(0xffff, 10); + classic.writeUInt32LE(0xffffffff, 12); + classic.writeUInt32LE(0xffffffff, 16); + } + return Buffer.concat([archive.subarray(0, end), record, locator, classic]); +}; diff --git a/src/domain/desktop/zipDirectory.ts b/src/domain/desktop/zipDirectory.ts new file mode 100644 index 0000000000..0e5a959e49 --- /dev/null +++ b/src/domain/desktop/zipDirectory.ts @@ -0,0 +1,207 @@ +import { crc32 } from "node:zlib"; + +import { + ZIP_EOCD_BYTES, + ZIP_MAX_COMMENT_BYTES, + zipEndOfCentralDirectoryCandidates, +} from "../zipEndOfCentralDirectory.js"; + +const CENTRAL_HEADER = 0x02014b50; +const CENTRAL_HEADER_BYTES = 46; +const ZIP64_LOCATOR = 0x07064b50; +const ZIP64_LOCATOR_BYTES = 20; +const ZIP64_END = 0x06064b50; +const ZIP64_END_BYTES = 56; +const UTF8_NAME = 0x800; +const UNICODE_PATH = 0x7075; +const UNIX_TYPE = 0o170000; +const UNIX_DIRECTORY = 0o040000; +const MSDOS_DIRECTORY = 0x10; +const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** An archive's central directory read a record at a time, bounded to the records it keeps. */ +export interface ZipDirectory { + /** Where the directory begins; the bytes before it hold the entries. */ + readonly start: number; + /** The directory offset the end record declares, which readers compare with `start` to allow for prepended data. */ + readonly declaredOffset: number; + /** The first file entries' records, as stored, in directory order. */ + readonly records: readonly Uint8Array[]; + /** File entries the directory lists, directories left out. */ + readonly count: number; +} + +/** Where the directory ends, how long it is and the offset it declares, from the end record and any Zip64 one. */ +const directoryEnd = (view: DataView) => { + const window = Math.max( + 0, + view.byteLength - ZIP_EOCD_BYTES - ZIP_MAX_COMMENT_BYTES, + ); + const found = zipEndOfCentralDirectoryCandidates( + new Uint8Array( + view.buffer, + view.byteOffset + window, + view.byteLength - window, + ), + ).next(); + if (found.done === true) return undefined; + const end = window + found.value.offset; + const { totalEntries, directorySize, directoryOffset } = found.value; + const classic = { end, size: directorySize, declaredOffset: directoryOffset }; + const record = end - ZIP64_LOCATOR_BYTES - ZIP64_END_BYTES; + // Like Python's zipfile, a Zip64 locator is followed whether or not the classic record defers to it. + if ( + record < 0 || + view.getUint32(end - ZIP64_LOCATOR_BYTES, true) !== ZIP64_LOCATOR + ) + return classic; + // Read just before its locator, where zipfile and zip.js look when the locator's own offset holds no Zip64 record. + if (view.getUint32(record, true) === ZIP64_END) + return { + end: record, + size: Number(view.getBigUint64(record + 40, true)), + declaredOffset: Number(view.getBigUint64(record + 48, true)), + }; + // Without one the classic values stand, as zip.js reads them, unless they are sentinels deferring to it. + const deferred = + totalEntries === 0xffff || + directorySize === 0xffffffff || + directoryOffset === 0xffffffff; + return deferred ? undefined : classic; +}; + +/** Where one record's name and extra field lie in the archive, and its flags. */ +interface RecordName { + readonly name: number; + readonly extra: number; + readonly comment: number; + readonly flags: number; +} + +/** An Info-ZIP Unicode Path when zip.js uses it: version 1, no UTF-8 flag, and the CRC-32 of the stored name. */ +const unicodePath = ( + bytes: Uint8Array, + view: DataView, + { name, extra, comment, flags }: RecordName, +): Uint8Array | undefined => { + for (let at = extra; at + 4 <= comment;) { + const length = view.getUint16(at + 2, true); + if (view.getUint16(at, true) === UNICODE_PATH) + return length >= 5 && + at + 4 + length <= comment && + bytes[at + 4] === 1 && + (flags & UTF8_NAME) === 0 && + view.getUint32(at + 5, true) === crc32(bytes.subarray(name, extra)) + ? bytes.subarray(at + 9, at + 4 + length) + : undefined; + at += 4 + length; + } + return undefined; +}; + +/** + * Whether an entry's name, as REA decodes it, ends in a slash; undefined when + * it must be UTF-8 and is not. A plain CP437 name is judged by its last byte, + * so most records are read without allocating. + */ +const endsInSlash = ( + bytes: Uint8Array, + view: DataView, + record: RecordName, +): boolean | undefined => { + const unicode = + record.comment > record.extra + ? unicodePath(bytes, view, record) + : undefined; + const utf8 = + unicode ?? + ((record.flags & UTF8_NAME) === 0 + ? undefined + : bytes.subarray(record.name, record.extra)); + if (utf8 === undefined) + return record.extra > record.name && bytes[record.extra - 1] === 0x2f; + try { + return STRICT_UTF8.decode(utf8).endsWith("/"); + } catch { + return undefined; + } +}; + +/** + * Read an archive's central directory a record at a time, keeping the first + * `keep` file entries' records and counting the rest, so no more than those is + * ever parsed into entries. Directories are told apart as zip.js tells them: a + * final slash, a Unix directory mode, or the MS-DOS attribute. Undefined when + * the directory cannot be found or a record in it is malformed. + */ +export const scanZipDirectory = ( + bytes: Uint8Array, + keep: number, +): ZipDirectory | undefined => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const located = directoryEnd(view); + if (located === undefined) return undefined; + const start = located.end - located.size; + if (start < 0) return undefined; + const records: Uint8Array[] = []; + let count = 0; + for (let at = start; at < located.end;) { + if ( + at + CENTRAL_HEADER_BYTES > located.end || + view.getUint32(at, true) !== CENTRAL_HEADER + ) + return undefined; + const name = at + CENTRAL_HEADER_BYTES; + const extra = name + view.getUint16(at + 28, true); + const comment = extra + view.getUint16(at + 30, true); + const next = comment + view.getUint16(at + 32, true); + if (next > located.end) return undefined; + const slash = endsInSlash(bytes, view, { + name, + extra, + comment, + flags: view.getUint16(at + 8, true), + }); + if (slash === undefined) return undefined; + const external = view.getUint32(at + 38, true); + const directory = + slash || + ((external >>> 16) & UNIX_TYPE) === UNIX_DIRECTORY || + (view.getUint16(at + 4, true) >> 8 === 0 && + (external & MSDOS_DIRECTORY) !== 0); + if (!directory) { + count += 1; + if (records.length < keep) records.push(bytes.subarray(at, next)); + } + at = next; + } + return { start, declaredOffset: located.declaredOffset, records, count }; +}; + +/** The kept records and an end record for them alone, which follow the entries in the archive a reader is given. */ +export const boundedDirectoryTail = (directory: ZipDirectory): Uint8Array => { + const size = directory.records.reduce( + (total, record) => total + record.length, + 0, + ); + const tail = new Uint8Array(size + ZIP_EOCD_BYTES); + let at = 0; + for (const record of directory.records) { + tail.set(record, at); + at += record.length; + } + const end = new DataView(tail.buffer, size); + end.setUint32(0, 0x06054b50, true); + end.setUint16(8, directory.records.length, true); + end.setUint16(10, directory.records.length, true); + end.setUint32(12, size, true); + // A declared offset past 32 bits cannot be right for a view-sized archive; the start stands in for it. + end.setUint32( + 16, + directory.declaredOffset <= 0xffffffff + ? directory.declaredOffset + : directory.start, + true, + ); + return tail; +}; diff --git a/src/domain/desktop/zipView.ts b/src/domain/desktop/zipView.ts new file mode 100644 index 0000000000..e76a46c21f --- /dev/null +++ b/src/domain/desktop/zipView.ts @@ -0,0 +1,200 @@ +import { createHash } from "node:crypto"; + +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + detectStateFormat, + textualView, + type DesktopStateFormat, +} from "./stateFormats.js"; +import { locatePointer } from "./viewPointer.js"; + +const MIB = 1024 * 1024; +/** Entries a zip view lists; past them it counts entries and is truncated. */ +export const MAX_ZIP_ENTRIES = 1000; +/** Largest entry read, uncompressed. */ +const MAX_ZIP_ENTRY_BYTES = 16 * MIB; +/** Uncompressed bytes read from one archive, the same as the largest file the state view reads. */ +const MAX_ZIP_BYTES = 64 * MIB; +/** Most an entry may expand over its compressed size; a zip bomb expands about a thousand times. */ +const MAX_ZIP_RATIO = 100; + +/** Why an archive has no view; the exported reader words it the same. */ +export const UNREADABLE_ZIP = "not a readable zip archive"; +/** Why an entry whose data fails its checks has no view. */ +export const DAMAGED_ZIP_ENTRY = "its compressed data is damaged"; + +/** What the central directory says of one entry, before it is read. */ +export interface ZipEntryHeader { + readonly name: string; + readonly size: number; + readonly compressedSize: number; + readonly method: number; + readonly encrypted: boolean; +} + +/** Why an entry is not read, checked in this order by both readers, or undefined; `spent` is the archive's bytes read so far. */ +export const zipEntryRefusal = ( + header: ZipEntryHeader, + spent: number, +): string | undefined => { + if (header.encrypted) return "encrypted, so it is not read"; + if (header.method !== 0 && header.method !== 8) + return `compressed with method ${String(header.method)}, which the view does not read`; + if (header.size > MAX_ZIP_ENTRY_BYTES) + return `larger than the ${String(MAX_ZIP_ENTRY_BYTES / MIB)} MiB an entry's view reads`; + if (header.size > MAX_ZIP_RATIO * header.compressedSize) + return `expands more than ${String(MAX_ZIP_RATIO)} times, so it is not read`; + if (spent + header.size > MAX_ZIP_BYTES) + return `past the ${String(MAX_ZIP_BYTES / MIB)} MiB read from one archive`; + return undefined; +}; + +/** An entry that was not read: its name and size, and why. */ +export const unreadZipEntry = ( + header: ZipEntryHeader, + error: string, +): JsonValue => ({ + name: header.name, + size: header.size, + sha256: null, + format: null, + view: null, + view_error: error, +}); + +/** An entry's view for its format; one whose reader needs a tool, a worker or another archive opened is not read. */ +const entryView = ( + format: DesktopStateFormat, + bytes: Uint8Array, +): Result => { + switch (format) { + case "text": + case "json": + case "keyfile": + case "xml": + return textualView(format, bytes); + case "sqlite": + return err("a SQLite database inside an archive is not read"); + case "dconf": + return err("a dconf database inside an archive is not read"); + case "zip": + return err("an archive inside an archive is not read"); + case "binary": + return ok(null); + } +}; + +/** An entry read from its uncompressed bytes by the readers and limits a file's view uses. */ +export const zipEntry = ( + header: ZipEntryHeader, + bytes: Uint8Array, +): JsonValue => { + const format = detectStateFormat(header.name, bytes); + const read = entryView(format, bytes); + return { + name: header.name, + size: header.size, + sha256: createHash("sha256").update(bytes).digest("hex"), + format, + view: read.ok ? read.value : null, + view_error: read.ok ? null : read.error, + }; +}; + +/** A zip archive's view: its file entries in archive order, up to the listing limit, and how many it has. */ +export const zipView = ( + entries: readonly JsonValue[], + count: number, +): JsonValue => ({ + entries: [...entries], + entry_count: count, + truncated: count > entries.length, +}); + +const isRecord = ( + value: JsonValue | null | undefined, +): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Whether an entry's view is missing or holds only part of it. */ +const entryLeavesOut = (entry: JsonValue): boolean => + isRecord(entry) && + (typeof entry.view_error === "string" || + ((entry.format === "text" || entry.format === "xml") && + isRecord(entry.view) && + entry.view.truncated === true)); + +/** Whether a zip view leaves anything out: entries past its listing, or what an entry's view lacks. */ +export const zipLeavesOut = (view: JsonValue | null): boolean => + isRecord(view) && + (view.truncated === true || + (Array.isArray(view.entries) && view.entries.some(entryLeavesOut))); + +/** The entry a pointer into a zip view goes into, `none` when it names an entry the view lacks, or undefined when it names none. */ +export const zipEntryAt = ( + view: JsonValue | null, + pointer: string, +): Record | "none" | undefined => { + const [first, second] = pointer.slice(1).split("/"); + if (first !== "entries" || second === undefined) return undefined; + const entry = locatePointer(view, `/entries/${second}`); + if (entry.kind !== "value") return "none"; + return isRecord(entry.value) ? entry.value : undefined; +}; + +/** Formats an entry's view is read in; any other entry's `view` is `null` for want of one. */ +const VIEW_FORMATS: ReadonlySet = new Set([ + "text", + "json", + "keyfile", + "xml", +]); + +/** Why an entry has no view a check can read: binary, unread, or of a format with none; undefined when it has one. */ +export const entryWithoutView = ( + entry: Record, +): string | undefined => { + if (typeof entry.view_error === "string") + return `has no view: ${entry.view_error}`; + if (entry.format === "binary") return "is binary and has no view"; + return VIEW_FORMATS.has(entry.format ?? null) ? undefined : "has no view"; +}; + +/** The entry whose view a pointer goes into when it has none, so its `view` of `null` is no JSON value, and why. */ +export const viewlessEntryAt = ( + view: JsonValue | null, + pointer: string, +): { readonly name: string; readonly reason: string } | undefined => { + const entry = zipEntryAt(view, pointer); + if (typeof entry !== "object" || pointer.split("/")[3] !== "view") + return undefined; + const reason = entryWithoutView(entry); + return reason === undefined + ? undefined + : { name: typeof entry.name === "string" ? entry.name : "", reason }; +}; + +/** + * Whether a zip view leaves out what a pointer reaches: for a pointer into one + * listed entry, only that entry; for an entry not listed, the listing; else anything. + */ +export const zipLeavesOutAt = ( + view: JsonValue | null, + pointer: string, +): boolean => { + const entry = zipEntryAt(view, pointer); + if (entry === undefined) return zipLeavesOut(view); + return entry === "none" + ? isRecord(view) && view.truncated === true + : entryLeavesOut(entry); +}; + +/** Whether an array in a zip view may lack members: the listing when it is cut, or an array in an entry whose view is. */ +export const zipArrayLeavesOut = ( + view: JsonValue | null, + array: string, +): boolean => + array === "/entries" + ? isRecord(view) && view.truncated === true + : zipLeavesOutAt(view, array); diff --git a/src/domain/xmlDocument.ts b/src/domain/xmlDocument.ts index 300f88136e..532abce214 100644 --- a/src/domain/xmlDocument.ts +++ b/src/domain/xmlDocument.ts @@ -20,6 +20,32 @@ export class XmlDocumentParseError extends Error { } } +/** Read XML syntax as a stream, throwing at the first error; it holds only the open elements. */ +const checkXmlSyntax = (source: string, doctype: "allow" | "reject"): void => { + const syntax = new SaxesParser({ + xmlns: true, + defaultXMLVersion: "1.0", + forceXMLVersion: true, + }); + syntax.on("doctype", () => { + if (doctype === "reject") throw new XmlDocumentParseError("doctype"); + }); + syntax.write(source).close(); +}; + +/** Whether XML passes the syntax check `parseXmlDocument` makes, without building a document. */ +export const isWellFormedXml = ( + source: string, + policy: { readonly doctype: "allow" | "reject" }, +): boolean => { + try { + checkXmlSyntax(source, policy.doctype); + return true; + } catch { + return false; + } +}; + /** One boundary owns strict XML syntax and the source-located DOM projection. */ export const parseXmlDocument = ( source: string, @@ -29,16 +55,7 @@ export const parseXmlDocument = ( }, ): Result => { try { - const syntax = new SaxesParser({ - xmlns: true, - defaultXMLVersion: "1.0", - forceXMLVersion: true, - }); - syntax.on("doctype", () => { - if (policy.doctype === "reject") - throw new XmlDocumentParseError("doctype"); - }); - syntax.write(source).close(); + checkXmlSyntax(source, policy.doctype); // Saxes owns well-formedness outside DTDs. The DOM parser additionally // checks DTD syntax and builds source-located nodes; its advisory warnings // (including literal U+FFFD) are not byte-decoding or syntax failures. diff --git a/tests/boundary/cli/desktopCaptureCli.test.ts b/tests/boundary/cli/desktopCaptureCli.test.ts index e79ae1191c..22b232eae3 100644 --- a/tests/boundary/cli/desktopCaptureCli.test.ts +++ b/tests/boundary/cli/desktopCaptureCli.test.ts @@ -1,7 +1,62 @@ +import { join } from "node:path"; + import { expect, it } from "vitest"; +import { + EDITED_NOTE, + seedEndCapture, + writeCaptureFolder, +} from "../../../src/application/desktop/desktopCapture.fixture.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { serveCli as serve } from "./cliServeHarness.js"; +it("exports and drafts between the checkpoints the command names", async () => { + const capture = await writeCaptureFolder(seedEndCapture(), undefined, [ + EDITED_NOTE, + ]); + const ends = ["--seed-checkpoint", "seed", "--end-checkpoint", "end"]; + const bundle = join(await createTestTempDirectory("desktop-cli-"), "bundle"); + + const exported = await serve([ + "export-desktop-application", + capture, + bundle, + ...ends, + "--json", + ]); + const drafted = await serve([ + "describe-desktop-application-task", + capture, + ...ends, + "--json", + ]); + const refused = await serve([ + "describe-desktop-application-task", + capture, + "--end-checkpoint", + "saved", + "--json", + ]); + + expect( + parseEvidence(JSON.parse(exported.stdout)).normalized_result, + ).toMatchObject({ + checkpoints: { seed: "seed", end: "end" }, + seed_files: [".config/notes/settings.ini", "Documents/today.txt"], + }); + expect( + parseEvidence(JSON.parse(drafted.stdout)).normalized_result, + ).toMatchObject({ checkpoints: { baseline: "seed", final: "end" } }); + expect(JSON.parse(refused.stdout)).toMatchObject({ + code: "invalid_request", + details: { + operation: "describe_desktop_application_task", + issues: [{ path: ["end_checkpoint"] }], + }, + }); +}); + it.each([ [["--exclude", ""], "exclude"], [["--state-path", ""], "statePath"], diff --git a/tests/boundary/providers/desktopExportPython.test.ts b/tests/boundary/providers/desktopExportPython.test.ts index 72cd6a9b84..57ec3b6621 100644 --- a/tests/boundary/providers/desktopExportPython.test.ts +++ b/tests/boundary/providers/desktopExportPython.test.ts @@ -10,8 +10,11 @@ import { } from "../../../src/domain/desktop/appExportPython.js"; import { detectStateFormat, + isJsonText, jsonView, + textualView, } from "../../../src/domain/desktop/stateFormats.js"; +import { safeParseJson } from "../../../src/domain/safeJson.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; /** The exported module's view format for each file, run with only its standard library imports. */ @@ -21,7 +24,7 @@ const pythonFormats = (files: readonly (readonly [string, string])[]) => { .join("\n"); const program = `${imports}${PYTHON_HELPERS} for name, text in json.load(sys.stdin): - print(_text_view(name, text)["format"]) + print(_text_view(name, text, len(text.encode()))["format"]) `; return execFileSync("python3", ["-I", "-B", "-c", program], { input: JSON.stringify(files), @@ -75,7 +78,7 @@ const pythonUtf16Respelled = (files: readonly string[]) => { .filter((line) => !line.includes("agentenv_protocol")) .join("\n"); const program = `${imports} -CAPTURED_HOME = "/home/ada" +SEED_HOME = "/home/ada" ${PYTHON_HELPERS} for data in json.load(sys.stdin): print(_respelled_utf16(bytes.fromhex(data), Path("/home/user")).hex()) @@ -145,7 +148,7 @@ const taskbarWindow = (bin: string, windowClass: string): string => { "-I", "-B", "-c", - `${imports}\nCAPTURED_HOME = "/home/ada"\n${PYTHON_HELPERS}\nprint(_taskbar_window(sys.argv[1]))`, + `${imports}\nSEED_HOME = "/home/ada"\n${PYTHON_HELPERS}\nprint(_taskbar_window(sys.argv[1]))`, windowClass, ], { @@ -189,7 +192,7 @@ const pythonJson = (texts: readonly string[]) => { .join("\n"); const program = `${imports}${PYTHON_HELPERS} for text in json.load(sys.stdin): - print(json.dumps(_text_view("a.json", text))) + print(json.dumps(_text_view("a.json", text, len(text.encode())))) `; return execFileSync("python3", ["-I", "-B", "-c", program], { input: JSON.stringify(texts), @@ -224,6 +227,159 @@ it("refuses the JSON numbers REA's state view refuses, as JSON with an error rat }); }); +it("refuses JSON nested deeper than the 500 levels REA's view keeps, reading the boundary itself", () => { + const nested = (levels: number) => + `${"[".repeat(levels)}${"]".repeat(levels)}`; + const texts = [nested(501), nested(502), nested(1100)]; + + const python = pythonJson(texts); + + expect(texts.map((text) => jsonView(text).ok)).toEqual([true, false, false]); + texts.forEach((text, index) => { + const typescript = jsonView(text); + expect(python[index]).toEqual( + typescript.ok + ? { format: "json", view: typescript.value, truncated: false } + : { format: "json", view: null, error: typescript.error }, + ); + }); +}); + +/** The exported module's format, view and error for each named text, and REA's for the same bytes. */ +const textViews = (files: readonly (readonly [string, string])[]) => { + const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + const program = `${imports}${PYTHON_HELPERS} +for name, text in json.load(sys.stdin): + view = _text_view(name, text, len(text.encode())) + print(json.dumps({"format": view["format"], "view": view["view"], "error": view.get("error")})) +`; + const python = execFileSync("python3", ["-I", "-B", "-c", program], { + input: JSON.stringify(files), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as unknown); + const rea = files.map(([name, text]) => { + const bytes = new TextEncoder().encode(text); + const format = detectStateFormat(name, bytes); + if (format !== "text" && format !== "json" && format !== "keyfile") + throw new Error(`${name} reads as ${format}`); + const read = textualView(format, bytes); + return read.ok + ? { format, view: read.value, error: null } + : { format, view: null, error: read.error }; + }); + return { python, rea }; +}; + +it("reads text nested past Python's recursion limit as JSON only when all of it is, as REA does", () => { + const brackets = (open: number, close: number) => + `${"[".repeat(open)}${"]".repeat(close)}`; + const files = [ + ["deep.txt", brackets(1100, 1099)], + ["deep.json", brackets(1100, 1099)], + ["deep.txt", brackets(1100, 1100)], + ["deep.json", `${brackets(1100, 1100)} x`], + ] as const; + + const { python, rea } = textViews(files); + + expect(python).toEqual(rea); + expect(rea.map(({ format, error }) => [format, error])).toEqual([ + ["text", null], + ["text", null], + ["json", "nested deeper than the 500 levels a JSON view keeps"], + ["text", null], + ]); +}); + +it("refuses JSON and key files over 4 MiB as REA does, telling JSON apart from text without parsing it", () => { + const objects = `[{}${",{}".repeat(1_400_000)}]`; + const files = [ + ["objects.json", objects], + ["objects", `${objects.slice(0, -1)}}`], + ["settings", `[ui]\n${"wrap=false\n".repeat(400_000)}`], + ["log", `[0] ${"x".repeat(4_200_000)}`], + ] as const; + + const { python, rea } = textViews(files); + + expect(python).toEqual(rea); + const refused = "larger than the 4 MiB limit for JSON and key file views"; + expect(rea.map(({ format, error }) => [format, error])).toEqual([ + ["json", refused], + ["text", null], + ["keyfile", refused], + ["text", null], + ]); +}); + +it("checks text for JSON the way REA does, token by token, without parsing it", () => { + const [bom, nbsp, lineSeparator] = [0xfeff, 0xa0, 0x2028].map((code) => + String.fromCharCode(code), + ); + const texts = [ + ["-0", "01", "1.", "1E+5", "-", "[1.5e+]", "truex", "[NaN]"], + ['"\\u00e9"', '"\\u00g9"', '"\\x"', '"\t"', '"a', '"\\uD800"'], + ["", " [1] ", "[1] x", "{}{}", `${bom}[1]`, `[1]${nbsp}`], + [`["${lineSeparator}"]`, "[]", "[1,]", "[,1]", "[1 2]", "[1]]", "[[1]"], + ["[}", "{}", '{"a":1}', '{"a" 1}', "{a:1}", '{"a":1,}', '{"a"::1}'], + ["{1:2}", '{"a":[1,{"b":null}],"c":-1.5e-3}'], + ].flat(); + const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + + const python = JSON.parse( + execFileSync( + "python3", + [ + "-I", + "-B", + "-c", + `${imports}${PYTHON_HELPERS}\nprint(json.dumps([_is_json(text) for text in json.load(sys.stdin)]))`, + ], + { input: JSON.stringify(texts), encoding: "utf8" }, + ), + ) as boolean[]; + + const parsed = texts.map((text) => safeParseJson(text).ok); + expect(texts.map((text) => isJsonText(text))).toEqual(parsed); + expect(python).toEqual(parsed); + expect(parsed.filter(Boolean)).toHaveLength(10); +}); + +it("checks JSON depth with one iterator per level, copying no wide list", () => { + const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + const program = `${imports} +import tracemalloc +${PYTHON_HELPERS} +wide = [None] * 800_000 +tracemalloc.start() +deep = _deeper_than(wide, 500) +peak = tracemalloc.get_traced_memory()[1] +tracemalloc.stop() +nested = [] +for _ in range(501): + nested = [nested] +print(json.dumps({"deep": deep, "peak": peak, "edge": _deeper_than(nested, 501), "over": _deeper_than(nested, 500)})) +`; + + const result = JSON.parse( + execFileSync("python3", ["-I", "-B", "-c", program], { encoding: "utf8" }), + ) as { deep: boolean; peak: number; edge: boolean; over: boolean }; + + expect(result).toMatchObject({ deep: false, edge: false, over: true }); + // Queuing each member of the 800,000-item list traced about 51 MB. + expect(result.peak).toBeLessThan(1024 * 1024); +}); + it("spells infinite and negative-zero SQLite reals as REA's SQLite reader does", () => { const imports = PYTHON_IMPORTS.split("\n") .filter((line) => !line.includes("agentenv_protocol")) diff --git a/tests/boundary/providers/desktopStateHeap.test.ts b/tests/boundary/providers/desktopStateHeap.test.ts new file mode 100644 index 0000000000..2f3431aa98 --- /dev/null +++ b/tests/boundary/providers/desktopStateHeap.test.ts @@ -0,0 +1,142 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const ROOT = join(import.meta.dirname, "../../.."); + +/** What a script using one export of a desktop state module prints, run in a 32 MiB V8 old space. */ +const inSmallHeap = async (module: string, name: string, body: string) => { + const folder = await createTestTempDirectory("desktop-state-heap-"); + const script = join(folder, "run.mjs"); + const url = pathToFileURL(join(ROOT, "src/domain/desktop", module)).href; + await writeFile( + script, + `import { ${name} } from ${JSON.stringify(url)};\n${body}`, + ); + const output = execFileSync( + process.execPath, + [ + "--max-old-space-size=32", + "--max-semi-space-size=1", + "--import", + "tsx", + script, + ], + { cwd: ROOT, encoding: "utf8", timeout: 120_000 }, + ); + return JSON.parse(output) as unknown; +}; + +it("checks JSON depth with one iterator per level, so a wide array fits a small heap", async () => { + // Queuing a pair per member needed more than 64 MiB here; one iterator per level runs in 18 MiB. + const depths = await inSmallHeap( + "stateFormats.ts", + "deeperThan", + `const wide = new Array(800_000).fill(null); +let nested = []; +for (let level = 0; level < 501; level += 1) nested = [nested]; +console.log(JSON.stringify({ wide: deeperThan(wide, 500), edge: deeperThan(nested, 501), over: deeperThan(nested, 500) })); +`, + ); + + expect(depths).toEqual({ wide: false, edge: false, over: true }); +}, 180_000); + +it("pairs elements with a long tag and many attributes in a small heap, spelling a selector only once chosen", async () => { + // Spelling the tag into a selector per attribute needed more than 140 MiB here. + const pairing = await inSmallHeap( + "viewPointer.ts", + "pairMembers", + `const tag = "t".repeat(40_000); +const element = (id) => { + const attributes = { id }; + for (let index = 0; index < 2000; index += 1) attributes["a" + index] = String(index); + return { name: tag, attributes, children: [] }; +}; +const members = [element("1"), element("2")]; +const { newer, older } = pairMembers(members, members); +console.log(JSON.stringify({ keys: newer.keys.map((key) => key.replace(tag, "T")), partners: newer.partners, same: older.keys.every((key, index) => key === newer.keys[index]) })); +`, + ); + + expect(pairing).toEqual({ + keys: ["[name=T][@id=1]", "[name=T][@id=2]"], + partners: [0, 1], + same: true, + }); +}, 180_000); + +it("refuses JSON over the parsed-view limit in a small heap, telling it apart from text without building it", async () => { + // Parsing the entry to detect its format needed more than 80 MiB here. + const entries = await inSmallHeap( + "zipView.ts", + "zipEntry", + `const read = (name, text) => { + const bytes = new TextEncoder().encode(text); + const entry = zipEntry({ name, size: bytes.length, compressedSize: bytes.length, method: 0, encrypted: false }, bytes); + return [entry.format, entry.view_error ?? entry.view.line_count]; +}; +console.log(JSON.stringify([ + read("objects.json", "[{}" + ",{}".repeat(1_400_000) + "]"), + read("objects.txt", "[{}" + ",{}".repeat(1_400_000) + "}"), +])); +`, + ); + + expect(entries).toEqual([ + ["json", "larger than the 4 MiB limit for JSON and key file views"], + ["text", 1], + ]); +}, 180_000); + +it("scrubs UTF-16 XML and JSON seeds of a million elements in a small heap, telling their format without building them", async () => { + // Building the XML document to choose the scrubber needed more than 768 MiB here, and parsing the JSON more than 64 MiB; the syntax checks and copying only changes run in 24 MiB. + const outcomes = await inSmallHeap( + "seedScrub.ts", + "scrubSeed", + `const scrub = (home, text, tail, clean) => { + const seed = Buffer.from("\\uFEFF" + text + tail, "utf16le"); + const scrubbed = scrubSeed(seed, { home, format: "binary" }, { home: "/home/ada", hostname: "e81232be457d", user: "ada" }); + const expected = Buffer.concat([seed.subarray(0, -2 * tail.length), Buffer.from(clean, "utf16le")]); + return { exact: scrubbed.bytes?.equals(expected), holdsIdentity: scrubbed.holdsIdentity }; +}; +console.log(JSON.stringify([ + scrub(".config/wide.xml", "\\n" + "".repeat(1_000_000), "ada@e81232be457d", "user@localhost"), + scrub(".config/wide.json", "[{}" + ",{}".repeat(1_000_000), ',"ada@e81232be457d"]', ',"user@localhost"]'), +])); +`, + ); + + expect(outcomes).toEqual([ + { exact: true, holdsIdentity: false }, + { exact: true, holdsIdentity: false }, + ]); +}, 180_000); + +it("compares objects of selector keys with arrays in a small heap, however many field sets or spellings they use", async () => { + // An index per field set or spelling needed more than 48 MiB for either comparison here. + const removed = await inSmallHeap( + "stateChanges.ts", + "diffStateViews", + `const fields = ["f0", "f1", "f2", "f3", "f4", "f5", "f6"]; +const members = Array.from({ length: 5000 }, (_, i) => Object.fromEntries(fields.map((f) => [f, f + "-" + i]))); +const combinations = {}; +for (let mask = 1; mask < 128; mask += 1) + combinations[fields.filter((_, bit) => mask & (1 << bit)).map((f) => "[" + f + "=" + f + "-0]").join("")] = mask; +const named = Array.from({ length: 800 }, (_, i) => ({ name: "n" + i })); +const repeated = {}; +for (let count = 1; count <= 160; count += 1) repeated["[name=n0]".repeat(count)] = count; +console.log(JSON.stringify([ + diffStateViews("json", { items: combinations }, { items: members }), + diffStateViews("json", { items: repeated }, { items: named }), +].map((difference) => difference.removed_count))); +`, + ); + + expect(removed).toEqual([0, 0]); +}, 180_000); diff --git a/tests/boundary/providers/desktopXmlPython.test.ts b/tests/boundary/providers/desktopXmlPython.test.ts new file mode 100644 index 0000000000..b3c699e483 --- /dev/null +++ b/tests/boundary/providers/desktopXmlPython.test.ts @@ -0,0 +1,198 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { DesktopTaskCheck } from "../../../src/domain/desktop/appTask.js"; +import { xmlView } from "../../../src/domain/desktop/stateFormats.js"; +import { zipArchive } from "../../../src/domain/desktop/zipArchive.fixture.js"; +import { + expectSameCheckResults, + expectSameFileViews, + pythonXmlViews, + typescriptFileView, +} from "../../support/desktopExportPython.js"; + +const FIXTURES = join(import.meta.dirname, "../../fixtures"); +const ENDS = ["before", "after"] as const; + +/** Inkscape's saves of the poster, before and after the headline was changed. */ +const posters = () => + Promise.all( + ENDS.map(async (end) => ({ + name: "poster.svg", + bytes: await readFile(join(FIXTURES, "inkscape", end, "poster.svg")), + })), + ); + +/** LibreOffice Writer's saves of the flyer, before and after the date was changed, zipped with their mimetype. */ +const flyers = () => + Promise.all( + ENDS.map(async (end) => ({ + name: "flyer.odt", + bytes: zipArchive([ + { + name: "mimetype", + data: "application/vnd.oasis.opendocument.text", + method: 0, + }, + { + name: "content.xml", + data: await readFile( + join(FIXTURES, "libreoffice", end, "content.xml"), + ), + }, + ]), + })), + ); + +const POSTER = "Pictures/poster.svg"; +const FLYER = "Documents/flyer.odt"; + +/** The poster's checks: its headline's text, then a setting Inkscape keeps in a namespaced attribute. */ +const POSTER_CHECKS: DesktopTaskCheck[] = [ + { + id: "headline", + kind: "value_equals", + file: POSTER, + pointer: + "/root/children/[name=g][@inkscape:label=Layer 1]/children/[name=text][@id=headline]/children/[name=tspan]/children/0", + value: "Summer Bake Sale", + }, + { + id: "units", + kind: "value_equals", + file: POSTER, + pointer: + "/root/children/[name=sodipodi:namedview]/attributes/inkscape:document-units", + value: "mm", + }, +]; + +const DATE: DesktopTaskCheck = { + id: "date", + kind: "value_equals", + file: FLYER, + pointer: + "/entries/[name=content.xml]/view/root/children/[name=office:body]/children/[name=office:text]/children/[name=text:p][@text:style-name=Subtitle]/children/0", + value: "Monday 24 November", +}; + +const passes = (outcomes: readonly { passed: boolean }[] | undefined) => + outcomes?.map(({ passed }) => passed); + +describe("namespaced XML in the exported module", () => { + it("reads an Inkscape SVG as REA does: prefixed names, the default namespace and its declarations", async () => { + const files = await posters(); + await expectSameFileViews(files); + + const [, after] = files; + if (after === undefined) throw new Error("fixture"); + const { view } = await typescriptFileView(after); + expect(view).toMatchObject({ + root: { + name: "svg", + attributes: { + xmlns: "http://www.w3.org/2000/svg", + "xmlns:svg": "http://www.w3.org/2000/svg", + "xmlns:sodipodi": + "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd", + "sodipodi:docname": "poster.svg", + }, + children: [{ name: "defs" }, { name: "sodipodi:namedview" }, {}], + }, + truncated: false, + }); + }); + + it("checks the poster's headline by name: failing before the edit and passing after, in both evaluators", async () => { + const [before, after] = await expectSameCheckResults( + POSTER, + POSTER_CHECKS, + await posters(), + ); + + expect(passes(before)).toEqual([false, true]); + expect(before?.[0]?.description).toContain( + 'is "Spring Bake Sale", not "Summer Bake Sale"', + ); + expect(passes(after)).toEqual([true, true]); + }); + + it("reads a Writer document's content.xml as REA does, in its archive", async () => { + const files = await flyers(); + await expectSameFileViews(files); + + const [, after] = files; + if (after === undefined) throw new Error("fixture"); + const { view } = await typescriptFileView(after); + expect(view).toMatchObject({ + entries: [ + { name: "mimetype", format: "text" }, + { + name: "content.xml", + format: "xml", + view: { + root: { + name: "office:document-content", + attributes: { + "xmlns:text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0", + "office:version": "1.3", + }, + }, + }, + }, + ], + }); + }); + + it("checks the flyer's date by name: failing before the edit and passing after, in both evaluators", async () => { + const [before, after] = await expectSameCheckResults( + FLYER, + [DATE], + await flyers(), + ); + + expect(passes(before)).toEqual([false]); + expect(before?.[0]?.description).toContain( + 'is "Monday 17 November", not "Monday 24 November"', + ); + expect(passes(after)).toEqual([true]); + }); + + it("splits text, CDATA and comments, spells xml: and xmlns names, and refuses documents, as REA does", () => { + const texts = [ + '', + "xzwv", + ' ', + "", + "", + '', + '', + '', + "", + "", + ]; + + const ours = texts.map((text) => xmlView(text) ?? null); + + expect(pythonXmlViews(texts)).toEqual(ours); + expect(ours.slice(4)).toEqual([null, null, null, null, null, null]); + expect(ours[1]).toMatchObject({ + root: { children: ["x", "y", "zw", "v", "q", "r"] }, + }); + }); + + it("keeps REA's node and depth limits on prefixed names", () => { + const declared = 'xmlns:p="urn:u"'; + const texts = [ + `${"".repeat(5000)}`, + `${"".repeat(70)}${"".repeat(70)}`, + ]; + + const ours = texts.map((text) => xmlView(text) ?? null); + + expect(pythonXmlViews(texts)).toEqual(ours); + expect(ours).toMatchObject([{ truncated: true }, { truncated: true }]); + }); +}); diff --git a/tests/boundary/providers/desktopZipHeap.test.ts b/tests/boundary/providers/desktopZipHeap.test.ts new file mode 100644 index 0000000000..498735ca4e --- /dev/null +++ b/tests/boundary/providers/desktopZipHeap.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect, it } from "vitest"; + +import { emptyEntriesArchive } from "../../../src/domain/desktop/zipArchive.fixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const ROOT = join(import.meta.dirname, "../../.."); + +it("reads a 600,000-entry archive's view in a small heap, parsing no more of its directory than it lists", async () => { + const folder = await createTestTempDirectory("desktop-zip-heap-"); + const archive = join(folder, "wide.zip"); + await writeFile(archive, emptyEntriesArchive(600_000)); + const script = join(folder, "read.mjs"); + const reader = pathToFileURL( + join(ROOT, "src/application/desktop/DesktopZipView.ts"), + ).href; + await writeFile( + script, + `import { readFileSync } from "node:fs"; +import { readZipView } from ${JSON.stringify(reader)}; +const read = await readZipView(readFileSync(process.argv[2])); +if (!read.ok) throw new Error(read.error); +console.log(JSON.stringify({ listed: read.value.entries.length, count: read.value.entry_count, truncated: read.value.truncated })); +`, + ); + + // zip.js parsing every record needed between 48 and 64 MiB here; the bounded directory reads in 24 MiB. + const output = execFileSync( + process.execPath, + [ + "--max-old-space-size=36", + "--max-semi-space-size=1", + "--import", + "tsx", + script, + archive, + ], + { cwd: ROOT, encoding: "utf8", timeout: 120_000 }, + ); + + expect(JSON.parse(output)).toEqual({ + listed: 1000, + count: 600_000, + truncated: true, + }); +}, 180_000); diff --git a/tests/boundary/providers/desktopZipPython.test.ts b/tests/boundary/providers/desktopZipPython.test.ts new file mode 100644 index 0000000000..deaf1d0ae1 --- /dev/null +++ b/tests/boundary/providers/desktopZipPython.test.ts @@ -0,0 +1,477 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { crc32 } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { readZipView } from "../../../src/application/desktop/DesktopZipView.js"; +import type { DesktopTaskCheck } from "../../../src/domain/desktop/appTask.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { detectStateFormat } from "../../../src/domain/desktop/stateFormats.js"; +import { + emptyEntriesArchive, + odtArchive, + withZip64End, + zipArchive, + type ZipMember, +} from "../../../src/domain/desktop/zipArchive.fixture.js"; +import { zipEntryRefusal } from "../../../src/domain/desktop/zipView.js"; +import { + expectSameCheckResults, + expectSameFileViews, + exportedPython, + pythonFileViews, + writeFiles, +} from "../../support/desktopExportPython.js"; + +const MIB = 1024 * 1024; +const FIXTURES = join(import.meta.dirname, "../../fixtures/freecad"); + +/** Archives as FreeCAD documents under the home. */ +const saved = (archives: readonly Buffer[]) => + archives.map((bytes) => ({ name: "plate.FCStd", bytes })); + +const expectSameViews = (archives: readonly Buffer[]) => + expectSameFileViews(saved(archives)); + +/** REA's view of an archive, once the exported module's is shown to be the same. */ +const sameView = async (archive: Buffer) => { + await expectSameViews([archive]); + const read = await readZipView(archive); + if (!read.ok) throw new Error(read.error); + return read.value; +}; + +/** The exported module's zip view of an archive file, and the most memory reading it traced. */ +const pythonPeak = async (archive: Buffer) => + JSON.parse( + exportedPython( + `import tracemalloc +tracemalloc.start() +view = _view(Path(json.load(sys.stdin)[0]))["view"] +peak = tracemalloc.get_traced_memory()[1] +tracemalloc.stop() +print(json.dumps({"view": view, "peak": peak}))`, + await writeFiles(saved([archive])), + ), + ) as { + view: { entries: Record[]; entry_count: number }; + peak: number; + }; + +const sqliteHeader = Buffer.concat([ + Buffer.from("SQLite format 3\0"), + Buffer.alloc(84), +]); + +describe("zip views in the exported module", () => { + it("lists entries in archive order with the formats and views REA reads", async () => { + const members: ZipMember[] = [ + { name: "settings.json", data: '{"zoom": 1.5, "tags": ["a"]}' }, + { name: "app.ini", data: "[ui]\nwrap=false\n" }, + { name: "doc/", data: "" }, + { name: "doc/page.xml", data: 'Hello' }, + { name: "notes.txt", data: "one\ntwo\n", method: 0 }, + { name: "café.txt", data: "naïve\n" }, + { name: "image.png", data: Buffer.from([0x89, 0x50, 0, 1, 2]) }, + { name: "inner.zip", data: zipArchive([{ name: "x", data: "y" }]) }, + { name: "store.db", data: sqliteHeader }, + { name: "big.json", data: '{"id": 9007199254740993}' }, + ]; + + await expectSameViews([zipArchive(members)]); + const read = await readZipView(zipArchive(members)); + expect(read).toMatchObject({ + ok: true, + value: { entry_count: 9, truncated: false }, + }); + if (!read.ok) return; + expect(read.value).toMatchObject({ + entries: [ + { name: "settings.json", format: "json", view: { zoom: 1.5 } }, + { name: "app.ini", format: "keyfile" }, + { name: "doc/page.xml", format: "xml", view_error: null }, + { name: "notes.txt", format: "text", view: { lines: ["one", "two"] } }, + { name: "café.txt", format: "text" }, + { name: "image.png", format: "binary", view: null, view_error: null }, + { + name: "inner.zip", + format: "zip", + view_error: "an archive inside an archive is not read", + }, + { + name: "store.db", + format: "sqlite", + view_error: "a SQLite database inside an archive is not read", + }, + { + name: "big.json", + format: "json", + view_error: + "a number in it cannot be read exactly, such as an integer above 2^53", + }, + ], + }); + }); + + it("refuses the same entries for the same reasons, and an archive it cannot read", async () => { + // A short declared size with the checksum of that prefix still leaves data the entry does not declare. + const helCrc = crc32(Buffer.from("hel")); + const archive = zipArchive([ + { name: "secret.txt", data: "x", flags: 1 }, + { name: "bzip2.txt", data: "x", method: 12 }, + { name: "huge.txt", data: "x", method: 0, declaredSize: 17 * MIB }, + { name: "bomb.txt", data: "a".repeat(200_000) }, + { name: "crc.txt", data: "hello", crc: 1 }, + { name: "short.txt", data: "hello", declaredSize: 3 }, + { name: "prefix.txt", data: "hello", declaredSize: 3, crc: helCrc }, + { + name: "prefix-stored.txt", + data: "hello", + method: 0, + declaredSize: 3, + crc: helCrc, + }, + { name: "fine.txt", data: "ok\n" }, + ]); + const damaged = Buffer.concat([ + Buffer.from("PK\u0003\u0004"), + Buffer.alloc(60, 7), + ]); + + await expectSameViews([archive, damaged]); + const read = await readZipView(archive); + expect(read.ok && read.value).toMatchObject({ + entries: [ + { view_error: "encrypted, so it is not read", sha256: null }, + { + view_error: "compressed with method 12, which the view does not read", + }, + { view_error: "larger than the 16 MiB an entry's view reads" }, + { view_error: "expands more than 100 times, so it is not read" }, + { view_error: "its compressed data is damaged" }, + { view_error: "its compressed data is damaged" }, + { name: "prefix.txt", view_error: "its compressed data is damaged" }, + { + name: "prefix-stored.txt", + view_error: "its compressed data is damaged", + }, + { format: "text", view_error: null }, + ], + }); + expect(await readZipView(damaged)).toEqual({ + ok: false, + error: "not a readable zip archive", + }); + }); + + it("lists at most 1,000 entries and counts the rest", async () => { + const archive = zipArchive( + Array.from({ length: 1002 }, (_, index) => ({ + name: `n/${String(index)}.txt`, + data: String(index), + method: 0, + })), + ); + + await expectSameViews([archive]); + const read = await readZipView(archive); + expect(read.ok && read.value).toMatchObject({ + entry_count: 1002, + truncated: true, + }); + expect( + read.ok && (read.value as { entries: unknown[] }).entries, + ).toHaveLength(1000); + }); + + it("stops reading an archive at 64 MiB of entries, as the exported reader does", () => { + const headers = [ + [{ size: 10, compressedSize: 10 }, 64 * MIB - 10], + [{ size: 11, compressedSize: 11 }, 64 * MIB - 10], + [{ size: 16 * MIB, compressedSize: 1 * MIB }, 0], + [{ size: 0, compressedSize: 0 }, 64 * MIB], + ] as const; + + const ours = headers.map( + ([sizes, spent]) => + zipEntryRefusal( + { name: "a", method: 8, encrypted: false, ...sizes }, + spent, + ) ?? null, + ); + const theirs = JSON.parse( + exportedPython( + `from types import SimpleNamespace +print(json.dumps([_zip_refusal(SimpleNamespace(flag_bits=0, compress_type=8, file_size=size, compress_size=packed), spent) + for (size, packed, spent) in json.load(sys.stdin)]))`, + headers.map(([sizes, spent]) => [ + sizes.size, + sizes.compressedSize, + spent, + ]), + ), + ) as unknown; + + expect(ours).toEqual([ + null, + "past the 64 MiB read from one archive", + null, + null, + ]); + expect(theirs).toEqual(ours); + }); +}); + +describe("zip directories in the exported module", () => { + it("tells directories apart as zip.js does: by a final slash, a Unix mode or the MS-DOS attribute", async () => { + const archive = zipArchive([ + { name: "a.txt", data: "a\n" }, + { name: "slash/", data: "" }, + { name: "unix", data: "", madeBy: 0x0314, external: 0o040755 * 0x10000 }, + { name: "dos", data: "", madeBy: 20, external: 0x10 }, + { + name: "unix-file", + data: "u\n", + madeBy: 0x0314, + external: 0o100644 * 0x10000, + }, + ]); + + expect(await sameView(archive)).toMatchObject({ + entries: [{ name: "a.txt" }, { name: "unix-file" }], + entry_count: 2, + truncated: false, + }); + }); + + it("finds the directory through a Zip64 locator whether or not the classic end record defers to it, as zipfile does", async () => { + const members = [ + { name: "a.txt", data: "a\n" }, + { name: "d/" }, + { name: "b.json", data: '{"b": 1}' }, + ]; + + // A classic record keeping its own values once left both readers short of the directory. + for (const sentinels of [true, false]) + expect( + await sameView(withZip64End(zipArchive(members), sentinels)), + ).toMatchObject({ + entries: [{ name: "a.txt" }, { name: "b.json", view: { b: 1 } }], + entry_count: 2, + }); + }); + + it("keeps the classic values when bytes before the end record only look like a Zip64 locator", async () => { + // The last name ends in a locator signature and 16 more bytes, which sit just before the end record. + const name = `c-PK\u0006\u0007${"0".repeat(16)}`; + const archive = zipArchive([ + { name: "a.txt", data: "a\n" }, + { name, data: "c", flags: 0x800 }, + ]); + + expect(await sameView(archive)).toMatchObject({ + entries: [{ name: "a.txt" }, { name }], + entry_count: 2, + }); + }); + + it("lists the first 1,000 of 50,000 entries holding no more of the directory than those", async () => { + const archive = emptyEntriesArchive(50_000); + await expectSameViews([archive]); + + const { view, peak } = await pythonPeak(archive); + + expect(view.entries).toHaveLength(1000); + expect(view.entry_count).toBe(50_000); + // Parsing every record before the listing limit applied traced about 25 MiB here. + expect(peak).toBeLessThan(4 * MIB); + }); +}); + +describe("zip entries the exported module cannot fully read", () => { + it("reads an entry at most a byte past its declared size, so a forged small size holds no memory", async () => { + // 32 MiB of zeros deflate to about 32 KiB; the entry declares one byte. + const archive = zipArchive([ + { + name: "bomb.txt", + data: Buffer.alloc(32 * MIB), + declaredSize: 1, + crc: 0, + }, + ]); + await expectSameViews([archive]); + + const { view, peak } = await pythonPeak(archive); + + expect(view.entries[0]?.view_error).toBe("its compressed data is damaged"); + // Reading the stream whole before clipping it traced about 77 MiB here. + expect(peak).toBeLessThan(4 * MIB); + }); + + it("refuses JSON and key file entries over 4 MiB before building their values, as REA does", async () => { + const objects = `[{}${",{}".repeat(1_400_000)}]`; + const archive = zipArchive([ + { name: "objects.json", data: objects, method: 0 }, + { name: "objects.txt", data: `${objects.slice(0, -1)}}`, method: 0 }, + { + name: "settings.ini", + data: `[ui]\n${"wrap=false\n".repeat(400_000)}`, + method: 0, + }, + ]); + const refused = "larger than the 4 MiB limit for JSON and key file views"; + + expect(await sameView(archive)).toMatchObject({ + entries: [ + { format: "json", view: null, view_error: refused }, + { format: "text", view: { line_count: 1 }, view_error: null }, + { format: "keyfile", view: null, view_error: refused }, + ], + }); + const { peak } = await pythonPeak( + zipArchive([{ name: "objects.json", data: objects, method: 0 }]), + ); + // Parsing the 4,200,004-byte entry before refusing it traced about 110 MB here. + expect(peak).toBeLessThan(4 * objects.length); + }, 60_000); + + it("keeps a JSON entry nested too deeply on that entry and reads its siblings, as REA does", async () => { + const nested = (levels: number) => + `${"[".repeat(levels)}${"]".repeat(levels)}`; + const archive = zipArchive([ + { name: "Document.xml", data: "" }, + { name: "deep.json", data: nested(1100) }, + { name: "edge.json", data: nested(501) }, + ]); + + expect(await sameView(archive)).toMatchObject({ + entries: [ + { name: "Document.xml", format: "xml", view_error: null }, + { + name: "deep.json", + format: "json", + view: null, + view_error: "nested deeper than the 500 levels a JSON view keeps", + }, + { name: "edge.json", format: "json", view_error: null }, + ], + }); + }); +}); + +/** A FreeCAD document zipped as FreeCAD saves it, with a shape and a thumbnail beside it. */ +const fcstd = async (version: string, end: "before" | "after") => + zipArchive([ + { + name: "Document.xml", + data: await readFile(join(FIXTURES, version, end, "Document.xml")), + }, + { + name: "PartShape.brp", + data: "CASCADE Topology V1, (c) Matra-Datavision\n", + }, + { + name: "thumbnails/Thumbnail.png", + data: Buffer.from([0x89, 0x50, 0]), + method: 0, + }, + ]); + +const DOCUMENT = "/entries/[name=Document.xml]/view/root/children"; +const FILE = "Documents/plate.FCStd"; + +/** The FreeCAD task's checks, by name: they hold in both versions' files. */ +const HOLE_CHECKS: DesktopTaskCheck[] = [ + { + id: "cut", + kind: "value_equals", + file: FILE, + pointer: `${DOCUMENT}/[name=Objects]/children/[name=Object][@type=Part::Cut]/attributes/type`, + value: "Part::Cut", + }, + { + id: "radius", + kind: "value_equals", + file: FILE, + pointer: `${DOCUMENT}/[name=ObjectData]/children/[name=Object][@name=Hole]/children/[name=Properties]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value`, + value: 5, + tolerance: 0.01, + }, + { + id: "tool", + kind: "value_equals", + file: FILE, + pointer: `${DOCUMENT}/[name=ObjectData]/children/[name=Object][@name=PlateWithHole]/children/[name=Properties]/children/[name=Property][@name=Tool]/children/[name=Link]/attributes/value`, + value: "Hole", + }, +]; + +describe("FreeCAD documents", () => { + it.each(["0.21", "1.0"])( + "checks the hole in FreeCAD %s by name: failing before the edit and passing after, in both evaluators", + async (version) => { + const archives = saved([ + await fcstd(version, "before"), + await fcstd(version, "after"), + ]); + await expectSameFileViews(archives); + + const [before, after] = await expectSameCheckResults( + FILE, + HOLE_CHECKS, + archives, + ); + expect(before?.map(({ passed }) => passed)).toEqual([ + false, + false, + false, + ]); + expect(after?.map(({ passed }) => passed)).toEqual([true, true, true]); + expect(after?.[1]?.description).toContain( + 'is "5.0000000000000000", within 0.01 of 5', + ); + }, + ); +}); + +describe("OpenDocument text", () => { + it("reads content.xml from an .odt as REA does, its namespaced names included", async () => { + const odt = odtArchive("Buy milk"); + const read = await readZipView(odt); + const [python] = await pythonFileViews([{ name: "list.odt", bytes: odt }]); + + expect(detectStateFormat("list.odt", odt)).toBe("zip"); + expect(read.ok && read.value).toMatchObject({ + entries: [ + { + name: "mimetype", + format: "text", + view: { lines: ["application/vnd.oasis.opendocument.text"] }, + }, + { + name: "content.xml", + format: "xml", + view: { + root: { + name: "office:document-content", + children: [ + { + name: "office:body", + children: [ + { + name: "office:text", + children: [{ name: "text:p", children: ["Buy milk"] }], + }, + ], + }, + ], + }, + }, + }, + { name: "META-INF/manifest.xml", format: "xml" }, + ], + }); + expect(python?.view).toEqual(read.ok ? read.value : null); + }); +}); diff --git a/tests/fixtures/freecad/0.21/after/Document.xml b/tests/fixtures/freecad/0.21/after/Document.xml new file mode 100644 index 0000000000..b869fbc412 --- /dev/null +++ b/tests/fixtures/freecad/0.21/after/Document.xml @@ -0,0 +1,222 @@ + + + + + <_Property name="FileName" type="App::PropertyString" status="50331649"/> + <_Property name="Tip" type="App::PropertyLink" status="33554433"/> + <_Property name="TransientDir" type="App::PropertyString" status="50331649"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_Property name="History" type="Part::PropertyShapeHistory" status="234881024"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/freecad/0.21/before/Document.xml b/tests/fixtures/freecad/0.21/before/Document.xml new file mode 100644 index 0000000000..36a24b3a62 --- /dev/null +++ b/tests/fixtures/freecad/0.21/before/Document.xml @@ -0,0 +1,119 @@ + + + + + <_Property name="FileName" type="App::PropertyString" status="50331649"/> + <_Property name="Tip" type="App::PropertyLink" status="33554433"/> + <_Property name="TransientDir" type="App::PropertyString" status="50331649"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/freecad/1.0/after/Document.xml b/tests/fixtures/freecad/1.0/after/Document.xml new file mode 100644 index 0000000000..bd310155dd --- /dev/null +++ b/tests/fixtures/freecad/1.0/after/Document.xml @@ -0,0 +1,272 @@ + + + + + + <_Property name="FileName" type="App::PropertyString" status="50331649"/> + <_Property name="Tip" type="App::PropertyLink" status="33554433"/> + <_Property name="TransientDir" type="App::PropertyString" status="50331649"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_Property name="History" type="Part::PropertyShapeHistory" status="234881024"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/freecad/1.0/before/Document.xml b/tests/fixtures/freecad/1.0/before/Document.xml new file mode 100644 index 0000000000..11346edf6a --- /dev/null +++ b/tests/fixtures/freecad/1.0/before/Document.xml @@ -0,0 +1,151 @@ + + + + + + <_Property name="FileName" type="App::PropertyString" status="50331649"/> + <_Property name="Tip" type="App::PropertyLink" status="33554433"/> + <_Property name="TransientDir" type="App::PropertyString" status="50331649"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/freecad/README.md b/tests/fixtures/freecad/README.md new file mode 100644 index 0000000000..df68be4d8d --- /dev/null +++ b/tests/fixtures/freecad/README.md @@ -0,0 +1,17 @@ +# FreeCAD documents + +Each `Document.xml` is unchanged from a `.FCStd` file that a real FreeCAD saved +during a desktop capture: `0.21` from FreeCAD 0.21 (`0.21R33771`) and `1.0` from +FreeCAD 1.0 (`1.0R`), both on Ubuntu. A `.FCStd` file is a zip archive, and +`Document.xml` is the entry that records the document's objects. + +`before` holds one `Part::Box` named `Plate`, 160 × 100 × 5 mm. `after` is the +same document once the user cut a 10 mm hole through it and saved: it adds a +`Part::Cylinder` named `Hole` with `Radius` `5.0000000000000000`, and a +`Part::Cut` named `PlateWithHole` whose `Base` links `Plate` and whose `Tool` +links `Hole`. + +The two versions write the same objects in different places: FreeCAD 1.0 +records more properties, so the hole's `Radius` sits at another position among +its siblings. Tests zip these files into `.FCStd` archives and check them by +name rather than by position. diff --git a/tests/fixtures/inkscape/README.md b/tests/fixtures/inkscape/README.md new file mode 100644 index 0000000000..4f7ec711b9 --- /dev/null +++ b/tests/fixtures/inkscape/README.md @@ -0,0 +1,13 @@ +# Inkscape drawings + +Each `poster.svg` is unchanged from a file that a real Inkscape saved as +Inkscape SVG: Inkscape 1.2.2 on Ubuntu 24.04, from +`inkscape poster.svg --export-type=svg`. The saved file declares the default +SVG namespace, `svg`, `inkscape` and `sodipodi`, names the document with +`sodipodi:docname`, adds a `sodipodi:namedview` whose settings are `inkscape:` +attributes, and keeps `xml:space="preserve"` on its text. + +`before` is an A4 poster with a layer `Layer 1` holding a banner and two +lines of text; the `text` with `id="headline"` reads `Spring Bake Sale`. +`after` is the same poster once the headline was changed to +`Summer Bake Sale`, and nothing else. diff --git a/tests/fixtures/inkscape/after/poster.svg b/tests/fixtures/inkscape/after/poster.svg new file mode 100644 index 0000000000..7731967be9 --- /dev/null +++ b/tests/fixtures/inkscape/after/poster.svg @@ -0,0 +1,57 @@ + + + + + + + Summer Bake Sale + Saturday at the school hall + + diff --git a/tests/fixtures/inkscape/before/poster.svg b/tests/fixtures/inkscape/before/poster.svg new file mode 100644 index 0000000000..8066cfecee --- /dev/null +++ b/tests/fixtures/inkscape/before/poster.svg @@ -0,0 +1,57 @@ + + + + + + + Spring Bake Sale + Saturday at the school hall + + diff --git a/tests/fixtures/libreoffice/README.md b/tests/fixtures/libreoffice/README.md new file mode 100644 index 0000000000..d4239c5296 --- /dev/null +++ b/tests/fixtures/libreoffice/README.md @@ -0,0 +1,13 @@ +# LibreOffice Writer documents + +Each `content.xml` is unchanged from an `.odt` file that a real LibreOffice +Writer saved: LibreOffice 24.2.7.2 on Ubuntu 24.04, from +`soffice --headless --convert-to odt`. An `.odt` file is a zip archive, and +`content.xml` is the entry that holds the document's text; it declares 35 +namespace prefixes on its root and names every element and attribute with one, +as `text:p` and `text:style-name`. + +`before` is a flyer with a `Title` paragraph `Summer Bake Sale`, a `Subtitle` +paragraph `Monday 17 November` and a body paragraph. `after` is the same flyer +once the date was changed to `Monday 24 November`, and nothing else. Tests zip +each file into an `.odt` archive with its `mimetype`. diff --git a/tests/fixtures/libreoffice/after/content.xml b/tests/fixtures/libreoffice/after/content.xml new file mode 100644 index 0000000000..ff7e91e178 --- /dev/null +++ b/tests/fixtures/libreoffice/after/content.xml @@ -0,0 +1,2 @@ + +Summer Bake SaleMonday 24 NovemberCakes, biscuits and bread from 10 o'clock in the school hall. \ No newline at end of file diff --git a/tests/fixtures/libreoffice/before/content.xml b/tests/fixtures/libreoffice/before/content.xml new file mode 100644 index 0000000000..8af1d0ab45 --- /dev/null +++ b/tests/fixtures/libreoffice/before/content.xml @@ -0,0 +1,2 @@ + +Summer Bake SaleMonday 17 NovemberCakes, biscuits and bread from 10 o'clock in the school hall. \ No newline at end of file diff --git a/tests/support/desktopExportPython.ts b/tests/support/desktopExportPython.ts new file mode 100644 index 0000000000..f4c1f421a5 --- /dev/null +++ b/tests/support/desktopExportPython.ts @@ -0,0 +1,149 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { readZipView } from "../../src/application/desktop/DesktopZipView.js"; +import type { DesktopTaskCheck } from "../../src/domain/desktop/appTask.js"; +import { + evaluateTaskChecks, + type TaskCheckOutcome, + type TaskFileView, +} from "../../src/domain/desktop/appTaskChecks.js"; +import { + PYTHON_HELPERS, + PYTHON_IMPORTS, +} from "../../src/domain/desktop/appExportPython.js"; +import { PYTHON_TASK_EVALUATOR } from "../../src/domain/desktop/appTaskPython.js"; +import { + detectStateFormat, + textualView, +} from "../../src/domain/desktop/stateFormats.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +/** A file as an app saved it: its name under the home and its bytes. */ +export interface SavedFile { + readonly name: string; + readonly bytes: Uint8Array; +} + +/** The exported module's standard library imports, its helpers and the verifier's evaluator, as one program. */ +const PYTHON = `${PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n")}${PYTHON_HELPERS}${PYTHON_TASK_EVALUATOR}`; + +/** Run a driver after the exported module's helpers and evaluator, with JSON on its stdin; its stdout. */ +export const exportedPython = (driver: string, input: unknown): string => + execFileSync("python3", ["-I", "-B", "-c", `${PYTHON}\n${driver}`], { + input: JSON.stringify(input), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + +/** One JSON value per line the driver printed. */ +const printed = (output: string): Value[] => + output + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Value); + +/** Write each file into a fresh folder, as data/get finds it under the home, keeping its suffix; their paths. */ +export const writeFiles = async (files: readonly SavedFile[]) => { + const folder = await createTestTempDirectory("desktop-export-python-"); + return Promise.all( + files.map(async ({ name, bytes }, index) => { + const path = join(folder, `${String(index)}-${name}`); + await writeFile(path, bytes); + return path; + }), + ); +}; + +/** The exported module's data/get entry for each file: its format and view, or its error. */ +export const pythonFileViews = async (files: readonly SavedFile[]) => + printed>( + exportedPython( + `for path in json.load(sys.stdin): + print(json.dumps(_view(Path(path))))`, + await writeFiles(files), + ), + ); + +/** The verifier's results for each file, read by the exported module as the home path `home`. */ +const pythonCheckResults = async ( + home: string, + checks: readonly DesktopTaskCheck[], + files: readonly SavedFile[], +) => + printed<{ id: string; result: boolean; description: string }[]>( + exportedPython( + `request = json.load(sys.stdin) +for path in request["paths"]: + print(json.dumps(evaluate(request["checks"], {request["home"]: _view(Path(path))})))`, + { home, checks, paths: await writeFiles(files) }, + ), + ); + +/** The exported module's XML view of each text, or null where it refuses one. */ +export const pythonXmlViews = (texts: readonly string[]) => + printed( + exportedPython( + `for text in json.load(sys.stdin): + print(json.dumps(_xml_view(text)))`, + texts, + ), + ); + +/** REA's state view of a zip, text, JSON, key file or XML file, in the shape data/get serves it. */ +export const typescriptFileView = async ({ + name, + bytes, +}: SavedFile): Promise => { + const format = detectStateFormat(name, bytes); + if (format === "sqlite" || format === "dconf" || format === "binary") + throw new Error(`${name} is ${format}, which these tests do not read`); + const read = + format === "zip" ? await readZipView(bytes) : textualView(format, bytes); + return read.ok + ? { format, view: read.value } + : { format, view: null, error: read.error }; +}; + +/** Both readers give each file the same format and view, or the same error. */ +export const expectSameFileViews = async (files: readonly SavedFile[]) => { + const fromPython = await pythonFileViews(files); + for (const [index, file] of files.entries()) { + const ours = await typescriptFileView(file); + const theirs = fromPython[index] ?? {}; + expect(theirs.format).toBe(ours.format); + expect(theirs.view).toEqual(ours.view); + expect(theirs.error).toBe(ours.error); + } +}; + +/** Both evaluators give the same results and descriptions on each file read as the home path `home`; REA's. */ +export const expectSameCheckResults = async ( + home: string, + checks: readonly DesktopTaskCheck[], + files: readonly SavedFile[], +): Promise<(readonly TaskCheckOutcome[])[]> => { + const fromPython = await pythonCheckResults(home, checks, files); + return Promise.all( + files.map(async (file, index) => { + const ours = evaluateTaskChecks( + checks, + new Map([[home, await typescriptFileView(file)]]), + ); + expect(fromPython[index]).toEqual( + ours.map(({ id, passed, description }) => ({ + id, + result: passed, + description, + })), + ); + return ours; + }), + ); +};