mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-11-07 06:19:43 -10:00
feat(live): add visual input, proactive assistance and memory (#11369)
* feat(live): add visual input, proactive memory and desktop controls * fix(live): preserve upstream checks and finalize migration validation * feat(live): open active configuration from host settings * fix(live): address lifecycle and proactive review findings * feat(live): add subagent controls and diagnostics with review fixes --------- Co-authored-by: 菲兹 <[email protected]> Co-authored-by: Shaojin Wen <[email protected]>
This commit is contained in:
247 files changed
+67548
-2084
No files matched your search
@@ -102,13 +102,18 @@ jobs:
|
||||
echo '::error::Packaged app declares an Input Monitoring usage description.'
|
||||
exit 1
|
||||
fi
|
||||
for unused_permission in NSBluetoothAlwaysUsageDescription NSBluetoothPeripheralUsageDescription NSCameraUsageDescription; do
|
||||
for unused_permission in NSBluetoothAlwaysUsageDescription NSBluetoothPeripheralUsageDescription; do
|
||||
if /usr/libexec/PlistBuddy -c "Print :$unused_permission" "$info_plist" >/dev/null 2>&1; then
|
||||
echo "::error::Packaged app declares unused permission $unused_permission."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
/usr/libexec/PlistBuddy -c 'Print :NSMicrophoneUsageDescription' "$info_plist" >/dev/null
|
||||
camera_usage="$(/usr/libexec/PlistBuddy -c 'Print :NSCameraUsageDescription' "$info_plist")"
|
||||
if [ -z "${camera_usage//[[:space:]]/}" ]; then
|
||||
echo '::error::Packaged app must describe its camera use.'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
assert_live_host_entitlements() {
|
||||
local entitlements_file="$1"
|
||||
@@ -121,6 +126,7 @@ jobs:
|
||||
"com.apple.security.cs.allow-jit",
|
||||
"com.apple.security.cs.allow-unsigned-executable-memory",
|
||||
"com.apple.security.device.audio-input",
|
||||
"com.apple.security.device.camera",
|
||||
];
|
||||
const actual = Object.keys(value).sort();
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# Qwen Live visual input — Source × Mode
|
||||
|
||||
## Goal
|
||||
|
||||
Make visual input explicit instead of asking the model to infer whether an
|
||||
image came from the desktop or camera. Visual input has two independent axes:
|
||||
|
||||
- Source: `screen` or `camera`
|
||||
- Mode: `on-demand` or `live-feed`
|
||||
|
||||
The selected pair is authoritative. The model never inspects or claims to see
|
||||
the unselected source.
|
||||
|
||||
## Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"visualInput": {
|
||||
"source": "screen",
|
||||
"mode": "on-demand",
|
||||
"fps": 1,
|
||||
"cameraResolution": { "width": 1280, "height": 720 },
|
||||
"cameraSnapshotResolution": "native",
|
||||
"liveResolution": { "width": 1280, "height": 720 },
|
||||
"snapshotResolution": "native"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
These defaults make visual capture private and pull-based until the user or
|
||||
model needs it. `qwen-live init` persists these defaults without prompting for
|
||||
them. The configuration file and environment variables can set the startup
|
||||
values; the Host orb can change Source and Mode at runtime without rewriting
|
||||
the file.
|
||||
|
||||
Environment overrides:
|
||||
|
||||
- `QWEN_LIVE_VISUAL_SOURCE`
|
||||
- `QWEN_LIVE_VISUAL_MODE`
|
||||
- `QWEN_LIVE_VISUAL_FPS`
|
||||
- `QWEN_LIVE_CAMERA_RESOLUTION`
|
||||
- `QWEN_LIVE_CAMERA_SNAPSHOT_RESOLUTION`
|
||||
- `QWEN_LIVE_VISUAL_LIVE_RESOLUTION`
|
||||
- `QWEN_LIVE_VISUAL_SNAPSHOT_RESOLUTION`
|
||||
|
||||
FPS is bounded to 0.1–10. Camera and Live Feed resolutions are width/height
|
||||
pairs and default to 1280×720. Screen `snapshotResolution` and Camera
|
||||
`cameraSnapshotResolution` independently accept a pair or `native`.
|
||||
|
||||
## Mode semantics
|
||||
|
||||
### Live Feed
|
||||
|
||||
The Host samples the selected source continuously at the configured FPS and
|
||||
resolution. Each bounded JPEG travels as `host.visual_frame`, is validated
|
||||
again by the daemon, and becomes `input_image_buffer.append` on the active
|
||||
Omni Realtime connection. Frames are best-effort and remain scoped by call
|
||||
epoch. Realtime rejects image frames until at least one microphone audio frame
|
||||
has been appended; periodic sampling naturally recovers after audio starts.
|
||||
|
||||
- Screen uses the in-process native Appshot implementation on the foreground
|
||||
non-Host window.
|
||||
- Camera uses one preload-owned MediaStream, requested at the configured
|
||||
`cameraResolution`, shared by transport and preview.
|
||||
- The `appshot` model tool is disabled in this mode.
|
||||
|
||||
Before the first microphone frame is available, the daemon retains only the
|
||||
latest sampled frame and sends it immediately after audio starts. This keeps
|
||||
the provider's required audio-before-image ordering without losing the newest
|
||||
view during connection startup.
|
||||
|
||||
### On Demand
|
||||
|
||||
The foreground model calls `appshot` when current visual information is needed.
|
||||
Active visual Proactive tasks independently request periodic private captures
|
||||
for their Monitor sessions in this mode. For Appshot, the daemon sends a correlated
|
||||
`host.capture_visual` request for the selected source. The Host captures one
|
||||
frame and returns `host.visual_capture_result`. `LiveSession` registers the
|
||||
captured file as an asset and returns the source metadata and asset handle via
|
||||
the original `function_call_output` continuation path. It does not append the
|
||||
snapshot to Realtime, commit an audio buffer, or change VAD mode.
|
||||
|
||||
Screen results also carry bounded app/window/accessibility metadata and keep
|
||||
the native PNG as a handoff asset. Camera results keep an independently captured
|
||||
still JPEG at `cameraSnapshotResolution` as a handoff asset. When pixel-level interpretation is needed, the model delegates
|
||||
the user's visual question with that asset in `input_refs`. A source or mode
|
||||
change rejects any obsolete pending capture.
|
||||
|
||||
## Resolution and payload policy
|
||||
|
||||
Live Feed provider input is canonical raw JPEG base64, never a data URL, and is
|
||||
capped at 190 KiB decoded. Every JPEG sent to Omni is also proportionally fitted
|
||||
within 1920×1080, the provider's 1080p input boundary.
|
||||
|
||||
- Live Feed first scales within `liveResolution`, lowers JPEG quality, and may
|
||||
reduce dimensions further when necessary.
|
||||
- Camera Appshot captures a full still image, then fits its asset within
|
||||
`cameraSnapshotResolution` when configured. Native requests the device's
|
||||
available still-image size; a supported video-constraint fallback restores
|
||||
the preview after capture. Failure to obtain native capture is explicit.
|
||||
- Camera assets are limited to 8 MiB and stay on the Host-local asset path.
|
||||
Their transport preview remains a bounded 1080p/190 KiB JPEG. Private
|
||||
Monitor frames use the preview/Live resolution and never trigger still
|
||||
capture. Screen PNG assets keep their captured dimensions.
|
||||
- Screen snapshot transport first fits its configured boundary and the 1080p
|
||||
cap, then lowers quality. An oversized minimum-quality transport frame
|
||||
fails explicitly. Live Feed retains its additional downscaling fallback.
|
||||
|
||||
No media payload, transcript, API key, or credential is written to diagnostics.
|
||||
Screen PNGs and camera JPEG handoff assets are private temporary files managed
|
||||
by the existing Appshot capture service.
|
||||
|
||||
## Host UI and permissions
|
||||
|
||||
Whenever the Host is connected, the orb or setup panel shows two compact
|
||||
controls, `Source` and `Mode`, including when the selected source is not ready.
|
||||
Clicking either opens an English text menu. Camera Source also shows a mirrored
|
||||
preview whenever the usable orb is visible, labelled `Camera · On Demand` or
|
||||
`Camera · Live Feed`. The selected camera stream remains local while idle;
|
||||
Live Feed uploads frames to Realtime only during an active call. On Demand
|
||||
Appshot returns a local handoff asset, while an active Proactive task may
|
||||
continuously request bounded Monitor frames. Stopping the call disposes those
|
||||
tasks; the idle camera preview stays local.
|
||||
|
||||
Readiness is source-specific:
|
||||
|
||||
- Both sources require microphone, audio self-checks, and the shortcut.
|
||||
- Screen requires Accessibility, Screen Recording, and Appshot readiness.
|
||||
- Camera requires Camera permission; missing Screen permissions do not block it.
|
||||
|
||||
Changing Source requests only the new source's permissions. If a call is
|
||||
active, the Host keeps the working source selected until the new source is
|
||||
ready, then applies the requested source atomically; this prevents the
|
||||
permission prompt itself from stopping the call. Stop, disconnect, renderer
|
||||
loss, epoch change, mode/source change, and Host shutdown all tear down the
|
||||
obsolete timer, stream, or pending request.
|
||||
|
||||
## Prompt contract
|
||||
|
||||
The system instructions define exactly one selected source and mode. A silent
|
||||
`[VISUAL_INPUT] source=… mode=….` context item updates that contract after an
|
||||
orb change.
|
||||
|
||||
- Live Feed: answer visual questions from recent selected-source frames; never
|
||||
call `appshot`.
|
||||
- On Demand: call `appshot`; use its Screen metadata when sufficient, or hand
|
||||
off the returned Screen/Camera asset for pixel-level interpretation.
|
||||
- Never combine, guess, or claim to see the unselected source. Ask the user to
|
||||
switch Source in the orb when they request it.
|
||||
|
||||
## Protocol compatibility
|
||||
|
||||
Visual input and playback identities use Host protocol v9 so an older Host cannot appear
|
||||
connected while silently rejecting the new capture commands:
|
||||
|
||||
- optional `host.welcome.visualInput`
|
||||
- `host.visual_frame`
|
||||
- `host.visual_settings`
|
||||
- `host.capture_visual`
|
||||
- `host.visual_capture_result`
|
||||
|
||||
The built-in `qwen serve` daemon can omit `visualInput`; the v9 Host then exposes
|
||||
no Source/Mode controls and uses `host.capture_visual` with source `screen` for
|
||||
Appshot. The qwen-live and CLI protocol type files remain byte-identical.
|
||||
|
||||
## Diagnostics and verification
|
||||
|
||||
`qwen-live --debug` logs privacy-safe configuration, connection, call, capture,
|
||||
frame acceptance, and provider error metadata to stderr. The Host executable
|
||||
also accepts `--live-debug` for capture-side dimensions, byte counts, and error
|
||||
codes. (`--debug` is reserved by Electron.)
|
||||
|
||||
Automated coverage includes config defaults/validation, init exclusion,
|
||||
protocol parsing and size limits, runtime setting persistence, correlated
|
||||
capture, source/mode frame filtering, Realtime audio-first image delivery,
|
||||
prompt routing, Host lifecycle, camera preview architecture, and package
|
||||
typecheck/build.
|
||||
@@ -0,0 +1,219 @@
|
||||
# Qwen Live Proactive
|
||||
|
||||
## Goal
|
||||
|
||||
Port the in-call Proactive capability from `qwen-omni-realtime-agent` into
|
||||
Qwen Live. The port supports only the DashScope Realtime monitor backend,
|
||||
uses the foreground Realtime endpoint, API key, and model, and is enabled by
|
||||
default. Tasks are scoped to one Live call and are never persisted.
|
||||
|
||||
The public capability consists of six foreground-model tools:
|
||||
|
||||
- `create_proactive_monitor`
|
||||
- `create_live_narration`
|
||||
- `create_proactive_timer`
|
||||
- `update_proactive_task`
|
||||
- `cancel_proactive_task`
|
||||
- `list_proactive_tasks`
|
||||
|
||||
When Proactive is disabled, none of these tools or their routing instructions
|
||||
are exposed to the foreground model.
|
||||
|
||||
Tool results are authoritative transactions. Mutation receipts contain the
|
||||
committed result and a fresh snapshot of the active task pool, while the text
|
||||
returned to the voice model is a speech-safe rendering that omits internal
|
||||
task ids and private error details. A failed mutation therefore cannot be
|
||||
mistaken for a completed action.
|
||||
|
||||
## Configuration
|
||||
|
||||
`config.json` contains one `proactive` object. Only settings read by the
|
||||
DashScope Realtime Proactive path are included:
|
||||
|
||||
```json
|
||||
{
|
||||
"proactive": {
|
||||
"enabled": true,
|
||||
"monitor": {
|
||||
"sessionRecycleEvals": 60
|
||||
},
|
||||
"scheduler": {
|
||||
"evalIntervalSec": 2,
|
||||
"maxConcurrentTasks": 4,
|
||||
"maxFailuresPerTask": 3,
|
||||
"repeat": {
|
||||
"cooldownSec": 3,
|
||||
"maxWaitTtsSec": 30,
|
||||
"clearBufferOnResume": true
|
||||
}
|
||||
},
|
||||
"vision": {
|
||||
"fps": 1,
|
||||
"windowSizeSec": 10,
|
||||
"minEvalDurationSec": 0
|
||||
},
|
||||
"audio": {
|
||||
"windowSizeSec": 60,
|
||||
"minEvalDurationSec": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`QWEN_LIVE_PROACTIVE_ENABLED` is the environment override for the master
|
||||
switch. Monitor endpoint, credentials, and model come from `realtime`; the
|
||||
text-only Monitor does not inherit or send the foreground voice. Microphone
|
||||
input remains fixed at 16 kHz mono PCM16.
|
||||
|
||||
## Task lifecycle
|
||||
|
||||
Each Live call owns one task manager and scheduler. A timer task uses a local
|
||||
generation-fenced timer. A perception task owns one independent manual-mode
|
||||
DashScope Realtime WebSocket and moves through provisioning, running,
|
||||
delivering, and a terminal state. Updating a running perception task replaces
|
||||
its monitor connection; cancelling, stopping the call, or a fatal foreground
|
||||
failure closes it. Late callbacks are rejected by both call epoch and task
|
||||
generation.
|
||||
|
||||
Perception tasks support two contracts:
|
||||
|
||||
- Event monitor: one future observable condition, optionally repeated. A
|
||||
repeated event rearms only after cooldown and a later `wait` result.
|
||||
- Live narration: continuing, novelty-sensitive descriptions. It stays active
|
||||
until cancelled and does not require a false edge between distinct updates.
|
||||
|
||||
One-shot tasks become completed only after their announcement is actually
|
||||
played. Repeated tasks stay running while any earlier notification is queued
|
||||
or announcing. Each notification has an independent delivery id and ACK;
|
||||
acknowledging one never retires a later event or resets ongoing observation.
|
||||
Cooldown is measured from detection and keeps the configured false-edge rule.
|
||||
Updating or cancelling a task invalidates all of its pending notifications.
|
||||
|
||||
The same genuine microphone turn may continue through multiple tool results:
|
||||
list then cancel, Appshot then create, or validation error then corrected
|
||||
arguments. Continuations inherit only that turn's existing tool capability;
|
||||
synthetic Proactive turns and repair-result continuations cannot gain it.
|
||||
Task-list receipts expose timer remaining duration, reminder content,
|
||||
monitoring condition/focus, response guidance, repeat state, and pending-event
|
||||
count without exposing internal ids.
|
||||
|
||||
The last successfully created or updated task may be referenced without a
|
||||
title only from the immediately adjacent genuine microphone turn. That
|
||||
one-shot context is never consumed by background, receipt, or repair turns.
|
||||
For safety, an implicit update can only change `repeat` from false to true; an
|
||||
implicit cancel must have no arguments. Failed mutations preserve the prior
|
||||
adjacent context, while a successful cancel clears it.
|
||||
|
||||
If a successful direct reply audibly promises future monitoring/reminding, or
|
||||
claims that a Proactive task was cancelled, but made no Proactive mutation
|
||||
call, Qwen Live asks the same foreground model to re-evaluate that turn in one
|
||||
silent, response-scoped repair round. The repair reads only the assistant's
|
||||
own transcript, never ASR text. It may call one allowlisted mutation tool
|
||||
(cancel-only for a cancellation claim); all other calls are rejected without
|
||||
side effects. New user speech invalidates a stale repair. Only the subsequent
|
||||
authoritative tool-receipt continuation may speak.
|
||||
|
||||
## DashScope monitor protocol
|
||||
|
||||
Every perception task has a text-output-only Realtime session configured with
|
||||
manual turn detection. The connection receives the fixed monitor system prompt
|
||||
and one standing user instruction. Microphone PCM and selected-source JPEG
|
||||
frames are appended continuously. At each evaluation boundary the client:
|
||||
|
||||
1. appends 100 ms of silence so the server-side audio buffer is non-empty;
|
||||
2. sends `input_audio_buffer.commit`;
|
||||
3. waits for the matching `input_audio_buffer.committed` event;
|
||||
4. sends one `response.create`.
|
||||
|
||||
The only valid final actions are `wait`, `Reply: <text>`, and
|
||||
`Func_call:<...>`. `Reply:` produces a Proactive event. `wait` is silent, and
|
||||
monitor-proposed function calls are recognized but never executed. Invalid
|
||||
output fails closed and recycles that monitor connection. Connections recycle
|
||||
after the configured number of evaluations while retaining recent media for
|
||||
the configured window.
|
||||
|
||||
## Media routing
|
||||
|
||||
Microphone PCM is copied to every active task that selected audio. Visual
|
||||
evidence always follows Qwen Live's currently selected Screen or Camera source:
|
||||
|
||||
- Live Feed frames are copied from the existing Host frame path and sampled no
|
||||
faster than `proactive.vision.fps`.
|
||||
- In On Demand mode, the scheduler privately requests frames from the existing
|
||||
correlated Host capture path while at least one active vision task exists.
|
||||
This does not invoke the foreground `appshot` tool or commit the foreground
|
||||
audio buffer.
|
||||
|
||||
Every selected modality must have real recent evidence before evaluation;
|
||||
transport padding silence never counts as evidence. Optional minimum-duration
|
||||
settings add warm-up requirements. Audio and vision keep their own configured
|
||||
windows both for warm-up and for replay after reconnect, including in a
|
||||
combined task. Changing the selected visual source replaces
|
||||
each vision Monitor session so committed frames and late results from the old
|
||||
source cannot cross into the new source timeline.
|
||||
|
||||
## Announcement FIFO
|
||||
|
||||
There is no Turn Arbiter. Triggered events enter a durable FIFO lane in the
|
||||
existing injection gate. The lane observes the same three foreground barriers:
|
||||
user speech, an in-flight foreground response, and Host playback.
|
||||
|
||||
Only one Proactive item may be submitted to the foreground Realtime model at a
|
||||
time. Submission itself closes the gate synchronously, covering the interval
|
||||
before `response.created`. The model receives a structured
|
||||
`[PROACTIVE_EVENT]` context item and produces a natural spoken response from
|
||||
the monitor evidence plus the user's requested response guidance.
|
||||
|
||||
Foreground waiting and playback do not pause repeated Monitor evaluation.
|
||||
Newly detected events append to the same FIFO, including multiple events from
|
||||
one task. Cancel/update/failure removes all affected records before notifying
|
||||
the gate, retracting queued tail items before the current item to avoid
|
||||
synchronously admitting another obsolete event.
|
||||
|
||||
The next FIFO item is released only after both boundaries for the current item
|
||||
have occurred, in either order:
|
||||
|
||||
- the foreground Realtime response is done;
|
||||
- the Host reports playback completed.
|
||||
|
||||
Audio chunks and playback receipts carry a monotonically increasing output id.
|
||||
When the Host advertises `outputAudioEndMarkerV1`, `response.done` sends an
|
||||
explicit end marker and seals that id; later response audio receives a new id.
|
||||
The Host acknowledges each id only after both its marker and all scheduled
|
||||
frames have drained, while the daemon keeps the injection gate closed until all
|
||||
consecutive outputs have completed. Clearing playback retires every outstanding
|
||||
id, so late started/completed receipts cannot acknowledge a newer Proactive
|
||||
delivery. Hosts without the capability retain the legacy drain behavior. If the
|
||||
user starts talking after the model response is done but before its queued audio
|
||||
finishes, that delivery is put back at the head of the FIFO and retried after
|
||||
the user's foreground turn.
|
||||
|
||||
Queued time does not consume the delivery-ack timeout. The timeout begins when
|
||||
foreground Realtime emits `response.created`, so generation and the Host
|
||||
playback acknowledgement share one bounded delivery window. Duplicate start
|
||||
signals do not extend that deadline. A stale epoch, cancelled task, or old
|
||||
delivery generation can never acknowledge a newer item.
|
||||
|
||||
## Failure boundaries
|
||||
|
||||
- Monitor setup and consecutive evaluation errors fail only the affected task.
|
||||
- A permanently failed task queues one fixed, speech-safe notification naming
|
||||
the task; technical details stay in diagnostics and the notice cannot recurse
|
||||
into task failure handling.
|
||||
- A rejected announcement submission remains queued for retry.
|
||||
- A delivery acknowledgement timeout fails that delivery and releases the FIFO,
|
||||
including when the Host never reports playback start or completion.
|
||||
- Cancelled or audio-less announcement responses never acknowledge delivery;
|
||||
user-interrupted announcements are retried at the front of the FIFO.
|
||||
- Foreground Realtime or Host-call failure tears down the whole call-scoped
|
||||
Proactive runtime.
|
||||
- No media payload, transcript, API key, monitor prompt, or task evidence is
|
||||
written to debug logs; logs contain ids, states, dimensions, byte counts, and
|
||||
sanitized error metadata only.
|
||||
|
||||
## Verification
|
||||
|
||||
Automated tests cover config defaults and validation, init output, tool
|
||||
visibility and CRUD, monitor manual commit ordering, action parsing, generation
|
||||
fences, timer replacement, perception warm-up, repeat false-edge behavior,
|
||||
call cleanup, and strict FIFO delivery across response/playback ordering.
|
||||
@@ -0,0 +1,91 @@
|
||||
# Qwen Live audit corrections
|
||||
|
||||
## Problem and scope
|
||||
|
||||
The local Live extension rejects Proactive tool chains that belong to one user
|
||||
turn, can leave the notification gate waiting after reordered provider events,
|
||||
uses the vision window for audio evidence, and omits useful task-list details.
|
||||
Repeated monitors also stop observing as soon as their first event is queued.
|
||||
Camera snapshots reuse the preview stream and cannot independently request
|
||||
their capture resolution.
|
||||
|
||||
This change fixes those behaviors. The source Monitor prompt and the Proactive
|
||||
repair prompt remain intact. Publishing, release feeds, general UI redesign,
|
||||
and ACP installation policy are outside this change.
|
||||
|
||||
## User-turn authority and delivery gate
|
||||
|
||||
The existing Realtime tool capability represents authority inherited from a
|
||||
real microphone turn. Proactive tools may execute in that turn's tool-result
|
||||
continuations, including list then cancel, Appshot then create, and validation
|
||||
failure then retry. Synthetic notifications and repair-result continuations
|
||||
must not gain this authority. Response lineage retains the originating input
|
||||
item while the user-turn capability is valid.
|
||||
|
||||
The input-commit callback explicitly reports whether a direct response still
|
||||
needs to be created. The Injector consumes that fact instead of independently
|
||||
assuming every commit precedes response creation. A late completion belonging
|
||||
to an earlier turn must not release a genuinely pending newer turn.
|
||||
|
||||
## Continuous observation and notification FIFO
|
||||
|
||||
Repeated perception tasks remain running while their events are queued or
|
||||
announcing. Each event has its own delivery id, status, and acknowledgement
|
||||
timer. Acknowledging an earlier event retires only that event; it does not
|
||||
pause, reset, or complete its monitor. One-shot tasks retain their existing
|
||||
delivering-to-completed lifecycle.
|
||||
|
||||
Observation and evaluation continue independently of foreground playback.
|
||||
Event monitors still require a false observation before another true match;
|
||||
configured cooldown suppresses duplicate alerts. Narration retains the source
|
||||
prompt's novelty behavior. Cooldown runs from detection, not from playback.
|
||||
|
||||
Cancel, update, failure, and call teardown invalidate all affected queued
|
||||
deliveries. All records are removed before invoking callbacks; queued tail
|
||||
items are retracted before releasing a current item so synchronous FIFO
|
||||
advancement cannot submit a cancelled event.
|
||||
|
||||
Audio and vision evidence use their own configured windows for warm-up and
|
||||
Monitor reconnect replay. Task-list receipts retain remaining timer duration,
|
||||
reminder contents, monitoring condition/focus, response guidance, repeat state,
|
||||
and any queued-delivery count, while still hiding internal ids.
|
||||
|
||||
## Independent camera snapshots
|
||||
|
||||
`visualInput.cameraSnapshotResolution` defaults to `native` and also accepts a
|
||||
width/height pair. `QWEN_LIVE_CAMERA_SNAPSHOT_RESOLUTION` overrides it.
|
||||
`cameraResolution` controls preview and Live Feed; `snapshotResolution`
|
||||
controls Screen snapshots. Init writes defaults without another prompt.
|
||||
|
||||
For user Appshot requests, the Host obtains a still image from the selected
|
||||
camera track and fits it within the requested snapshot bounds. Native uses the
|
||||
device's available still-image size; when still-image capture is unavailable,
|
||||
a supported video-constraint capture may be used with explicit frame readiness
|
||||
and preview restoration. Capture failures must not masquerade as native 720p.
|
||||
|
||||
The high-resolution JPEG is stored as the camera handoff asset. Its bounded
|
||||
transport preview still follows the existing 1080p/190 KiB provider limit.
|
||||
The full asset crosses only trusted Host-local IPC and retains the existing
|
||||
8 MiB asset limit. Proactive private captures do not invoke high-resolution
|
||||
still capture or reconfigure the camera for each monitor frame.
|
||||
|
||||
Configuration flows through the standalone daemon, synchronized Host type
|
||||
copies, Host welcome/settings parsing, correlated capture request, preload
|
||||
camera capture, and main-process asset storage. Fields are optional on the v9
|
||||
wire format; local Host and daemon builds must be updated together.
|
||||
|
||||
## Files and verification
|
||||
|
||||
- Realtime session and orchestrator: user-turn continuation and commit gate.
|
||||
- Proactive scheduler, task manager, Monitor, receipts: independent event
|
||||
delivery, modality windows, and complete task information.
|
||||
- Config/init/daemon and Host coordinator/protocol/camera capture: snapshot
|
||||
resolution, full asset, and bounded transport preview.
|
||||
- README and the existing visual/Proactive design documents: current behavior.
|
||||
|
||||
The reproducible baseline and verification matrix live in
|
||||
`.qwen/e2e-tests/2026-09-05-qwen-live-audit-fixes.md`. Verification uses source
|
||||
protocol simulations and built package artifacts, followed by targeted unit
|
||||
tests, type checks, builds, and independent review. The global qwen command is
|
||||
not available and has no standalone Live Proactive surface; runtime simulations
|
||||
exercise the affected code directly without recording user media.
|
||||
@@ -0,0 +1,121 @@
|
||||
# Qwen Live memory
|
||||
|
||||
## Scope
|
||||
|
||||
Port the memory subsystem from qwen-omni-realtime-agent into standalone Qwen
|
||||
Live: library storage and management, dialogue recording and segmentation,
|
||||
working-memory edits, LTM/STM preload and consolidation, hybrid dialogue and
|
||||
visual-observation retrieval, and optional visual observation. Memory is enabled
|
||||
by default and stores its data beneath the Live data directory, normally
|
||||
`~/.qwen-live/memories`.
|
||||
|
||||
The orb provides Memory and Visual memory switches, library selection, New,
|
||||
and Rename. Library ids are stable and names never become filesystem paths.
|
||||
Browser session archives, hidden prompt inspectors, library deletion, Frontier,
|
||||
and publishing are outside this port.
|
||||
|
||||
## Runtime and storage
|
||||
|
||||
One daemon owns MemoryStore and shared model clients. Each Live call attaches
|
||||
one MemorySession to the selected library. The library retains the prototype's
|
||||
SQLite v1 schema and meta.json shape; no existing prototype data is imported
|
||||
automatically. Node's built-in SQLite/FTS5 requires Node >=22.13. Jieba search
|
||||
tokenization is used consistently for index and query text.
|
||||
|
||||
The recorder stores final dialogue text, segments and interrupted-answer
|
||||
markers. Synthetic Proactive, backend speech and repair turns do not become
|
||||
user dialogue. Raw audio and camera frames are not persisted. LTM and STM are
|
||||
preloaded once per attachment, while retrieved context and the ordered WM list
|
||||
change through tools. All ids, paths, database statements and model patches are
|
||||
validated; directories/files use private permissions.
|
||||
|
||||
Dialogue retrieval retains FTS OR/AND ranking, vector fallback and RRF fusion,
|
||||
soft time-range boosting, unsegmented tail search, and both raw/rendered
|
||||
budgets. Env retrieval has a separate index, keyword-first ranking and temporal
|
||||
spacing. Embeddings use text-embedding-v4 by default, cache queries, backfill
|
||||
missing rows asynchronously, and degrade to keyword search on unavailable or
|
||||
slow providers. Model identity is included when selecting stored vectors.
|
||||
|
||||
Closing an attachment flushes dialogue and schedules WM consolidation. Model
|
||||
work is serialized per library. Each WM snapshot version is applied at most
|
||||
once, including repeated OFF/ON within a call. Updater patches follow the
|
||||
source LTM/STM operation ordering and never write visual observations. Shutdown
|
||||
waits for the configured bounded grace period; abandoned work is reported and
|
||||
does not silently claim successful consolidation.
|
||||
|
||||
## Model configuration
|
||||
|
||||
Port only effective memory settings, in camelCase groups: retrieve, preload,
|
||||
updater, observer, wm and segment. The master default is true. Observer capture
|
||||
defaults to false and its periodic interval defaults to 60 seconds.
|
||||
|
||||
Updater and Observer use ordinary compatible Chat Completions, not the Live
|
||||
Realtime model or voice. The public default is qwen3.7-plus, using the
|
||||
current DashScope API key and matching compatible endpoint; optional baseUrl
|
||||
and apiKeyEnv overrides preserve alternate deployments. The prototype's
|
||||
internal gateway and private credential name are not hardcoded. The user
|
||||
confirmed this choice. Init asks whether to enable Memory and, when enabled,
|
||||
asks for this model name. The orb exposes the same consolidation-model setting.
|
||||
|
||||
## Memory tools and prompt publication
|
||||
|
||||
Preserve the source omnibio and omniretrieve schemas, descriptions and memory
|
||||
guidance. Both are local receipt tools operating only with genuine-user-turn
|
||||
authority. Memory sections are untrusted data and cannot authorize tools.
|
||||
|
||||
The four sections are always ordered user_profile, recent, retrieved,
|
||||
personalized_user_memories. A successful retrieval replaces retrieved as a
|
||||
whole; a valid empty result clears it, while invalid/failed requests preserve
|
||||
it. Tool receipts carry status/counts, and the text itself is sent through
|
||||
instructions. A continuation response.create includes the latest instructions;
|
||||
the durable session.update is deferred until response idle. Socket event
|
||||
handlers never await session.updated. Disabling memory immediately revokes
|
||||
memory tool handling and stages removal of all memory instructions.
|
||||
|
||||
## Orb settings and isolation
|
||||
|
||||
The daemon is authoritative for settings and publishes memory state over the
|
||||
existing authenticated Host WebSocket. Actions carry correlation ids and
|
||||
explicit results. Accepted switch/selection changes persist by atomically
|
||||
merging only memory preferences into the Live config file.
|
||||
|
||||
Source-compatible behavior: Memory and Visual memory switches apply
|
||||
to the active call; Rename changes metadata at any time. Select/New are locked
|
||||
while a call is starting/active/stopping and become available after it ends,
|
||||
preventing old-library context from leaking into a newly selected library.
|
||||
The user confirmed this boundary. New creates and selects a
|
||||
library; turning Memory off preserves its selection and stored contents.
|
||||
|
||||
The Host uses an English overlay panel with inline name editing, Save/Cancel,
|
||||
and a bounded scrolling library list. Input drafts, focus and selection survive
|
||||
normal audio/status updates. No browser prompt dialog or extra management HTTP
|
||||
server is needed. Built-in qwen serve omits the optional memory capability.
|
||||
|
||||
## Visual observations
|
||||
|
||||
When enabled, observation follows the orb's current Screen/Camera source.
|
||||
Live Feed reuses the latest frame. On Demand privately captures a bounded frame
|
||||
at the observation cadence without invoking foreground Appshot or creating an
|
||||
asset. The first available frame is observed promptly; stale frames are
|
||||
discarded. Observer records only a cleaned description in stm_env and its
|
||||
indexes. A source marker accompanies the original Observer prompt so screen
|
||||
content is not presented as the user's physical room.
|
||||
|
||||
Capture/model completions are fenced by attachment, library and source
|
||||
generation. Switching a source or turning memory/visual observation off
|
||||
invalidates outstanding observation work. Historical visual retrieval remains
|
||||
available while live visual observation is off.
|
||||
|
||||
## Verification
|
||||
|
||||
Port the prototype's key behavioral tests: SQLite interoperability and private
|
||||
files, library isolation, recorder sequencing, WM operation order, LTM/STM
|
||||
patch validation and expiry, ranking and budgets, embedding fallback, visual
|
||||
deduplication, consolidation idempotence, and late callback rejection.
|
||||
Integration tests cover live transcript ownership, safe prompt publication,
|
||||
toggle/selection lifecycle and all orb actions. Model/network/camera behavior
|
||||
uses local controlled fixtures rather than recording user media.
|
||||
|
||||
The detailed baseline and results are kept in
|
||||
`.qwen/e2e-tests/2026-09-05-qwen-live-memory.md`. Existing Proactive, Appshot and
|
||||
audio regression suites remain required.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Live orb startup, sources and compact edges
|
||||
|
||||
## Request and current gap
|
||||
|
||||
The orb currently shares a 384 by 480 native collision rectangle with setup,
|
||||
uses a subtle 1.025 speaking scale, starts idle, and places camera preview far
|
||||
above the orb. Settings order and its combined mode explanation are unclear.
|
||||
|
||||
The user confirmed three peer settings groups: Audio Source (microphone),
|
||||
Video Source (Screen/Camera), and Capture Mode (On Demand/Live Feed). They also
|
||||
confirmed that hiding camera preview only changes the local small window,
|
||||
not the selected source or transmitted frames.
|
||||
|
||||
## Changes
|
||||
|
||||
- Increase speaking motion while reserving its maximum animated extent and
|
||||
keeping reduced-motion support. Do not restore a drop shadow or rebuild
|
||||
orb/video nodes on state updates.
|
||||
- Automatically start one call when a newly launched Host first becomes fully
|
||||
ready: authenticated connection, renderer/native services, source-specific
|
||||
permissions, audio checks and provider availability. Consume the startup
|
||||
intention before dispatch. An existing call or explicit start/stop/new/quit
|
||||
consumes it; failure, renderer reload and reconnect must not repeatedly
|
||||
start calls after the user stopped.
|
||||
- Present Audio Source, Video Source and Capture Mode as equal settings
|
||||
groups, in that order. Use a recognizable sliders Settings icon. Explain
|
||||
only the selected mode, using authoritative state rather than optimistic
|
||||
selection before acknowledgement.
|
||||
- Make preview default-visible whenever Camera is selected. A small floating
|
||||
show/hide button remains reachable even when the preview is hidden. Hiding
|
||||
preserves the video node and camera input; selecting Screen or quitting
|
||||
retains the existing capture shutdown behavior.
|
||||
- Place preview closer to the orb, with caption-aware spacing but a fixed orb
|
||||
anchor. Keep controls, captions and the animated orb within the declared
|
||||
compact visible envelope.
|
||||
- Use mode-specific native collision bounds rather than the entire transparent
|
||||
window: setup/settings use the full panel, while orb uses its compact content
|
||||
envelope (including room for controls and maximum animation). Preview has its
|
||||
own envelope. Persist the user's desired resting position, not temporary
|
||||
repositioning required to show a full Settings panel. Close Settings restores
|
||||
the resting position. Preserve old saved x/y files and clamp against available
|
||||
displays on restore/topology change. Do not reposition on every caption.
|
||||
- macOS clamps the actual native top edge to the menu bar. After placement,
|
||||
read back native bounds and compensate `logical - actual` on the voice
|
||||
surface through a Host-local offset event. Drag deltas begin at the displayed
|
||||
logical origin. Setup/Settings are not transformed; full-frame placement
|
||||
resets the offset and closing restores it. Persist only logical desired
|
||||
coordinates and ignore older state offsets after dedicated events arrive.
|
||||
|
||||
## Ownership and scope
|
||||
|
||||
Changes are confined to live-host renderer, native window policy, a one-shot
|
||||
startup helper, shared Host-local UI types/geometry, tests and READMEs. No wire
|
||||
protocol or model/Memory/Proactive behavior changes. Configuration, media and
|
||||
model traffic used by tests are synthetic; startup auto-call will not be tested
|
||||
against the user's real daemon. Existing user work and configuration stay intact.
|
||||
|
||||
## Verification
|
||||
|
||||
Reproduce the old full-frame clamp and old mode explanations first. Test the
|
||||
one-shot startup state machine, manual-stop suppression, edge coordinates,
|
||||
temporary panel clamping/restoration, preview hide without capture API calls,
|
||||
and stable node identity. Measure actual renderer geometry in a browser with
|
||||
the native-size viewport; run native lifecycle tests and a safe isolated window
|
||||
smoke where possible. Finish with build/typecheck and two clean reviews.
|
||||
@@ -0,0 +1,37 @@
|
||||
# Live status placement and cumulative audit
|
||||
|
||||
## Requested adjustment
|
||||
|
||||
Move the status pill below the orb, retain hover-only controls above it, use
|
||||
a translucent status background, replace the Settings sliders icon with a
|
||||
round cog, and make the Settings scrollbar visible on opening. Preserve
|
||||
automatic startup, compact edge clamping, macOS top compensation, preview-only
|
||||
visibility controls, and all existing Memory/Proactive behavior.
|
||||
|
||||
## Layout
|
||||
|
||||
Keep the orb anchor and its maximum animation envelope stable. Move status
|
||||
and background-permission action below the orb with an explicit gap. Expand
|
||||
the shared compact collision envelope to include the status, so neither
|
||||
progress/error text nor the permission action is clipped at screen edges.
|
||||
The caption and camera preview remain above the floating controls.
|
||||
|
||||
Use a symmetric toothed cog with a circular hub, inside a round Settings
|
||||
button. Status translucency must remain readable on both light and dark
|
||||
desktop backgrounds. Settings owns a persistent scrollbar track/thumb and
|
||||
stable gutter, without changing macOS system scrollbar preferences.
|
||||
|
||||
## Verification and audit
|
||||
|
||||
Add geometry/style regression checks and inspect the actual browser/native
|
||||
renderer at the 384 by 480 viewport. Then audit all accumulated changes from
|
||||
baseline `f7b0b88b2fc0124d55d04b7797c8f8f342fadd63`, including untracked files,
|
||||
following AGENTS.md and CONTRIBUTING.md. Check parameter producers/consumers,
|
||||
daemon-route ownership, authenticated protocol boundaries, capture/audio state,
|
||||
Memory persistence and model calls, Proactive FIFO, startup/quit, CLI parity,
|
||||
documentation and dependencies. Reproduce confirmed problems before fixing.
|
||||
|
||||
Run scoped suites from package directories, root build/typecheck/bundle,
|
||||
changed-file formatting/lint and boundary checks. Do not run full preflight's
|
||||
clean/reinstall on this mixed local dependency tree. Do not change user config,
|
||||
Memory data or live device permissions. No GitHub writes or release work.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Qwen Live stable orb and settings
|
||||
|
||||
## Problem and evidence
|
||||
|
||||
The September 5 investigation reproduced full-tree redraws on unchanged Host
|
||||
state, focus/animation resets, preview-slot replacement, dropped clicks when
|
||||
closing menus, menus clipped by 11px, and a 52px jump between listening and
|
||||
thinking. Native logic also repositions on every active update and retains
|
||||
stale mouse-interactivity state across overlay replacement.
|
||||
|
||||
The user approved a stable orb with click-open settings, remembered dragging,
|
||||
hover-only controls, Command+E, a persistent gray stopped state, explicit quit,
|
||||
and removal of shadow and clipped animation.
|
||||
|
||||
## Behavior
|
||||
|
||||
- First launch places the overlay at the bottom-right of a display. Setup and
|
||||
orb share one saved window position, retained across transitions/restarts.
|
||||
Restore clamps to available work areas when displays change.
|
||||
- The setup view focuses on connection and required authorization. Source is
|
||||
selectable there because Camera must not require Screen permissions. Mode,
|
||||
Memory and input device preferences live in the normal Settings panel.
|
||||
- The orb, toolbar, status, caption and preview slot are created once. Updates
|
||||
only change text, values, attributes and visibility. No transcript update
|
||||
may recreate the orb or reattach an unchanged camera preview.
|
||||
- A fixed orb anchor does not move when status/captions appear. Animation has
|
||||
bounded scale and reserved space. No orb drop shadow is rendered.
|
||||
- Hovering the orb reveals Microphone, Voice output, Start/End call, Settings
|
||||
and Quit controls. Leaving their interaction area waits 1 second before a
|
||||
fade; reentry cancels hiding. Keyboard focus and an open settings panel keep
|
||||
controls available. Call stop never hides the window and makes the orb gray.
|
||||
- Setup header and orb can be dragged. Dragging must not accidentally start a
|
||||
call. Window position is updated only for explicit dragging, initial restore,
|
||||
or display topology changes, not every live status update.
|
||||
- Settings contains Source, Mode, input device selection, and the existing
|
||||
Memory controls as an embedded stable section. It closes by Escape or
|
||||
outside interaction, without destroying the clicked control. Async actions
|
||||
show pending/failure feedback and retain editor drafts/focus.
|
||||
- Command+E uses the existing global shortcut path for starting/ending calls.
|
||||
Preserve explicit custom configuration, while checking the user's current
|
||||
shortcut and documenting the default.
|
||||
- End call stops interaction, leaving the app and gray orb available. Quit
|
||||
shuts down the connected standalone Live daemon gracefully and closes Host;
|
||||
owned session/adapter/media resources are released. It must not terminate
|
||||
unrelated Qwen Code serve processes or sessions. Legacy WebShell connections
|
||||
may end only their Live call and close Host, never shut down the shared server.
|
||||
- An idle camera preview remains local only, as before. No user camera/mic or
|
||||
model API is used for automated verification.
|
||||
|
||||
## Implementation boundaries
|
||||
|
||||
Renderer changes stay in live-host, retaining Memory action validation and
|
||||
call-time locking. Native Host owns bounds persistence, pointer/drag routing,
|
||||
visibility and trusted IPC. Quit is an authenticated, instance-scoped operation
|
||||
handled by the standalone daemon's existing cleanup path, not an OS PID kill.
|
||||
The renderer never receives credentials or filesystem paths for daemon access.
|
||||
|
||||
Primary files: live-host renderer/main.ts, style.css, memory-panel.ts and new
|
||||
settings component; preload/index.ts; shared/host-api.ts; main/index.ts and
|
||||
overlay position helpers; Host daemon connection; standalone daemon/host
|
||||
coordinator and matched wire types; package tests and READMEs.
|
||||
|
||||
Reuse previous bug reproductions; add position persistence, graceful quit,
|
||||
hover timing, stable DOM, settings dismissal and layout boundary regressions.
|
||||
No issue/PR creation, release, or unrelated refactor is included.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Live bilingual text, input animation and movable settings
|
||||
|
||||
## User-visible scope
|
||||
|
||||
Increase normal microphone-level orb animation without changing captured audio.
|
||||
Add Simplified Chinese and English to the last Settings group. The first init
|
||||
question selects these languages with left/right keys, before the setup banner
|
||||
or overwrite prompt; all subsequent fixed wizard text follows that choice.
|
||||
Use one editable, browser-safe typed catalogue with English/Chinese pairs for
|
||||
Live Host, setup, native menus, status/error messages and CLI user guidance.
|
||||
User data, device/model names, transcripts, provider error detail and model
|
||||
prompts remain unchanged. Existing configs default to English; a fresh init
|
||||
offers Chinese first. The setting is independent of Memory and may change
|
||||
during a call without reconnecting media or changing model settings.
|
||||
|
||||
## Ownership and integration
|
||||
|
||||
The canonical dictionary is `packages/qwen-live/src/i18n/messages.ts`, exported
|
||||
through the package's public `./i18n` entry. Host build/type aliases compile this
|
||||
same pure module into its standalone bundle, without Electron dependencies in
|
||||
root workspaces or a new runtime package installation. No duplicate dictionary
|
||||
or cross-package relative production import is introduced.
|
||||
|
||||
Top-level Live config stores `language: 'en' | 'zh-CN'`. The authenticated Host
|
||||
protocol advertises an optional language setting, distinct from Memory, and
|
||||
uses correlated request/result messages. Writes atomically merge with config,
|
||||
preserving credentials and unrelated preferences, and only confirmed state is
|
||||
applied. Legacy daemons do not acquire permission to write standalone config;
|
||||
Host can retain its local language preference when no supporting daemon is
|
||||
present, while a connected standalone daemon's setting is authoritative.
|
||||
|
||||
Display messages use stable IDs and parameter substitution. Fixed native and
|
||||
daemon failures visible in UI are included; unowned external detail is left
|
||||
unchanged. Switching language updates text/ARIA/title in existing DOM nodes,
|
||||
without rebuilding controls, losing drafts, changing device IDs or stopping
|
||||
camera preview. Tests enumerate catalogue keys and placeholders in both locales.
|
||||
|
||||
## Settings placement and dragging
|
||||
|
||||
Reuse the current authenticated renderer drag channel with a title-bar drag
|
||||
handle, excluding buttons and inputs. Retain shared desired position unless
|
||||
the user chooses otherwise; temporary opening clamps must not overwrite it.
|
||||
Constrain CSS tracks and nested content so long names or translated strings
|
||||
cannot widen the panel. Verify both native bounds and actual painted DOM bounds.
|
||||
Ensure native full-panel placement completes before displaying Settings, with
|
||||
cancel/disconnect guards, if the edge-opening timing reproduction confirms it.
|
||||
Keep Escape, outside click and focus return. No real device/provider action is
|
||||
needed for validation.
|
||||
|
||||
## Animation
|
||||
|
||||
Use a bounded nonlinear visual gain for normal small peaks, fast attack and
|
||||
slower release. Keep silence at base scale and reset on mute/stop. The maximal
|
||||
visual envelope, including outline, remains inside the existing motion area;
|
||||
do not increase input gain, alter PCM or change daemon call-state semantics.
|
||||
|
||||
## Verification
|
||||
|
||||
Independent pre-fix probes already show no Settings drag events and only
|
||||
0.67/1.34px diameter increase at microphone peaks 0.05/0.1. Init and Settings
|
||||
have no language choice. Use source/DOM/native fixtures for UI because the
|
||||
global CLI cannot expose this standalone native overlay. Test real terminal
|
||||
left/right selection with isolated config and no installer/provider calls;
|
||||
test runtime persistence, invalid/stale requests, disconnect, and reload.
|
||||
Run scoped package tests, root build/typecheck/bundle, Host build, applicable
|
||||
format/lint checks and two final review passes. No clean/reinstall, release,
|
||||
user config modification or model/device use.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Subagents pinning, compact presentation, theme and diagnostics
|
||||
|
||||
## User request
|
||||
|
||||
Keep the Subagents surface present while its list or a task detail is open,
|
||||
until explicit Close/Escape. When neither is open, hiding must depend on real
|
||||
pointer/keyboard interaction and clear stale hover state. Preserve the existing
|
||||
drag/edge invariants. Discuss a few smaller summary layouts before choosing a
|
||||
permanent replacement. The user chose compact option C: a 132×62 bot/title
|
||||
summary with dot/running and check/completed counts. A gentle 1.4s opacity pulse
|
||||
marks a running task only while connected; reduced motion is static. Waiting
|
||||
uses a conditional amber marker, with exact counts in localized accessible text
|
||||
and tooltips. Large summary counts truncate at 999+ without losing exact totals.
|
||||
Add a final Theme setting after Language with System,
|
||||
Light and Dark, default System, applying to all Host windows. Improve debug
|
||||
diagnostics for Proactive and separate permission waiting from failure status.
|
||||
|
||||
## Interaction ownership
|
||||
|
||||
Native SubagentsWindows owns one frameless window for summary/list/detail.
|
||||
An explicit open is pinned; blur, Settings, disconnect and orb dragging do not
|
||||
close it. Task selection changes that same panel to detail; Back returns to
|
||||
the list even when disconnected. Close/Escape hides the panel without affecting
|
||||
tasks. A new daemon instance invalidates old selection. Expanded headers are
|
||||
draggable; list and detail share 330×430 bounds, so navigation preserves their
|
||||
position without resizing into the adjacent orb. Display changes clamp the
|
||||
panel to the current screen. No macOS title bar, second detail window or new detail-position
|
||||
preference is used. Content updates never change window bounds.
|
||||
|
||||
Hover and keyboard focus are separate from explicit open state. Missing mouse
|
||||
leave or stale focus must not make a collapsed summary stay forever. Use current
|
||||
native cursor containment as a bounded backstop while a transient surface is
|
||||
visible; keyboard-held visibility ends on window blur or pointer interaction.
|
||||
The orb and side panel own separate keyboard holds, so periodic state updates
|
||||
cannot release another window's focused control. Real pointer entry reasserts
|
||||
hover intent even after native cursor checks closed a stale renderer state.
|
||||
Do not run polling when hidden or when a pinned view already determines state.
|
||||
|
||||
## Theme
|
||||
|
||||
Theme is Host-local display configuration (`system | light | dark`) persisted
|
||||
privately, independently of daemon language and memory preferences. The main
|
||||
process owns Electron nativeTheme and broadcasts both preferred and resolved
|
||||
appearance. Renderer only applies the resolved appearance; switching theme
|
||||
does not replace nodes, restart media or update model settings. Every fixed
|
||||
display label remains in the paired i18n catalogue. Brand orb gradients remain
|
||||
stable; backgrounds, text, borders, statuses and scrollbars receive light/dark
|
||||
tokens with readable contrast.
|
||||
|
||||
## Proactive and diagnostics
|
||||
|
||||
The recent session log confirms a Proactive injection and response creation,
|
||||
followed by a cancelled response 15 ms before VAD speech-start, then a fatal
|
||||
unattributed transcript. Independent actual-source replay reproduces the
|
||||
cancel-before-VAD failure; a bounded 250 ms grace on an active, unfinished
|
||||
Proactive delivery lets the subsequent VAD requeue it at the FIFO head. Explicit
|
||||
cancellation is not retried; expiry keeps the existing failure. Missing ASR
|
||||
commit is a separate failure, not justified by that cancellation race. Do not
|
||||
weaken tool authority or fabricate audio commits. Log safe IDs/phase, evidence counts/durations,
|
||||
evaluation accept/reject reasons, trigger/admission, foreground response and
|
||||
playback barriers, retries, terminal errors and subagent state transitions in
|
||||
`--debug` stderr. No raw microphone/video/transcript/prompt/key payloads.
|
||||
|
||||
`Needs you` is reserved for tasks genuinely waiting for user input/approval.
|
||||
Failed/interrupted tasks show explicit Error/Interrupted, not a request for
|
||||
permission. Historical failed counters remain separate from current waiting.
|
||||
|
||||
## Verification
|
||||
|
||||
Independent actual-class/DOM probes reproduce pin loss and sticky hover before
|
||||
fixes. Test open list/detail/close permutations, cursor leave without IPC,
|
||||
keyboard focus, blur, drag, disconnect, delayed load and teardown. Theme tests
|
||||
cover System changes, persistence failures, all windows, media invariance and
|
||||
both appearances. Debug tests assert observability and secret/content omission.
|
||||
Builds and tests are serialized because the user's machine recently restarted.
|
||||
Use no real devices/provider requests or user preference writes for tests.
|
||||
@@ -0,0 +1,91 @@
|
||||
# Subagents: side summary, compact list and inline detail
|
||||
|
||||
Updated by the user-approved refinement in
|
||||
`2026-09-07-live-subagents-refinement.md`: one frameless panel and compact
|
||||
icon/count summary, replacing the original independent detail window.
|
||||
|
||||
## Confirmed behavior
|
||||
|
||||
Implement proposal C. The hover entry explicitly reads `Subagents` /
|
||||
`子智能体`, not just anonymous counts. It summarizes working, completed and
|
||||
needs-attention tasks. A click opens a compact task list; selecting a task
|
||||
opens detail in that same panel with status, original request, recent
|
||||
activity, public intermediate output and result. All fixed text uses the
|
||||
existing paired language catalogue. No cancel, permission approval or task
|
||||
creation controls are added in this read-only feature.
|
||||
|
||||
Counts represent logical tasks, not model connections, monitor evaluations,
|
||||
backend sessions or steering instructions joined to an existing task. Only an
|
||||
authoritative successful terminal event counts as completed. Failed, cancelled
|
||||
and interrupted entries stay distinct. A monitor's condition match, queued
|
||||
notification and delivered notification are independent of its task lifetime.
|
||||
|
||||
History covers the current Live daemon run with bounded retained detail. Voice
|
||||
End call stops Proactive as before but retains its final history. Harness jobs
|
||||
and observation continue until their terminal event or daemon shutdown, without
|
||||
opening a model/audio connection. Requests requiring permission while no voice
|
||||
call exists are recorded without granting new permissions.
|
||||
|
||||
## Runtime and protocol
|
||||
|
||||
A daemon-lifetime ledger in LiveSession receives normalized backend events and
|
||||
Proactive task observations before the speech Injector's throttling. Backend
|
||||
event pumps have a single daemon-level owner; voice is an optional consumer.
|
||||
Public agent text, plan and tool updates become a new observation-only activity
|
||||
event, never a model instruction. Thought chunks, arbitrary raw objects,
|
||||
credentials and binary/image payloads are not forwarded.
|
||||
|
||||
Shared pure contract is the public `@qwen-code/qwen-live/subagents` module,
|
||||
compiled into Host through the same style of aliases as i18n. It defines a
|
||||
strict bounded snapshot validator. Optional `subagentsV1` welcome/state field
|
||||
and an independent `host.subagents` update avoid marking legacy daemons as
|
||||
supporting this feature and avoid republishing audio/capture state per chunk.
|
||||
|
||||
Snapshot: revision, counts, retained tasks. A task has stable id, kind, title,
|
||||
status, created/updated timestamps, backend/source metadata, latest activity,
|
||||
bounded public output, recent typed events, and Proactive notification counts.
|
||||
Retain at most 32 task details and a 240KiB JSON budget, trim terminal history
|
||||
before active detail and explicitly report omitted records/truncated output.
|
||||
Counts remain independent of visible retention. One bounded update interval
|
||||
coalesces text chunks; every new revision remains authoritative.
|
||||
|
||||
## Native geometry and interaction
|
||||
|
||||
Keep the orb's 384×480 canvas and existing saved logical position unchanged.
|
||||
Use one separate frameless BrowserWindow for summary, list and detail.
|
||||
It has a dedicated inert preload, no audio/camera acquisition or general
|
||||
Live actions. Every IPC verifies sender identity and validates task IDs.
|
||||
|
||||
Summary is 132×62 with a bot icon, explicit Subagents title and dot/check counts.
|
||||
The running dot pulses only with active tasks and respects reduced motion.
|
||||
List and detail both use 330×430 with internal scrolling, so navigation does
|
||||
not enlarge the panel across the orb or shift its position.
|
||||
Choose left/right once on opening from the orb's actual visible rectangle and
|
||||
current display work area; prefer left near the usual bottom-right position.
|
||||
Clamp the full window, fall back above/below when necessary, and never move the
|
||||
orb to fit a task surface. Content updates do not resize or reposition windows.
|
||||
During orb drag, hide only the collapsed summary; a later hover reanchors it.
|
||||
Crossing from orb to summary allows about 1s of dismissal grace. Clicking pins
|
||||
the panel until Escape/Close. Settings, blur and disconnection do not close
|
||||
expanded content. List/detail headers are draggable; Back preserves location
|
||||
and clamps the changed size to the display. New output does not move the panel.
|
||||
Closing the panel does not stop tasks or quit Host.
|
||||
|
||||
Language switches and task revisions update stable DOM by ID, preserve selected
|
||||
task/scroll position, and do not follow new output unless already at the bottom.
|
||||
Repeated open focuses existing detail; late results cannot reopen a closed
|
||||
window. Quit closes all surfaces and aborts observers. Display removal clamps
|
||||
windows to a surviving work area without altering unrelated orb preference.
|
||||
|
||||
## Verification
|
||||
|
||||
Independent baseline recorded in `.qwen/e2e-tests/subagents-baseline-2026-09-07.md`:
|
||||
stopping voice aborts backend subscriptions; a later completion stays queued
|
||||
until a new call; Proactive dispose clears tasks; Host has no subagent surface.
|
||||
Use fake backends/realtime and real loopback Host protocol, no user config or
|
||||
devices. Verify hangup continuity, one observer, queued/joined tasks, exact
|
||||
terminal counts, public activity filtering/bounds, permissions and cleanup.
|
||||
Verify geometry with actual primary/right-hand display metadata, opposite
|
||||
screen edges, drag, close/reopen, Settings, and status updates. Native smoke
|
||||
uses inert services. Run package suites, build/typecheck/bundle, boundary checks
|
||||
and two clean review passes. No release or dependency reinstall.
|
||||
@@ -0,0 +1,80 @@
|
||||
# Live status, docking and audio device compatibility
|
||||
|
||||
## Requested behavior and baseline
|
||||
|
||||
Cancelled Proactive monitors belong to the Completed aggregate, while details
|
||||
retain their Cancelled outcome. Cancelled harness jobs and timers do not become
|
||||
successful. Subagents must avoid the whole orb dock, including its wider lower
|
||||
Listening bar. Input/output mute states must be visible in that bar. New users
|
||||
see setup and orb at bottom-right; saved user positions must not be replaced.
|
||||
|
||||
The independent actual-source probe reproduces cancelled monitors as
|
||||
completed=0/cancelled=1, unchanged Listening text in all mute combinations,
|
||||
34×32 pixels of Subagents/status overlap, and a fresh setup at margins 0/14
|
||||
instead of 20/20. Host output already uses a default-rate AudioContext with
|
||||
24 kHz AudioBuffers; there is no forced 24 kHz hardware clock to remove.
|
||||
|
||||
The user confirmed Bluetooth headphones and disruption immediately at startup,
|
||||
before model speech. Microphone activation can switch Bluetooth to hands-free
|
||||
mode. This cannot be resolved solely by resampling response PCM. Tests must
|
||||
not claim to reproduce physical Bluetooth behavior without device validation.
|
||||
|
||||
## Design
|
||||
|
||||
### Status and task totals
|
||||
|
||||
Keep detailed terminal states truthful. A small internal monitor identity set
|
||||
survives detail eviction and lets cancelled monitors contribute once to
|
||||
Completed rather than Cancelled. It is cleaned on archival. No protocol field
|
||||
or model/tool semantic change is needed. Mute labels use the central paired
|
||||
i18n catalogue and a compact secondary line inside the existing 248×32 bar;
|
||||
the primary call/error/permission state remains available.
|
||||
|
||||
### Dock and first-run geometry
|
||||
|
||||
Use the complete visible orb dock envelope, not only its motion rectangle,
|
||||
when placing Subagents. Keep pointer-hit/hover ownership separate if the
|
||||
placement envelope is larger than the interactive area. Prefer an inward side
|
||||
with room for the expanded 330×430 panel, re-evaluate a side if expansion no
|
||||
longer fits, and clamp the panel to the screen without covering the dock when
|
||||
a nonoverlapping placement is possible. Pinned list/detail navigation preserves
|
||||
the same bounds; task-state updates do not move a user-dragged panel.
|
||||
|
||||
First-run default is bottom-right on the startup display, with 20px margin for
|
||||
the currently shown layout. Saved coordinates and manual dragging override the
|
||||
default. Settings/preview changes should preserve the chosen orb position and
|
||||
should not persist a temporary screen clamp as a new user preference.
|
||||
|
||||
### Audio
|
||||
|
||||
Official reference inspected 2026-09-08:
|
||||
https://help.aliyun.com/zh/model-studio/realtime and its session.update link
|
||||
https://help.aliyun.com/zh/model-studio/client-events#26a8302028sjm .
|
||||
|
||||
Qwen3.5 Omni Plus/Flash Realtime support `audio.input.format` and
|
||||
`audio.output.format` with PCM/WAV and 8/16/24/48 kHz; defaults are 16 kHz input,
|
||||
24 kHz output. Set PCM input 16000 and output 24000 explicitly for documented
|
||||
3.5 Plus/Flash model names. Older model names retain legacy PCM format fields
|
||||
and their fixed 24 kHz output. Do not add unsupported top-level sample_rate
|
||||
fields, change VAD/tool flow, or send a second monitor voice/output config.
|
||||
|
||||
Keep device-native AudioContext output and 24 kHz source buffers. Web Audio
|
||||
performs source-to-context resampling (including upsampling to 44.1/48/96 kHz)
|
||||
without forcing a system device rate or introducing a second media clock.
|
||||
Log source/context rates safely. Muting input must stop tracks and release the
|
||||
capture graph; unmuting rebuilds it without ending the call. Stale callbacks
|
||||
must not upload samples or corrupt levels, and device changes while muted must
|
||||
not reacquire a microphone.
|
||||
|
||||
The user is being asked whether an unselected microphone should prefer Mac's
|
||||
built-in input while retaining system output and explicit saved input choices.
|
||||
No system-wide audio setting is changed and no real media is opened for tests.
|
||||
|
||||
## Verification and scope
|
||||
|
||||
Follow the independent baseline/test plan in
|
||||
`.qwen/e2e-tests/2026-09-08-live-followups.md`. Use offline actual-source tests,
|
||||
renderer fixtures, safe WebSocket mocks and device fakes. Run focused tests from
|
||||
package directories, builds/typechecks serially, then two clean review passes.
|
||||
Preserve all existing dirty changes. No reinstall, publishing, broad refactor,
|
||||
real provider requests, or writes to user preferences.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Open the active Live configuration from Settings
|
||||
|
||||
## Problem and scope
|
||||
|
||||
The Host Settings panel edits selected preferences but cannot open the complete
|
||||
standalone Live configuration. The daemon loads `config.json` under its own
|
||||
resolved data directory; a separately launched Host cannot infer that directory
|
||||
from its environment. This change adds one native-editor action, without a new
|
||||
editor preference, config editor, live reload, or wire-protocol version.
|
||||
|
||||
## Design
|
||||
|
||||
The standalone daemon adds its absolute `configPath` to the existing private
|
||||
discovery record. The Host validates this optional field (absolute path,
|
||||
`config.json` basename, bounded length, no NUL) and includes it in the discovery
|
||||
identity. The connection exposes the path only after the matching nonce welcome
|
||||
and while the socket is ready. Older daemons and built-in `qwen serve` omit the
|
||||
field and do not acquire standalone configuration authority.
|
||||
|
||||
The renderer receives only `canOpenConfig`, not a filesystem path or contents.
|
||||
A no-argument `openConfig()` preload action reaches a trusted-sender IPC handler.
|
||||
The main process rejects inactive renderers, unavailable connections and Quit,
|
||||
checks the advertised target is a regular non-symlink file, then uses Electron
|
||||
`shell.openPath`. The operating system's JSON-file association selects the IDE
|
||||
or text editor. No shell command is composed and renderer arguments cannot
|
||||
select a file. Missing, inaccessible or unsafe targets and native-open failures
|
||||
produce localized errors without exposing filesystem errors or file contents.
|
||||
No config is created or overwritten.
|
||||
|
||||
Settings gains a compact, right-aligned `Open config.json ↗` action at the top
|
||||
of its scrollable content, separate from the draggable header. An adjacent
|
||||
status/hint explains the default editor and restart requirement, reports errors,
|
||||
and identifies unsupported connections. Pending opens are deduplicated; the
|
||||
existing Settings geometry, language/theme order, focus trap and drag behavior
|
||||
remain unchanged. All English and Chinese display text stays in the shared Live
|
||||
message catalog. Opening a file never stops the call or changes preferences.
|
||||
|
||||
## Affected components
|
||||
|
||||
- Standalone daemon/discovery: publish the authoritative configuration path.
|
||||
- Host discovery/connection: validate and scope the optional capability.
|
||||
- Host main/preload/public API: privileged open action and boolean availability.
|
||||
- Settings, shared message catalog and Host/Live READMEs: discoverable bilingual
|
||||
action, status and usage documentation.
|
||||
- Focused discovery, connection, native IPC, renderer and localization tests.
|
||||
|
||||
## Verification and open questions
|
||||
|
||||
Use isolated discovery records, a local WebSocket fixture, the native IPC test
|
||||
harness and a built-renderer browser fixture. Do not open the user's real config
|
||||
in tests, read its secrets or start real media/provider sessions. Verify legacy
|
||||
connections, custom directories, nonce/renderer rejection, missing/unsafe files,
|
||||
open failures, bilingual pending/error states and unchanged drag bounds.
|
||||
Native editor choice is controlled by the user's OS association; actual editor
|
||||
launch is not required for the deterministic regression suite. No blocking
|
||||
design questions remain.
|
||||
@@ -0,0 +1,58 @@
|
||||
# Qwen Live extension on the merged M5 baseline
|
||||
|
||||
## Scope
|
||||
|
||||
Prepare the local visual input, Proactive, Memory, language/theme, subagent
|
||||
status and desktop interaction work as one follow-up change to merged M5.
|
||||
This is a local migration and review; publishing, a remote branch and a GitHub
|
||||
PR are explicitly outside this preparation step.
|
||||
|
||||
The initial checkout was PR #10769 at `f7b0b88b2f`. M5 subsequently merged as
|
||||
`829385a14e`. The migration targets the fetched main snapshot `078b924989`.
|
||||
The original local work is retained in the backup branch
|
||||
`backup/qwen-live-enhancements-20260908` at `ea4e78064e`; only that local delta
|
||||
is transplanted onto `feat/qwen-live-enhancements`, not the old PR history.
|
||||
|
||||
## Conflict decisions and invariants
|
||||
|
||||
Three files overlap the upstream playback fixes. Both IPC receipt handlers
|
||||
retain the upstream trusted-renderer guard, add renderer-readiness validation,
|
||||
and retain the extension's epoch/outputId shape. The daemon binds actual Host
|
||||
receipts to the extended session handlers. There is only one receipt path:
|
||||
duplicate upstream-only session methods are not kept beside the extended ones.
|
||||
|
||||
The extension's provisional playback hold is used only for accepted,
|
||||
non-muted output until the real Host receipt arrives. The v9 coordinator's
|
||||
output identity, completion markers, epoch checks and session callbacks clear
|
||||
it; it must not regress to the old M5 unbound-completion latch. Proactive
|
||||
delivery acknowledgement is settled before the next FIFO item is released.
|
||||
|
||||
Automatically merged CLI changes must preserve the newer upstream runtime
|
||||
ownership, discovery and pre-start checks. The shared qwen serve integration
|
||||
remains screen-only and does not gain standalone Memory or shutdown authority.
|
||||
Upstream release workflows and unrelated main changes remain untouched.
|
||||
|
||||
## Publication hygiene
|
||||
|
||||
Only source, tests, design documents, README, manifests and lockfiles are in the
|
||||
candidate diff. Private configuration, captured audio/images, memory databases,
|
||||
runtime logs and build outputs remain outside Git. Pattern scan hits are
|
||||
reviewed test sentinels, not production credentials. Adapted Memory/Proactive
|
||||
source retains the prototype's Apache-2.0 Alibaba copyright and identifies the
|
||||
TypeScript modifications. Both npm and pnpm locks must agree with manifests;
|
||||
unrelated package versions/metadata must not be downgraded during migration.
|
||||
|
||||
## Validation plan
|
||||
|
||||
Run builds and heavy test groups serially. Verify qwen-live and Host typechecks,
|
||||
the repository build/bundle, focused CLI compatibility tests, complete Live and
|
||||
Host tests, inert realtime/ACP integration tests, lockfile/format/lint checks,
|
||||
and an independent source/transport replay. Reuse prior before/after UI evidence
|
||||
but clearly distinguish it from tests executed against this migrated branch.
|
||||
No tests may acquire real microphone/camera input, call paid providers or change
|
||||
user preferences. Windows/Linux and physical Bluetooth behavior remain manual
|
||||
verification items, not implied by a macOS mocked test pass.
|
||||
|
||||
Final local PR metadata follows the repository template and records the large
|
||||
cross-package feature scope for maintainer review. The user elected one PR;
|
||||
this is not a core-only refactor or a reason to silently omit features.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Selected-display capture for Live
|
||||
|
||||
[English](2026-09-09-live-display-capture.md) | [简体中文](2026-09-09-live-display-capture.zh-CN.md)
|
||||
|
||||
## Scope and baseline
|
||||
|
||||
Screen currently routes every frame through Appshot's foreground-window/AX
|
||||
capture. The user wants Proactive monitors and Screen Live Feed to see an entire
|
||||
selected display. Keep the foreground Appshot tool's existing window behavior;
|
||||
do not widen On Demand visual-memory observation as an incidental change.
|
||||
Existing uncommitted subagent controls are the baseline and must be preserved.
|
||||
|
||||
## Design
|
||||
|
||||
- Add a native display-only path alongside Appshot, sharing the serial capture
|
||||
queue. Enumerate active displays with UUID, label, dimensions and primary
|
||||
status. Persist a display UUID in `visualInput.screenDisplayId`; default
|
||||
`primary` means the system's primary display. Explicit unavailable UUIDs fail
|
||||
closed without selecting another display or falling back to a window.
|
||||
- The native screenshot covers the whole selected display, including desktop,
|
||||
menu bar, Dock and other apps, excluding Live Host's own windows. Use a
|
||||
display ScreenCaptureKit filter on macOS 14+ and selected-display bounds with
|
||||
a composed window list on macOS 12/13. Display capture does not read AX.
|
||||
Bound native output to the existing realtime 1920x1080 envelope, preserving
|
||||
the whole image/aspect ratio; retain current FPS, JPEG and transport limits.
|
||||
- Add a Display selector beneath Video Source in Settings. Show primary and
|
||||
connected displays; retain an unavailable selected entry with an explicit
|
||||
error. New fixed text stays in the bilingual catalogue. No new init question.
|
||||
Use the existing authenticated visual-settings/config persistence channel.
|
||||
- Extend protocol v9 additively: `displayCaptureV1` advertises support;
|
||||
`host.capture_visual` requests an explicit `screenScope: display`, and
|
||||
display results/frames carry the resolved `displayId`. New full-display
|
||||
requests must never silently downgrade against an old Host. Normal Appshot
|
||||
requests omit the scope and retain the window result/asset/AX semantics.
|
||||
Mirror the ten optional type fields in the canonical CLI protocol file to
|
||||
retain the existing byte-identity check; CLI runtime/parser behavior and
|
||||
capability advertisement remain unchanged.
|
||||
- Monitor On Demand capture explicitly requests display scope. Memory's
|
||||
existing On Demand window capture remains separate. Live Feed always uses
|
||||
display capture for Screen; Camera behavior stays unchanged. Display-setting
|
||||
changes invalidate pending capture and reset monitor visual buffers just as
|
||||
source changes do. Host capture-generation fences discard old in-flight frames
|
||||
on settings/topology changes; incoming explicit display IDs are validated.
|
||||
- Preserve the existing Appshot-related permission flow for On Demand tools.
|
||||
Full-display capture itself only requires Screen Recording; never ask for AX
|
||||
from the new native display operation, and do not broaden OS permissions.
|
||||
|
||||
## Verification and boundaries
|
||||
|
||||
Baseline: first try global qwen, then a safe source/mock fixture if unavailable.
|
||||
Verify split capture routing, missing display rejection, display identity and
|
||||
stale frames, original Appshot/Camera/memory behavior, native API availability,
|
||||
config migration/defaults, picker persistence and bilingual layout. Build both
|
||||
native architectures; serialize heavy builds/tests. Never capture the user's
|
||||
private desktop or invoke a model merely for verification. A physical multi-
|
||||
display/lock-screen test remains explicitly unverified unless performed safely
|
||||
with user-visible fixture content. No commit/push is requested in this turn.
|
||||
|
||||
## Summary
|
||||
|
||||
Add a separate full-display capture path for monitors and Screen Live Feed;
|
||||
preserve window capture for the user's Appshot tool and On Demand visual memory.
|
||||
Settings selects a display, with its UUID persisted and the primary display as
|
||||
the default. If an explicitly selected display disconnects, stop capture with an
|
||||
error rather than silently switching displays. Retain frame-rate, dimension and
|
||||
transport limits: full coverage does not mean native resolution. Clear old
|
||||
visual buffers and discard stale frames when the display changes. Fail explicitly
|
||||
when an old Host lacks support; never substitute a window image for a display
|
||||
image. Continue on top of the existing uncommitted changes without automatic push.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Live 的选定显示器采集
|
||||
|
||||
[English](2026-09-09-live-display-capture.md) | [简体中文](2026-09-09-live-display-capture.zh-CN.md)
|
||||
|
||||
## 范围与基线
|
||||
|
||||
Screen 当前通过 Appshot 的前台窗口／AX 采集路径处理每一帧。用户希望 Proactive
|
||||
monitor 和 Screen Live Feed 能看到所选显示器的完整画面。保留前台 Appshot 工具现有
|
||||
的窗口行为；不能借此附带扩大 On Demand 视觉记忆观察的范围。现有未提交的子智能体
|
||||
控制修改是本次工作的基线，必须保留。
|
||||
|
||||
## 设计
|
||||
|
||||
- 在 Appshot 旁新增原生的纯显示器采集路径，共用串行采集队列。枚举活动显示器的
|
||||
UUID、名称、尺寸及是否为主显示器。将显示器 UUID 保存到
|
||||
`visualInput.screenDisplayId`；默认值 `primary` 表示系统主显示器。明确指定的
|
||||
UUID 不可用时，停止采集并报错，不能改选其他显示器或回退到窗口。
|
||||
- 原生截图覆盖选定显示器的完整画面，包括桌面、菜单栏、Dock 和其他应用，但排除
|
||||
Live Host 自己的窗口。macOS 14+ 使用 ScreenCaptureKit 的显示器过滤器；macOS
|
||||
12/13 使用选定显示器的边界和组合窗口列表。显示器采集不读取 AX。将原生输出限制
|
||||
在现有的实时 1920x1080 范围内，同时保留完整画面和宽高比；继续遵守当前的 FPS、
|
||||
JPEG 和传输限制。
|
||||
- 在 Settings 的 Video Source 下方增加 Display 选择器。显示主显示器及已连接的
|
||||
显示器；已选显示器不可用时仍保留该选项，并显示明确错误。新增固定文本统一进入
|
||||
双语文案目录。不增加 init 问题。使用现有经过认证的视觉设置／配置持久化通道。
|
||||
- 以增量方式扩展协议 v9：`displayCaptureV1` 声明支持该能力；
|
||||
`host.capture_visual` 通过显式的 `screenScope: display` 请求整屏采集，显示器
|
||||
结果／帧携带解析后的 `displayId`。新整屏请求面对旧 Host 时绝不能悄悄降级。普通
|
||||
Appshot 请求不传 scope，保留窗口结果／资源／AX 语义。在规范的 CLI 协议文件中
|
||||
同步这十个可选类型字段，以保留现有的字节一致性检查；CLI 运行时／解析器行为及
|
||||
能力声明保持不变。
|
||||
- Monitor 的 On Demand 采集显式请求 display scope。Memory 现有的 On Demand
|
||||
窗口采集保持独立。Live Feed 在 Screen 源下始终使用显示器采集；Camera 行为不变。
|
||||
显示器设置发生变化时，像输入源切换一样使待处理采集失效，并重置 monitor 视觉
|
||||
缓冲。Host 的采集代次校验在设置／拓扑变化时丢弃旧的在途帧；校验收到的显式
|
||||
显示器 ID。
|
||||
- 保留 On Demand 工具现有的 Appshot 权限流程。整屏采集本身只需要屏幕录制权限；
|
||||
新的原生显示器操作绝不请求 AX，也不扩大操作系统权限范围。
|
||||
|
||||
## 验证与边界
|
||||
|
||||
基线：先尝试全局 qwen；不可用时再使用安全的源码／模拟夹具。验证采集路由分离、
|
||||
不可用显示器拒绝、显示器身份与过期帧、原有 Appshot／Camera／memory 行为、原生
|
||||
API 可用性、配置迁移／默认值、选择器持久化及双语布局。构建两种原生架构；串行运行
|
||||
重型构建／测试。绝不能仅为验证而采集用户的私人桌面或调用模型。真实多显示器／锁屏
|
||||
测试，除非已使用用户可见的安全夹具内容完成，否则必须明确标为未验证。本轮工作不
|
||||
要求提交或推送。
|
||||
|
||||
## 摘要
|
||||
|
||||
新增独立的完整显示器采集路径，用于 monitor 和 Screen Live Feed；保留用户 Appshot
|
||||
工具以及 On Demand 视觉记忆的窗口截图行为。设置里可选择显示器，配置保存 UUID，默认
|
||||
主显示器；明确选中的显示器断开时停止采集并报错，不能悄悄换屏。保留帧率、尺寸及传输
|
||||
上限，完整范围不代表原生分辨率。显示器切换时清除旧视觉缓冲并丢弃过期帧。旧 Host
|
||||
缺少能力时明确失败，不能用窗口图冒充整屏。全部更改在原未提交修改上继续，不自动推送。
|
||||
@@ -0,0 +1,46 @@
|
||||
# Monitor delivery diagnostics and capture-event isolation
|
||||
|
||||
[English](2026-09-09-live-monitor-diagnostics.md) | [简体中文](2026-09-09-live-monitor-diagnostics.zh-CN.md)
|
||||
|
||||
The user reports that real Screen monitoring still does not trigger and capture
|
||||
appears to move the orb. The full-display feature was previously verified with
|
||||
inert fixtures/native compilation, not an actual desktop-to-model session.
|
||||
|
||||
A real synthetic probe through the production Monitor and configured DashScope
|
||||
model returned wait / Reply / wait for green / red / green frames. This proves
|
||||
basic visual delivery works for the configured model, not that the user's
|
||||
screen capture or actual condition works. No private desktop/audio was sent.
|
||||
|
||||
An independent reproduction confirmed that a colorSpace-only display event
|
||||
invalidated a pending frame and interrupted drag despite unchanged geometry.
|
||||
Only geometry-relevant metrics and display add/remove should invalidate capture.
|
||||
For those events, clamp the current logical orb position only if necessary;
|
||||
keep first-launch anchoring and ordinary saved-position/layout behavior.
|
||||
|
||||
Existing debug modes gain metadata only: a SHA256 JPEG prefix connects Host
|
||||
capture, daemon receipt and actual Monitor socket write; per-commit image/audio
|
||||
counts report submitted input, and action classes distinguish wait/reply/tool
|
||||
proposal/invalid output without logging text. Packet counters reset at the same
|
||||
commit/clear/recycle boundaries as actual media. Queue admission is not reported
|
||||
as successful socket delivery. Provider credentials and raw media stay omitted.
|
||||
Native display, explicit positioning and native window-move diagnostics identify
|
||||
the cause of motion without themselves moving windows.
|
||||
|
||||
Validate metadata/counters, capture invalidation, drag/geometry, all existing
|
||||
live behavior, build and lint; repeat the independent reproduction. A separate
|
||||
local native probe is prepared for a user-approved safe display, saving at most
|
||||
three private local frames and event bounds, never sending desktop images to a
|
||||
provider. Until that run occurs, actual screenshot-to-movement and the user's
|
||||
failed visual condition remain unconfirmed. Do not change model prompts or
|
||||
protocol based on those unconfirmed causes. No PR submission is part of this fix.
|
||||
|
||||
## Summary
|
||||
|
||||
The real model correctly returned wait / reply / wait for a green / red / green
|
||||
sequence of synthetic images; this does not verify the user's actual desktop
|
||||
path. The reproduced defect is that non-geometric display events drop frames and
|
||||
interrupt dragging, so only this confirmed mechanism is fixed. Add hashes,
|
||||
actually submitted frame/audio counts, model action classifications and window
|
||||
coordinate logs, without recording images or speech by default. The real desktop
|
||||
capture probe requires safe content prepared by the user for local verification;
|
||||
it does not automatically send desktop content to a model.
|
||||
@@ -0,0 +1,35 @@
|
||||
# Monitor 投递诊断与采集事件隔离
|
||||
|
||||
[English](2026-09-09-live-monitor-diagnostics.md) | [简体中文](2026-09-09-live-monitor-diagnostics.zh-CN.md)
|
||||
|
||||
用户反馈真实 Screen 监控仍然不触发，采集似乎还会导致悬浮球移动。此前整屏功能只通过
|
||||
无外部副作用的夹具／原生编译验证，未验证实际的桌面到模型会话。
|
||||
|
||||
通过生产版 Monitor 和已配置的 DashScope 模型进行的真实合成探针，对绿／红／绿
|
||||
画面返回 wait / Reply / wait。这证明该模型的基本视觉投递可用，但不能证明用户的
|
||||
屏幕采集或实际监控条件有效。没有发送私人桌面或音频。
|
||||
|
||||
独立复现确认：即使几何尺寸不变，仅 colorSpace 变化的显示器事件也会使待处理帧失效
|
||||
并打断拖拽。只有与几何相关的指标变化以及显示器添加／移除才应使采集失效。针对这些
|
||||
事件，仅在必要时限制悬浮球当前逻辑位置；保留首次启动锚定，以及通常的已保存位置／
|
||||
布局行为。
|
||||
|
||||
现有 debug 模式仅增加元数据：通过 JPEG 的 SHA256 前缀关联 Host 采集、daemon
|
||||
接收及 Monitor 实际的 socket 写入；每次 commit 的图像／音频计数反映已提交输入，
|
||||
动作分类区分 wait／reply／工具提案／无效输出，而不记录文本。数据包计数器的重置
|
||||
边界与实际媒体的 commit／clear／recycle 一致。进入队列不算 socket 投递成功。仍然
|
||||
不记录提供方凭据和原始媒体。原生显示器、显式定位及原生窗口移动诊断用于查明位移
|
||||
原因，诊断本身不得移动窗口。
|
||||
|
||||
验证元数据／计数器、采集失效、拖拽／几何行为及所有现有 Live 行为，完成构建和 lint，
|
||||
并重复独立复现。另已准备独立的本地原生探针，用于用户授权的安全显示器画面：最多
|
||||
保留三帧私密的本地图像以及事件边界，绝不向提供方发送桌面图像。在执行该探针之前，
|
||||
实际截图与位移的因果关系以及用户视觉条件未触发的问题仍未确认。不要基于这些未确认
|
||||
的原因修改模型提示词或协议。本次修复不包含提交 PR。
|
||||
|
||||
## 摘要
|
||||
|
||||
真实模型对纯测试图的绿／红／绿序列正确返回等待／提醒／等待；这不代表用户真实桌面链路
|
||||
已验证。已经复现的缺陷是非几何显示器事件会丢帧、打断拖拽，因此只修这一已确认机制。
|
||||
补充哈希、实际送入的帧／音频计数、模型动作分类及窗口坐标日志，不默认记录画面或语音。
|
||||
真实桌面采集探针需用户准备安全画面后本地验证，不自动把桌面发给模型。
|
||||
@@ -0,0 +1,82 @@
|
||||
# Live review corrections
|
||||
|
||||
## Scope
|
||||
|
||||
Address confirmed PR #11369 review defects on `3f0a30c2bf`, retaining the three
|
||||
maintainer main merges. The review is partial: addressing its findings does not
|
||||
certify unrelated parts of this feature PR. Isolated regression evidence must
|
||||
precede fixes; builds and affected-package verification follow them.
|
||||
|
||||
## Decisions
|
||||
|
||||
- Keep v9 camera readiness strict and update stale CLI hello fixtures. Camera
|
||||
is requested functionality: packaging requires its nonempty usage description
|
||||
and exactly the four existing entitlements, rejecting all other permissions.
|
||||
- Cache shutdown authority by authenticated daemon identity, not WebSocket
|
||||
lifetime. Identity/credential changes revoke it; failed Quit stays pinned to
|
||||
the original target. Shared-daemon retries still send a real stop frame.
|
||||
- Keep the failed-Quit UI and stopped media, with a truthful connection state.
|
||||
Only a matching shutdown receipt or signal-zero ESRCH for the authenticated
|
||||
PID proves shutdown. HTTP/network failures or ambiguous process probes do not.
|
||||
Preserve the failure cause but log only non-sensitive classifications.
|
||||
- Full readiness recovery dominates visual-only recovery. Visual changes cannot
|
||||
cancel microphone/audio/shortcut updates. Rejected mode sends report failure
|
||||
without optimistically changing the acknowledged mode.
|
||||
- Clear output bookkeeping at both stop boundaries. Replay buffered submission
|
||||
residue without inventing job ownership or reopening resolved permissions.
|
||||
Ignore a late tool result only after that exact response failed nonfatally
|
||||
and while its Realtime connection remains usable.
|
||||
- Pair Host output clears with injector state and retry deferred Proactive
|
||||
repair once every response authority has settled. Preserve FIFO, direct-user
|
||||
adjacency and tool-authority boundaries.
|
||||
- Reset monitor recycle state at its authoritative ready event. Keep the entire
|
||||
SFT prompt and non-executable Func_call behavior; distinguish it from wait in
|
||||
content-free diagnostics. Retain nominal warm-up and add continuous observed
|
||||
time for slow successful capture; expired evidence resets that interval.
|
||||
- Advertise exact adjacent-update/cancel restrictions. Map only known argument
|
||||
errors to actionable model receipts. Reuse PCM/instruction-size constants,
|
||||
classify oversized initial instructions as configuration errors, and remove
|
||||
an image-drop reason with no producer.
|
||||
|
||||
## Deferred maintenance
|
||||
|
||||
Keep post-call harness observation: the backend treats some typed 404/session
|
||||
not-found errors as recoverable during runtime replacement, so an arbitrary
|
||||
retry cap is unsafe. Current visual prompts, JPEG validation, wire enums and
|
||||
tool schema/allowlist copies show no drift. Broad unification stays separate
|
||||
maintenance; the review report records the parity evidence.
|
||||
|
||||
## 中文说明
|
||||
|
||||
### 范围
|
||||
|
||||
基于 `3f0a30c2bf` 修复 PR #11369 已确认的问题，保留维护者三次合入 main 的历史。
|
||||
原审查只覆盖部分内容，处理这些意见不等于为整个功能 PR 作完整认证。先用隔离回归
|
||||
复现，再修复，并完成构建和受影响包的验证。
|
||||
|
||||
### 决策
|
||||
|
||||
- 保留严格的 v9 摄像头就绪校验，更新过期 CLI hello 测试替身。摄像头是已要求的
|
||||
功能；打包检查要求非空用途说明和现有四项权限，拒绝所有额外权限。
|
||||
- 退出权限按已认证 daemon 身份缓存，而非按 WebSocket 存活期。身份或凭据改变时
|
||||
撤销；失败重试固定原目标。共享 daemon 的重试仍必须真正发送 stop 帧。
|
||||
- 退出失败时保留错误与重试界面，媒体保持停止，连接状态不能误报 ready。
|
||||
只有匹配回执或对已认证 PID 的零信号探测得到 ESRCH，才能证明已退出；网络、HTTP
|
||||
失败和不明确的进程探测都不是证明。保留失败原因，日志只记录不敏感的分类。
|
||||
- 完整就绪恢复优先于纯视觉恢复；视觉变化不能取消麦克风、音频或快捷键更新。
|
||||
模式发送失败应提示，不提前改变已确认的模式。
|
||||
- 两种停止终态都清理播放状态；重放提交残留事件，但不猜测任务归属、不复活已处理
|
||||
的授权。只有同一响应已发生非致命失败、连接仍可用时，才忽略迟到工具结果。
|
||||
- Host 清空音频时同步 injector；所有响应类型都结束后重试延后的 Proactive 修复，
|
||||
同时保持 FIFO、直接用户相邻引用与工具权限边界。
|
||||
- monitor 回收状态只在权威 ready 事件中重置。完整保留 SFT prompt 和 Func_call
|
||||
不执行语义，仅在不含内容的诊断中区分它与 wait。保留正常帧率暖启动，并允许慢速
|
||||
成功采集按连续观测时间满足门槛；证据过期后重置计时。
|
||||
- 工具描述明确相邻更新／取消限制；模型回执仅映射已知参数错误。复用 PCM 和指令
|
||||
长度常量，初始指令过长归为配置错误，删除没有生产者的丢帧原因类型。
|
||||
|
||||
### 延期维护
|
||||
|
||||
保持通话结束后的后台观察：后端在运行时替换期间把部分带类型的 404／session-not-found
|
||||
视为可恢复错误，不能据此加入任意重试上限。视觉提示词、JPEG 校验、线上枚举及工具
|
||||
schema／白名单副本当前未发现漂移；大范围统一留作独立维护，一致性证据记录在审查报告中。
|
||||
@@ -0,0 +1,171 @@
|
||||
# Live subagent controls and approvals
|
||||
|
||||
[English](2026-09-09-live-subagent-controls.md) | [简体中文](2026-09-09-live-subagent-controls.zh-CN.md)
|
||||
|
||||
## Current behavior and scope
|
||||
|
||||
Live can observe multiple Harness sessions and runs one evaluator per monitor.
|
||||
However, monitor admission defaults to four, and the 32-record detail cache can
|
||||
evict active tasks. Subagents is read-only. Session-level cancellation can stop
|
||||
the wrong task when invoked through an old job handle. Permission requests are
|
||||
retained after hangup but have no native approval controls. Codex ACP versions
|
||||
inspected locally advertise an explicit asking mode, not `default`.
|
||||
|
||||
Remove Live's application-level active-task admission cap, retain every active
|
||||
task's bounded detail, and add paginated management, exact-task stop, explicit
|
||||
approval controls and truthful text receipts to Omni. Backend/service quotas,
|
||||
per-session queue limits and physical resources still apply; no global backend
|
||||
settings, sandbox bypass, model call or existing user configuration is changed.
|
||||
Independent Harness work uses separate sessions; continuing an existing session
|
||||
retains its existing queue/steer behavior.
|
||||
|
||||
## Contract and transport
|
||||
|
||||
Keep protocol v9 compatibility. A new optional `subagentsControlV1` welcome
|
||||
capability advertises standalone management. The existing subagents snapshot
|
||||
stream remains a bounded change signal and legacy read-only view. New Host
|
||||
management uses an authenticated loopback `POST /live/subagents` endpoint,
|
||||
requiring the same bearer token and instance nonce as standalone shutdown.
|
||||
Requests and responses are size-bounded and validated. It is daemon-scoped,
|
||||
not call-epoch-scoped, so Harness work remains manageable after End call.
|
||||
Requests are limited to 4 KiB and management responses to 1 MiB; the legacy
|
||||
snapshot remains 240 KiB. Pending unassigned approvals have their own snapshot
|
||||
count so the collapsed summary can warn without inventing a task. Approval
|
||||
descriptions retain up to 4096 characters; an incomplete description is marked
|
||||
and cannot be approved here, although an offered Deny remains available.
|
||||
|
||||
Shared browser-safe contracts live with the subagent types:
|
||||
|
||||
- Request: `list` with offset and optional selected task ID; `stop` with exact
|
||||
task ID; `permission` with exact pending request handle and offered decision.
|
||||
- Page: bounded snapshot, offset, retained total and optional selected detail.
|
||||
Task metadata may advertise `canStop`, a fixed disabled reason and a bounded
|
||||
list of pending permissions. Unassigned permission requests are exposed
|
||||
separately rather than attributed to an unrelated job.
|
||||
- Result: page or an explicit operation outcome (`stopping`, `stopped`,
|
||||
`already_ended`, `allowed`, `denied`); failures use owned error codes.
|
||||
|
||||
Host keeps all credentials in main. Native IPC accepts only these typed
|
||||
operations from its current subagent renderer. Mutations echo the rendered
|
||||
daemon instance, checked before dispatch; stale replies and old instance/row
|
||||
actions cannot act on a replacement daemon's reused job/request counters.
|
||||
Host refreshes the current page on bounded state updates while expanded,
|
||||
coalesces in-flight refreshes, and stops fetching when the panel is closed.
|
||||
|
||||
## Runtime behavior
|
||||
|
||||
- Remove the monitor admission gate and generated maxConcurrentTasks setting.
|
||||
Accept the legacy field without enforcing it so existing init files do not
|
||||
retain the old cap. Per-task failure/media limits remain unchanged.
|
||||
- Retain all active ledger details; evict only old terminal history. Page at
|
||||
the existing 32-row/message-size boundary, with selected detail loaded on
|
||||
demand. Never solve unlimited active tasks by unbounding a single payload.
|
||||
- Stop Proactive by immutable task ID and use its existing cleanup/delivery
|
||||
invalidation path, not a title lookup that can match a replacement task.
|
||||
- Add targeted adaptor cancellation. Qwen uses the existing exact
|
||||
removePendingPrompt API. ACP removes a matching queued item or cancels only
|
||||
its matching active ref/generation. Unknown refs and unsupported adaptors
|
||||
fail safely; they never fall back to cancelling another current task.
|
||||
Cancelling queued B must not clear running A's state or steal A's output.
|
||||
- Preserve cancellation-request versus terminal-confirmation semantics. A
|
||||
manual request and its eventual outcome generate small, owned text receipts.
|
||||
Pending receipts survive End call, queue while foreground speech is busy,
|
||||
and are removed only after the complete text is accepted by Realtime. Use
|
||||
distinct silent injector control items so ordinary 6000-character batch
|
||||
truncation or speech-only acceptance cannot falsely acknowledge a receipt.
|
||||
- Advertise an asking mode only after selecting and awaiting an explicitly
|
||||
supported ACP mode. Prefer advertised `default`, or the verified Codex
|
||||
`read-only` / `Ask for approval` combination. Never select full access.
|
||||
Unknown/unavailable mode negotiation reports a warning, not guaranteed
|
||||
manual approval. Changes apply only to sessions Live creates.
|
||||
- Present real broker requests and their supported options, including while
|
||||
the call is stopped. Revalidate the exact pending handle on approval; do not
|
||||
turn an ordinary filesystem denial into a fabricated permission request.
|
||||
The user's specific file error remains unconfirmed until matching evidence
|
||||
arrives; compatibility and missing UI controls are independently testable.
|
||||
|
||||
## UI and validation
|
||||
|
||||
Use the current pinned floating panel, without native dialogs or changed drag
|
||||
bounds. Add compact Stop controls, inline result/error feedback, Previous/Next
|
||||
pagination and approval actions in details. Close still only closes the panel.
|
||||
All fixed English/Chinese labels belong to the existing message catalogue.
|
||||
Cover >32 active tasks, more than four monitors, same-title replacement, queued
|
||||
versus active jobs, duplicate/stale stop, no active call, full text delivery,
|
||||
unsupported capability and permission replay/resolution. No real media or paid
|
||||
provider is needed for regression tests.
|
||||
|
||||
## Summary
|
||||
|
||||
### Current behavior and scope
|
||||
|
||||
Live already observes multiple Harness sessions, and each monitor has its own
|
||||
evaluator. However, monitors default to a limit of four, and the 32-entry detail
|
||||
cache can evict active tasks. The management panel is read-only; session-level
|
||||
cancellation through an old job can stop a newer task. Permission requests remain
|
||||
after hangup without native approval buttons. The locally inspected Codex ACP
|
||||
offers an explicit manual approval mode instead of `default`.
|
||||
|
||||
Remove Live's own active-task count limit, retain bounded details for all active
|
||||
tasks, and add paginated management, exact stopping, explicit approval and
|
||||
truthful text receipts to Omni. Backend quotas, per-session queues and physical
|
||||
resource limits still apply. Do not change global backend settings, bypass the
|
||||
sandbox, call a model or change existing user configuration. Independent parallel
|
||||
work uses different Harness sessions; existing sessions keep their queue/append
|
||||
semantics.
|
||||
|
||||
### Contract and transport
|
||||
|
||||
Keep v9 compatibility and advertise standalone management through optional
|
||||
`subagentsControlV1`. Preserve the snapshot stream as a bounded change signal and
|
||||
legacy read-only view. Management uses loopback `POST /live/subagents`, validates
|
||||
both the bearer token and instance nonce, and bounds and validates request and
|
||||
response bodies. Its scope is the daemon, not the call epoch, so background
|
||||
Harness work remains manageable after End call. Shared types cover pagination,
|
||||
stopping by ID, approval by request handle and explicit operation outcomes;
|
||||
unassigned permissions must not be attributed to other tasks.
|
||||
|
||||
Credentials stay in Host main; IPC accepts typed operations only from the current
|
||||
subagent renderer. Mutation requests carry the displayed instance identity,
|
||||
preventing old instances/rows from acting on counters reused by a new daemon.
|
||||
Coalesce state-driven refreshes for the current page while expanded, and stop
|
||||
fetching when collapsed.
|
||||
|
||||
### Runtime and UI
|
||||
|
||||
- Remove monitor count limits and maxConcurrentTasks from newly generated
|
||||
configuration. Read the legacy field compatibly but no longer enforce it.
|
||||
Per-task failure, media and transport limits remain unchanged.
|
||||
- Retain every active detail and evict only old terminal history. Keep the
|
||||
32-entry page and payload limits, loading selected detail on demand; never
|
||||
implement “unlimited tasks” by enlarging a single message.
|
||||
- Cancel Proactive by immutable ID, reusing cleanup and delivery invalidation;
|
||||
title matching must not stop a replacement task with the same title.
|
||||
- Harness cancellation is exact: Qwen uses existing removePendingPrompt; ACP
|
||||
removes only the matching queued item or cancels the matching current
|
||||
ref/generation. Unsupported or unknown IDs fail explicitly without falling
|
||||
back to another current task. Cancelling queued B must not clear running A's
|
||||
state or take its output.
|
||||
- Distinguish requested stop from an actual terminal state. User actions and
|
||||
final outcomes produce bounded text receipts, retained across hangup and queued
|
||||
while the foreground is busy. Confirm delivery only after Realtime accepts the
|
||||
complete text. Separate silent control items prevent false acknowledgements
|
||||
through ordinary 6000-character batch truncation or speech-only success.
|
||||
- Claim manual approval is enabled only when explicitly supported and
|
||||
successfully selected. Prefer advertised default, or the verified Codex
|
||||
read-only / Ask for approval combination; never choose full access. Warn on
|
||||
unknown/failed selection, and apply only to sessions newly created by Live.
|
||||
- Show actual pending approval requests and backend-offered options in details;
|
||||
these remain actionable after hangup, with handle revalidation before acting.
|
||||
Never invent an approvable request from an ordinary write denial. The actual
|
||||
reported error still needs log confirmation; mode compatibility and missing UI
|
||||
controls can be verified independently.
|
||||
- Retain the pinned, draggable floating panel; add compact Stop controls, inline
|
||||
feedback, Previous/Next pagination and approval actions without native dialogs.
|
||||
Close only closes the panel. All fixed text stays in the English/Chinese
|
||||
message catalogue.
|
||||
|
||||
Tests cover more than 32 active tasks / four monitors, old IDs, same-title
|
||||
replacement, queued and running tasks, duplicate operations, hangup and complete
|
||||
text delivery, legacy capabilities and permission replay/resolution. No real
|
||||
media or paid model is used.
|
||||
@@ -0,0 +1,125 @@
|
||||
# Live 子智能体控制与审批
|
||||
|
||||
[English](2026-09-09-live-subagent-controls.md) | [简体中文](2026-09-09-live-subagent-controls.zh-CN.md)
|
||||
|
||||
## 当前行为与范围
|
||||
|
||||
Live 可以观察多个 Harness 会话，并为每个 monitor 运行一个 evaluator。但 monitor
|
||||
默认只允许四个，且 32 条记录的详情缓存可能淘汰活动任务。Subagents 目前只读。通过
|
||||
旧 job handle 发起的会话级取消可能停止错误的任务。挂断后仍会保留权限请求，但没有
|
||||
原生审批控件。本机检查的 Codex ACP 版本声明的是明确的询问模式，而非 `default`。
|
||||
|
||||
移除 Live 应用层的活动任务数量上限，保留每个活动任务的有界详情，增加分页管理、
|
||||
精确任务停止、显式审批控件，以及向 Omni 投递真实的文本回执。后端／服务配额、单会话
|
||||
队列限制和物理资源仍然适用；不修改全局后端设置、不绕过沙箱、不调用模型，也不改变
|
||||
用户现有配置。独立 Harness 工作使用单独的会话；继续现有会话时仍保留原有的排队／
|
||||
steer 行为。
|
||||
|
||||
## 契约与传输
|
||||
|
||||
保持协议 v9 兼容。通过新增可选的 welcome 能力 `subagentsControlV1` 声明独立管理
|
||||
能力。原有子智能体快照流仍作为有界的变更信号和旧版只读视图。新的 Host 管理使用
|
||||
经过认证的环回 `POST /live/subagents` 端点，要求与独立关闭相同的 bearer token
|
||||
和实例 nonce。请求与响应都有大小限制，并进行校验。端点归属 daemon，而不是通话
|
||||
epoch，因此 End call 后仍可管理 Harness 工作。请求上限为 4 KiB，管理响应上限为
|
||||
1 MiB；旧版快照仍为 240 KiB。尚未关联任务的待审批请求拥有独立的快照计数，使收起的
|
||||
摘要可以提示用户而不虚构任务。审批描述最多保留 4096 个字符；不完整的描述必须标记，
|
||||
且不能在此批准，但如果后端提供 Deny，仍可拒绝。
|
||||
|
||||
浏览器安全的共享契约与子智能体类型放在一起：
|
||||
|
||||
- 请求：`list` 携带 offset 及可选的选中任务 ID；`stop` 携带精确任务 ID；
|
||||
`permission` 携带精确的待审批请求 handle 和后端提供的决策选项。
|
||||
- 分页：有界快照、offset、保留记录总数，以及可选的选中任务详情。任务元数据可以
|
||||
声明 `canStop`、固定的禁用原因，以及有界的待审批权限列表。未关联任务的权限
|
||||
请求单独展示，不能归到无关 job。
|
||||
- 结果：分页或明确的操作结果（`stopping`、`stopped`、`already_ended`、`allowed`、
|
||||
`denied`）；失败使用本系统定义的错误码。
|
||||
|
||||
Host 将所有凭据保留在主进程。原生 IPC 只接受当前子智能体 renderer 发出的这些
|
||||
类型化操作。变更请求回传界面所展示的 daemon 实例标识，并在分派前校验；过期响应和
|
||||
旧实例／旧列表行的操作不能作用于替换后的 daemon 所复用的 job／请求计数器。展开时，
|
||||
Host 根据有界状态更新刷新当前页，合并尚未完成的刷新请求；面板关闭后停止获取。
|
||||
|
||||
## 运行时行为
|
||||
|
||||
- 移除 monitor 准入门槛，以及新生成配置中的 maxConcurrentTasks 设置。兼容接受
|
||||
旧字段但不执行，以免现有 init 文件保留旧上限。每任务的失败／媒体限制不变。
|
||||
- 保留所有活动账本详情；仅淘汰旧的终态历史。按现有的 32 行／消息大小边界分页，
|
||||
按需加载选中详情。绝不能以取消单条负载上限来实现不限活动任务数量。
|
||||
- 按不可变的任务 ID 停止 Proactive，使用现有的清理／投递失效路径，而非可能匹配
|
||||
到替换任务的标题查找。
|
||||
- 增加精确的 adaptor 取消。Qwen 使用现有的精确 removePendingPrompt API。
|
||||
ACP 移除匹配的排队项，或只取消与当前 ref／generation 匹配的活动项。未知 ref
|
||||
及不支持的 adaptor 必须安全失败；绝不回退到取消另一个当前任务。取消排队的 B
|
||||
不能清空运行中 A 的状态或窃取 A 的输出。
|
||||
- 保留“已请求取消”与“已确认终态”的区别。手动请求及其最终结果生成简短的、本系统
|
||||
定义的文本回执。待投递回执跨 End call 保留，在前台播报忙时排队，只有完整文本被
|
||||
Realtime 接受后才移除。使用独立的静默 injector control 项，避免普通的
|
||||
6000 字符批次截断或仅语音接受被误认为回执已送达。
|
||||
- 只有选择了明确支持的 ACP 模式并等待设置成功后，才声明询问模式可用。优先使用
|
||||
已声明的 `default`，或已验证的 Codex `read-only` / `Ask for approval` 组合。
|
||||
绝不选择 full access。未知／不可用的模式协商给出警告，而不是保证人工审批可用。
|
||||
修改仅适用于 Live 创建的会话。
|
||||
- 展示真实 broker 请求及其支持的选项，包括通话停止期间。批准时重新校验精确的
|
||||
待审批 handle；不能把普通文件系统拒绝虚构成权限请求。用户遇到的具体文件错误在
|
||||
获得匹配证据前仍未确认；兼容性和缺失的 UI 控件可以独立测试。
|
||||
|
||||
## UI 与验证
|
||||
|
||||
沿用当前固定展开的悬浮面板，不使用原生弹窗，也不修改拖拽边界。增加紧凑的 Stop
|
||||
控件、行内结果／错误反馈、Previous/Next 分页，以及详情里的审批操作。Close 仍只
|
||||
关闭面板。所有固定中英文标签归入现有消息目录。覆盖 >32 个活动任务、超过四个
|
||||
monitor、同名替换、排队与活动 job、重复／过期停止、无活动通话、完整文本投递、
|
||||
不支持的能力以及权限重播／处理。回归测试无需真实媒体或付费提供方。
|
||||
|
||||
## 摘要
|
||||
|
||||
### 当前行为与范围
|
||||
|
||||
Live 已能观察多个 Harness 会话，每个 monitor 也有独立 evaluator；但 monitor 默认
|
||||
上限为四个，32 条详情缓存还会淘汰活动任务。管理面板只读，旧 job 的会话级取消可能
|
||||
误停新任务。挂断后的授权请求仍被保留，却没有原生审批按钮。本机核对的 Codex ACP
|
||||
提供明确的人工审批模式，而不是 `default`。
|
||||
|
||||
移除 Live 自身的活动任务数量门槛，保留全部活动任务的有限详情，增加分页管理、精确
|
||||
停止、显式审批及对 Omni 的真实文本回执。后端限额、单会话队列及物理资源限制仍适用；
|
||||
不改全局后端设置、不绕过沙箱、不发模型请求、不修改用户已有配置。独立并行工作使用
|
||||
不同 Harness 会话，既有会话仍遵守原排队／追加语义。
|
||||
|
||||
### 契约与传输
|
||||
|
||||
保持 v9 兼容，以可选 `subagentsControlV1` 声明独立管理能力。原快照流保留为有限的
|
||||
变更信号与旧版只读视图。新管理使用环回 `POST /live/subagents`，同时校验 bearer
|
||||
token 和实例 nonce，限制并验证请求／响应体。归属为 daemon 而不是通话 epoch，结束
|
||||
通话后仍可管理后台 Harness。共享类型包括分页查询、按 ID 停止、按请求 handle 审批，
|
||||
及明确的操作结果；不能把未关联授权归到其他任务。
|
||||
|
||||
凭据只留在 Host 主进程，IPC 只接受当前子智能体 renderer 的类型化操作。修改请求
|
||||
携带画面对应的实例标识，拒绝旧实例／旧列表行对新 daemon 复用计数器的操作。
|
||||
展开时按状态更新合并刷新当前页，收起后停止获取。
|
||||
|
||||
### 运行时与界面
|
||||
|
||||
- 移除 monitor 数量门槛与新配置中的 maxConcurrentTasks；兼容读取旧字段但不再
|
||||
执行其上限。每任务失败、媒体和传输限制不变。
|
||||
- 保留全部活动详情，只淘汰旧终态历史；沿用每页 32 条及消息体限制，按需获取选中
|
||||
详情，不能以放大单个消息的方式实现“无限任务”。
|
||||
- Proactive 按不可变 ID 取消，复用清理与投递失效流程，不能按标题误停同名新任务。
|
||||
- Harness 使用精确取消：Qwen 调现有 removePendingPrompt；ACP 只移除匹配排队项
|
||||
或取消匹配的当前 ref／generation。不支持或未知 ID 明确失败，不能退回取消另一个
|
||||
当前任务。取消排队 B 不得清空运行 A 的状态或拿走其输出。
|
||||
- 区分已请求停止与真正终态。用户操作及最终结果产生有限的文本回执；挂断期间保留，
|
||||
前台忙时排队，完整文本被 Realtime 接受后才确认送达。独立静默 control 项避免普通
|
||||
6000 字符批次截断或仅语音成功造成错误确认。
|
||||
- 只在明确支持且设置成功后声称人工审批已启用。优先声明的 default，或已核对的
|
||||
Codex read-only／Ask for approval 组合，绝不选择 full access。未知／设置失败时
|
||||
给出警告，仅作用于 Live 新建的会话。
|
||||
- 在详情显示真实待审批请求及后端提供的选项；挂断后仍可处理，操作前重查 handle。
|
||||
不能把普通写入拒绝虚构成可批准请求。实际现场错误仍需日志确认，模式兼容及界面
|
||||
缺失可以独立验证。
|
||||
- 沿用固定且可拖动的悬浮面板，增加紧凑 Stop、行内反馈、前后页和审批动作，不开
|
||||
原生弹窗。Close 只关闭面板，所有固定文案继续集中在中英文消息表。
|
||||
|
||||
测试覆盖超过 32 个活动任务／四个 monitor、旧 ID、同名替换、排队与运行任务、重复
|
||||
操作、挂断和完整文本投递、旧版本能力以及授权重播／处理；不使用真实媒体或付费模型。
|
||||
@@ -0,0 +1,63 @@
|
||||
# Debug-only visual Monitor request archives
|
||||
|
||||
[English](2026-09-10-live-monitor-request-archive.md) | [简体中文](2026-09-10-live-monitor-request-archive.zh-CN.md)
|
||||
|
||||
## Scope
|
||||
|
||||
The user explicitly requests temporary JSON/image/audio records for every
|
||||
inference of a visual Monitor, with its directory in each log message, retaining
|
||||
only the latest ten Monitors. This extends existing metadata-only debugging;
|
||||
normal runs must never start this recorder. Pure audio Monitors remain excluded.
|
||||
Existing code/worktree and the audio/position bug fixes are separate concerns.
|
||||
|
||||
## Recording contract
|
||||
|
||||
- A daemon started with debug logging initializes a private managed directory
|
||||
under the OS temporary directory, prunes it at startup and when a visual
|
||||
Monitor is created, and passes the recorder to that Monitor only. Use a new
|
||||
independent random directory per logical Monitor instance, not per WebSocket
|
||||
recycle. No new init question or persistent user configuration is needed.
|
||||
- Record the actual successfully sent inputs, not merely queued/captured data.
|
||||
Each commit/response request gets `request.json`, ordered JPEG files and an
|
||||
input WAV (mono signed PCM16, 16 kHz), including protocol silence. JSON records
|
||||
media ordering, sequence, offsets/hashes and transport/evaluation identity.
|
||||
Session instructions and text-conversation context are retained, with references
|
||||
to earlier requests rather than repeatedly copying media history. Store a
|
||||
bounded response/status record too, useful for distinguishing wait, reply and
|
||||
errors. API keys and authorization headers are never recorded.
|
||||
- Frame/audio input transactions may be assembled in memory until commit, with
|
||||
a bounded debug-only pending-byte budget. File I/O is serialized asynchronously;
|
||||
write/permission/disk/queue failure disables that recorder and logs an explicit
|
||||
error, never interrupts the live model or delays playback. No silent truncation:
|
||||
a failed/incomplete recording is clearly marked.
|
||||
- Directories use 0700 and files 0600. Retention only deletes validated, owned,
|
||||
marked Monitor directories immediately beneath this managed root; reject
|
||||
symlink roots and never traverse unrelated temporary paths. Order by creation
|
||||
time, not later inference writes. Evicted active Monitors continue running but
|
||||
cannot recreate their deleted archive. State explicitly when no longer retained.
|
||||
- Debug startup and every request log show the absolute Monitor directory and
|
||||
request directory/status. Documentation warns that these debug files contain
|
||||
real screen/camera and (for combined Monitors) microphone content. Retention is
|
||||
ten Monitor directories, not ten requests; long-running traces may be large.
|
||||
|
||||
## Verification
|
||||
|
||||
Baseline debug currently emits metadata only and creates no media archive.
|
||||
Use global qwen help first, then safe deterministic fake-provider/temporary-dir
|
||||
fixtures. Verify debug off/no files; visual vs audio-only; exact sent-byte order,
|
||||
WAV decode, rejected writes, interleaved next-evaluation input, recycle/reset,
|
||||
credentials omitted; startup/creation retention and no symlink/outside deletion;
|
||||
older active writer eviction; write failure nonfatal. Root runs serialized builds,
|
||||
package-local tests, independent verification and two clean self-audit passes.
|
||||
No actual microphone or private desktop is needed to test the implementation.
|
||||
|
||||
## Summary
|
||||
|
||||
Only debug startup enables real request archives for visual Monitors. Retain
|
||||
JSON, JPEG and WAV for each inference, recording content and order actually
|
||||
written to the model connection rather than treating queued data as sent. Log
|
||||
the paths; normal runs do not save media. Keep only the ten most recently created
|
||||
Monitor directories. Older active Monitors continue after eviction but do not
|
||||
recreate their records. Files are readable/writable only by the current user;
|
||||
safe cleanup covers only directories created by this feature. Archive failure
|
||||
must not affect calls.
|
||||
@@ -0,0 +1,51 @@
|
||||
# 仅限 debug 的视觉 Monitor 请求归档
|
||||
|
||||
[English](2026-09-10-live-monitor-request-archive.md) | [简体中文](2026-09-10-live-monitor-request-archive.zh-CN.md)
|
||||
|
||||
## 范围
|
||||
|
||||
用户明确要求为视觉 Monitor 的每次推理保存临时的 JSON／图像／音频记录，每条日志
|
||||
包含其目录，并只保留最近十个 Monitor。这是在现有仅记录元数据的调试能力上的扩展；
|
||||
正常运行绝不能启动此记录器。纯音频 Monitor 仍不在范围内。现有代码／工作区及音频／
|
||||
位置问题修复是独立事项。
|
||||
|
||||
## 记录契约
|
||||
|
||||
- 以 debug 日志启动的 daemon，在操作系统临时目录下初始化私有的托管目录，在
|
||||
启动时及创建视觉 Monitor 时清理旧记录，并只向该 Monitor 传入记录器。每个逻辑
|
||||
Monitor 实例使用新的独立随机目录，而非每次 WebSocket 重建使用新目录。不增加
|
||||
init 问题或持久化用户配置。
|
||||
- 记录实际成功发送的输入，而不是仅排队／采集到的数据。每个 commit／response
|
||||
请求对应 `request.json`、按顺序保存的 JPEG 文件，以及输入 WAV（单声道有符号
|
||||
PCM16，16 kHz），包括协议静音。JSON 记录媒体顺序、序列、偏移／哈希及
|
||||
transport／evaluation 身份。保留会话指令与文本对话上下文，通过引用此前请求，
|
||||
避免反复复制媒体历史。同时保存有界的响应／状态记录，用于区分 wait、reply 和
|
||||
错误。绝不记录 API key 和 authorization header。
|
||||
- 帧／音频输入事务可在 commit 前于内存中组装，但必须限制仅用于 debug 的待处理
|
||||
字节预算。文件 I/O 异步串行执行；写入／权限／磁盘／队列故障应禁用该记录器并
|
||||
记录明确错误，绝不能中断实时模型或延迟播放。不允许静默截断：记录失败／不完整
|
||||
时必须明确标记。
|
||||
- 目录权限为 0700，文件权限为 0600。保留策略仅删除托管根目录直接下属、经过验证、
|
||||
归当前用户所有且带有标记的 Monitor 目录；拒绝符号链接根目录，绝不遍历无关的
|
||||
临时路径。按创建时间排序，而非后续推理写入时间。被清理的活动 Monitor 继续
|
||||
运行，但不能重新创建已删除的归档。不再保留记录时必须明确说明。
|
||||
- debug 启动日志及每次请求日志都显示绝对 Monitor 目录，以及请求目录／状态。
|
||||
文档警告这些 debug 文件包含真实屏幕／摄像头内容，以及组合模态 Monitor 的
|
||||
麦克风内容。保留数量是十个 Monitor 目录，而不是十个请求；长时间运行的轨迹
|
||||
可能很大。
|
||||
|
||||
## 验证
|
||||
|
||||
当前 debug 基线仅输出元数据，不生成媒体归档。先使用全局 qwen help，再使用安全、
|
||||
确定性的假提供方／临时目录夹具。验证 debug 关闭时没有文件、视觉与纯音频的区别、
|
||||
实际发送字节的准确顺序、WAV 解码、被拒绝的写入、交错到来的下一次 evaluation
|
||||
输入、recycle／reset、凭据排除；验证启动／创建时的保留策略、不删除符号链接／外部
|
||||
路径、旧活动写入者被清理，以及写入失败不致命。主代理串行执行构建、包内测试、独立
|
||||
验证以及连续两轮无问题的自审。测试此实现不需要真实麦克风或私人桌面。
|
||||
|
||||
## 摘要
|
||||
|
||||
仅在 debug 启动时为视觉 Monitor 开启真实请求留档：每次推理保留 JSON、JPEG 与 WAV，
|
||||
记录实际写入模型连接的内容及顺序，不把排队数据当作已发送。日志打印路径，默认运行不
|
||||
落盘媒体。只保留最近创建的十个 Monitor 目录，旧活动 Monitor 被清理后继续运行但不再
|
||||
重建记录。文件仅本用户可读写，安全清理只覆盖本功能创建的目录。留档失败不能影响通话。
|
||||
@@ -0,0 +1,49 @@
|
||||
# Continuous Live output audio
|
||||
|
||||
[English](2026-09-10-live-output-audio-continuity.md) | [简体中文](2026-09-10-live-output-audio-continuity.zh-CN.md)
|
||||
|
||||
## Observed defect
|
||||
|
||||
The user hears periodic sharp crackles during continuous speech on Bluetooth
|
||||
headphones. A safe Chromium OfflineAudioContext reproduction using the actual
|
||||
Host audio engine found a digital discontinuity independent of hardware: the
|
||||
same 24 kHz PCM tone played as chunks and as one buffer differs at chunk joins.
|
||||
At 44.1 kHz, 480-sample chunks can produce a sample near -0.804 instead of -0.400.
|
||||
Seventeen of twenty-five tested rate/partition combinations failed. Separately,
|
||||
a chunk arriving with 5 ms still queued received an unnecessary 5 ms gap.
|
||||
|
||||
This establishes defects in the playback path, not proof of the cause of every
|
||||
Bluetooth noise. Opening a Bluetooth headset microphone can still switch macOS
|
||||
to a lower-quality hands-free route; that is a separate device behavior.
|
||||
|
||||
## Change
|
||||
|
||||
Keep the AudioContext on the device's default clock. In current connections that
|
||||
negotiate output-end markers, continuously resample each `(epoch, outputId)`
|
||||
stream before constructing device-rate AudioBuffers. A windowed-sinc low-pass
|
||||
filter retains a short, bounded input history and lookahead across PCM chunks.
|
||||
The filter also rejects above-Nyquist content when a device runs below 24 kHz;
|
||||
linear interpolation would not provide this anti-aliasing property.
|
||||
|
||||
The final output marker flushes the retained tail. Playback completion waits for
|
||||
all scheduled sources including that tail; mute, clear, disconnect, context loss
|
||||
and mode changes discard the state. Epoch and output identity fencing remains
|
||||
unchanged. The phase-kernel cache is bounded at 1,025 entries. Connections without
|
||||
end markers retain the existing per-frame conversion and drain path so a filter
|
||||
tail cannot wait indefinitely for a marker those peers never send.
|
||||
|
||||
Schedule current connections at integer device-sample boundaries. Add the small
|
||||
startup lead only when no audio remains queued; never insert it between already
|
||||
contiguous chunks. No new configuration or forced hardware sample rate is added.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
Focused tests cover rate conversion, partition invariance, anti-alias rejection,
|
||||
empty/tiny outputs, completion, late markers, failed scheduling, mute/clear and
|
||||
legacy peers. The same Chromium offline oracle must pass against source and built
|
||||
preload, including a non-silent waveform check. No actual audio device or user
|
||||
microphone is needed for these tests.
|
||||
|
||||
Physical Bluetooth listening remains a user/runtime verification step. Genuine
|
||||
network underruns are not solved by this change. A late final marker can schedule
|
||||
the short retained tail after an earlier source has drained.
|
||||
@@ -0,0 +1,39 @@
|
||||
# 连续的 Live 输出音频
|
||||
|
||||
[English](2026-09-10-live-output-audio-continuity.md) | [简体中文](2026-09-10-live-output-audio-continuity.zh-CN.md)
|
||||
|
||||
## 已观察到的缺陷
|
||||
|
||||
用户通过蓝牙耳机聆听连续播报时，会周期性听到尖锐爆音。使用实际 Host 音频引擎的
|
||||
安全 Chromium OfflineAudioContext 复现发现，与硬件无关的数字信号存在不连续：
|
||||
同一段 24 kHz PCM 音调分块播放和作为单个缓冲播放，在分块连接处出现差异。
|
||||
在 44.1 kHz 下，480 个采样的分块可能产生接近 -0.804 而非 -0.400 的采样值。
|
||||
测试的二十五种采样率／分块组合中有十七种失败。另外，一个到达时仍有 5 ms 音频
|
||||
排队的分块，被额外插入了不必要的 5 ms 间隙。
|
||||
|
||||
这些结果确认了播放路径的缺陷，但并不能证明每一种蓝牙噪声都由此引起。打开蓝牙耳机
|
||||
的麦克风仍可能使 macOS 切换到音质较低的免提链路；这是独立的设备行为。
|
||||
|
||||
## 修改
|
||||
|
||||
保持 AudioContext 使用设备默认时钟。对于当前已协商输出结束标记的连接，在构造
|
||||
设备采样率的 AudioBuffer 之前，对每条 `(epoch, outputId)` 流进行连续重采样。
|
||||
加窗 sinc 低通滤波器跨 PCM 分块保留短而有界的输入历史及前瞻数据。当设备采样率低于
|
||||
24 kHz 时，滤波器也会抑制高于奈奎斯特频率的内容；线性插值不具备这种抗混叠特性。
|
||||
|
||||
最终输出标记会刷出保留的尾部。播放完成要等待所有已调度音源，包括该尾部；静音、
|
||||
清空、断开连接、context 丢失和模式切换都会丢弃此状态。epoch 与输出身份校验保持
|
||||
不变。相位滤波核缓存上限为 1,025 项。没有结束标记的连接保留现有的逐帧转换及排空
|
||||
路径，避免滤波器尾部无限等待这些对端永远不会发送的标记。
|
||||
|
||||
当前连接按整数设备采样边界进行调度。仅在没有音频排队时添加少量启动提前量；绝不能
|
||||
在已经连续的分块之间插入该间隙。不增加新配置，也不强制硬件采样率。
|
||||
|
||||
## 验证与限制
|
||||
|
||||
定向测试覆盖采样率转换、分块不变性、抗混叠抑制、空／极短输出、完成、延迟标记、
|
||||
调度失败、静音／清空及旧版对端。同一 Chromium 离线判定基准必须在源码及构建后的
|
||||
preload 上通过，其中包括非静音波形检查。这些测试不需要真实音频设备或用户麦克风。
|
||||
|
||||
真实蓝牙听感仍需要用户／运行时验证。此修改不解决真正的网络欠载。若最终标记较晚
|
||||
到达，可能在先前音源已排空之后才调度短暂保留的尾部。
|
||||
@@ -0,0 +1,61 @@
|
||||
# Live review follow-up and integration
|
||||
|
||||
[English](2026-09-10-live-review-follow-up.md) | [简体中文](2026-09-10-live-review-follow-up.zh-CN.md)
|
||||
|
||||
This follow-up addresses PR #11369's second review against `5136b2713f` and
|
||||
publishes the user's accumulated local Subagents controls, selected-display
|
||||
capture, audio continuity and debug Monitor archives. Preserve the existing PR
|
||||
and merge history; no force push or unrelated refactor is needed.
|
||||
|
||||
## Correctness boundaries
|
||||
|
||||
- A stale PID is not proof that the recorded daemon URL is unavailable. Host
|
||||
first attempts authenticated Quit. Only a matching receipt, or actual
|
||||
connection refusal together with ESRCH for the original PID, permits success.
|
||||
Failed shared Quit remains terminal for incoming media/state, but can retry
|
||||
its stop action. Never redirect a retry to an unverified replacement daemon.
|
||||
- A joined handoff may recover a missing job reference only from the exact
|
||||
message ID in its injection acknowledgement, scoped to the same session.
|
||||
Handle signals arriving before or after the receipt without orphaning the
|
||||
returned handle. Ordinary missing refs, conflicting refs and historical work
|
||||
cannot be guessed into a completed task; no-signal outcomes remain unknown.
|
||||
- Slow successful visual capture may warm by continuous elapsed observation,
|
||||
but a hole longer than three nominal frame intervals (at least one second)
|
||||
breaks continuity. This preserves slow-capture progress without allowing a
|
||||
stale start timestamp to qualify one isolated frame.
|
||||
- Reduced-motion rules must actually win the CSS cascade. Stopping is a muted
|
||||
state. Truncated status text must have a real interactive hover target without
|
||||
changing orb drag/layout geometry.
|
||||
- Unknown visual configuration keys fail loudly, including typos that would
|
||||
otherwise silently capture Screen. Keep the newly supported display selector.
|
||||
- Memory endpoint derivation failure cannot take down daemon setup or local
|
||||
memory. Use the established unavailable endpoint sentinel, not a fabricated URL.
|
||||
- HTTP Quit cleanup failures preserve the authenticated retry route and owned
|
||||
discovery. SIGINT/SIGTERM exit cannot retry, so it releases only its own record.
|
||||
Log bounded, redacted resource/cause details without changing HTTP error text.
|
||||
- Fixed Proactive repair rules share symbols across producer and exact-match
|
||||
renderer. Never expose arbitrary raw errors or loosen matching to prefixes.
|
||||
|
||||
## Verification
|
||||
|
||||
Reproduce before fixing using isolated loopback daemons, synthetic inputs and
|
||||
hidden test-owned Electron windows. Cover both positive and negative identity,
|
||||
permission and retry cases. Keep independent literal assertions for the
|
||||
documented 16 kHz input / 24 kHz output contract, and test the evaluation budget
|
||||
across multiple Monitor transport recycles. Mutation checks must use isolated
|
||||
copies, not the user's working files.
|
||||
|
||||
Run package-local tests and the Live integration suites on the integrated tree,
|
||||
then build/typecheck/bundle and build/typecheck Host separately. Review the entire
|
||||
pending diff, including untracked production/tests, in two clean passes. Keep
|
||||
physical Bluetooth playback and real screen-condition triggering explicitly
|
||||
outside synthetic verification claims.
|
||||
|
||||
## Deliberate deferrals
|
||||
|
||||
Do not cap backend event retries based on 404 or the text `session not found`:
|
||||
the SDK documents that runtime draining/replacement may produce the same
|
||||
recoverable response. A permanent retirement policy requires authoritative
|
||||
backend/session closure, not a guessed timeout. Preserve the fully ported Monitor
|
||||
prompt as requested; its non-executable Func_call behavior remains diagnosed.
|
||||
Schema/JPEG deduplication suggestions stay separate from correctness fixes.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Live 审查跟进与集成
|
||||
|
||||
[English](2026-09-10-live-review-follow-up.md) | [简体中文](2026-09-10-live-review-follow-up.zh-CN.md)
|
||||
|
||||
本次跟进处理 PR #11369 针对 `5136b2713f` 的第二轮审查，并发布用户累积在本地的
|
||||
Subagents 控制、选定显示器采集、音频连续性和 debug Monitor 归档修改。保留现有 PR
|
||||
及合并历史；不需要强制推送或无关重构。
|
||||
|
||||
## 正确性边界
|
||||
|
||||
- 过期 PID 不能证明已记录的 daemon URL 不可用。Host 首先尝试经过认证的 Quit。
|
||||
只有收到匹配回执，或实际连接被拒绝且原始 PID 同时返回 ESRCH，才可认定成功。
|
||||
共享 Quit 失败后仍不再接受传入的媒体／状态，但可以重试停止操作。绝不能将重试
|
||||
重定向到未经验证的替换 daemon。
|
||||
- joined handoff 只能通过其注入确认中的精确消息 ID 恢复缺失的 job 引用，且必须
|
||||
限定在同一会话。处理回执前后到达的信号，不能让返回的 handle 失去关联。普通的
|
||||
引用缺失、引用冲突和历史工作都不能靠猜测归为已完成任务；没有信号的结果仍保持
|
||||
unknown。
|
||||
- 成功但较慢的视觉采集，可以通过连续观察的经过时间完成预热；但超过三个标称帧
|
||||
间隔（至少一秒）的空档会中断连续性。这样既保留慢速采集的进展，又不会让过期的
|
||||
起始时间戳使单个孤立帧满足条件。
|
||||
- 减少动态效果的规则必须在 CSS 层叠中实际生效。Stopping 使用弱化的视觉状态。
|
||||
被截断的状态文本必须有真正可交互的悬停目标，同时不改变悬浮球拖拽／布局几何。
|
||||
- 未知的视觉配置键必须明确报错，包括原本会悄悄采集 Screen 的拼写错误。保留
|
||||
新支持的显示器选择器。
|
||||
- Memory endpoint 推导失败不能使 daemon 初始化或本地 memory 不可用。使用已有的
|
||||
unavailable endpoint 哨兵值，而非伪造 URL。
|
||||
- HTTP Quit 清理失败时保留经过认证的重试路由，以及本实例拥有的发现记录。
|
||||
SIGINT/SIGTERM 退出无法重试，因此只释放自己的记录。记录有界且经过脱敏的资源／
|
||||
原因详情，不改变 HTTP 错误文本。
|
||||
- 固定的 Proactive 修复规则在生成方和精确匹配的 renderer 之间共享符号。绝不暴露
|
||||
任意原始错误，也不能将匹配放宽为前缀匹配。
|
||||
|
||||
## 验证
|
||||
|
||||
使用隔离的环回 daemon、合成输入及隐藏的测试自有 Electron 窗口，先复现再修复。
|
||||
覆盖身份、权限和重试的正反例。对已文档化的 16 kHz 输入／24 kHz 输出契约保留
|
||||
独立的字面量断言，并跨多次 Monitor transport 重建测试 evaluation 预算。变异测试
|
||||
必须使用隔离副本，不能修改用户工作文件。
|
||||
|
||||
在集成后的代码树上运行包内测试和 Live 集成测试，然后执行构建／类型检查／打包，
|
||||
并单独构建／类型检查 Host。审查全部待提交差异，包括未跟踪的生产代码／测试，完成
|
||||
连续两轮无问题的自审。必须明确：合成验证不能证明真实蓝牙播放或真实屏幕条件触发。
|
||||
|
||||
## 明确延后的事项
|
||||
|
||||
不要根据 404 或 `session not found` 文本来限制后端事件重试：SDK 文档说明，
|
||||
runtime 排空／替换也可能产生同样的可恢复响应。永久停止重试的策略需要权威的
|
||||
backend／session 关闭信息，不能依赖猜测的超时。按用户要求保留完整移植的 Monitor
|
||||
提示词；其不可执行的 Func_call 行为仍保留在诊断记录中。Schema／JPEG 去重建议与
|
||||
正确性修复分开处理。
|
||||
@@ -73,6 +73,8 @@ export interface FakeDashScopeConnection {
|
||||
* Returns the response id.
|
||||
*/
|
||||
functionCall(call: FakeDashScopeFunctionCall): string;
|
||||
/** Script the next client response.create; does not simulate user input. */
|
||||
queueFunctionCall(call: FakeDashScopeFunctionCall): void;
|
||||
/**
|
||||
* Simulate a direct spoken answer: response.created →
|
||||
* response.audio.delta (base64) → response.audio.done → response.done.
|
||||
@@ -101,6 +103,8 @@ export interface FakeDashScopeServer {
|
||||
inbox: JsonObject[];
|
||||
/** Answer client `response.create` with created+done automatically. */
|
||||
autoAckResponses: boolean;
|
||||
/** Response id automatically sent for this exact inbox request. */
|
||||
autoResponseIdFor(request: JsonObject): string | undefined;
|
||||
waitForConnection(timeoutMs?: number): Promise<FakeDashScopeConnection>;
|
||||
waitForMessage(
|
||||
predicate: (message: JsonObject) => boolean,
|
||||
@@ -156,6 +160,7 @@ export async function startFakeDashScopeServer(): Promise<FakeDashScopeServer> {
|
||||
|
||||
const connections: FakeDashScopeConnection[] = [];
|
||||
const inbox: JsonObject[] = [];
|
||||
const autoResponseIds = new WeakMap<JsonObject, string>();
|
||||
let eventSeq = 0;
|
||||
let itemSeq = 0;
|
||||
let responseSeq = 0;
|
||||
@@ -166,6 +171,7 @@ export async function startFakeDashScopeServer(): Promise<FakeDashScopeServer> {
|
||||
connections,
|
||||
inbox,
|
||||
autoAckResponses: true,
|
||||
autoResponseIdFor: (request) => autoResponseIds.get(request),
|
||||
waitForConnection: (timeoutMs = 15_000) => {
|
||||
if (connections.length > 0) return Promise.resolve(connections[0]);
|
||||
return new Promise<FakeDashScopeConnection>((resolve, reject) => {
|
||||
@@ -261,6 +267,7 @@ export async function startFakeDashScopeServer(): Promise<FakeDashScopeServer> {
|
||||
response: { id: responseId, status },
|
||||
});
|
||||
};
|
||||
let queuedFunctionCall: FakeDashScopeFunctionCall | undefined;
|
||||
|
||||
const connection: FakeDashScopeConnection = {
|
||||
index: connections.length,
|
||||
@@ -313,6 +320,11 @@ export async function startFakeDashScopeServer(): Promise<FakeDashScopeServer> {
|
||||
finishResponse(responseId);
|
||||
return responseId;
|
||||
},
|
||||
queueFunctionCall: (call) => {
|
||||
if (queuedFunctionCall)
|
||||
throw new Error('A function call is already queued');
|
||||
queuedFunctionCall = call;
|
||||
},
|
||||
respondWithAudio: (pcm16) => {
|
||||
if (pcm16.byteLength === 0 || pcm16.byteLength % 2 !== 0) {
|
||||
throw new Error('respondWithAudio needs a non-empty PCM16 buffer');
|
||||
@@ -345,11 +357,16 @@ export async function startFakeDashScopeServer(): Promise<FakeDashScopeServer> {
|
||||
inbox.push(parsed);
|
||||
if (parsed['type'] === 'session.update') {
|
||||
sendJson({ type: 'session.updated', session: { id: 'sess-1' } });
|
||||
} else if (
|
||||
parsed['type'] === 'response.create' &&
|
||||
handle.autoAckResponses
|
||||
) {
|
||||
finishResponse(beginResponse());
|
||||
} else if (parsed['type'] === 'response.create') {
|
||||
if (queuedFunctionCall) {
|
||||
const call = queuedFunctionCall;
|
||||
queuedFunctionCall = undefined;
|
||||
connection.functionCall(call);
|
||||
} else if (handle.autoAckResponses) {
|
||||
const responseId = beginResponse();
|
||||
autoResponseIds.set(parsed, responseId);
|
||||
finishResponse(responseId);
|
||||
}
|
||||
}
|
||||
emitter.emit('message', parsed);
|
||||
});
|
||||
|
||||
@@ -13,10 +13,10 @@
|
||||
* hermetic env (tmp data/discovery dirs, fake DashScope endpoint, a real
|
||||
* `qwen serve` URL) and parses the single machine-readable stdout line,
|
||||
* following the `_daemon-harness.spawnDaemon` pattern.
|
||||
* - `FakeHost` speaks Host protocol v6 against the daemon's `/live/host`
|
||||
* - `FakeHost` speaks Host protocol v9 against the daemon's `/live/host`
|
||||
* WebSocket: discovery-file lookup, Bearer token + `x-qwen-live-nonce`
|
||||
* headers, `host.hello`, auto `host.pong`, auto success replies to
|
||||
* `host.capture_screen_context`, `host.action`, binary input
|
||||
* `host.capture_visual`, `host.action`, binary input
|
||||
* audio frames (8-byte BigUInt64BE epoch prefix + PCM16), and records
|
||||
* `host.welcome`/`host.state` plus raw output PCM frames.
|
||||
* - `bootLiveStack` assembles the full fixture: tmp workspace + HOME, a
|
||||
@@ -72,8 +72,9 @@ const SERVE_TOKEN = 'qwen-live-e2e-token';
|
||||
const LIVE_LISTENING_RE = /qwen-live listening on http:\/\/127\.0\.0\.1:(\d+)/;
|
||||
const DISPOSE_GRACE_MS = 10_000;
|
||||
const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host';
|
||||
const LIVE_HOST_PROTOCOL_VERSION = 7;
|
||||
const LIVE_HOST_PROTOCOL_VERSION = 9;
|
||||
const LIVE_INPUT_AUDIO_EPOCH_BYTES = 8;
|
||||
const LIVE_OUTPUT_AUDIO_HEADER_BYTES = 16;
|
||||
|
||||
// -- small async utilities ----------------------------------------------------
|
||||
|
||||
@@ -268,7 +269,7 @@ export async function spawnQwenLive(
|
||||
};
|
||||
}
|
||||
|
||||
// -- fake Host (protocol v6) ---------------------------------------------------
|
||||
// -- fake Host (protocol v9) ---------------------------------------------------
|
||||
|
||||
export interface FakeHostStateEntry {
|
||||
type: 'host.welcome' | 'host.state';
|
||||
@@ -287,7 +288,7 @@ export class FakeHost {
|
||||
readonly messages: JsonObject[] = [];
|
||||
/** host.welcome / host.state frames, normalized. */
|
||||
readonly states: FakeHostStateEntry[] = [];
|
||||
/** Raw binary output frames (bare PCM16). */
|
||||
/** Decoded PCM16 payloads from framed daemon output audio. */
|
||||
readonly audioFrames: Buffer[] = [];
|
||||
|
||||
private socket: WebSocket | undefined;
|
||||
@@ -319,8 +320,12 @@ export class FakeHost {
|
||||
: Array.isArray(data)
|
||||
? Buffer.concat(data)
|
||||
: Buffer.from(data as ArrayBuffer);
|
||||
this.audioFrames.push(frame);
|
||||
this.emitter.emit('audio', frame);
|
||||
if (frame.byteLength <= LIVE_OUTPUT_AUDIO_HEADER_BYTES) return;
|
||||
const pcm16 = Buffer.from(
|
||||
frame.subarray(LIVE_OUTPUT_AUDIO_HEADER_BYTES),
|
||||
);
|
||||
this.audioFrames.push(pcm16);
|
||||
this.emitter.emit('audio', pcm16);
|
||||
return;
|
||||
}
|
||||
let parsed: unknown;
|
||||
@@ -334,14 +339,18 @@ export class FakeHost {
|
||||
this.messages.push(message);
|
||||
if (message['type'] === 'host.ping') {
|
||||
this.send({ type: 'host.pong', pingId: message['pingId'] });
|
||||
} else if (message['type'] === 'host.capture_screen_context') {
|
||||
// The hello advertises `appshot: true`, so the daemon may request a
|
||||
// capture; settle it immediately (a real Host replies with a
|
||||
// screenshot it wrote to disk plus the accessibility dump).
|
||||
} else if (
|
||||
message['type'] === 'host.capture_visual' &&
|
||||
message['source'] === 'screen'
|
||||
) {
|
||||
this.send({
|
||||
type: 'host.screen_context_result',
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: message['requestId'],
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image: Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64'),
|
||||
width: 1280,
|
||||
height: 720,
|
||||
appName: 'FakeApp',
|
||||
windowTitle: 'Fake Window',
|
||||
accessibilityText: 'fake accessibility text',
|
||||
@@ -383,6 +392,7 @@ export class FakeHost {
|
||||
instanceNonce: this.hostInstanceNonce,
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -487,11 +497,7 @@ export class FakeHost {
|
||||
this.socketOrThrow().send(JSON.stringify(message));
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily materialize a real (1x1) PNG for `host.screen_context_result`:
|
||||
* the daemon registers the path as an asset, so it should exist on disk.
|
||||
* Written into the discovery dir, which the fixture already tears down.
|
||||
*/
|
||||
/** Materialize the PNG handoff asset returned with a visual capture. */
|
||||
private fakeScreenshotPath(): string {
|
||||
if (!this.screenshotPath) {
|
||||
const file = path.join(this.discoveryDir, 'fake-appshot.png');
|
||||
@@ -582,6 +588,39 @@ export async function waitForLiveLogEvents(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* For a single-tool continuation or a SPEAK_TO_USER item, wait for its next
|
||||
* response request to complete in Live, not just be sent by the fake provider.
|
||||
* The anchor must be from the current connection's inbox with no intervening
|
||||
* user turn. Handoff receipts do not themselves request a continuation.
|
||||
*/
|
||||
export async function waitForLiveResponseAfter(
|
||||
stack: Pick<LiveStack, 'fakeDash' | 'dataDir'>,
|
||||
anchor: JsonObject,
|
||||
authority: 'tool_continuation' | 'backend_speech',
|
||||
): Promise<void> {
|
||||
const anchorIndex = stack.fakeDash.inbox.indexOf(anchor);
|
||||
if (anchorIndex < 0) throw new Error('Response anchor is not in the inbox');
|
||||
const request = await stack.fakeDash.waitForMessage(
|
||||
(message) => message['type'] === 'response.create',
|
||||
{
|
||||
fromIndex: anchorIndex + 1,
|
||||
description: `${authority} response request`,
|
||||
},
|
||||
);
|
||||
const responseId = stack.fakeDash.autoResponseIdFor(request);
|
||||
if (!responseId) throw new Error('Expected an auto-acknowledged response');
|
||||
await waitForLiveLogEvents(
|
||||
stack.dataDir,
|
||||
(event) =>
|
||||
event.type === 'response.done' &&
|
||||
event.payload['responseId'] === responseId &&
|
||||
event.payload['authority'] === authority &&
|
||||
event.payload['status'] === 'completed',
|
||||
{ description: `${authority} ${responseId} completion` },
|
||||
);
|
||||
}
|
||||
|
||||
// -- full fixture ---------------------------------------------------------------
|
||||
|
||||
export interface BootLiveStackOptions {
|
||||
|
||||
@@ -8,13 +8,13 @@
|
||||
* qwen-live M1 — one full call, end to end, against real subprocesses:
|
||||
* a real `qwen serve` (model side backed by the fake OpenAI server), the
|
||||
* real `qwen-live` daemon binary, a fake DashScope realtime endpoint, and
|
||||
* a protocol-v6 FakeHost.
|
||||
* a protocol-v9 FakeHost.
|
||||
*
|
||||
* a. the discovery file exists with the documented fields;
|
||||
* b. FakeHost connect → hello → host.welcome;
|
||||
* c. `toggle` opens the realtime connection (auth header + model query),
|
||||
* sends session.update with the 8-tool surface, and the call reaches
|
||||
* `listening`;
|
||||
* sends session.update with the default 16-tool surface, and the call
|
||||
* reaches `listening`;
|
||||
* d. direct-answer path: Host input audio frames reach the provider as
|
||||
* input_audio_buffer.append, provider output audio reaches the Host as
|
||||
* bare PCM frames;
|
||||
@@ -55,13 +55,21 @@ const describeE2E = SKIP ? describe.skip : describe;
|
||||
|
||||
const EXPECTED_TOOL_NAMES = [
|
||||
'appshot',
|
||||
'cancel_proactive_task',
|
||||
'create_live_narration',
|
||||
'create_proactive_monitor',
|
||||
'create_proactive_timer',
|
||||
'handoff',
|
||||
'list_proactive_tasks',
|
||||
'omnibio',
|
||||
'omniretrieve',
|
||||
'remain_silent',
|
||||
'respond_permission',
|
||||
'session_create',
|
||||
'session_list',
|
||||
'session_monitor',
|
||||
'session_stop',
|
||||
'update_proactive_task',
|
||||
];
|
||||
|
||||
describeE2E('qwen-live M1 — end-to-end voice call', () => {
|
||||
@@ -88,7 +96,7 @@ describeE2E('qwen-live M1 — end-to-end voice call', () => {
|
||||
expect(record['url']).toBe(stack.live.url);
|
||||
expect(typeof record['token']).toBe('string');
|
||||
expect(String(record['token']).length).toBeGreaterThan(0);
|
||||
expect(record['protocolVersion']).toBe(7);
|
||||
expect(record['protocolVersion']).toBe(9);
|
||||
expect(record['pid']).toBe(stack.live.proc.pid);
|
||||
expect(String(record['instanceNonce'])).toMatch(/^[A-Za-z0-9_-]{16,256}$/);
|
||||
});
|
||||
@@ -101,6 +109,17 @@ describeE2E('qwen-live M1 — end-to-end voice call', () => {
|
||||
expect(Number.isInteger(welcome!.epoch)).toBe(true);
|
||||
expect(welcome!.status['available']).toBe(true);
|
||||
expect(welcome!.status['state']).toBe('idle');
|
||||
expect(
|
||||
stack.host.messages.find((entry) => entry['type'] === 'host.welcome')?.[
|
||||
'memory'
|
||||
],
|
||||
).toMatchObject({
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
libraryId: 'default',
|
||||
model: 'qwen3.7-plus',
|
||||
locked: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('toggle connects to the realtime provider and reaches listening', async () => {
|
||||
@@ -127,6 +146,14 @@ describeE2E('qwen-live M1 — end-to-end voice call', () => {
|
||||
);
|
||||
expect(typeof session['instructions']).toBe('string');
|
||||
expect(String(session['instructions']).length).toBeGreaterThan(0);
|
||||
for (const section of [
|
||||
'user_profile',
|
||||
'recent',
|
||||
'retrieved',
|
||||
'personalized_user_memories',
|
||||
]) {
|
||||
expect(String(session['instructions'])).toContain(`<${section}>`);
|
||||
}
|
||||
|
||||
const listening = await stack.host.waitForState(
|
||||
(entry) => entry.status['state'] === 'listening',
|
||||
@@ -161,12 +188,12 @@ describeE2E('qwen-live M1 — end-to-end voice call', () => {
|
||||
|
||||
it('hands off to the real serve daemon and injects the result back', async () => {
|
||||
const inboxIndex = stack.fakeDash.inbox.length;
|
||||
conn.speakTranscript('fix the failing test');
|
||||
conn.functionCall({
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: '{"task":"fix the failing test"}',
|
||||
callId: 'call-1',
|
||||
});
|
||||
conn.speakTranscript('fix the failing test');
|
||||
|
||||
// Receipt: the handoff was admitted by qwen serve.
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
deferred,
|
||||
startLiveCall,
|
||||
waitForLiveLogEvents,
|
||||
waitForLiveResponseAfter,
|
||||
type Deferred,
|
||||
type LiveStack,
|
||||
} from './qwen-live-harness.js';
|
||||
@@ -67,14 +68,20 @@ describeE2E('qwen-live M2 — injection window', () => {
|
||||
gates.set(task, gate);
|
||||
gateHandles.set(task, gate);
|
||||
const callId = `call-h${++callSeq}`;
|
||||
conn.functionCall({
|
||||
const fromIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: JSON.stringify({ task, ...extraArgs }),
|
||||
callId,
|
||||
});
|
||||
conn.speakTranscript(`Please run ${task}.`);
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === callId,
|
||||
{ timeoutMs: 30_000, description: `handoff receipt for ${task}` },
|
||||
{
|
||||
fromIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: `handoff receipt for ${task}`,
|
||||
},
|
||||
);
|
||||
const receipt = JSON.parse(
|
||||
functionCallOutputOf(receiptMessage)!.output,
|
||||
@@ -167,24 +174,42 @@ describeE2E('qwen-live M2 — injection window', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(complete)).toContain('finished inject-window-task');
|
||||
const spoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes('finished inject-window-task')
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(complete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spoken, 'backend_speech');
|
||||
});
|
||||
|
||||
it('batches multiple completions into one context injection', async () => {
|
||||
// A second backend session so two independent turns can complete.
|
||||
conn.functionCall({
|
||||
const createIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name: 'session_create',
|
||||
argumentsJson: JSON.stringify({ label: 'second workstream' }),
|
||||
callId: 'call-sc',
|
||||
});
|
||||
conn.speakTranscript('Create a second workstream.');
|
||||
const createdMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-sc',
|
||||
{ timeoutMs: 30_000, description: 'session_create receipt' },
|
||||
{
|
||||
fromIndex: createIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: 'session_create receipt',
|
||||
},
|
||||
);
|
||||
const created = JSON.parse(
|
||||
functionCallOutputOf(createdMessage)!.output,
|
||||
) as Record<string, unknown>;
|
||||
expect(created['status']).toBe('ok');
|
||||
const secondSession = String(created['handle']);
|
||||
await waitForLiveResponseAfter(stack, createdMessage, 'tool_continuation');
|
||||
|
||||
const receiptA = await gatedHandoff('batch-task-a');
|
||||
const receiptB = await gatedHandoff('batch-task-b', {
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
import {
|
||||
bootLiveStack,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
type LiveStack,
|
||||
} from './qwen-live-harness.js';
|
||||
|
||||
@@ -88,25 +89,46 @@ describeE2E('qwen-live M2 — permission relay', () => {
|
||||
|
||||
it('relays the permission ask to voice and delivers the allow vote', async () => {
|
||||
// Warmup: materialize the orchestrator's default serve session.
|
||||
conn.functionCall({
|
||||
const warmupIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: JSON.stringify({ task: 'perm-warmup' }),
|
||||
callId: 'call-w',
|
||||
});
|
||||
conn.speakTranscript('Run perm-warmup.');
|
||||
const warmupReceiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-w',
|
||||
{ timeoutMs: 30_000, description: 'the warmup handoff receipt' },
|
||||
{
|
||||
fromIndex: warmupIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: 'the warmup handoff receipt',
|
||||
},
|
||||
);
|
||||
const warmupReceipt = JSON.parse(
|
||||
functionCallOutputOf(warmupReceiptMessage)!.output,
|
||||
) as Record<string, unknown>;
|
||||
expect(warmupReceipt['status']).toBe('accepted');
|
||||
const warmupJob = String(warmupReceipt['job']);
|
||||
await stack.fakeDash.waitForMessage(
|
||||
const warmupComplete = await stack.fakeDash.waitForMessage(
|
||||
(message) =>
|
||||
contextTextOf(message)?.includes(`[COMPLETE ${warmupJob}]`) ?? false,
|
||||
{ timeoutMs: 30_000, description: 'the warmup [COMPLETE] injection' },
|
||||
{
|
||||
fromIndex: warmupIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: 'the warmup [COMPLETE] injection',
|
||||
},
|
||||
);
|
||||
const warmupSpoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes('warmup done')
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(warmupComplete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, warmupSpoken, 'backend_speech');
|
||||
|
||||
// Pin the approval mode of the orchestrator-created session so the
|
||||
// write below deterministically raises a permission_request.
|
||||
@@ -124,11 +146,12 @@ describeE2E('qwen-live M2 — permission relay', () => {
|
||||
|
||||
// The permission-triggering handoff.
|
||||
const inboxIndex = stack.fakeDash.inbox.length;
|
||||
conn.functionCall({
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: JSON.stringify({ task: 'perm-write-task' }),
|
||||
callId: 'call-p',
|
||||
});
|
||||
conn.speakTranscript('Run perm-write-task.');
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-p',
|
||||
{
|
||||
@@ -173,18 +196,21 @@ describeE2E('qwen-live M2 — permission relay', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(speakMessage)).toBeDefined();
|
||||
await waitForLiveResponseAfter(stack, speakMessage, 'backend_speech');
|
||||
|
||||
// The user says yes: respond_permission must deliver the vote to serve.
|
||||
conn.functionCall({
|
||||
const voteIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name: 'respond_permission',
|
||||
argumentsJson: '{"request_id":"req_1","decision":"allow"}',
|
||||
callId: 'call-2',
|
||||
});
|
||||
conn.speakTranscript('Yes, allow it.');
|
||||
const voteReceiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-2',
|
||||
{
|
||||
timeoutMs: 15_000,
|
||||
fromIndex: inboxIndex,
|
||||
fromIndex: voteIndex,
|
||||
description: 'the respond_permission receipt',
|
||||
},
|
||||
);
|
||||
@@ -192,6 +218,11 @@ describeE2E('qwen-live M2 — permission relay', () => {
|
||||
functionCallOutputOf(voteReceiptMessage)!.output,
|
||||
) as Record<string, unknown>;
|
||||
expect(voteReceipt['status']).toBe('delivered');
|
||||
await waitForLiveResponseAfter(
|
||||
stack,
|
||||
voteReceiptMessage,
|
||||
'tool_continuation',
|
||||
);
|
||||
|
||||
// Serve accepted the vote: the tool ran and the turn completed.
|
||||
const completeMessage = await stack.fakeDash.waitForMessage(
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
bootLiveStack,
|
||||
deferred,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
withTimeout,
|
||||
type Deferred,
|
||||
type LiveStack,
|
||||
@@ -56,15 +57,27 @@ describeE2E('qwen-live M2 — mid-turn steering', () => {
|
||||
callId: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
conn.functionCall({
|
||||
const fromIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name,
|
||||
argumentsJson: JSON.stringify(args),
|
||||
callId,
|
||||
});
|
||||
conn.speakTranscript(`Please ${name}: ${JSON.stringify(args)}`);
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === callId,
|
||||
{ timeoutMs: 30_000, description: `${name} receipt ${callId}` },
|
||||
{
|
||||
fromIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: `${name} receipt ${callId}`,
|
||||
},
|
||||
);
|
||||
if (name !== 'handoff')
|
||||
await waitForLiveResponseAfter(
|
||||
stack,
|
||||
receiptMessage,
|
||||
'tool_continuation',
|
||||
);
|
||||
return JSON.parse(functionCallOutputOf(receiptMessage)!.output) as Record<
|
||||
string,
|
||||
unknown
|
||||
@@ -168,6 +181,17 @@ describeE2E('qwen-live M2 — mid-turn steering', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(complete)).toContain('slow task finished');
|
||||
const spoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes('slow task finished')
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(complete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spoken, 'backend_speech');
|
||||
});
|
||||
|
||||
it('accepts a plain handoff to the now-idle session', async () => {
|
||||
@@ -184,6 +208,7 @@ describeE2E('qwen-live M2 — mid-turn steering', () => {
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
const inboxIndex = stack.fakeDash.inbox.length;
|
||||
const receipt = await handoff('call-s3', {
|
||||
task: 'one more quick task',
|
||||
session: sessionHandle,
|
||||
@@ -200,6 +225,7 @@ describeE2E('qwen-live M2 — mid-turn steering', () => {
|
||||
(message) =>
|
||||
contextTextOf(message)?.includes(`[COMPLETE ${job}]`) ?? false,
|
||||
{
|
||||
fromIndex: inboxIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: `[COMPLETE ${job}] for the idle handoff`,
|
||||
},
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import {
|
||||
bootAcpLiveStack,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
type AcpLiveStack,
|
||||
} from './qwen-live-harness.js';
|
||||
|
||||
@@ -57,11 +58,12 @@ describeE2E('qwen-live M4 — ACP backend call loop', () => {
|
||||
|
||||
it('hands off to the ACP backend and speaks the completion', async () => {
|
||||
const inboxIndex = stack.fakeDash.inbox.length;
|
||||
conn.functionCall({
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: JSON.stringify({ task: 'acp-call-task' }),
|
||||
callId: 'call-m4-1',
|
||||
});
|
||||
conn.speakTranscript('Run acp-call-task.');
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-m4-1',
|
||||
{
|
||||
@@ -86,18 +88,32 @@ describeE2E('qwen-live M4 — ACP backend call loop', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(complete)).toContain('acp call task complete');
|
||||
const spoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes('acp call task complete')
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(complete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spoken, 'backend_speech');
|
||||
});
|
||||
|
||||
it('lists the acp session with its backend name', async () => {
|
||||
conn.functionCall({
|
||||
const fromIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name: 'session_list',
|
||||
argumentsJson: '{}',
|
||||
callId: 'call-m4-2',
|
||||
});
|
||||
conn.speakTranscript('List my coding sessions.');
|
||||
const listMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-m4-2',
|
||||
{
|
||||
timeoutMs: 30_000,
|
||||
fromIndex,
|
||||
description: 'the session_list receipt',
|
||||
},
|
||||
);
|
||||
@@ -108,5 +124,6 @@ describeE2E('qwen-live M4 — ACP backend call loop', () => {
|
||||
const sessions = list['sessions'] as Array<Record<string, unknown>>;
|
||||
expect(sessions.length).toBeGreaterThanOrEqual(1);
|
||||
expect(sessions.every((row) => row['backend'] === 'qwen-acp')).toBe(true);
|
||||
await waitForLiveResponseAfter(stack, listMessage, 'tool_continuation');
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import {
|
||||
bootAcpLiveStack,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
type AcpLiveStack,
|
||||
} from './qwen-live-harness.js';
|
||||
|
||||
@@ -63,15 +64,28 @@ describeE2E('qwen-live M4 — multi-backend coexistence', () => {
|
||||
callId: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
conn.functionCall({
|
||||
const fromIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name,
|
||||
argumentsJson: JSON.stringify(args),
|
||||
callId,
|
||||
});
|
||||
conn.speakTranscript(`Please ${name}: ${JSON.stringify(args)}`);
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === callId,
|
||||
{ timeoutMs: 30_000, description: `${name} receipt ${callId}` },
|
||||
{
|
||||
fromIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: `${name} receipt ${callId}`,
|
||||
},
|
||||
);
|
||||
if (name !== 'handoff') {
|
||||
await waitForLiveResponseAfter(
|
||||
stack,
|
||||
receiptMessage,
|
||||
'tool_continuation',
|
||||
);
|
||||
}
|
||||
return JSON.parse(functionCallOutputOf(receiptMessage)!.output) as Record<
|
||||
string,
|
||||
unknown
|
||||
@@ -116,6 +130,17 @@ describeE2E('qwen-live M4 — multi-backend coexistence', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(complete)).toContain(marker);
|
||||
const spoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes(marker)
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(complete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spoken, 'backend_speech');
|
||||
}
|
||||
|
||||
// session_list shows both backends.
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import {
|
||||
bootAcpLiveStack,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
type AcpLiveStack,
|
||||
} from './qwen-live-harness.js';
|
||||
|
||||
@@ -78,11 +79,12 @@ describeE2E('qwen-live M4 — ACP permission relay', () => {
|
||||
|
||||
it('relays the ask to voice and the allow vote resolves the RPC', async () => {
|
||||
const inboxIndex = stack.fakeDash.inbox.length;
|
||||
conn.functionCall({
|
||||
conn.queueFunctionCall({
|
||||
name: 'handoff',
|
||||
argumentsJson: JSON.stringify({ task: 'perm-acp-task' }),
|
||||
callId: 'call-p',
|
||||
});
|
||||
conn.speakTranscript('Run perm-acp-task.');
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-p',
|
||||
{
|
||||
@@ -109,7 +111,7 @@ describeE2E('qwen-live M4 — ACP permission relay', () => {
|
||||
);
|
||||
expect(contextTextOf(permissionMessage)).toContain('respond_permission');
|
||||
// …plus the spoken ask.
|
||||
await stack.fakeDash.waitForMessage(
|
||||
const spokenAsk = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
@@ -124,13 +126,15 @@ describeE2E('qwen-live M4 — ACP permission relay', () => {
|
||||
description: 'the spoken permission ask',
|
||||
},
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spokenAsk, 'backend_speech');
|
||||
|
||||
// The user says yes: the vote must resolve the parked RPC.
|
||||
conn.functionCall({
|
||||
conn.queueFunctionCall({
|
||||
name: 'respond_permission',
|
||||
argumentsJson: '{"request_id":"req_1","decision":"allow"}',
|
||||
callId: 'call-v',
|
||||
});
|
||||
conn.speakTranscript('Yes, allow it.');
|
||||
const voteReceiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === 'call-v',
|
||||
{
|
||||
@@ -143,6 +147,11 @@ describeE2E('qwen-live M4 — ACP permission relay', () => {
|
||||
functionCallOutputOf(voteReceiptMessage)!.output,
|
||||
) as Record<string, unknown>;
|
||||
expect(voteReceipt['status']).toBe('delivered');
|
||||
await waitForLiveResponseAfter(
|
||||
stack,
|
||||
voteReceiptMessage,
|
||||
'tool_continuation',
|
||||
);
|
||||
|
||||
// The turn completes and the file landed.
|
||||
const complete = await stack.fakeDash.waitForMessage(
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
bootAcpLiveStack,
|
||||
deferred,
|
||||
startLiveCall,
|
||||
waitForLiveResponseAfter,
|
||||
withTimeout,
|
||||
type AcpLiveStack,
|
||||
type Deferred,
|
||||
@@ -53,15 +54,28 @@ describeE2E('qwen-live M4 — ACP steering', () => {
|
||||
callId: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
conn.functionCall({
|
||||
const fromIndex = stack.fakeDash.inbox.length;
|
||||
conn.queueFunctionCall({
|
||||
name,
|
||||
argumentsJson: JSON.stringify(args),
|
||||
callId,
|
||||
});
|
||||
conn.speakTranscript(`Please ${name}: ${JSON.stringify(args)}`);
|
||||
const receiptMessage = await stack.fakeDash.waitForMessage(
|
||||
(message) => functionCallOutputOf(message)?.callId === callId,
|
||||
{ timeoutMs: 30_000, description: `${name} receipt ${callId}` },
|
||||
{
|
||||
fromIndex,
|
||||
timeoutMs: 30_000,
|
||||
description: `${name} receipt ${callId}`,
|
||||
},
|
||||
);
|
||||
if (name !== 'handoff') {
|
||||
await waitForLiveResponseAfter(
|
||||
stack,
|
||||
receiptMessage,
|
||||
'tool_continuation',
|
||||
);
|
||||
}
|
||||
return JSON.parse(functionCallOutputOf(receiptMessage)!.output) as Record<
|
||||
string,
|
||||
unknown
|
||||
@@ -171,6 +185,17 @@ describeE2E('qwen-live M4 — ACP steering', () => {
|
||||
},
|
||||
);
|
||||
expect(contextTextOf(complete)).toContain('slow acp task finished');
|
||||
const spoken = await stack.fakeDash.waitForMessage(
|
||||
(message) => {
|
||||
const text = contextTextOf(message);
|
||||
return (
|
||||
text?.startsWith('[SPEAK_TO_USER] ') === true &&
|
||||
text.includes('slow acp task finished')
|
||||
);
|
||||
},
|
||||
{ fromIndex: stack.fakeDash.inbox.indexOf(complete) + 1 },
|
||||
);
|
||||
await waitForLiveResponseAfter(stack, spoken, 'backend_speech');
|
||||
});
|
||||
|
||||
it('accepts a plain handoff to the now-idle acp session', async () => {
|
||||
|
||||
Generated
+238
-1
@@ -3725,6 +3725,242 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@node-rs/jieba": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba/-/jieba-2.0.2.tgz",
|
||||
"integrity": "sha512-aONN6nwpbwHKenEzCcYUbm6ZFHWEs7N5eas7zwWFs3c4MmEdN79m9Si4PvOxCp285I2M+g4MfLyUm9WcYaQi7Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@node-rs/jieba-android-arm-eabi": "2.0.2",
|
||||
"@node-rs/jieba-android-arm64": "2.0.2",
|
||||
"@node-rs/jieba-darwin-arm64": "2.0.2",
|
||||
"@node-rs/jieba-darwin-x64": "2.0.2",
|
||||
"@node-rs/jieba-freebsd-x64": "2.0.2",
|
||||
"@node-rs/jieba-linux-arm-gnueabihf": "2.0.2",
|
||||
"@node-rs/jieba-linux-arm64-gnu": "2.0.2",
|
||||
"@node-rs/jieba-linux-arm64-musl": "2.0.2",
|
||||
"@node-rs/jieba-linux-x64-gnu": "2.0.2",
|
||||
"@node-rs/jieba-linux-x64-musl": "2.0.2",
|
||||
"@node-rs/jieba-win32-arm64-msvc": "2.0.2",
|
||||
"@node-rs/jieba-win32-ia32-msvc": "2.0.2",
|
||||
"@node-rs/jieba-win32-x64-msvc": "2.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-android-arm-eabi": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-android-arm-eabi/-/jieba-android-arm-eabi-2.0.2.tgz",
|
||||
"integrity": "sha512-5w+349/6X+0MkW0DMCLmtmjbCx9YXKMqMzSGH9A/XMP3sSy+MgzgMQ5UwEteu2YFnRve8V0mJzKQpG0R7TmT7A==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-android-arm64": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-android-arm64/-/jieba-android-arm64-2.0.2.tgz",
|
||||
"integrity": "sha512-tqNVsZ6VVzkbwWZAQ7zcOwJLtRKANW+Oa+uj8R+PWQSVXjy8Xs6KzfpDC5r+euFdNOiSjKPuZv4VbYSSTXed8g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-darwin-arm64": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-darwin-arm64/-/jieba-darwin-arm64-2.0.2.tgz",
|
||||
"integrity": "sha512-JIpC+9p3E67OPzvvLcxI9TUHfQL6Xuti+e4zbio8Kc7gUZ3EYJ+USRHGS6EixhsWDDZKFhNThl3ztTk9ZqBjzg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-darwin-x64": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-darwin-x64/-/jieba-darwin-x64-2.0.2.tgz",
|
||||
"integrity": "sha512-E7xPjd3L4oSPl9VSZJC6yN8niFYYL9NmmxKRHW14kAoa9TOr07xeiXtfRWajq95bgfvxHB9ymMkYKHCzGlTR4Q==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-freebsd-x64": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-freebsd-x64/-/jieba-freebsd-x64-2.0.2.tgz",
|
||||
"integrity": "sha512-FHfveI/E/uLNgVGWLobFhWLrh09+DM2g3UBq8YuDJ22tnBefjGaakoTIjd9NHQ03sxAAkhlucR8QLTrFN6cMUg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-linux-arm-gnueabihf": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-linux-arm-gnueabihf/-/jieba-linux-arm-gnueabihf-2.0.2.tgz",
|
||||
"integrity": "sha512-Hl2+3GOff5WmUkialLs0HXwUycacxIJlygMTvGnl3au6UnAxdj9KyF9+n+HQpRRLZqE0aExPQEBxvnshlwl/mg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-linux-arm64-gnu": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-linux-arm64-gnu/-/jieba-linux-arm64-gnu-2.0.2.tgz",
|
||||
"integrity": "sha512-tM0Gdh37ZhHpol8O3REGaWILdPYYoXfDuw+XfXhq1ppNbfwG5fm+DmoW7n4qg4nQM7jGM2t67BbmlENwxiAlCQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-linux-arm64-musl": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-linux-arm64-musl/-/jieba-linux-arm64-musl-2.0.2.tgz",
|
||||
"integrity": "sha512-7BrJjtsiuHdKzaVedWusiMzS9dUJXPTZBAunuRly/dyTvQyJqRJ3/N6RdgNRXaF49zuEz8JrtA9+aCn0t+ajTg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-linux-x64-gnu": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-linux-x64-gnu/-/jieba-linux-x64-gnu-2.0.2.tgz",
|
||||
"integrity": "sha512-514+0NFGCZp2e9lrnVRmyfe1/Cd+zUV3IUD0pWYB2gHfmowNrHPHL95WZosJMDp96C6LYbJZuNTRKifm0PyVBA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-linux-x64-musl": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-linux-x64-musl/-/jieba-linux-x64-musl-2.0.2.tgz",
|
||||
"integrity": "sha512-KFlbnGoGoX58qLKeWutPyupTbbYPGgC4DAGVjEu8ctYVh4UyjeRHCxlY7ipC7NIW8I3YzKQRZ/yFMzmiDYX7UQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-win32-arm64-msvc": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-win32-arm64-msvc/-/jieba-win32-arm64-msvc-2.0.2.tgz",
|
||||
"integrity": "sha512-EI3JLL01kf6pP3mAUoYnDrt8S+FY2W2nWpjtiK7ucs46tITLkYDx9wmAwDsOnoCkxs4CETzLytPhLZ12lzR5Qw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-win32-ia32-msvc": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-win32-ia32-msvc/-/jieba-win32-ia32-msvc-2.0.2.tgz",
|
||||
"integrity": "sha512-FtTx1cth53zZqdZTCsTDTlg0rTGDvFZyaMCLEa3FcjrDD8DeWNavNcJbLmProS9YlZPdCvArPzH/etaoiTKimA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@node-rs/jieba-win32-x64-msvc": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@node-rs/jieba-win32-x64-msvc/-/jieba-win32-x64-msvc-2.0.2.tgz",
|
||||
"integrity": "sha512-9c08mSvOoluteKy0AiyAy+x4uvWB45Q3fheg51gcIxYGxl+Lk0+Xq8JnQxs/f6nLH+mNaOWAmX4DtORQMWabxA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
@@ -28766,6 +29002,7 @@
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@agentclientprotocol/sdk": "^0.14.1",
|
||||
"@node-rs/jieba": "2.0.2",
|
||||
"@qwen-code/sdk": "file:../sdk-typescript",
|
||||
"ansi-regex": "^6.2.2",
|
||||
"prompts": "^2.4.2",
|
||||
@@ -28784,7 +29021,7 @@
|
||||
"vitest": "^3.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
"node": ">=22.13"
|
||||
}
|
||||
},
|
||||
"packages/qwen-live/node_modules/@types/node": {
|
||||
|
||||
@@ -16,6 +16,8 @@ import {
|
||||
LIVE_HOST_BUNDLE_ID,
|
||||
LIVE_HOST_PROTOCOL_VERSION,
|
||||
LIVE_INPUT_AUDIO_EPOCH_BYTES,
|
||||
LIVE_OUTPUT_AUDIO_EPOCH_BYTES,
|
||||
LIVE_OUTPUT_AUDIO_HEADER_BYTES,
|
||||
type LiveDaemonMessage,
|
||||
type LiveHostHello,
|
||||
} from './types.js';
|
||||
@@ -60,9 +62,25 @@ class FakeSocket extends EventEmitter {
|
||||
.filter((value): value is string => typeof value === 'string')
|
||||
.map((value) => JSON.parse(value) as LiveDaemonMessage);
|
||||
}
|
||||
|
||||
outputFrames(): Array<{ epoch: number; outputId: number; audio: Buffer }> {
|
||||
return this.sent
|
||||
.filter((value): value is Uint8Array => typeof value !== 'string')
|
||||
.map((value) => {
|
||||
const frame = Buffer.from(value);
|
||||
return {
|
||||
epoch: Number(frame.readBigUInt64BE(0)),
|
||||
outputId: Number(
|
||||
frame.readBigUInt64BE(LIVE_OUTPUT_AUDIO_EPOCH_BYTES),
|
||||
),
|
||||
audio: frame.subarray(LIVE_OUTPUT_AUDIO_HEADER_BYTES),
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const coordinators: LiveHostCoordinator[] = [];
|
||||
const TEST_JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64');
|
||||
|
||||
function readyHello(overrides: Partial<LiveHostHello> = {}): LiveHostHello {
|
||||
return {
|
||||
@@ -73,6 +91,7 @@ function readyHello(overrides: Partial<LiveHostHello> = {}): LiveHostHello {
|
||||
instanceNonce: 'host_instance_nonce_0001',
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -221,7 +240,7 @@ describe('LiveHostCoordinator', () => {
|
||||
expect(
|
||||
socket
|
||||
.messages()
|
||||
.some((message) => message.type === 'host.capture_screen_context'),
|
||||
.some((message) => message.type === 'host.capture_visual'),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
@@ -297,24 +316,29 @@ describe('LiveHostCoordinator', () => {
|
||||
sessionId: 'coordinator-1',
|
||||
});
|
||||
|
||||
await expect(value.captureScreenContext('worker-1')).rejects.toThrow(
|
||||
await expect(value.captureVisualContext('worker-1')).rejects.toThrow(
|
||||
'active Live session',
|
||||
);
|
||||
const capture = value.captureScreenContext('coordinator-1');
|
||||
const capture = value.captureVisualContext('coordinator-1');
|
||||
const request = socket
|
||||
.messages()
|
||||
.find((message) => message.type === 'host.capture_screen_context');
|
||||
.find((message) => message.type === 'host.capture_visual');
|
||||
expect(request).toMatchObject({
|
||||
type: 'host.capture_screen_context',
|
||||
type: 'host.capture_visual',
|
||||
epoch: call.epoch,
|
||||
source: 'screen',
|
||||
});
|
||||
if (!request || request.type !== 'host.capture_screen_context') {
|
||||
if (!request || request.type !== 'host.capture_visual') {
|
||||
throw new Error('Missing Appshot request');
|
||||
}
|
||||
socket.receive({
|
||||
type: 'host.screen_context_result',
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: request.requestId,
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image: TEST_JPEG,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
appName: 'Google Chrome',
|
||||
windowTitle: 'LIVE_APP_A',
|
||||
accessibilityText: 'AXWindow LIVE_APP_A',
|
||||
@@ -375,17 +399,21 @@ describe('LiveHostCoordinator', () => {
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
const capture = value.captureScreenContext('coordinator-1');
|
||||
const capture = value.captureVisualContext('coordinator-1');
|
||||
const request = socket
|
||||
.messages()
|
||||
.find((message) => message.type === 'host.capture_screen_context');
|
||||
if (!request || request.type !== 'host.capture_screen_context') {
|
||||
.find((message) => message.type === 'host.capture_visual');
|
||||
if (!request || request.type !== 'host.capture_visual') {
|
||||
throw new Error('Missing Appshot request');
|
||||
}
|
||||
socket.receive({
|
||||
type: 'host.screen_context_result',
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: request.requestId,
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image: TEST_JPEG,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
appName: 'TextEdit',
|
||||
accessibilityText: 'APPSHOT-MARKER-AMBER-4827',
|
||||
screenshotPath: '/private/tmp/qwen-live-appshot/test.png',
|
||||
@@ -410,7 +438,7 @@ describe('LiveHostCoordinator', () => {
|
||||
workspaceCwd: '/conversations/live-1',
|
||||
sessionId: 'coordinator-1',
|
||||
});
|
||||
const capture = value.captureScreenContext('coordinator-1');
|
||||
const capture = value.captureVisualContext('coordinator-1');
|
||||
const settled = capture.catch((error: unknown) => error);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
@@ -527,6 +555,42 @@ describe('LiveHostCoordinator', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('reports a pre-camera Host hello as an incompatible protocol', () => {
|
||||
const value = coordinator();
|
||||
const socket = new FakeSocket();
|
||||
value.attachHost(socket as unknown as WebSocket, value.daemonInstanceNonce);
|
||||
const legacyHello = readyHello({ protocolVersion: 7 }) as unknown as {
|
||||
permissions: Record<string, unknown>;
|
||||
};
|
||||
delete legacyHello.permissions['camera'];
|
||||
|
||||
socket.receive(legacyHello);
|
||||
|
||||
expect(socket.closeCode).toBe(4006);
|
||||
expect(value.getStatus()).toMatchObject({
|
||||
available: false,
|
||||
blocker: 'host_version',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a v9 Host hello that omits camera readiness', () => {
|
||||
const value = coordinator();
|
||||
const socket = new FakeSocket();
|
||||
value.attachHost(socket as unknown as WebSocket, value.daemonInstanceNonce);
|
||||
const invalidHello = readyHello() as unknown as {
|
||||
permissions: Record<string, unknown>;
|
||||
};
|
||||
delete invalidHello.permissions['camera'];
|
||||
|
||||
socket.receive(invalidHello);
|
||||
|
||||
expect(socket.closeCode).toBe(1002);
|
||||
expect(value.getStatus()).toMatchObject({
|
||||
available: false,
|
||||
blocker: 'host_missing',
|
||||
});
|
||||
});
|
||||
|
||||
it('welcomes one compatible, fully-authorized Host', () => {
|
||||
const value = coordinator();
|
||||
const socket = connectReady(value);
|
||||
@@ -839,8 +903,8 @@ describe('LiveHostCoordinator', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects the removed permission and session actions', () => {
|
||||
const removedActions = [
|
||||
it('rejects removed Host messages', () => {
|
||||
const removedMessages = [
|
||||
{
|
||||
type: 'host.action',
|
||||
action: 'request_permission',
|
||||
@@ -851,13 +915,19 @@ describe('LiveHostCoordinator', () => {
|
||||
action: 'open_session',
|
||||
locator: { workspaceCwd: '/work/one', sessionId: 'session-1' },
|
||||
},
|
||||
{
|
||||
type: 'host.screen_context_result',
|
||||
requestId: 'capture-1',
|
||||
success: false,
|
||||
error: 'removed',
|
||||
},
|
||||
];
|
||||
|
||||
for (const action of removedActions) {
|
||||
for (const message of removedMessages) {
|
||||
const value = coordinator();
|
||||
const socket = connectReady(value);
|
||||
|
||||
socket.receive(action);
|
||||
socket.receive(message);
|
||||
|
||||
expect(socket.closeCode).toBe(1002);
|
||||
expect(socket.messages()).toContainEqual({
|
||||
@@ -888,6 +958,27 @@ describe('LiveHostCoordinator', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('frames one output generation until playback is cleared', () => {
|
||||
const value = coordinator();
|
||||
const socket = connectReady(value);
|
||||
const call = value.start('resume');
|
||||
|
||||
expect(value.sendOutputAudio(call.epoch, Buffer.from([1, 0]))).toBe(true);
|
||||
expect(value.sendOutputAudio(call.epoch, Buffer.from([2, 0]))).toBe(true);
|
||||
expect(socket.outputFrames()).toEqual([
|
||||
{ epoch: call.epoch, outputId: 1, audio: Buffer.from([1, 0]) },
|
||||
{ epoch: call.epoch, outputId: 1, audio: Buffer.from([2, 0]) },
|
||||
]);
|
||||
|
||||
value.clearOutput(call.epoch);
|
||||
expect(value.sendOutputAudio(call.epoch, Buffer.from([3, 0]))).toBe(true);
|
||||
expect(socket.outputFrames().at(-1)).toEqual({
|
||||
epoch: call.epoch,
|
||||
outputId: 2,
|
||||
audio: Buffer.from([3, 0]),
|
||||
});
|
||||
});
|
||||
|
||||
it('forwards bounded PCM only for an active, unmuted call', () => {
|
||||
const onInputAudio = vi.fn();
|
||||
const value = coordinator({ handlers: { onInputAudio } });
|
||||
@@ -977,8 +1068,7 @@ describe('LiveHostCoordinator', () => {
|
||||
expect(onInputAudio).not.toHaveBeenCalled();
|
||||
|
||||
socket.receiveAudio(second.epoch, [2, 0]);
|
||||
expect(onInputAudio).toHaveBeenCalledOnce();
|
||||
expect(onInputAudio).toHaveBeenCalledWith({
|
||||
expect(onInputAudio).toHaveBeenCalledExactlyOnceWith({
|
||||
epoch: second.epoch,
|
||||
callId: second.callId,
|
||||
pcm16: Buffer.from([2, 0]),
|
||||
@@ -1118,8 +1208,7 @@ describe('LiveHostCoordinator', () => {
|
||||
requirements: { provider: 'checking', appshot: 'unavailable' },
|
||||
});
|
||||
expect(value.getStatus().callId).toBeUndefined();
|
||||
expect(onStop).toHaveBeenCalledOnce();
|
||||
expect(onStop).toHaveBeenCalledWith({
|
||||
expect(onStop).toHaveBeenCalledExactlyOnceWith({
|
||||
epoch: call.epoch,
|
||||
callId: call.callId,
|
||||
});
|
||||
@@ -1148,8 +1237,7 @@ describe('LiveHostCoordinator', () => {
|
||||
requirements: { provider: 'checking', screenRecording: 'denied' },
|
||||
});
|
||||
expect(value.getStatus().callId).toBeUndefined();
|
||||
expect(onStop).toHaveBeenCalledOnce();
|
||||
expect(onStop).toHaveBeenCalledWith({
|
||||
expect(onStop).toHaveBeenCalledExactlyOnceWith({
|
||||
epoch: call.epoch,
|
||||
callId: call.callId,
|
||||
});
|
||||
|
||||
@@ -11,12 +11,14 @@ import {
|
||||
LIVE_HOST_BUNDLE_ID,
|
||||
LIVE_HOST_PROTOCOL_VERSION,
|
||||
LIVE_INPUT_AUDIO_EPOCH_BYTES,
|
||||
LIVE_OUTPUT_AUDIO_EPOCH_BYTES,
|
||||
LIVE_OUTPUT_AUDIO_HEADER_BYTES,
|
||||
type LiveAppshotReadiness,
|
||||
type LiveDaemonMessage,
|
||||
type LiveHostAction,
|
||||
type LiveHostHello,
|
||||
type LiveHostShortcutResult,
|
||||
type LiveHostScreenContextResult,
|
||||
type LiveHostVisualCaptureResult,
|
||||
type LiveHostStatus,
|
||||
type LiveHostMessage,
|
||||
type LiveMuteUpdate,
|
||||
@@ -31,8 +33,11 @@ const DEFAULT_HELLO_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
|
||||
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 15_000;
|
||||
const DEFAULT_SHORTCUT_TIMEOUT_MS = 5_000;
|
||||
const MAX_HOST_TEXT_BYTES = 64 * 1024;
|
||||
const MAX_HOST_TEXT_BYTES = 512 * 1024;
|
||||
const MAX_HOST_AUDIO_BYTES = 64 * 1024;
|
||||
const MAX_HOST_VISUAL_IMAGE_BYTES = 190 * 1024;
|
||||
const MAX_HOST_VISUAL_BASE64_LENGTH =
|
||||
Math.ceil(MAX_HOST_VISUAL_IMAGE_BYTES / 3) * 4;
|
||||
const MAX_HOST_AUDIO_WIRE_BYTES =
|
||||
LIVE_INPUT_AUDIO_EPOCH_BYTES + MAX_HOST_AUDIO_BYTES;
|
||||
const MAX_DAEMON_AUDIO_BYTES = 256 * 1024;
|
||||
@@ -114,7 +119,7 @@ export interface LiveHostCoordinatorOptions {
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export interface LiveScreenContextCapture {
|
||||
export interface LiveVisualCapture {
|
||||
appName: string;
|
||||
windowTitle?: string;
|
||||
accessibilityText: string;
|
||||
@@ -124,7 +129,7 @@ export interface LiveScreenContextCapture {
|
||||
interface PendingAppshot {
|
||||
epoch: number;
|
||||
timer: NodeJS.Timeout;
|
||||
resolve: (capture: LiveScreenContextCapture) => void;
|
||||
resolve: (capture: LiveVisualCapture) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
@@ -155,6 +160,10 @@ function isBoundedString(value: unknown, maxLength = MAX_ID_LENGTH): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isNonNegativeSafeInteger(value: unknown): value is number {
|
||||
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
function isPermissionState(value: unknown): value is LivePermissionState {
|
||||
return (
|
||||
value === 'granted' || value === 'denied' || value === 'not_determined'
|
||||
@@ -162,17 +171,25 @@ function isPermissionState(value: unknown): value is LivePermissionState {
|
||||
}
|
||||
|
||||
function parseHello(value: Record<string, unknown>): LiveHostHello | undefined {
|
||||
const protocolVersion = value['protocolVersion'];
|
||||
const permissions = value['permissions'];
|
||||
const selfChecks = value['selfChecks'];
|
||||
const cameraPermission = isObject(permissions)
|
||||
? permissions['camera']
|
||||
: undefined;
|
||||
const legacyHelloWithoutCamera =
|
||||
protocolVersion !== LIVE_HOST_PROTOCOL_VERSION &&
|
||||
cameraPermission === undefined;
|
||||
if (
|
||||
value['type'] !== 'host.hello' ||
|
||||
typeof value['protocolVersion'] !== 'number' ||
|
||||
!Number.isInteger(value['protocolVersion']) ||
|
||||
typeof protocolVersion !== 'number' ||
|
||||
!Number.isInteger(protocolVersion) ||
|
||||
!isBoundedString(value['hostVersion'], MAX_VERSION_LENGTH) ||
|
||||
!isBoundedString(value['bundleId']) ||
|
||||
!isBoundedString(value['instanceNonce']) ||
|
||||
!isObject(permissions) ||
|
||||
!isPermissionState(permissions['microphone']) ||
|
||||
(!isPermissionState(cameraPermission) && !legacyHelloWithoutCamera) ||
|
||||
!isPermissionState(permissions['accessibility']) ||
|
||||
!isPermissionState(permissions['screenRecording']) ||
|
||||
!isObject(selfChecks) ||
|
||||
@@ -183,7 +200,15 @@ function parseHello(value: Record<string, unknown>): LiveHostHello | undefined {
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return value as unknown as LiveHostHello;
|
||||
return {
|
||||
...(value as unknown as LiveHostHello),
|
||||
permissions: {
|
||||
...permissions,
|
||||
camera: isPermissionState(cameraPermission)
|
||||
? cameraPermission
|
||||
: 'not_determined',
|
||||
} as LiveHostHello['permissions'],
|
||||
};
|
||||
}
|
||||
|
||||
function parseAction(
|
||||
@@ -226,6 +251,84 @@ function parseAction(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isBoundedVisualImage(image: unknown): image is string {
|
||||
if (
|
||||
typeof image !== 'string' ||
|
||||
image.length === 0 ||
|
||||
image.length > MAX_HOST_VISUAL_BASE64_LENGTH ||
|
||||
image.length % 4 !== 0 ||
|
||||
!/^[A-Za-z0-9+/]*={0,2}$/.test(image)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const jpeg = Buffer.from(image, 'base64');
|
||||
return (
|
||||
jpeg.byteLength >= 4 &&
|
||||
jpeg.byteLength <= MAX_HOST_VISUAL_IMAGE_BYTES &&
|
||||
jpeg[0] === 0xff &&
|
||||
jpeg[1] === 0xd8 &&
|
||||
jpeg[jpeg.byteLength - 2] === 0xff &&
|
||||
jpeg[jpeg.byteLength - 1] === 0xd9 &&
|
||||
jpeg.toString('base64') === image
|
||||
);
|
||||
}
|
||||
|
||||
function parseVisualCaptureResult(
|
||||
value: Record<string, unknown>,
|
||||
): LiveHostVisualCaptureResult | undefined {
|
||||
const requestId = value['requestId'];
|
||||
if (
|
||||
value['type'] !== 'host.visual_capture_result' ||
|
||||
!isBoundedString(requestId)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (value['success'] === false && isBoundedString(value['error'], 1_024)) {
|
||||
return {
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: requestId as string,
|
||||
success: false,
|
||||
error: value['error'] as string,
|
||||
};
|
||||
}
|
||||
const width = value['width'];
|
||||
const height = value['height'];
|
||||
if (
|
||||
value['success'] !== true ||
|
||||
value['source'] !== 'screen' ||
|
||||
!isBoundedVisualImage(value['image']) ||
|
||||
typeof width !== 'number' ||
|
||||
!Number.isSafeInteger(width) ||
|
||||
width <= 0 ||
|
||||
typeof height !== 'number' ||
|
||||
!Number.isSafeInteger(height) ||
|
||||
height <= 0 ||
|
||||
!isBoundedString(value['appName'], 512) ||
|
||||
(value['windowTitle'] !== undefined &&
|
||||
!isBoundedString(value['windowTitle'], 2_048)) ||
|
||||
typeof value['accessibilityText'] !== 'string' ||
|
||||
value['accessibilityText'].length > MAX_APPSHOT_TEXT_LENGTH ||
|
||||
!isBoundedString(value['screenshotPath'], 4_096)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: requestId as string,
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image: value['image'],
|
||||
width,
|
||||
height,
|
||||
appName: value['appName'] as string,
|
||||
...(value['windowTitle']
|
||||
? { windowTitle: value['windowTitle'] as string }
|
||||
: {}),
|
||||
accessibilityText: value['accessibilityText'],
|
||||
screenshotPath: value['screenshotPath'] as string,
|
||||
};
|
||||
}
|
||||
|
||||
function parseHostMessage(text: string): LiveHostMessage | undefined {
|
||||
let value: unknown;
|
||||
try {
|
||||
@@ -236,6 +339,9 @@ function parseHostMessage(text: string): LiveHostMessage | undefined {
|
||||
if (!isObject(value)) return undefined;
|
||||
if (value['type'] === 'host.hello') return parseHello(value);
|
||||
if (value['type'] === 'host.action') return parseAction(value);
|
||||
if (value['type'] === 'host.visual_capture_result') {
|
||||
return parseVisualCaptureResult(value);
|
||||
}
|
||||
if (value['type'] === 'host.pong' && isBoundedString(value['pingId'])) {
|
||||
return { type: 'host.pong', pingId: value['pingId'] as string };
|
||||
}
|
||||
@@ -261,50 +367,27 @@ function parseHostMessage(text: string): LiveHostMessage | undefined {
|
||||
};
|
||||
}
|
||||
}
|
||||
if (value['type'] === 'host.screen_context_result') {
|
||||
const requestId = value['requestId'];
|
||||
if (!isBoundedString(requestId)) return undefined;
|
||||
if (value['success'] === false && isBoundedString(value['error'], 1_024)) {
|
||||
return {
|
||||
type: 'host.screen_context_result',
|
||||
requestId: requestId as string,
|
||||
success: false,
|
||||
error: value['error'] as string,
|
||||
};
|
||||
}
|
||||
if (
|
||||
value['success'] === true &&
|
||||
isBoundedString(value['appName'], 512) &&
|
||||
(value['windowTitle'] === undefined ||
|
||||
isBoundedString(value['windowTitle'], 2_048)) &&
|
||||
typeof value['accessibilityText'] === 'string' &&
|
||||
value['accessibilityText'].length <= MAX_APPSHOT_TEXT_LENGTH &&
|
||||
isBoundedString(value['screenshotPath'], 4_096)
|
||||
) {
|
||||
return {
|
||||
type: 'host.screen_context_result',
|
||||
requestId: requestId as string,
|
||||
success: true,
|
||||
appName: value['appName'] as string,
|
||||
...(value['windowTitle']
|
||||
? { windowTitle: value['windowTitle'] as string }
|
||||
: {}),
|
||||
accessibilityText: value['accessibilityText'],
|
||||
screenshotPath: value['screenshotPath'] as string,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (
|
||||
value['type'] === 'host.playback_started' &&
|
||||
typeof value['epoch'] === 'number'
|
||||
isNonNegativeSafeInteger(value['epoch']) &&
|
||||
isNonNegativeSafeInteger(value['outputId'])
|
||||
) {
|
||||
return { type: 'host.playback_started', epoch: value['epoch'] };
|
||||
return {
|
||||
type: 'host.playback_started',
|
||||
epoch: value['epoch'],
|
||||
outputId: value['outputId'],
|
||||
};
|
||||
}
|
||||
if (
|
||||
value['type'] === 'host.playback_completed' &&
|
||||
typeof value['epoch'] === 'number'
|
||||
isNonNegativeSafeInteger(value['epoch']) &&
|
||||
isNonNegativeSafeInteger(value['outputId'])
|
||||
) {
|
||||
return { type: 'host.playback_completed', epoch: value['epoch'] };
|
||||
return {
|
||||
type: 'host.playback_completed',
|
||||
epoch: value['epoch'],
|
||||
outputId: value['outputId'],
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -366,6 +449,8 @@ export class LiveHostCoordinator {
|
||||
private pendingStartMode?: 'new';
|
||||
private actionGeneration = 0;
|
||||
private nextEpoch = 0;
|
||||
private nextOutputId = 0;
|
||||
private activeOutputId?: number;
|
||||
private inputMuted = false;
|
||||
private outputMuted = false;
|
||||
private lastCallError?: string;
|
||||
@@ -775,9 +860,7 @@ export class LiveHostCoordinator {
|
||||
);
|
||||
}
|
||||
|
||||
captureScreenContext(
|
||||
callerSessionId: string,
|
||||
): Promise<LiveScreenContextCapture> {
|
||||
captureVisualContext(callerSessionId: string): Promise<LiveVisualCapture> {
|
||||
const call = this.call;
|
||||
const host = this.host;
|
||||
if (
|
||||
@@ -796,7 +879,7 @@ export class LiveHostCoordinator {
|
||||
);
|
||||
}
|
||||
const requestId = randomUUID();
|
||||
return new Promise<LiveScreenContextCapture>((resolve, reject) => {
|
||||
return new Promise<LiveVisualCapture>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingAppshots.delete(requestId);
|
||||
reject(new Error('Live Host Appshot timed out.'));
|
||||
@@ -810,9 +893,10 @@ export class LiveHostCoordinator {
|
||||
});
|
||||
if (
|
||||
!this.sendHost({
|
||||
type: 'host.capture_screen_context',
|
||||
type: 'host.capture_visual',
|
||||
requestId,
|
||||
epoch: call.epoch,
|
||||
source: 'screen',
|
||||
})
|
||||
) {
|
||||
this.rejectPendingAppshot(
|
||||
@@ -862,9 +946,18 @@ export class LiveHostCoordinator {
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
socket.send(pcm16, { binary: true });
|
||||
const outputId = this.activeOutputId ?? this.allocateOutputId();
|
||||
const frame = Buffer.allocUnsafe(
|
||||
LIVE_OUTPUT_AUDIO_HEADER_BYTES + pcm16.byteLength,
|
||||
);
|
||||
frame.writeBigUInt64BE(BigInt(epoch), 0);
|
||||
frame.writeBigUInt64BE(BigInt(outputId), LIVE_OUTPUT_AUDIO_EPOCH_BYTES);
|
||||
frame.set(pcm16, LIVE_OUTPUT_AUDIO_HEADER_BYTES);
|
||||
socket.send(frame, { binary: true });
|
||||
this.activeOutputId = outputId;
|
||||
writeLiveHostDiagnostic('output_audio_sent', {
|
||||
epoch,
|
||||
outputId,
|
||||
bytes: pcm16.byteLength,
|
||||
socketBufferedBytes: socket.bufferedAmount,
|
||||
});
|
||||
@@ -873,6 +966,7 @@ export class LiveHostCoordinator {
|
||||
|
||||
clearOutput(epoch: number): void {
|
||||
if (this.call && this.call.epoch !== epoch) return;
|
||||
this.activeOutputId = undefined;
|
||||
writeLiveHostDiagnostic('clear_output_sent', { epoch });
|
||||
this.sendHost({ type: 'host.clear_output', epoch });
|
||||
}
|
||||
@@ -956,6 +1050,7 @@ export class LiveHostCoordinator {
|
||||
host_disconnected: 'Qwen Live Host disconnected.',
|
||||
host_version: 'Qwen Live Host is not protocol-compatible.',
|
||||
microphone_permission: 'Microphone permission is required.',
|
||||
camera_permission: 'Camera permission is required.',
|
||||
accessibility_permission: 'Accessibility permission is required.',
|
||||
screen_recording_permission: 'Screen Recording permission is required.',
|
||||
audio_input: 'Live Host audio input self-check failed.',
|
||||
@@ -1006,20 +1101,22 @@ export class LiveHostCoordinator {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.type === 'host.screen_context_result') {
|
||||
this.handleScreenContextResult(message);
|
||||
if (message.type === 'host.visual_capture_result') {
|
||||
this.handleVisualCaptureResult(message);
|
||||
return;
|
||||
}
|
||||
if (message.type === 'host.shortcut_result') {
|
||||
this.handleShortcutResult(message);
|
||||
return;
|
||||
}
|
||||
// v7 playback receipts: accepted but not forwarded to the built-in
|
||||
// Live session coordinator (which uses byte estimation). The
|
||||
// standalone qwen-live daemon wires these to its injector.
|
||||
// Standalone extensions are not advertised by the built-in Live daemon
|
||||
// and are unreachable because this parser intentionally omits them.
|
||||
if (
|
||||
message.type === 'host.playback_started' ||
|
||||
message.type === 'host.playback_completed'
|
||||
message.type === 'host.playback_completed' ||
|
||||
message.type === 'host.visual_frame' ||
|
||||
message.type === 'host.visual_settings' ||
|
||||
message.type === 'host.memory_action'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
@@ -1051,8 +1148,8 @@ export class LiveHostCoordinator {
|
||||
pending.resolve(status);
|
||||
}
|
||||
|
||||
private handleScreenContextResult(
|
||||
message: LiveHostScreenContextResult,
|
||||
private handleVisualCaptureResult(
|
||||
message: LiveHostVisualCaptureResult,
|
||||
): void {
|
||||
const pending = this.pendingAppshots.get(message.requestId);
|
||||
if (!pending) return;
|
||||
@@ -1070,6 +1167,18 @@ export class LiveHostCoordinator {
|
||||
pending.reject(new Error(message.error));
|
||||
return;
|
||||
}
|
||||
if (message.source !== 'screen') {
|
||||
pending.reject(
|
||||
new Error('Qwen Live Host returned a non-screen Appshot.'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!message.screenshotPath) {
|
||||
pending.reject(
|
||||
new Error('Qwen Live Host did not persist the requested Appshot.'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
pending.resolve({
|
||||
appName: message.appName,
|
||||
...(message.windowTitle ? { windowTitle: message.windowTitle } : {}),
|
||||
@@ -1085,8 +1194,8 @@ export class LiveHostCoordinator {
|
||||
(lease.hello && lease.hello.instanceNonce !== hello.instanceNonce)
|
||||
) {
|
||||
this.lastHostFailure = 'host_version';
|
||||
lease.socket.close(4006, 'Incompatible Live Host.');
|
||||
this.detachHost(lease, 'host_version');
|
||||
lease.socket.close(4006, 'Incompatible Live Host.');
|
||||
return;
|
||||
}
|
||||
lease.hello = hello;
|
||||
@@ -1394,6 +1503,14 @@ export class LiveHostCoordinator {
|
||||
return true;
|
||||
}
|
||||
|
||||
private allocateOutputId(): number {
|
||||
if (this.nextOutputId >= Number.MAX_SAFE_INTEGER) {
|
||||
this.nextOutputId = 0;
|
||||
}
|
||||
this.nextOutputId += 1;
|
||||
return this.nextOutputId;
|
||||
}
|
||||
|
||||
private notifyInactive(): void {
|
||||
if (this.call) return;
|
||||
for (const resolve of this.inactiveWaiters) resolve();
|
||||
|
||||
@@ -268,7 +268,10 @@ describe('LiveHostInstaller', () => {
|
||||
});
|
||||
|
||||
it('launches an existing verified installation without downloading', async () => {
|
||||
const inspectInstalled = vi.fn(async () => ({ version: '0.1.0' }));
|
||||
const inspectInstalled = vi.fn(async () => ({
|
||||
version: '0.1.0',
|
||||
protocolVersion: LIVE_HOST_PROTOCOL_VERSION,
|
||||
}));
|
||||
const installLatest = vi.fn();
|
||||
const launch = vi.fn(async () => {});
|
||||
const installer = new LiveHostInstaller({
|
||||
@@ -287,15 +290,43 @@ describe('LiveHostInstaller', () => {
|
||||
expect(launch).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('replaces an installed Host with an incompatible protocol', async () => {
|
||||
const installLatest = vi.fn(async () => ({
|
||||
version: '0.2.0',
|
||||
protocolVersion: LIVE_HOST_PROTOCOL_VERSION,
|
||||
}));
|
||||
const installer = new LiveHostInstaller({
|
||||
platform: 'darwin',
|
||||
architecture: 'arm64',
|
||||
inspectInstalled: async () => ({
|
||||
version: '0.1.0',
|
||||
protocolVersion: LIVE_HOST_PROTOCOL_VERSION - 1,
|
||||
}),
|
||||
installLatest,
|
||||
launch: async () => {},
|
||||
});
|
||||
|
||||
await expect(installer.ensureInstalled()).resolves.toEqual({
|
||||
state: 'installed',
|
||||
version: '0.2.0',
|
||||
});
|
||||
expect(installLatest).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('coalesces concurrent installs and exposes progress', async () => {
|
||||
let finish: ((value: { version: string }) => void) | undefined;
|
||||
let finish:
|
||||
| ((value: { version: string; protocolVersion: number }) => void)
|
||||
| undefined;
|
||||
const installLatest = vi.fn(
|
||||
async (
|
||||
_architecture: 'arm64' | 'x64',
|
||||
onStatus: (status: { state: 'downloading'; progress: number }) => void,
|
||||
) => {
|
||||
onStatus({ state: 'downloading', progress: 0.5 });
|
||||
return await new Promise<{ version: string }>((resolve) => {
|
||||
return await new Promise<{
|
||||
version: string;
|
||||
protocolVersion: number;
|
||||
}>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
},
|
||||
@@ -316,7 +347,10 @@ describe('LiveHostInstaller', () => {
|
||||
progress: 0.5,
|
||||
});
|
||||
});
|
||||
finish?.({ version: '0.1.0' });
|
||||
finish?.({
|
||||
version: '0.1.0',
|
||||
protocolVersion: LIVE_HOST_PROTOCOL_VERSION,
|
||||
});
|
||||
await expect(first).resolves.toMatchObject({ state: 'installed' });
|
||||
await expect(second).resolves.toMatchObject({ state: 'installed' });
|
||||
expect(installLatest).toHaveBeenCalledOnce();
|
||||
|
||||
@@ -70,6 +70,7 @@ export interface LiveHostReleaseManifest {
|
||||
|
||||
interface InstalledLiveHost {
|
||||
version: string;
|
||||
protocolVersion: number;
|
||||
}
|
||||
|
||||
export interface LiveHostInstallerDeps {
|
||||
@@ -183,6 +184,11 @@ async function inspectApp(appPath: string): Promise<InstalledLiveHost> {
|
||||
if (!VERSION_PATTERN.test(version)) {
|
||||
throw new Error('Qwen Live Host version is invalid.');
|
||||
}
|
||||
const rawProtocolVersion = await readBundleValue(
|
||||
appPath,
|
||||
'QwenLiveProtocolVersion',
|
||||
).catch(() => '0');
|
||||
const protocolVersion = Number(rawProtocolVersion);
|
||||
await run('/usr/bin/codesign', [
|
||||
'--verify',
|
||||
'--deep',
|
||||
@@ -204,7 +210,16 @@ async function inspectApp(appPath: string): Promise<InstalledLiveHost> {
|
||||
throw new Error('Qwen Live Host signing identity is invalid.');
|
||||
}
|
||||
await run('/usr/sbin/spctl', ['-a', '-t', 'exec', appPath]);
|
||||
return { version };
|
||||
return {
|
||||
version,
|
||||
protocolVersion: Number.isSafeInteger(protocolVersion)
|
||||
? protocolVersion
|
||||
: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function isCompatibleInstalledHost(host: InstalledLiveHost): boolean {
|
||||
return host.protocolVersion === LIVE_HOST_PROTOCOL_VERSION;
|
||||
}
|
||||
|
||||
async function inspectInstalledHost(): Promise<InstalledLiveHost | undefined> {
|
||||
@@ -360,7 +375,10 @@ async function installLatestHost(
|
||||
await fsp.mkdir(extractedPath, { mode: 0o700 });
|
||||
await run('/usr/bin/ditto', ['-x', '-k', archivePath, extractedPath]);
|
||||
const candidate = await inspectApp(candidatePath);
|
||||
if (candidate.version !== manifest.version) {
|
||||
if (
|
||||
candidate.version !== manifest.version ||
|
||||
!isCompatibleInstalledHost(candidate)
|
||||
) {
|
||||
throw new Error('Live Host package version does not match its manifest.');
|
||||
}
|
||||
onStatus({ state: 'installing', version: manifest.version });
|
||||
@@ -375,7 +393,10 @@ async function installLatestHost(
|
||||
await fsp.rename(stagingPath, LIVE_HOST_APP_PATH);
|
||||
installedCandidate = true;
|
||||
const installed = await inspectApp(LIVE_HOST_APP_PATH);
|
||||
if (installed.version !== manifest.version) {
|
||||
if (
|
||||
installed.version !== manifest.version ||
|
||||
!isCompatibleInstalledHost(installed)
|
||||
) {
|
||||
throw new Error('Installed Live Host version is invalid.');
|
||||
}
|
||||
if (movedExisting) {
|
||||
@@ -438,9 +459,10 @@ export class LiveHostInstaller {
|
||||
this.status = { state: 'checking' };
|
||||
try {
|
||||
const installed = await this.inspectInstalled();
|
||||
this.status = installed
|
||||
? { state: 'installed', version: installed.version }
|
||||
: { state: 'missing' };
|
||||
this.status =
|
||||
installed && isCompatibleInstalledHost(installed)
|
||||
? { state: 'installed', version: installed.version }
|
||||
: { state: 'missing' };
|
||||
} catch (error) {
|
||||
this.setError(errorMessage(error), true);
|
||||
}
|
||||
@@ -465,6 +487,12 @@ export class LiveHostInstaller {
|
||||
const installed = await this.inspectInstalled();
|
||||
if (!installed)
|
||||
return this.setError('Qwen Live Host is not installed.', true);
|
||||
if (!isCompatibleInstalledHost(installed)) {
|
||||
return this.setError(
|
||||
`Qwen Live Host protocol v${installed.protocolVersion} is incompatible; v${LIVE_HOST_PROTOCOL_VERSION} is required.`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
this.status = { state: 'launching', version: installed.version };
|
||||
await this.launchHost();
|
||||
this.status = { state: 'installed', version: installed.version };
|
||||
@@ -484,7 +512,9 @@ export class LiveHostInstaller {
|
||||
this.status = { state: 'checking' };
|
||||
const installed = force ? undefined : await this.inspectInstalled();
|
||||
const ready =
|
||||
installed ??
|
||||
(installed && isCompatibleInstalledHost(installed)
|
||||
? installed
|
||||
: undefined) ??
|
||||
(await this.installLatest(currentArchitecture, (status) => {
|
||||
this.status = { ...status };
|
||||
}));
|
||||
|
||||
@@ -9,6 +9,7 @@ import type { Settings } from '../../config/settings.js';
|
||||
import { LiveHostCoordinator } from './live-host-coordinator.js';
|
||||
import { LiveHostInstaller } from './live-host-installer.js';
|
||||
import { LiveSetupController } from './live-setup-controller.js';
|
||||
import { LIVE_HOST_PROTOCOL_VERSION } from './types.js';
|
||||
|
||||
function createHarness(options: { initiallyEnabled?: boolean } = {}) {
|
||||
const initiallyEnabled = options.initiallyEnabled ?? false;
|
||||
@@ -38,7 +39,10 @@ function createHarness(options: { initiallyEnabled?: boolean } = {}) {
|
||||
const setEnabled = vi.fn(async (next: boolean) => {
|
||||
enabled = next;
|
||||
});
|
||||
const installLatest = vi.fn(async () => ({ version: '0.1.0' }));
|
||||
const installLatest = vi.fn(async () => ({
|
||||
version: '0.1.0',
|
||||
protocolVersion: LIVE_HOST_PROTOCOL_VERSION,
|
||||
}));
|
||||
const installer = new LiveHostInstaller({
|
||||
platform: 'darwin',
|
||||
architecture: 'arm64',
|
||||
|
||||
@@ -4,9 +4,70 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
export const LIVE_HOST_PROTOCOL_VERSION = 7 as const;
|
||||
export const LIVE_HOST_PROTOCOL_VERSION = 9 as const;
|
||||
export const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host' as const;
|
||||
|
||||
export type LiveVisualSource = 'screen' | 'camera';
|
||||
export type LiveVisualMode = 'on-demand' | 'live-feed';
|
||||
|
||||
export interface LiveMemoryState {
|
||||
enabled: boolean;
|
||||
visualEnabled: boolean;
|
||||
libraryId: string;
|
||||
model: string;
|
||||
libraries: Array<{ id: string; name: string }>;
|
||||
locked: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export type LiveMemoryAction =
|
||||
| { action: 'set_enabled'; enabled: boolean }
|
||||
| { action: 'set_visual_enabled'; enabled: boolean }
|
||||
| { action: 'select'; libraryId: string }
|
||||
| { action: 'create'; name: string }
|
||||
| { action: 'rename'; libraryId: string; name: string }
|
||||
| { action: 'set_model'; model: string };
|
||||
|
||||
export type LiveHostMemoryAction = LiveMemoryAction & {
|
||||
type: 'host.memory_action';
|
||||
requestId: string;
|
||||
epoch: number;
|
||||
};
|
||||
|
||||
export type LiveMemoryResult =
|
||||
| {
|
||||
type: 'host.memory_result';
|
||||
requestId: string;
|
||||
ok: true;
|
||||
memory: LiveMemoryState;
|
||||
}
|
||||
| {
|
||||
type: 'host.memory_result';
|
||||
requestId: string;
|
||||
ok: false;
|
||||
error: string;
|
||||
memory?: LiveMemoryState;
|
||||
};
|
||||
|
||||
export interface LiveVisualInput {
|
||||
source: LiveVisualSource;
|
||||
mode: LiveVisualMode;
|
||||
screenDisplayId?: string;
|
||||
fps: number;
|
||||
cameraWidth?: number;
|
||||
cameraHeight?: number;
|
||||
cameraSnapshotWidth?: number;
|
||||
cameraSnapshotHeight?: number;
|
||||
liveWidth: number;
|
||||
liveHeight: number;
|
||||
snapshotWidth?: number;
|
||||
snapshotHeight?: number;
|
||||
}
|
||||
export const LIVE_INPUT_AUDIO_EPOCH_BYTES = 8;
|
||||
export const LIVE_OUTPUT_AUDIO_EPOCH_BYTES = 8;
|
||||
export const LIVE_OUTPUT_AUDIO_ID_BYTES = 8;
|
||||
export const LIVE_OUTPUT_AUDIO_HEADER_BYTES =
|
||||
LIVE_OUTPUT_AUDIO_EPOCH_BYTES + LIVE_OUTPUT_AUDIO_ID_BYTES;
|
||||
|
||||
export type LiveState =
|
||||
| 'unavailable'
|
||||
@@ -23,6 +84,7 @@ export type LiveBlocker =
|
||||
| 'host_disconnected'
|
||||
| 'host_version'
|
||||
| 'microphone_permission'
|
||||
| 'camera_permission'
|
||||
| 'accessibility_permission'
|
||||
| 'screen_recording_permission'
|
||||
| 'audio_input'
|
||||
@@ -66,6 +128,7 @@ export interface LiveStatus {
|
||||
Record<
|
||||
| 'host'
|
||||
| 'microphone'
|
||||
| 'camera'
|
||||
| 'accessibility'
|
||||
| 'screenRecording'
|
||||
| 'audioInput'
|
||||
@@ -88,12 +151,17 @@ export type LivePermissionState = 'granted' | 'denied' | 'not_determined';
|
||||
|
||||
export interface LiveHostHello {
|
||||
type: 'host.hello';
|
||||
displayCaptureV1?: true;
|
||||
protocolVersion: number;
|
||||
hostVersion: string;
|
||||
bundleId: string;
|
||||
instanceNonce: string;
|
||||
capabilities?: {
|
||||
outputAudioEndMarkerV1: true;
|
||||
};
|
||||
permissions: {
|
||||
microphone: LivePermissionState;
|
||||
camera: LivePermissionState;
|
||||
accessibility: LivePermissionState;
|
||||
screenRecording: LivePermissionState;
|
||||
};
|
||||
@@ -132,59 +200,149 @@ export interface LiveHostShortcutResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export type LiveHostScreenContextResult =
|
||||
| {
|
||||
type: 'host.screen_context_result';
|
||||
requestId: string;
|
||||
success: true;
|
||||
appName: string;
|
||||
windowTitle?: string;
|
||||
accessibilityText: string;
|
||||
screenshotPath: string;
|
||||
}
|
||||
| {
|
||||
type: 'host.screen_context_result';
|
||||
requestId: string;
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export interface LiveHostPlaybackStarted {
|
||||
type: 'host.playback_started';
|
||||
epoch: number;
|
||||
outputId: number;
|
||||
}
|
||||
|
||||
export interface LiveHostPlaybackCompleted {
|
||||
type: 'host.playback_completed';
|
||||
epoch: number;
|
||||
outputId: number;
|
||||
}
|
||||
|
||||
export interface LiveHostVisualFrame {
|
||||
type: 'host.visual_frame';
|
||||
epoch: number;
|
||||
source: LiveVisualSource;
|
||||
image: string;
|
||||
screenScope?: 'display';
|
||||
displayId?: string;
|
||||
}
|
||||
|
||||
export interface LiveHostVisualSettings {
|
||||
type: 'host.visual_settings';
|
||||
epoch: number;
|
||||
source: LiveVisualSource;
|
||||
mode: LiveVisualMode;
|
||||
screenDisplayId?: string;
|
||||
permissions: {
|
||||
camera: LivePermissionState;
|
||||
accessibility: LivePermissionState;
|
||||
screenRecording: LivePermissionState;
|
||||
};
|
||||
appshot: boolean;
|
||||
}
|
||||
|
||||
export type LiveHostVisualCaptureResult =
|
||||
| {
|
||||
type: 'host.visual_capture_result';
|
||||
requestId: string;
|
||||
success: true;
|
||||
source: 'screen';
|
||||
screenScope?: 'display';
|
||||
displayId?: string;
|
||||
image: string;
|
||||
width: number;
|
||||
height: number;
|
||||
appName: string;
|
||||
windowTitle?: string;
|
||||
accessibilityText: string;
|
||||
screenshotPath?: string;
|
||||
}
|
||||
| {
|
||||
type: 'host.visual_capture_result';
|
||||
requestId: string;
|
||||
success: true;
|
||||
source: 'camera';
|
||||
image: string;
|
||||
width: number;
|
||||
height: number;
|
||||
screenshotPath?: string;
|
||||
}
|
||||
| {
|
||||
type: 'host.visual_capture_result';
|
||||
requestId: string;
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type LiveLanguageState = { language: 'en' | 'zh-CN' };
|
||||
export type LiveHostLanguageAction = {
|
||||
type: 'host.language_action';
|
||||
requestId: string;
|
||||
epoch: number;
|
||||
language: LiveLanguageState['language'];
|
||||
};
|
||||
export type LiveLanguageResult =
|
||||
| {
|
||||
type: 'host.language_result';
|
||||
requestId: string;
|
||||
ok: true;
|
||||
uiLanguageV1: LiveLanguageState;
|
||||
}
|
||||
| {
|
||||
type: 'host.language_result';
|
||||
requestId: string;
|
||||
ok: false;
|
||||
error: string;
|
||||
uiLanguageV1?: LiveLanguageState;
|
||||
};
|
||||
|
||||
export type LiveHostMessage =
|
||||
| LiveHostHello
|
||||
| LiveHostAction
|
||||
| LiveHostMemoryAction
|
||||
| LiveHostPong
|
||||
| LiveHostShortcutResult
|
||||
| LiveHostScreenContextResult
|
||||
| LiveHostPlaybackStarted
|
||||
| LiveHostPlaybackCompleted;
|
||||
| LiveHostPlaybackCompleted
|
||||
| LiveHostVisualFrame
|
||||
| LiveHostVisualSettings
|
||||
| LiveHostVisualCaptureResult;
|
||||
|
||||
export type LiveDaemonMessage =
|
||||
| {
|
||||
type: 'host.welcome';
|
||||
protocolVersion: typeof LIVE_HOST_PROTOCOL_VERSION;
|
||||
daemonInstanceNonce: string;
|
||||
daemonShutdownV1?: true;
|
||||
displayCaptureV1?: true;
|
||||
uiLanguageV1?: LiveLanguageState;
|
||||
heartbeatIntervalMs: number;
|
||||
epoch: number;
|
||||
capabilities?: {
|
||||
outputAudioEndMarkerV1: true;
|
||||
};
|
||||
visualInput?: LiveVisualInput;
|
||||
memory?: LiveMemoryState;
|
||||
status: LiveHostStatus;
|
||||
}
|
||||
| { type: 'host.state'; epoch: number; status: LiveHostStatus }
|
||||
| {
|
||||
type: 'host.state';
|
||||
epoch: number;
|
||||
uiLanguageV1?: LiveLanguageState;
|
||||
visualInput?: LiveVisualInput;
|
||||
memory?: LiveMemoryState;
|
||||
status: LiveHostStatus;
|
||||
}
|
||||
| LiveMemoryResult
|
||||
| LiveLanguageResult
|
||||
| { type: 'host.ping'; pingId: string }
|
||||
| { type: 'host.clear_output'; epoch: number }
|
||||
| { type: 'host.output_audio_finished'; epoch: number; outputId: number }
|
||||
| { type: 'host.set_shortcut'; requestId: string; shortcut: string }
|
||||
| {
|
||||
type: 'host.capture_screen_context';
|
||||
type: 'host.capture_visual';
|
||||
requestId: string;
|
||||
epoch: number;
|
||||
source: LiveVisualSource;
|
||||
screenScope?: 'display';
|
||||
screenDisplayId?: string;
|
||||
snapshotWidth?: number;
|
||||
snapshotHeight?: number;
|
||||
persistAsset?: boolean;
|
||||
}
|
||||
| {
|
||||
type: 'host.error';
|
||||
|
||||
@@ -61,6 +61,7 @@ class FakeSocket extends EventEmitter {
|
||||
instanceNonce: 'host_instance_nonce_0001',
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
|
||||
@@ -42523,6 +42523,7 @@ class FakeLiveHostSocket extends EventEmitter {
|
||||
instanceNonce,
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -42678,7 +42679,7 @@ describe('Live Appshot server integration', () => {
|
||||
);
|
||||
const captureHandler = setup.captureHandler;
|
||||
expect(captureHandler).toEqual(expect.any(Function));
|
||||
const capture = vi.spyOn(setup.coordinator, 'captureScreenContext');
|
||||
const capture = vi.spyOn(setup.coordinator, 'captureVisualContext');
|
||||
const discovery = await import('./live/discovery.js');
|
||||
const assertPublisher = vi.spyOn(
|
||||
discovery,
|
||||
|
||||
@@ -1591,7 +1591,7 @@ export function createServeApp(
|
||||
liveBoundRuntime = runtime;
|
||||
try {
|
||||
setScreenHandler.call(runtime.bridge, ({ callerSessionId }) =>
|
||||
liveCoordinator.captureScreenContext(callerSessionId),
|
||||
liveCoordinator.captureVisualContext(callerSessionId),
|
||||
);
|
||||
setTaskHandler.call(runtime.bridge, (info) =>
|
||||
liveTaskService.handle(info),
|
||||
|
||||
+228
-27
@@ -1,18 +1,18 @@
|
||||
# Qwen Live Host
|
||||
|
||||
Qwen Live Host 是 WebShell Live Voice 在 macOS 上的必需组件。它只承载小浮层、
|
||||
Electron 全局快捷键、麦克风输入、扬声器输出和内置原生 Appshot。Host 不打开
|
||||
WebShell 或 Session 窗口；具体对话和任务仍在现有 WebShell 中跟进。没有浏览器
|
||||
Qwen Live Host 是独立 Qwen Live daemon 和 WebShell Live Voice 在 macOS 上的原生组件。它承载小浮层、
|
||||
Electron 全局快捷键、麦克风输入、可选摄像头输入、扬声器输出和内置原生 Appshot。Host 不打开
|
||||
WebShell 或 Session 窗口；对话由连接的 daemon 管理，编码任务由配置的后端执行。没有浏览器
|
||||
麦克风或浏览器快捷键降级方案。
|
||||
|
||||
## 用户要求
|
||||
|
||||
- macOS 12 或更高版本。
|
||||
- 本机运行的 Qwen Code WebShell。
|
||||
- 本机运行的独立 `qwen-live` daemon 或 Qwen Code WebShell。
|
||||
- 可调用 `qwen3.5-omni-plus-realtime` 的 DashScope API key。
|
||||
|
||||
Live Voice 仅在 macOS WebShell 中提供，默认关闭。普通 CLI/TUI、`--no-web`
|
||||
daemon 和其他操作系统不会显示入口。
|
||||
WebShell 内置 Live Voice 默认关闭；独立 `qwen-live` 可直接从命令行启动并连接 Host。
|
||||
原生音视频功能目前要求 macOS。
|
||||
|
||||
## 首次启用
|
||||
|
||||
@@ -23,8 +23,9 @@ daemon 和其他操作系统不会显示入口。
|
||||
镜像不可用时回退到独立的 GitHub `live-host-latest` feed。下载后校验 manifest、
|
||||
SHA-256、bundle identity、Developer ID 签名和 Gatekeeper，然后原子安装到
|
||||
`/Applications/Qwen Live Host.app` 并启动。
|
||||
4. 按 Host 引导完成麦克风、辅助功能和屏幕录制授权。授权只能由用户在 macOS
|
||||
完成；全部 readiness 通过前 Live 不可使用。
|
||||
4. 按 Host 引导完成麦克风以及当前视觉源需要的授权：Screen 需要辅助功能和屏幕
|
||||
录制，Camera 需要摄像头。授权只能由用户在 macOS 完成；当前 Source 的 readiness
|
||||
通过前 Live 不可使用。
|
||||
|
||||
API key 只写入用户级设置。WebShell 只能看到“已配置”状态，不会读取或回显 key。
|
||||
关闭 Live 会停止当前通话、撤下快捷键和 Host discovery，但不会卸载 Host 或删除
|
||||
@@ -74,12 +75,12 @@ locator。Host 只连接 loopback 地址并校验协议版本和 daemon nonce。
|
||||
bearer token，不要打印、复制或共享其内容。
|
||||
|
||||
Live 被禁用、discovery 不存在或 daemon 断开时，Host 的全局快捷键、音频和 Appshot
|
||||
readiness 保持 dormant。只有 v6 daemon 完成 welcome 后这些服务才启动；断开时会立即
|
||||
readiness 保持 dormant。只有 v9 daemon 完成 welcome 后这些服务才启动；断开时会立即
|
||||
清理音频 context 并解注册快捷键。
|
||||
|
||||
活跃 Live Session 的屏幕上下文只走 Session-local 的内置
|
||||
`capture_screen_context` 通道。它不会修改、隐藏或跳过普通 Qwen 工具及用户配置的
|
||||
MCP；Appshot 本身不依赖这些能力。
|
||||
活跃 Live Session 的视觉上下文只走 Session-local 的内置视觉通道。它不会修改、隐藏
|
||||
或跳过普通 Qwen 工具及用户配置的 MCP；Screen 捕获使用的 Appshot 本身也不依赖这些
|
||||
能力。
|
||||
|
||||
## 快捷键
|
||||
|
||||
@@ -89,33 +90,233 @@ Electron `globalShortcut` 注册普通 accelerator，不请求 Input Monitoring
|
||||
解注册旧值并持久化；冲突或非法值会保留旧快捷键并返回设置错误。退出或断开 daemon
|
||||
也会解注册当前 accelerator。
|
||||
|
||||
浮层和菜单栏中的“新对话”会显式创建新的无项目对话；开始、停止当前通话是独立动作。
|
||||
菜单栏中的“新对话”会显式创建新的无项目对话；开始、停止当前通话是独立动作。
|
||||
|
||||
## 内置 Appshot 和三项授权
|
||||
## 悬浮球与设置
|
||||
|
||||
| 权限 | 授权主体 | 用途 |
|
||||
| -------- | -------------- | ------------------------------- |
|
||||
| 麦克风 | Qwen Live Host | 采集 Live 对话音频 |
|
||||
| 辅助功能 | Qwen Live Host | 读取前台窗口的可访问性树 |
|
||||
| 屏幕录制 | Qwen Live Host | 为显式 Appshot 请求采集窗口图像 |
|
||||
初始化框和悬浮球首次分别按当前可见尺寸放在启动屏幕右下角，保留 20px 边距。
|
||||
拖动初始化框标题栏或小球可移动，
|
||||
位置保存到 Host 用户数据目录中的 `overlay-position.json`，下次启动恢复；显示器移除后
|
||||
会调整到可见区域。小球按动画、工具栏和字幕的紧凑区域限位，不再被初始化框的透明空白
|
||||
挡住。打开 Settings 或预览时会临时调整到完整可见的位置，关闭后恢复记忆位置；临时调整
|
||||
不会覆盖拖动记录。状态刷新不会重新定位窗口。
|
||||
|
||||
鼠标移到小球上显示麦克风、播报、`Start call`／`End call`、`Settings` 和 `Quit Host`
|
||||
按钮；移出后等待 1 秒淡出，移回或键盘聚焦会保持可用。工具栏位于球上方，为底部贴边
|
||||
留出空间。`Command+E` 启动／结束通话。
|
||||
Host 启动后会等待连接、当前来源权限和自检就绪，再自动开始一次交互；已有通话时不会
|
||||
重复启动。手动启停／新对话／退出、自动启动失败、重连或 renderer 重载均不会再次触发
|
||||
自动启动。重新启动 Host 才产生下一次自动启动意图。
|
||||
结束后小球变灰并留在原位，不再自动隐藏。小球动画、摄像头预览和设置控件保持挂载，
|
||||
状态／字幕变化不会重建它们；小球没有投影。半透明状态条位于小球下方，控制按钮悬浮在
|
||||
上方；圆形齿轮按钮打开 Settings，面板默认显示滚动条及轨道。
|
||||
麦克风关闭或扬声器静音时，状态条第二行显示 `Mic off`／`Speaker muted`（支持中文），
|
||||
同时保留第一行的通话状态或授权入口。麦克风静音会释放输入设备，取消静音后重新收音。
|
||||
|
||||
初始化页只处理连接、Source 和对应权限，不要求 Camera 用户先授权 Screen。
|
||||
日常设置集中在 `Settings`：Audio Source（麦克风）、Video Source（Screen／Camera）、
|
||||
Capture Mode（On Demand／Live Feed）三个同级设置组，以及独立 daemon 支持的 Memory。
|
||||
模式说明随已确认的选项更新。设置支持 Esc、外部点击关闭，编辑草稿保留。
|
||||
顶部的 `Open config.json ↗`／`打开 config.json ↗` 会使用系统为 JSON 文件关联的
|
||||
默认 IDE／文本编辑器，打开当前独立 daemon 实际使用的配置（默认
|
||||
`~/.qwen-live/config.json`，也支持 daemon 的 `QWEN_LIVE_DATA_DIR`）。保存后需重启
|
||||
Qwen Live 才应用手动修改。旧 daemon 或内置 `qwen serve` 不提供此入口能力；文件
|
||||
缺失、不是常规文件（包括符号链接）或编辑器打开失败时会提示，不自动创建或覆盖配置。
|
||||
设置标题栏可以拖动，与小球共享位置记忆；打开时先等待原生窗口完成屏内定位再显示，
|
||||
避免边缘处先露出被裁切的面板。用户说话的小音量视觉响应已增强，保留有界动画和缓慢
|
||||
回落，不会提高发送给模型的音频音量。
|
||||
|
||||
Settings 倒数第二组为 `Language`／`语言`（其后是 Theme），支持 `简体中文` 和 `English`。
|
||||
独立 daemon 确认后立即切换，并保存到 `~/.qwen-live/config.json` 顶层 `language`
|
||||
（`zh-CN`／`en`）；旧配置未设置时保持英文。通话中也可修改，不重连媒体、不丢编辑草稿。
|
||||
旧 WebShell 连接仅保存 Host 本地语言偏好，不写独立 Live 的配置。首次未连接时使用
|
||||
本地缓存，连接 standalone 后以 daemon 设置为准。语言不影响模型提示词／回答或用户
|
||||
自定义名称。`qwen-live init` 的第一项也可通过左右方向键选择语言，后续问题随之切换。
|
||||
|
||||
全部固定 Live 展示文案统一维护在
|
||||
[`packages/qwen-live/src/i18n/messages.ts`](../qwen-live/src/i18n/messages.ts)，
|
||||
每个键并列 `en` 和 `zh-CN`。修改后分别重建 Live 和 Host。Host 的构建别名直接编译
|
||||
同一份纯文本模块，打包后不需要 qwen-live 运行时依赖；开发构建仍应在完整仓库内进行。
|
||||
默认是 Screen + On Demand；配置决定每次 daemon 启动的初值，Source／Mode 切换只影响
|
||||
当前运行实例。
|
||||
|
||||
### 子智能体状态
|
||||
|
||||
悬停或用键盘聚焦小球时，侧面显示明确标注 `Subagents`／`子智能体` 的摘要：
|
||||
紧凑图标计数显示进行中和已完成；有运行任务时小点柔和闪烁（遵循系统减少动态效果设置），
|
||||
需要用户输入时才显示提醒标记。悬停提示与辅助功能标签保留完整计数。
|
||||
点击展开列表，再点击任务在同一个无边框悬浮面板中显示详情，查看原始委托、
|
||||
实际运行状态、最新活动、公开中间文本、可用的计划／工具更新及最终结果。
|
||||
`Back`／`返回` 回到列表；列表与详情的标题区均可拖动，返回时保留位置并限制在屏幕内。
|
||||
不再打开带 macOS 标题栏的独立详情窗口；关闭面板不会取消任务。
|
||||
|
||||
摘要离开后约 1 秒收起，鼠标移到摘要上会保持；点击展开后一直保留，直到点击关闭或 Esc。
|
||||
拖动小球时只隐藏未展开的摘要，下次悬停按新位置贴靠。展开的列表／详情不会因失焦、
|
||||
打开 Settings、拖动小球或连接中断而消失。任务面板不会挤压、移动或调整小球大小，
|
||||
状态更新不翻边、不重排已打开列表的点击目标，也不会把用户正在看的输出滚到底部。
|
||||
自动展开的位置会避开小球、工具栏、预览和下方状态条的完整可见区域；原侧面不足以容纳
|
||||
展开面板时选择有空间的另一侧，不把面板夹回状态条上。
|
||||
|
||||
进行中包含排队／等待输入任务；已完成包含成功任务及已取消的 Proactive monitor。
|
||||
monitor 详情仍显示已取消，不伪装成成功；取消的 timer／harness、失败和中断仍独立计数。
|
||||
需关注只表示正在等待用户输入／授权，不包括失败或中断。持续 monitor 的每轮判断或通知不会增加子智能体数量，
|
||||
通知排队／送达与任务结束是两个状态。追加到既有后台任务的指令不重复计数。
|
||||
|
||||
结束语音后 Proactive 停止采样并保留结束记录，后台 harness 任务继续运行及更新。
|
||||
无通话时收到权限请求只登记等待，不新增自动批准；可在任务详情中按后端实际提供的
|
||||
范围允许或拒绝，未确认所属任务的请求单独显示。普通文件系统拒绝不会被虚构成授权请求。
|
||||
未关联请求也会点亮摘要的待处理标记。描述过长时请到后端完整查看后授权，仍可在这里拒绝。
|
||||
Live 为支持的 Codex ACP 新会话选择明确提供的 `Ask for approval` 模式；不改全局权限、
|
||||
不关闭沙箱，模式不支持或设置失败时记录警告。
|
||||
|
||||
列表和详情提供 `Stop`／`停止`，只停止对应任务；后端尚未确认时显示正在停止，不会提前
|
||||
宣称已结束，也不会用旧任务 ID 取消同一会话的新任务。停止请求与最终结果以静默文字
|
||||
反馈给主 Omni，前台忙时排队，挂断后保留到本次 daemon 的下一次通话。`Close` 仅关闭面板。
|
||||
Live 不再限制活动 Harness／monitor 数量；独立并行的 Harness 任务使用独立会话，
|
||||
同一会话保留追加／排队语义，后端自身的队列、配额以及机器和 API 资源限制仍适用。
|
||||
|
||||
任务历史只保留本次 daemon 运行；全部活动任务保留有限详情，已结束任务仅保留最近 32 条。
|
||||
使用上一页／下一页访问任务，每页最多 32 条，单个快照上限 240 KiB；超限时明确提示
|
||||
省略／截断，总计数仍覆盖省略任务。此功能通过可选能力协商，仅在
|
||||
支持的独立 Live daemon 连接上显示，不影响旧版 Host 或 WebShell。
|
||||
|
||||
Settings 最后一个选项 `Theme`／`主题` 位于 Language 后，支持跟随系统（默认）、浅色和深色。
|
||||
主题保存在 Host 本地，与 daemon 的模型／Memory 配置无关；小球控件、设置和子智能体面板
|
||||
同时更新，切换不会重建媒体或中断通话。
|
||||
|
||||
`Quit Host` 请求当前独立 Live daemon 完成通话、后端、Memory 和 discovery 清理后退出
|
||||
Host；不会关闭另外运行的 `qwen serve`。旧 WebShell 连接只结束 Live 并退出 Host。
|
||||
从未连接到已认证实例时只退出 Host；同一独立实例重连期间仍请求原实例退出。
|
||||
退出未获确认时保留界面并显示错误、保持媒体停止，再次点击只重试原实例，
|
||||
不会悄悄切到另一个 daemon。正常结束通话不会执行这条完整退出路径。
|
||||
只有匹配的退出回执，或系统明确确认原 daemon PID 已不存在，才允许完成退出；
|
||||
404、连接重置或拒绝连接都不单独视为退出成功。
|
||||
清理失败时 daemon 保留同实例的退出控制入口和 discovery，但拒绝新通话及普通请求；
|
||||
重试只处理尚未成功关闭的资源，全部完成后才确认退出。
|
||||
|
||||
连接独立 Qwen Live daemon 时，Settings 的 `Memory` 区域提供记忆开关、
|
||||
独立的 `Visual memory` 开关、选择记忆库、`New`／`Rename`，以及
|
||||
`Consolidation model` 设置（默认 `qwen3.7-plus`）。通话中可以开关和改名；
|
||||
选择／新建记忆库和修改模型需要先结束通话。操作由 daemon 确认并写回 Live 配置，
|
||||
内联编辑草稿不会被音频状态刷新打断。库默认存储在 `~/.qwen-live/memories`。
|
||||
daemon 断开时 Settings 关闭，未保存的编辑草稿保留到重新连接后再次打开。
|
||||
`qwen-live init` 也会询问是否启用 Memory 和整理模型名，详细参数见
|
||||
[Memory 文档](../qwen-live/README.md#memory)。旧的 WebShell 内置 Live 没有声明 Memory
|
||||
能力时，Host 不显示这一入口。
|
||||
|
||||
Camera 被选中后，气泡球上方默认显示镜像的本地预览小窗；没有字幕时更靠近小球。
|
||||
球旁的小眼睛按钮可以隐藏／显示预览，独立于悬停工具栏。隐藏只影响本地小窗，不会停止
|
||||
Camera 输入、视频帧上传或后台监控；切回 Screen 或 Quit 才关闭摄像头。再次选择 Camera
|
||||
默认重新显示预览。空闲时画面只留在本机；
|
||||
Live Feed 在通话中按配置 FPS（默认 1）持续发送有界 JPEG。On Demand 的前台模型
|
||||
通过 `appshot` 获取单次截图；若存在视觉 Proactive 任务，后台 Monitor 仍会按自己的
|
||||
FPS 持续采样。任务等待播报时继续观察，新事件进入 FIFO；结束通话会停止全部监控。
|
||||
|
||||
`visualInput.cameraResolution` 控制预览／Live Feed 采集，默认 1280×720。
|
||||
`visualInput.cameraSnapshotResolution` 独立控制 Camera Appshot，默认 `native`，
|
||||
也可设置 `{ "width": 1920, "height": 1080 }`；环境变量为
|
||||
`QWEN_LIVE_CAMERA_SNAPSHOT_RESOLUTION=native` 或 `WIDTHxHEIGHT`。Host 优先从同一
|
||||
camera track 拍摄静态照片；设备不支持时尝试临时调整视频采集约束，等新尺寸画面就绪后
|
||||
截图并恢复预览。无法满足原生采集时明确报错，不会把预览的 720p 冒充原生照片。
|
||||
Camera 高分辨率 JPEG 单独保存为 handoff asset（上限 8 MiB），不通过 Host WebSocket
|
||||
传输；后台 Monitor 不调用这条高分辨率拍照路径。`snapshotResolution` 继续用于 Screen。
|
||||
|
||||
Screen Live Feed 与视觉 Proactive monitor 使用独立的完整显示器采集路径，包含桌面、
|
||||
菜单栏、Dock 和其他应用，但排除 Live Host 自身窗口。Settings 的 Video Source 下可选
|
||||
`Display`／`显示器`，选择保存到 `config.json` 的 `visualInput.screenDisplayId`。
|
||||
默认 `primary` 跟随系统主显示器，也可保存某块显示器的 UUID；明确选择的显示器断开后
|
||||
报错，不自动换屏。切换显示器会丢弃过期截图并清空 monitor 旧视觉缓冲。无需重新 init。
|
||||
两条持续画面路径都使用 `liveResolution`，默认等比放进 1280×720；完整范围不代表原生像素。
|
||||
所有送入 Omni 的 JPEG 和 Host 传输预览受 1080p／190 KiB 上限约束，
|
||||
Camera 原图 asset 与 Screen 的 PNG asset 不受该小图上限影响。
|
||||
前台 Appshot 工具与 On Demand 视觉记忆仍使用原来的前台窗口截图，不读取整屏；Camera 不变。
|
||||
|
||||
停止通话、切换 Source/Mode、daemon 断开或 Host 退出都会清理不再使用的通话采集。
|
||||
停止通话后，Camera Source 的可见小窗可以继续显示本地预览，此时不会上传画面；
|
||||
切回 Screen、断开 daemon 或退出 Host 会关闭摄像头。
|
||||
Camera Source 需要摄像头权限；Screen On Demand 的 Appshot 需要辅助功能和屏幕录制权限，
|
||||
Screen Live Feed 仅需屏幕录制权限。未选中的
|
||||
来源权限不会阻止 Live。通话中切换到尚未授权的来源时，Host 会先保留当前可用来源，
|
||||
授权成功后再一次性完成切换；授权取消或失败不会让正在工作的来源提前失效。
|
||||
|
||||
开发诊断可在终端启动开发版或应用可执行文件并传入 `--live-debug`。不要给 Electron
|
||||
Host 传 `--debug`，该参数会被 Electron 当成已废弃的 Node 调试参数并在启动前退出。
|
||||
日志只包含状态、readiness blocker、尺寸、字节数
|
||||
和错误码，不包含图片、音频、API key 或转写内容：
|
||||
|
||||
```bash
|
||||
cd packages/live-host
|
||||
npm start -- --live-debug
|
||||
|
||||
"release/mac-arm64/Qwen Live Host.app/Contents/MacOS/Qwen Live Host" --live-debug
|
||||
```
|
||||
|
||||
Proactive 判断、通知排队／播报、harness 任务和 Realtime 生命周期日志由 **daemon** 输出，
|
||||
需在另一个终端运行 `qwen-live --debug`。Host 的 `--live-debug` 不会替代 daemon 的日志开关。
|
||||
|
||||
排查 monitor 输入时，用 `frameHash`（JPEG 字节的 SHA256 前 16 位）对应 Host 的
|
||||
`visual_snapshot_captured`／`visual_frame_sent`、daemon 的截图／帧记录，以及
|
||||
`proactive.monitor_image_sent`。后者只表示实际写入模型连接的帧，不把排队当作已发送。
|
||||
`proactive.monitor_commit` 显示本次实际发送的图片数、音频字节数和时长（含协议要求的静音），
|
||||
`monitor_committed` 表示服务端确认提交，`monitor_action` 区分 wait／reply／function_call／invalid。
|
||||
这些日志不包含图像、音频或模型输出原文。
|
||||
|
||||
daemon 的 debug 模式另外为视觉 Monitor 保存真实请求，目录为系统临时目录下的
|
||||
`qwen-live-monitor-debug/`。每个 Monitor 一个目录，每次推理保存 `request.json`、
|
||||
实际送出的 JPEG、含协议静音的 16 kHz `input.wav`，以及结果 `response.json`。
|
||||
`proactive.monitor_debug_started` 和 `proactive.monitor_request_saved` 日志给出绝对路径。
|
||||
仅 daemon debug 开启；Host 的 `--live-debug` 单独启用不会录制，纯音频 Monitor 也不录制。
|
||||
启动及新建 Monitor 时清理，只保留最近创建的 10 个 Monitor（不是最近 10 次请求）。
|
||||
被清理的 Monitor 继续运行但停止录制；文件仅当前用户可访问。内容包含真实屏幕／摄像头、
|
||||
任务文本和混合 Monitor 的麦克风输入，虽然不保存连接凭据，画面或音频中的秘密不会被脱敏。
|
||||
录制失败会单独报错而不影响通话；长时间 debug 可能占用较多磁盘，诊断完请关闭 debug。
|
||||
完整格式与清理规则见 [Qwen Live README](../qwen-live/README.md)。
|
||||
|
||||
`native_display_changed` 记录原生显示器事件与几何信息；`overlay_position` 记录 Host 主动
|
||||
定位的原因及前后坐标，`overlay_native_moved` 记录原生窗口移动。非几何显示器事件不会再
|
||||
丢弃正在采集的帧或中断拖拽；真正需要边界修正时才调整小球位置。
|
||||
|
||||
## 内置 Appshot 与来源相关授权
|
||||
|
||||
| 权限 | 授权主体 | 用途 |
|
||||
| -------- | -------------- | -------------------------------------- |
|
||||
| 麦克风 | Qwen Live Host | 采集 Live 对话音频 |
|
||||
| 摄像头 | Qwen Live Host | Camera Source 的预览、实时帧或单帧截图 |
|
||||
| 辅助功能 | Qwen Live Host | 读取前台窗口的可访问性树 |
|
||||
| 屏幕录制 | Qwen Live Host | Screen Source 的实时帧或单帧截图 |
|
||||
|
||||
Appshot 是 Host 的内部核心能力。内置原生模块选择最前面的非 Host 普通窗口，通过
|
||||
macOS 原生 API 返回应用信息、窗口标题、AX 文本和 PNG。模型只能调用一次无参数、只读的
|
||||
`capture_screen_context`；不能通过这条通道指定窗口、坐标或动作。
|
||||
macOS 原生 API 返回应用信息、窗口标题、AX 文本和 PNG。模型侧的 `appshot` 是无参数、
|
||||
只读工具，捕获气泡球当前选中的 Source；不能通过工具参数临时指定另一个来源、窗口、
|
||||
坐标或动作。Live Feed 模式禁用该工具，On Demand 模式才允许调用。
|
||||
|
||||
Host 激活时只读取一次权限状态，用户显式点击授权或 Host 再次激活时才重新检查，不做
|
||||
后台轮询。每次真实 Appshot 都在 Host 进程内重新验证两项权限。授权丢失会令捕获失败并
|
||||
使 Live fail closed。整个流程不启动或探测任何外置屏幕工具。
|
||||
Host 激活后会在 Screen 为当前或待切换来源时定期刷新 Appshot 权限，以便授权完成后
|
||||
自动恢复或切换；每次真实 Screen 捕获还会在 Host 进程内重新验证两项权限。当前来源的
|
||||
授权丢失会令捕获失败并使 Live fail closed。整个流程不启动或探测任何外置屏幕工具。
|
||||
|
||||
## 音频和 fail-closed
|
||||
|
||||
`devicechange`、输入 track ended/mute、播放失败或音频帧无法交给 daemon 时，Host 会先
|
||||
Omni 响应以单声道 16-bit、24 kHz PCM 接收。播放 AudioContext 不指定采样率，使用当前
|
||||
系统输出设备的默认时钟（如 44.1／48／96 kHz），不强制更改设备采样率。
|
||||
当前协商了输出结束标记的连接，对每条响应进行连续、带抗混叠滤波的流式重采样，再放入
|
||||
设备采样率的 AudioBuffer，按整数采样点连续排程，避免逐块转换的衔接尖峰及无谓间隙。
|
||||
结束标记到达时输出短暂的滤波尾部。未协商结束标记的旧连接保持原有 Web Audio
|
||||
逐帧转换和播放排空逻辑，不会等待不存在的标记。
|
||||
`--live-debug` 日志中的 `output_context_ready` 显示源／输出上下文采样率及是否重采样。
|
||||
|
||||
蓝牙耳机的麦克风被打开时，macOS 可能将耳机切换到免提通话模式，影响同时播放的音乐／视频；
|
||||
这与模型 PCM 采样率是两回事。可在 Audio Source 选择 Mac 内置麦克风，输出仍使用蓝牙耳机。
|
||||
关闭麦克风时 Host 立即停止输入 track 并释放捕获上下文，而不只是丢弃录音数据；
|
||||
静音期间设备变化不会重新打开麦克风，取消静音后才重新收音。
|
||||
|
||||
`devicechange` 会在收音时检查／替换输入，在空闲时重新自检；静音通话不重新获取输入。
|
||||
输入 track ended、播放失败或音频帧无法交给 daemon 时，Host 会先
|
||||
将 input/output 标记为 unavailable、停止当前通话并清理旧 context，再重新执行自检。
|
||||
麦克风重新授权后只有实际输入自检通过才会恢复 ready。overlay renderer、preload 加载、
|
||||
页面加载失败或 renderer 无响应也会执行 fail-closed。
|
||||
|
||||
任一权限、自检、快捷键或 provider 配置失败时，Live 都保持不可用。Host readiness
|
||||
不会连接 Realtime；只有用户开始对话时才建立 provider WebSocket。限流或配额错误不做
|
||||
不会连接 Realtime；首次就绪自动开始或用户手动开始对话时才建立 provider WebSocket。限流或配额错误不做
|
||||
自动重试或后台探测，用户稍后可手工重试。
|
||||
|
||||
## 卸载
|
||||
@@ -124,4 +325,4 @@ Host 激活时只读取一次权限状态，用户显式点击授权或 Host 再
|
||||
2. 如果用户曾手工添加 Login Item，在系统设置中将其移除。
|
||||
3. 从 `/Applications` 删除 **Qwen Live Host.app**。
|
||||
4. 在 WebShell 的 **设置 → 实验性功能 → Qwen Live** 中关闭功能。
|
||||
5. 如不再需要，可在“隐私与安全性”中撤销 Host 的麦克风、辅助功能和屏幕录制权限。
|
||||
5. 如不再需要，可在“隐私与安全性”中撤销 Host 的麦克风、摄像头、辅助功能和屏幕录制权限。
|
||||
@@ -8,5 +8,7 @@
|
||||
<true/>
|
||||
<key>com.apple.security.device.audio-input</key>
|
||||
<true/>
|
||||
<key>com.apple.security.device.camera</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -11,6 +11,7 @@ directories:
|
||||
files:
|
||||
- 'dist/main.cjs'
|
||||
- 'dist/preload.cjs'
|
||||
- 'dist/subagents-preload.cjs'
|
||||
- 'dist/renderer/**/*'
|
||||
- 'package.json'
|
||||
|
||||
@@ -20,6 +21,7 @@ extraMetadata:
|
||||
asar: true
|
||||
|
||||
electronFuses:
|
||||
resetAdHocDarwinSignature: true
|
||||
runAsNode: false
|
||||
enableNodeOptionsEnvironmentVariable: false
|
||||
enableNodeCliInspectArguments: false
|
||||
@@ -34,8 +36,9 @@ mac:
|
||||
minimumSystemVersion: '12.0.0'
|
||||
extendInfo:
|
||||
LSUIElement: true
|
||||
QwenLiveProtocolVersion: 6
|
||||
QwenLiveProtocolVersion: 9
|
||||
NSMicrophoneUsageDescription: 'Qwen Live Host uses the microphone for live voice conversations with Qwen Code.'
|
||||
NSCameraUsageDescription: 'Qwen Live Host uses the camera to share visual context with Qwen Live when you turn it on.'
|
||||
hardenedRuntime: true
|
||||
gatekeeperAssess: false
|
||||
entitlements: 'build/entitlements.mac.plist'
|
||||
|
||||
Generated
+493
@@ -12,12 +12,14 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron/asar": "3.4.1",
|
||||
"@types/jsdom": "28.0.3",
|
||||
"@types/node": "^25.0.8",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@types/ws": "^8.18.1",
|
||||
"electron": "^39.2.7",
|
||||
"electron-builder": "^26.0.12",
|
||||
"esbuild": "^0.25.0",
|
||||
"jsdom": "26.1.0",
|
||||
"semver": "^7.7.2",
|
||||
"tsx": "^4.20.3",
|
||||
"typescript": "^5.0.0",
|
||||
@@ -27,6 +29,142 @@
|
||||
"node": ">=22.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/css-color": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz",
|
||||
"integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/css-calc": "^2.1.3",
|
||||
"@csstools/css-color-parser": "^3.0.9",
|
||||
"@csstools/css-parser-algorithms": "^3.0.4",
|
||||
"@csstools/css-tokenizer": "^3.0.3",
|
||||
"lru-cache": "^10.4.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/css-color/node_modules/lru-cache": {
|
||||
"version": "10.4.3",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
|
||||
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/@csstools/color-helpers": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz",
|
||||
"integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-calc": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz",
|
||||
"integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-color-parser": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz",
|
||||
"integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/color-helpers": "^5.1.0",
|
||||
"@csstools/css-calc": "^2.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-parser-algorithms": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz",
|
||||
"integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-tokenizer": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz",
|
||||
"integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@electron/asar": {
|
||||
"version": "3.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.4.1.tgz",
|
||||
@@ -1507,6 +1645,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/jsdom": {
|
||||
"version": "28.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-28.0.3.tgz",
|
||||
"integrity": "sha512-/HQ2uFoetFTXuye8vzIcHw2z6Fwi7Hi/qcgC+RoS9NCyewiqxhVGqlG+ViGB6lkax481R6dmhf1I7lIGlzJStQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"@types/tough-cookie": "*",
|
||||
"parse5": "^8.0.0",
|
||||
"undici-types": "^7.21.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/keyv": {
|
||||
"version": "3.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz",
|
||||
@@ -1547,6 +1698,13 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/tough-cookie": {
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
|
||||
"integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/ws": {
|
||||
"version": "8.18.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
|
||||
@@ -2275,6 +2433,34 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz",
|
||||
"integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/css-color": "^3.2.0",
|
||||
"rrweb-cssom": "^0.8.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz",
|
||||
"integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^14.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
@@ -2293,6 +2479,13 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/decimal.js": {
|
||||
"version": "10.6.0",
|
||||
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz",
|
||||
"integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/decompress-response": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
|
||||
@@ -2775,6 +2968,19 @@
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz",
|
||||
"integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/env-paths": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
|
||||
@@ -3377,6 +3583,19 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/html-encoding-sniffer": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz",
|
||||
"integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/http-cache-semantics": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz",
|
||||
@@ -3426,6 +3645,19 @@
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
@@ -3455,6 +3687,13 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
@@ -3536,6 +3775,72 @@
|
||||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom": {
|
||||
"version": "26.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz",
|
||||
"integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cssstyle": "^4.2.1",
|
||||
"data-urls": "^5.0.0",
|
||||
"decimal.js": "^10.5.0",
|
||||
"html-encoding-sniffer": "^4.0.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"nwsapi": "^2.2.16",
|
||||
"parse5": "^7.2.1",
|
||||
"rrweb-cssom": "^0.8.0",
|
||||
"saxes": "^6.0.0",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^5.1.1",
|
||||
"w3c-xmlserializer": "^5.0.0",
|
||||
"webidl-conversions": "^7.0.0",
|
||||
"whatwg-encoding": "^3.1.1",
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^14.1.1",
|
||||
"ws": "^8.18.0",
|
||||
"xml-name-validator": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"canvas": "^3.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"canvas": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/parse5": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
|
||||
"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/json-buffer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
||||
@@ -3897,6 +4202,13 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/nwsapi": {
|
||||
"version": "2.2.27",
|
||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.27.tgz",
|
||||
"integrity": "sha512-gQPNF78qebCQ6tvVFBYrvJdBNOrYZm90ZlXgpIFm06p6qHDHq/XC4TnJftN6OMbxVE0UTBAoRgcsDeJBBooITw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/object-keys": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
|
||||
@@ -3944,6 +4256,19 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",
|
||||
"integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^8.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
@@ -4173,6 +4498,16 @@
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/punycode": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
|
||||
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/pvtsutils": {
|
||||
"version": "1.3.6",
|
||||
"resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz",
|
||||
@@ -4383,6 +4718,13 @@
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/rrweb-cssom": {
|
||||
"version": "0.8.0",
|
||||
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz",
|
||||
"integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
@@ -4390,6 +4732,13 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sanitize-filename": {
|
||||
"version": "1.6.4",
|
||||
"resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.4.tgz",
|
||||
@@ -4410,6 +4759,19 @@
|
||||
"node": ">=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/saxes": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
|
||||
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"xmlchars": "^2.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=v12.22.7"
|
||||
}
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.8.5",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
|
||||
@@ -4604,6 +4966,13 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/symbol-tree": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
|
||||
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tar": {
|
||||
"version": "7.5.22",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz",
|
||||
@@ -4732,6 +5101,26 @@
|
||||
"url": "https://github.com/sponsors/SuperchupuDev"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",
|
||||
"integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^6.1.86"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts-core": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz",
|
||||
"integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tmp": {
|
||||
"version": "0.2.7",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz",
|
||||
@@ -4752,6 +5141,32 @@
|
||||
"tmp": "^0.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
|
||||
"integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tldts": "^6.1.32"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
|
||||
"integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"punycode": "^2.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/truncate-utf8-bytes": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
|
||||
@@ -5464,6 +5879,19 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/w3c-xmlserializer": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz",
|
||||
"integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xml-name-validator": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/webcrypto-core": {
|
||||
"version": "1.9.2",
|
||||
"resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.9.2.tgz",
|
||||
@@ -5478,6 +5906,54 @@
|
||||
"tslib": "^2.8.1"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
|
||||
"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
|
||||
"deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-mimetype": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
|
||||
"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "14.2.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
|
||||
"integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "^5.1.0",
|
||||
"webidl-conversions": "^7.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz",
|
||||
@@ -5540,6 +6016,16 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/xml-name-validator": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz",
|
||||
"integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlbuilder": {
|
||||
"version": "15.1.1",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
|
||||
@@ -5550,6 +6036,13 @@
|
||||
"node": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlchars": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
|
||||
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
||||
@@ -23,12 +23,14 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron/asar": "3.4.1",
|
||||
"@types/jsdom": "28.0.3",
|
||||
"@types/node": "^25.0.8",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@types/ws": "^8.18.1",
|
||||
"electron": "^39.2.7",
|
||||
"electron-builder": "^26.0.12",
|
||||
"esbuild": "^0.25.0",
|
||||
"jsdom": "26.1.0",
|
||||
"semver": "^7.7.2",
|
||||
"tsx": "^4.20.3",
|
||||
"typescript": "^5.0.0",
|
||||
|
||||
@@ -4,7 +4,6 @@ const path = require('node:path');
|
||||
const UNUSED_PERMISSION_KEYS = [
|
||||
'NSBluetoothAlwaysUsageDescription',
|
||||
'NSBluetoothPeripheralUsageDescription',
|
||||
'NSCameraUsageDescription',
|
||||
];
|
||||
|
||||
module.exports = async function afterPack(context) {
|
||||
|
||||
@@ -7,6 +7,16 @@ import { build as viteBuild } from 'vite';
|
||||
|
||||
const appDir = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
const distDir = join(appDir, 'dist');
|
||||
const liveTextAlias = {
|
||||
'@qwen-code/qwen-live/subagents': join(
|
||||
appDir,
|
||||
'../qwen-live/src/subagents/types.ts',
|
||||
),
|
||||
'@qwen-code/qwen-live/i18n': join(
|
||||
appDir,
|
||||
'../qwen-live/src/i18n/messages.ts',
|
||||
),
|
||||
};
|
||||
rmSync(distDir, { recursive: true, force: true });
|
||||
mkdirSync(distDir, { recursive: true });
|
||||
|
||||
@@ -56,6 +66,8 @@ execFileSync(
|
||||
'-framework',
|
||||
'ApplicationServices',
|
||||
'-framework',
|
||||
'ColorSync',
|
||||
'-framework',
|
||||
'ImageIO',
|
||||
'-framework',
|
||||
'ScreenCaptureKit',
|
||||
@@ -67,6 +79,7 @@ execFileSync(
|
||||
|
||||
await esbuild({
|
||||
entryPoints: [join(appDir, 'src', 'main', 'index.ts')],
|
||||
alias: liveTextAlias,
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
target: 'node22',
|
||||
@@ -78,6 +91,7 @@ await esbuild({
|
||||
|
||||
await esbuild({
|
||||
entryPoints: [join(appDir, 'src', 'preload', 'index.ts')],
|
||||
alias: liveTextAlias,
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
target: 'node22',
|
||||
@@ -88,6 +102,16 @@ await esbuild({
|
||||
});
|
||||
|
||||
await viteBuild({ configFile: join(appDir, 'vite.config.ts') });
|
||||
await esbuild({
|
||||
entryPoints: [join(appDir, 'src', 'preload', 'subagents.ts')],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
target: 'node22',
|
||||
format: 'cjs',
|
||||
outfile: join(distDir, 'subagents-preload.cjs'),
|
||||
external: ['electron'],
|
||||
sourcemap: true,
|
||||
});
|
||||
|
||||
const license = join(appDir, '..', '..', 'LICENSE');
|
||||
cpSync(license, join(distDir, 'LICENSE'));
|
||||
|
||||
@@ -58,6 +58,91 @@ describe('built-in Appshot architecture', () => {
|
||||
assert.doesNotMatch(builderConfig, /from:\s*['"][^'"]+\.app['"]/u);
|
||||
});
|
||||
|
||||
it('keeps full-display capture separate from foreground Appshot and never reads accessibility', async () => {
|
||||
const source = await readFile(NATIVE_SOURCE, 'utf8');
|
||||
const capture = source.slice(
|
||||
source.indexOf('void ExecuteDisplayCapture('),
|
||||
source.indexOf('void CompleteDisplayCapture('),
|
||||
);
|
||||
assert.match(capture, /CGPreflightScreenCaptureAccess/u);
|
||||
assert.match(capture, /ResolveDisplay\(work->selection\)/u);
|
||||
assert.match(capture, /current->uuid != target->uuid/u);
|
||||
assert.doesNotMatch(
|
||||
capture,
|
||||
/AXIsProcessTrusted|CaptureAccessibilityTree|FindForegroundWindow|CapturePng\(/u,
|
||||
);
|
||||
assert.match(source, /CGGetActiveDisplayList/u);
|
||||
assert.match(source, /CGDisplayCreateUUIDFromDisplayID/u);
|
||||
assert.match(source, /screen\.localizedName/u);
|
||||
assert.match(source, /"listDisplays"/u);
|
||||
assert.match(source, /"captureDisplay"/u);
|
||||
});
|
||||
|
||||
it('covers full display bounds, desktop and system layers while excluding Host windows', async () => {
|
||||
const source = await readFile(NATIVE_SOURCE, 'utf8');
|
||||
const modern = source.slice(
|
||||
source.indexOf('CGImageRef CaptureDisplayWithScreenCaptureKit('),
|
||||
source.indexOf('std::vector<uint8_t> CaptureDisplayPng('),
|
||||
);
|
||||
const legacy = source.slice(
|
||||
source.indexOf('std::vector<uint8_t> CaptureDisplayPng('),
|
||||
source.indexOf('napi_value Boolean('),
|
||||
);
|
||||
assert.match(modern, /getShareableContentExcludingDesktopWindows:NO/u);
|
||||
assert.match(modern, /application\.processID == getpid\(\)/u);
|
||||
assert.match(
|
||||
modern,
|
||||
/initWithDisplay:selected_display\s+excludingApplications:excluded\s+exceptingWindows:@\[\]/u,
|
||||
);
|
||||
assert.match(modern, /macOS 14\.2[^]*includeMenuBar = YES/u);
|
||||
assert.match(modern, /filter\.pointPixelScale/u);
|
||||
assert.match(modern, /excluded\.count == 0/u);
|
||||
assert.doesNotMatch(modern, /initWithDesktopIndependentWindow|sourceRect/u);
|
||||
assert.match(
|
||||
legacy,
|
||||
/CGWindowListCopyWindowInfo\(\s*kCGWindowListOptionOnScreenOnly/u,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
legacy,
|
||||
/kCGWindowListExcludeDesktopElements|kCGWindowLayer|layer\.intValue/u,
|
||||
);
|
||||
assert.match(legacy, /pid\.intValue != getpid\(\)/u);
|
||||
assert.match(
|
||||
legacy,
|
||||
/CFArrayCreateMutable\(kCFAllocatorDefault, 0, nullptr\)/u,
|
||||
);
|
||||
assert.match(
|
||||
legacy,
|
||||
/CGWindowListCreateImageFromArray\(\s*target\.bounds, ids/u,
|
||||
);
|
||||
assert.match(source, /1920\.0 \/ width, 1080\.0 \/ height/u);
|
||||
assert.match(legacy, /bytes\.size\(\) > kMaxDisplayPngBytes/u);
|
||||
const build = await readFile(
|
||||
new URL('../../../scripts/build.mjs', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(build, /'ColorSync'/u);
|
||||
});
|
||||
|
||||
it('rejects ambiguous display UUIDs instead of selecting the first match', async () => {
|
||||
const source = await readFile(NATIVE_SOURCE, 'utf8');
|
||||
const resolve = source.slice(
|
||||
source.indexOf('std::optional<DisplayTarget> ResolveDisplay('),
|
||||
source.indexOf('std::string NormalizeText('),
|
||||
);
|
||||
assert.match(resolve, /std::optional<DisplayTarget> selected;/u);
|
||||
assert.match(
|
||||
resolve,
|
||||
/if \(selected\.has_value\(\)\) return std::nullopt;/u,
|
||||
);
|
||||
assert.match(
|
||||
resolve,
|
||||
/selection == "primary" && display_id == CGMainDisplayID\(\)/u,
|
||||
);
|
||||
assert.match(resolve, /return selected;/u);
|
||||
assert.doesNotMatch(resolve, /return DisplayTarget\{/u);
|
||||
});
|
||||
|
||||
it('locks down Electron runtime escape hatches in packaged builds', async () => {
|
||||
const builderConfig = await readFile(BUILDER_CONFIG, 'utf8');
|
||||
|
||||
|
||||
@@ -6,11 +6,22 @@ import { afterEach, describe, it } from 'node:test';
|
||||
import {
|
||||
AppshotCaptureService,
|
||||
validateNativeCapture,
|
||||
validateNativeDisplayCapture,
|
||||
} from '../appshot-capture.ts';
|
||||
import type { NativeAppshot } from '../native-appshot.ts';
|
||||
import {
|
||||
MAX_CAPTURE_ASSET_BYTES,
|
||||
MAX_INPUT_IMAGE_FRAME_BYTES,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: string[] = [];
|
||||
const PNG = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 1]);
|
||||
const DISPLAY_ID = '11223344-5566-7788-99aa-bbccddeeff00';
|
||||
const OTHER_DISPLAY_ID = '11223344-5566-7788-99aa-bbccddeeff11';
|
||||
const DISPLAY_PNG = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jp1sAAAAASUVORK5CYII=',
|
||||
'base64',
|
||||
);
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
@@ -29,11 +40,198 @@ function fakeNative(
|
||||
requestAccessibility: () => true,
|
||||
requestScreenRecording: () => true,
|
||||
captureAppshot,
|
||||
listDisplays: () => [],
|
||||
captureDisplay: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('AppshotCaptureService', () => {
|
||||
it('performs one in-process capture and returns a private PNG path', async () => {
|
||||
it('lists display identities without invoking capture or permission requests', () => {
|
||||
const native = fakeNative(async () => {
|
||||
throw new Error('Appshot should not run');
|
||||
});
|
||||
native.listDisplays = () => [
|
||||
{
|
||||
id: DISPLAY_ID.toUpperCase(),
|
||||
name: ' Studio Display ',
|
||||
width: 5120,
|
||||
height: 2880,
|
||||
primary: true,
|
||||
},
|
||||
];
|
||||
native.requestAccessibility = () => {
|
||||
throw new Error('No AX prompt');
|
||||
};
|
||||
native.requestScreenRecording = () => {
|
||||
throw new Error('No recording prompt');
|
||||
};
|
||||
const service = new AppshotCaptureService(undefined, () => native);
|
||||
assert.deepEqual(service.listDisplays(), [
|
||||
{
|
||||
id: DISPLAY_ID,
|
||||
name: 'Studio Display',
|
||||
width: 5120,
|
||||
height: 2880,
|
||||
primary: true,
|
||||
},
|
||||
]);
|
||||
native.listDisplays = () => [
|
||||
{
|
||||
id: DISPLAY_ID,
|
||||
name: 'Display',
|
||||
width: 5120,
|
||||
height: 2880,
|
||||
primary: true,
|
||||
},
|
||||
{
|
||||
id: DISPLAY_ID.toUpperCase(),
|
||||
name: 'Duplicate',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
primary: false,
|
||||
},
|
||||
];
|
||||
assert.throws(() => service.listDisplays(), /host.error.displayList/u);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('captures the exact display with no AX/window fallback and canonical identity', async () => {
|
||||
const selections: string[] = [];
|
||||
const native = fakeNative(async () => {
|
||||
throw new Error('No window fallback');
|
||||
});
|
||||
native.getPermissionState = () => {
|
||||
throw new Error('No AX readiness gate');
|
||||
};
|
||||
native.captureDisplay = async (selection) => {
|
||||
selections.push(selection);
|
||||
return { displayId: DISPLAY_ID.toUpperCase(), screenshot: DISPLAY_PNG };
|
||||
};
|
||||
const service = new AppshotCaptureService(undefined, () => native);
|
||||
assert.deepEqual(
|
||||
await service.captureDisplayFrame(DISPLAY_ID.toUpperCase()),
|
||||
{ displayId: DISPLAY_ID, screenshot: DISPLAY_PNG },
|
||||
);
|
||||
assert.deepEqual(await service.captureDisplayFrame(), {
|
||||
displayId: DISPLAY_ID,
|
||||
screenshot: DISPLAY_PNG,
|
||||
});
|
||||
assert.deepEqual(selections, [DISPLAY_ID, 'primary']);
|
||||
await assert.rejects(
|
||||
service.captureDisplayFrame('foreground'),
|
||||
/host.error.displayUnavailable/u,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.captureDisplayFrame(`${DISPLAY_ID}\n`),
|
||||
/host.error.displayUnavailable/u,
|
||||
);
|
||||
await assert.rejects(
|
||||
service.captureDisplayFrame(OTHER_DISPLAY_ID),
|
||||
/host.error.displayUnavailable/u,
|
||||
);
|
||||
assert.equal(selections.length, 3);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('shares one capture queue between display frames and original Appshot, including rejection', async () => {
|
||||
let finish!: () => void;
|
||||
const order: string[] = [];
|
||||
const native = fakeNative(async () => {
|
||||
order.push('window');
|
||||
return {
|
||||
appName: 'Editor',
|
||||
windowId: 1,
|
||||
accessibilityText: '- AXWindow',
|
||||
screenshot: PNG,
|
||||
};
|
||||
});
|
||||
native.captureDisplay = async () => {
|
||||
order.push('display');
|
||||
await new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
throw Object.assign(new Error('No display'), {
|
||||
code: 'DISPLAY_UNAVAILABLE',
|
||||
});
|
||||
};
|
||||
const service = new AppshotCaptureService(undefined, () => native);
|
||||
const first = service.captureDisplayFrame(DISPLAY_ID);
|
||||
const second = service.captureFrame();
|
||||
assert.deepEqual(order, ['display']);
|
||||
finish();
|
||||
await assert.rejects(first, /host.error.displayUnavailable/u);
|
||||
assert.equal((await second).appName, 'Editor');
|
||||
assert.deepEqual(order, ['display', 'window']);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('returns localized display errors without leaking native messages', async () => {
|
||||
const native = fakeNative(async () => {
|
||||
throw new Error('No fallback');
|
||||
});
|
||||
const service = new AppshotCaptureService(undefined, () => native);
|
||||
for (const [code, expected] of [
|
||||
['DISPLAY_PERMISSION', 'runtime.screenPermission'],
|
||||
['DISPLAY_UNAVAILABLE', 'host.error.displayUnavailable'],
|
||||
['unknown', 'host.error.displayCapture'],
|
||||
]) {
|
||||
native.captureDisplay = async () => {
|
||||
throw Object.assign(new Error('private backend detail'), { code });
|
||||
};
|
||||
await assert.rejects(
|
||||
service.captureDisplayFrame(),
|
||||
(error: Error) =>
|
||||
error.message.includes(expected!) &&
|
||||
!error.message.includes('private'),
|
||||
);
|
||||
}
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('rejects invalid display PNGs and dimensions before passing them to image decoding', () => {
|
||||
assert.deepEqual(
|
||||
validateNativeDisplayCapture(
|
||||
{ displayId: DISPLAY_ID, screenshot: DISPLAY_PNG },
|
||||
'primary',
|
||||
).screenshot,
|
||||
DISPLAY_PNG,
|
||||
);
|
||||
const oversizedDimensions = Buffer.from(DISPLAY_PNG);
|
||||
oversizedDimensions.writeUInt32BE(1921, 16);
|
||||
for (const screenshot of [
|
||||
PNG,
|
||||
Buffer.alloc(MAX_CAPTURE_ASSET_BYTES + 1),
|
||||
oversizedDimensions,
|
||||
])
|
||||
assert.throws(
|
||||
() =>
|
||||
validateNativeDisplayCapture(
|
||||
{ displayId: DISPLAY_ID, screenshot },
|
||||
DISPLAY_ID,
|
||||
),
|
||||
/host.error.displayCapture/u,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateNativeDisplayCapture(
|
||||
{ displayId: OTHER_DISPLAY_ID, screenshot: DISPLAY_PNG },
|
||||
DISPLAY_ID,
|
||||
),
|
||||
/host.error.displayUnavailable/u,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateNativeDisplayCapture(
|
||||
{ displayId: `${DISPLAY_ID}\n`, screenshot: DISPLAY_PNG },
|
||||
'primary',
|
||||
),
|
||||
/host.error.displayUnavailable/u,
|
||||
);
|
||||
});
|
||||
|
||||
it('performs one in-process capture and stores a private PNG', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-appshot-test-'));
|
||||
cleanup.push(directory);
|
||||
let captures = 0;
|
||||
@@ -50,32 +248,114 @@ describe('AppshotCaptureService', () => {
|
||||
});
|
||||
const service = new AppshotCaptureService(directory, () => native);
|
||||
|
||||
const result = await service.capture();
|
||||
const result = await service.captureFrame();
|
||||
const screenshotPath = await service.storePng(result.screenshot);
|
||||
|
||||
assert.equal(captures, 1);
|
||||
assert.equal(result.appName, 'TextEdit');
|
||||
assert.equal(result.windowTitle, 'LIVE_APP_A');
|
||||
assert.equal(result.accessibilityText, '- AXWindow title="LIVE_APP_A"');
|
||||
assert.deepEqual(await readFile(result.screenshotPath), PNG);
|
||||
const stat = await lstat(result.screenshotPath);
|
||||
assert.deepEqual(await readFile(screenshotPath), PNG);
|
||||
const stat = await lstat(screenshotPath);
|
||||
assert.equal(stat.mode & 0o077, 0);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('serializes capture so one Live request cannot fan out', async () => {
|
||||
it('queues capture so one Live request cannot fan out', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-appshot-busy-'));
|
||||
cleanup.push(directory);
|
||||
const native = fakeNative(async () => ({
|
||||
appName: 'Safari',
|
||||
windowId: 7,
|
||||
accessibilityText: '- AXWindow',
|
||||
screenshot: PNG,
|
||||
}));
|
||||
let finishFirst: (() => void) | undefined;
|
||||
let active = 0;
|
||||
let captures = 0;
|
||||
const native = fakeNative(async () => {
|
||||
captures += 1;
|
||||
active += 1;
|
||||
assert.equal(active, 1);
|
||||
if (captures === 1) {
|
||||
await new Promise<void>((resolve) => {
|
||||
finishFirst = resolve;
|
||||
});
|
||||
}
|
||||
active -= 1;
|
||||
return {
|
||||
appName: 'Safari',
|
||||
windowId: 7,
|
||||
accessibilityText: '- AXWindow',
|
||||
screenshot: PNG,
|
||||
};
|
||||
});
|
||||
const service = new AppshotCaptureService(directory, () => native);
|
||||
|
||||
const first = service.capture();
|
||||
await assert.rejects(service.capture(), /already in progress/u);
|
||||
await first;
|
||||
const first = service.captureFrame();
|
||||
const second = service.captureFrame();
|
||||
assert.equal(captures, 1);
|
||||
finishFirst?.();
|
||||
await Promise.all([first, second]);
|
||||
assert.equal(captures, 2);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('continues the capture queue after an earlier request fails', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-appshot-queue-'));
|
||||
cleanup.push(directory);
|
||||
let captures = 0;
|
||||
const native = fakeNative(async () => {
|
||||
captures += 1;
|
||||
if (captures === 1) throw new Error('capture failed');
|
||||
return {
|
||||
appName: 'Safari',
|
||||
windowId: 7,
|
||||
accessibilityText: '- AXWindow',
|
||||
screenshot: PNG,
|
||||
};
|
||||
});
|
||||
const service = new AppshotCaptureService(directory, () => native);
|
||||
|
||||
const first = service.captureFrame();
|
||||
const second = service.captureFrame();
|
||||
await assert.rejects(first, /capture failed/u);
|
||||
await assert.doesNotReject(second);
|
||||
assert.equal(captures, 2);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it('stores bounded JPEG and PNG assets as private files', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-appshot-store-'));
|
||||
cleanup.push(directory);
|
||||
const service = new AppshotCaptureService(directory, () =>
|
||||
fakeNative(async () => {
|
||||
throw new Error('unused');
|
||||
}),
|
||||
);
|
||||
const jpeg = Buffer.alloc(MAX_INPUT_IMAGE_FRAME_BYTES + 1);
|
||||
jpeg[0] = 0xff;
|
||||
jpeg[1] = 0xd8;
|
||||
jpeg[jpeg.length - 2] = 0xff;
|
||||
jpeg[jpeg.length - 1] = 0xd9;
|
||||
|
||||
const jpegPath = await service.storeJpeg(jpeg);
|
||||
const pngPath = await service.storePng(PNG);
|
||||
|
||||
assert.deepEqual(await readFile(jpegPath), jpeg);
|
||||
assert.deepEqual(await readFile(pngPath), PNG);
|
||||
assert.equal((await lstat(jpegPath)).mode & 0o077, 0);
|
||||
assert.equal((await lstat(pngPath)).mode & 0o077, 0);
|
||||
await assert.rejects(service.storeJpeg(Buffer.from('invalid')), /JPEG/u);
|
||||
await assert.rejects(
|
||||
service.storeJpeg(Buffer.alloc(MAX_CAPTURE_ASSET_BYTES + 1)),
|
||||
/JPEG/u,
|
||||
);
|
||||
await assert.rejects(service.storePng(Buffer.from('invalid')), /PNG/u);
|
||||
|
||||
const invalidPath = join(directory, 'invalid.png');
|
||||
const writer = service as unknown as {
|
||||
writePrivateCapture: (path: string, image: Uint8Array) => Promise<void>;
|
||||
};
|
||||
await assert.rejects(
|
||||
writer.writePrivateCapture(invalidPath, new Uint8Array()),
|
||||
/invalid screenshot file/u,
|
||||
);
|
||||
await assert.rejects(lstat(invalidPath), { code: 'ENOENT' });
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ function fakeNative(overrides: Partial<NativeAppshot> = {}): NativeAppshot {
|
||||
captureAppshot: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
listDisplays: () => [],
|
||||
captureDisplay: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -9,6 +9,19 @@ const audioEngine = readFileSync(
|
||||
join(appRoot, 'src', 'preload', 'audio-engine.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const audioOutputQueue = readFileSync(
|
||||
join(appRoot, 'src', 'preload', 'audio-output-queue.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const preload = readFileSync(
|
||||
join(appRoot, 'src', 'preload', 'index.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const main = readFileSync(join(appRoot, 'src', 'main', 'index.ts'), 'utf8');
|
||||
const daemonConnection = readFileSync(
|
||||
join(appRoot, 'src', 'main', 'daemon-connection.ts'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
describe('Live Host audio architecture', () => {
|
||||
it('matches the Codex virtual microphone graph for capture', () => {
|
||||
@@ -20,8 +33,9 @@ describe('Live Host audio architecture', () => {
|
||||
it('plays provider PCM on the device clock without a second media clock', () => {
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/context\.createBuffer\(1, samples, OUTPUT_SAMPLE_RATE\)/,
|
||||
/context\.createBuffer\(1, samples\.length, sampleRate\)/,
|
||||
);
|
||||
assert.match(audioEngine, /new StreamingOutputResampler\(/);
|
||||
assert.match(audioEngine, /source\.connect\(context\.destination\)/);
|
||||
assert.doesNotMatch(audioEngine, /sampleRate: OUTPUT_SAMPLE_RATE/);
|
||||
assert.doesNotMatch(audioEngine, /private outputDestination:/);
|
||||
@@ -33,6 +47,93 @@ describe('Live Host audio architecture', () => {
|
||||
assert.match(audioEngine, /context\?\.close\(\)/);
|
||||
});
|
||||
|
||||
it('preserves output identity from the wire through playback receipts', () => {
|
||||
assert.match(
|
||||
daemonConnection,
|
||||
/const frame = decodeOutputAudioFrame\(rawDataToBuffer\(data\)\)/u,
|
||||
);
|
||||
assert.match(main, /onOutputAudio: \(\{ audio, epoch, outputId \}\)/u);
|
||||
assert.match(
|
||||
main,
|
||||
/sendRendererCommand\('live:audio:play', \{ audio, epoch, outputId \}\)/u,
|
||||
);
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/play\(frame: Uint8Array, identity: PlaybackIdentity\)/u,
|
||||
);
|
||||
assert.match(audioEngine, /this\.onPlaybackStarted\(playbackIdentity\)/u);
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/this\.onPlaybackCompleted\(transition\.completed\)/u,
|
||||
);
|
||||
assert.match(
|
||||
preload,
|
||||
/ipcRenderer\.send\('live:audio:playback-started', identity\)/u,
|
||||
);
|
||||
assert.match(
|
||||
preload,
|
||||
/ipcRenderer\.send\('live:audio:playback-completed', identity\)/u,
|
||||
);
|
||||
assert.doesNotMatch(preload, /currentPlaybackEpoch/u);
|
||||
assert.match(
|
||||
main,
|
||||
/sendRequiredPlaybackReceipt\('playback_started', \(\) =>[\s\S]*daemon\.sendPlaybackStarted\(epoch, outputId\)/u,
|
||||
);
|
||||
assert.match(
|
||||
main,
|
||||
/sendRequiredPlaybackReceipt\('playback_completed', \(\) =>[\s\S]*daemon\.sendPlaybackCompleted\(epoch, outputId\)/u,
|
||||
);
|
||||
assert.match(
|
||||
main,
|
||||
/if \(!sent\) failRequiredDaemonMessage\(messageType\)/u,
|
||||
);
|
||||
});
|
||||
|
||||
it('gates output completion on the negotiated end marker and FIFO drain', () => {
|
||||
assert.match(
|
||||
daemonConnection,
|
||||
/capabilities: \{ outputAudioEndMarkerV1: true \}/u,
|
||||
);
|
||||
assert.match(
|
||||
daemonConnection,
|
||||
/case 'host\.output_audio_finished':[\s\S]*outputAudioEndMarkerV1 === true[\s\S]*onOutputAudioFinished/u,
|
||||
);
|
||||
assert.match(
|
||||
main,
|
||||
/syncOutputAudioEndMarkerMode\(\)[\s\S]*live:audio:set-output-end-marker-mode/u,
|
||||
);
|
||||
assert.match(
|
||||
main,
|
||||
/onOutputAudioFinished:[\s\S]*live:audio:output-finished/u,
|
||||
);
|
||||
assert.match(
|
||||
preload,
|
||||
/live:audio:set-output-end-marker-mode[\s\S]*setOutputEndMarkerMode/u,
|
||||
);
|
||||
assert.match(
|
||||
preload,
|
||||
/live:audio:output-finished[\s\S]*finishOutputAudio/u,
|
||||
);
|
||||
assert.match(audioEngine, /private outputQueue: Promise<void>/u);
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/this\.outputQueue\.catch\(\(\) => undefined\)\.then\(operation\)/u,
|
||||
);
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/finishOutputAudio\([\s\S]*this\.outputPlayback\.finish/u,
|
||||
);
|
||||
assert.match(
|
||||
audioOutputQueue,
|
||||
/output\.activeFrames !== 0[\s\S]*this\.endMarkerRequired && !output\.finished/u,
|
||||
);
|
||||
assert.match(
|
||||
audioEngine,
|
||||
/this\.outputGeneration \+= 1;\s*this\.outputPlayback\.clear\(\);[\s\S]*source\.stop\(\)/u,
|
||||
);
|
||||
assert.match(audioOutputQueue, /private readonly outputs = new Map/u);
|
||||
});
|
||||
|
||||
it('monitors both initial and replacement input tracks for device loss', () => {
|
||||
assert.match(audioEngine, /private monitorInputTracks\(/);
|
||||
assert.equal(audioEngine.match(/this\.monitorInputTracks\(/gu)?.length, 2);
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import { describe, it } from 'node:test';
|
||||
import ts from 'typescript';
|
||||
import type { HostAudioEngine } from '../../preload/audio-engine.ts';
|
||||
import { HostAudioLifecycle } from '../../preload/audio-lifecycle.ts';
|
||||
import * as inputPolicy from '../../preload/audio-input-policy.ts';
|
||||
import * as outputQueue from '../../preload/audio-output-queue.ts';
|
||||
import * as outputResampler from '../../preload/audio-output-resampler.ts';
|
||||
|
||||
const engineSource = ts.transpileModule(
|
||||
readFileSync(
|
||||
new URL('../../preload/audio-engine.ts', import.meta.url),
|
||||
'utf8',
|
||||
),
|
||||
{
|
||||
compilerOptions: {
|
||||
target: ts.ScriptTarget.ES2022,
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
},
|
||||
},
|
||||
).outputText;
|
||||
|
||||
function fixture(contextSampleRate = 48_000) {
|
||||
class Track extends EventTarget {
|
||||
enabled = true;
|
||||
stopped = false;
|
||||
getSettings() {
|
||||
return { sampleRate: 48_000, channelCount: 1 };
|
||||
}
|
||||
stop() {
|
||||
this.stopped = true;
|
||||
this.dispatchEvent(new Event('ended'));
|
||||
}
|
||||
}
|
||||
class Stream {
|
||||
readonly track = new Track();
|
||||
getTracks() {
|
||||
return [this.track];
|
||||
}
|
||||
getAudioTracks() {
|
||||
return this.getTracks();
|
||||
}
|
||||
}
|
||||
class Node {
|
||||
connected: unknown;
|
||||
disconnects = 0;
|
||||
connect(target: unknown) {
|
||||
this.connected = target;
|
||||
}
|
||||
disconnect() {
|
||||
this.disconnects++;
|
||||
}
|
||||
}
|
||||
class BufferSource extends Node {
|
||||
buffer: AudioBuffer | undefined;
|
||||
onended?: () => void;
|
||||
startedAt?: number;
|
||||
stopped = false;
|
||||
start(at: number) {
|
||||
this.startedAt = at;
|
||||
}
|
||||
stop() {
|
||||
this.stopped = true;
|
||||
this.onended?.();
|
||||
}
|
||||
}
|
||||
const contexts: Context[] = [];
|
||||
const worklets: Worklet[] = [];
|
||||
const streams: Stream[] = [];
|
||||
const constraints: MediaStreamConstraints[] = [];
|
||||
const ipc: Array<{ channel: string; value: unknown }> = [];
|
||||
const levels: number[] = [];
|
||||
const diagnostics: Array<{
|
||||
event: string;
|
||||
details: Readonly<Record<string, string | number | boolean | undefined>>;
|
||||
}> = [];
|
||||
const started: Array<{ epoch: number; outputId: number }> = [];
|
||||
const completed: Array<{ epoch: number; outputId: number }> = [];
|
||||
class Context {
|
||||
state = 'suspended';
|
||||
readonly sampleRate = contextSampleRate;
|
||||
readonly destination = {};
|
||||
currentTime = 0;
|
||||
readonly buffers: Array<{
|
||||
channels: number;
|
||||
length: number;
|
||||
rate: number;
|
||||
data: Float32Array;
|
||||
}> = [];
|
||||
readonly sources: BufferSource[] = [];
|
||||
readonly audioWorklet = { addModule: async () => {} };
|
||||
readonly virtualStreams: Stream[] = [];
|
||||
constructor(readonly options: AudioContextOptions) {
|
||||
contexts.push(this);
|
||||
}
|
||||
async resume() {
|
||||
this.state = 'running';
|
||||
}
|
||||
async close() {
|
||||
this.state = 'closed';
|
||||
}
|
||||
createBuffer(channels: number, length: number, rate: number) {
|
||||
const data = new Float32Array(length);
|
||||
this.buffers.push({ channels, length, rate, data });
|
||||
return { duration: length / rate, getChannelData: () => data };
|
||||
}
|
||||
createBufferSource() {
|
||||
const source = new BufferSource();
|
||||
this.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
createMediaStreamSource() {
|
||||
return new Node();
|
||||
}
|
||||
createMediaStreamDestination() {
|
||||
const stream = new Stream();
|
||||
this.virtualStreams.push(stream);
|
||||
return { stream, channelCount: 2 };
|
||||
}
|
||||
}
|
||||
class Worklet extends Node {
|
||||
readonly port: {
|
||||
onmessage?: (event: {
|
||||
data: { level: number; pcm16: ArrayBuffer };
|
||||
}) => void;
|
||||
} = {};
|
||||
constructor() {
|
||||
super();
|
||||
worklets.push(this);
|
||||
}
|
||||
frame(level = 0.5) {
|
||||
this.port.onmessage?.({
|
||||
data: { level, pcm16: new ArrayBuffer(320) },
|
||||
});
|
||||
}
|
||||
}
|
||||
const mediaDevices = Object.assign(new EventTarget(), {
|
||||
enumerateDevices: async () => [
|
||||
{ kind: 'audioinput', deviceId: 'mic-1', label: 'Microphone' },
|
||||
],
|
||||
getUserMedia: async (value: MediaStreamConstraints) => {
|
||||
constraints.push(value);
|
||||
const stream = new Stream();
|
||||
streams.push(stream);
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
const storage = new Map<string, string>();
|
||||
const exports: { HostAudioEngine?: typeof HostAudioEngine } = {};
|
||||
const modules: Record<string, unknown> = {
|
||||
electron: {
|
||||
ipcRenderer: {
|
||||
send: (channel: string, value: unknown) => ipc.push({ channel, value }),
|
||||
},
|
||||
},
|
||||
'@qwen-code/qwen-live/i18n': { liveMessage: (key: string) => key },
|
||||
'./audio-lifecycle.ts': { HostAudioLifecycle },
|
||||
'./audio-input-policy.ts': inputPolicy,
|
||||
'./audio-output-queue.ts': outputQueue,
|
||||
'./audio-output-resampler.ts': outputResampler,
|
||||
};
|
||||
runInNewContext(engineSource, {
|
||||
exports,
|
||||
require: (name: string) => {
|
||||
assert(Object.hasOwn(modules, name), `Unexpected import: ${name}`);
|
||||
return modules[name];
|
||||
},
|
||||
AudioContext: Context,
|
||||
AudioWorkletNode: Worklet,
|
||||
navigator: { mediaDevices },
|
||||
window: { location: { href: 'file:///synthetic/preload/index.js' } },
|
||||
localStorage: {
|
||||
getItem: (key: string) => storage.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => storage.set(key, value),
|
||||
removeItem: (key: string) => storage.delete(key),
|
||||
},
|
||||
URL,
|
||||
DOMException,
|
||||
DataView,
|
||||
Uint8Array,
|
||||
});
|
||||
assert(exports.HostAudioEngine);
|
||||
const engine = new exports.HostAudioEngine(
|
||||
(level) => levels.push(level),
|
||||
(event, details) => diagnostics.push({ event, details }),
|
||||
(identity) => started.push(identity),
|
||||
(identity) => completed.push(identity),
|
||||
);
|
||||
return {
|
||||
engine,
|
||||
contexts,
|
||||
worklets,
|
||||
streams,
|
||||
constraints,
|
||||
mediaDevices,
|
||||
ipc,
|
||||
levels,
|
||||
diagnostics,
|
||||
started,
|
||||
completed,
|
||||
};
|
||||
}
|
||||
|
||||
describe('Live Host audio engine', () => {
|
||||
it('checks readiness without opening microphone or output devices', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
assert.equal(h.constraints.length, 0);
|
||||
assert.equal(h.contexts.length, 0);
|
||||
const result = h.ipc.findLast(
|
||||
(entry) => entry.channel === 'live:audio:self-check',
|
||||
)?.value;
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(result)), {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
});
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
for (const rate of [44_100, 48_000, 96_000]) {
|
||||
it(`preserves 24 kHz PCM timing on a ${rate} Hz output clock`, async () => {
|
||||
const h = fixture(rate);
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 1, outputId: 1 };
|
||||
const frame = new Uint8Array(4800);
|
||||
new DataView(frame.buffer).setInt16(0, 16384, true);
|
||||
await h.engine.play(frame, identity);
|
||||
await h.engine.play(frame, identity);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(h.contexts.length, 1);
|
||||
const context = h.contexts[0];
|
||||
assert.equal(Object.hasOwn(context.options, 'sampleRate'), false);
|
||||
assert.equal(context.sampleRate, rate);
|
||||
assert.equal(context.buffers.length, 3);
|
||||
assert.ok(context.buffers.every((buffer) => buffer.rate === rate));
|
||||
assert.equal(
|
||||
context.buffers.reduce((length, buffer) => length + buffer.length, 0),
|
||||
rate / 5,
|
||||
);
|
||||
assert.equal(context.sources[0].connected, context.destination);
|
||||
assert.equal(context.sources[0].startedAt, 0.01);
|
||||
for (let index = 1; index < context.sources.length; index += 1) {
|
||||
assert.equal(
|
||||
context.sources[index].startedAt,
|
||||
Math.round(
|
||||
context.sources[index - 1].startedAt! * rate +
|
||||
context.buffers[index - 1].length,
|
||||
) / rate,
|
||||
);
|
||||
}
|
||||
assert.equal(h.started.length, 1);
|
||||
assert.equal(h.completed.length, 0);
|
||||
context.sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 0);
|
||||
context.sources[1].onended?.();
|
||||
assert.equal(h.completed.length, 0);
|
||||
context.sources[2].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
assert.equal(h.completed[0].outputId, identity.outputId);
|
||||
const diagnostic = h.diagnostics.find(
|
||||
(entry) => entry.event === 'output_context_ready',
|
||||
);
|
||||
assert.equal(diagnostic?.details.sourceSampleRate, 24_000);
|
||||
assert.equal(diagnostic?.details.contextSampleRate, rate);
|
||||
assert.equal(diagnostic?.details.resampling, true);
|
||||
assert.equal(
|
||||
h.diagnostics.filter(
|
||||
(entry) => entry.event === 'output_context_ready',
|
||||
).length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it('flushes a tiny output once and rejects late PCM and duplicate markers', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 2, outputId: 10 };
|
||||
const bytes = new Uint8Array([0xff, 0, 64, 0xff]);
|
||||
await h.engine.play(bytes.subarray(1, 3), identity);
|
||||
const context = h.contexts[0];
|
||||
assert.equal(context.sources.length, 0);
|
||||
assert.equal(h.started.length, 1);
|
||||
assert.equal(h.completed.length, 0);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(context.sources.length, 1);
|
||||
assert.deepEqual(Array.from(context.buffers[0].data), [0.5, 0.5]);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
await h.engine.play(bytes.subarray(1, 3), identity);
|
||||
assert.equal(context.sources.length, 1);
|
||||
assert.equal(h.completed.length, 0);
|
||||
context.sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
await h.engine.play(bytes.subarray(1, 3), identity);
|
||||
assert.equal(context.sources.length, 1);
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for a late marker and its retained tail after the sources drain', async () => {
|
||||
const h = fixture(16_000);
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 3, outputId: 1 };
|
||||
await h.engine.play(new Uint8Array(960), identity);
|
||||
const context = h.contexts[0];
|
||||
context.sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 0);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(context.sources.length, 2);
|
||||
assert.equal(h.completed.length, 0);
|
||||
assert.equal(
|
||||
context.buffers.reduce((length, buffer) => length + buffer.length, 0),
|
||||
320,
|
||||
);
|
||||
context.sources[1].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('completes sub-device-sample PCM at its marker without leaving a pending output', async () => {
|
||||
const h = fixture(8_000);
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 3, outputId: 2 };
|
||||
await h.engine.play(new Uint8Array([0, 0]), identity);
|
||||
assert.equal(h.contexts[0].sources.length, 0);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(h.contexts[0].sources.length, 0);
|
||||
assert.equal(h.started.length, 1);
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('clears playback and retained state if scheduling the terminal tail fails', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 3, outputId: 3 };
|
||||
await h.engine.play(new Uint8Array([0, 64]), identity);
|
||||
const context = h.contexts[0];
|
||||
context.createBuffer = () => {
|
||||
throw new Error('synthetic-buffer-failure');
|
||||
};
|
||||
await assert.rejects(
|
||||
h.engine.finishOutputAudio(identity),
|
||||
/synthetic-buffer-failure/,
|
||||
);
|
||||
assert.equal(context.state, 'closed');
|
||||
assert.equal(h.completed.length, 0);
|
||||
await h.engine.play(new Uint8Array([0, 0]), identity);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.deepEqual(Array.from(h.contexts[1].buffers[0].data), [0, 0]);
|
||||
h.contexts[1].sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('isolates retained audio by both epoch and output identity', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const first = { epoch: 3, outputId: 1 };
|
||||
const second = { epoch: 3, outputId: 2 };
|
||||
const nextEpoch = { epoch: 4, outputId: 1 };
|
||||
await h.engine.play(new Uint8Array([0, 64]), first);
|
||||
await h.engine.play(new Uint8Array([0, 224]), second);
|
||||
await h.engine.play(new Uint8Array([0, 0]), nextEpoch);
|
||||
const context = h.contexts[0];
|
||||
assert.equal(context.sources.length, 0);
|
||||
await h.engine.finishOutputAudio(second);
|
||||
await h.engine.finishOutputAudio(first);
|
||||
await h.engine.finishOutputAudio(nextEpoch);
|
||||
assert.deepEqual(
|
||||
context.buffers.map((buffer) => Array.from(buffer.data)),
|
||||
[
|
||||
[-0.25, -0.25],
|
||||
[0.5, 0.5],
|
||||
[0, 0],
|
||||
],
|
||||
);
|
||||
for (const source of context.sources) source.onended?.();
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(h.completed)), [
|
||||
second,
|
||||
first,
|
||||
nextEpoch,
|
||||
]);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('discards retained audio and pending finish operations when muted', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 5, outputId: 1 };
|
||||
await h.engine.play(new Uint8Array([0, 64]), identity);
|
||||
const staleContext = h.contexts[0];
|
||||
const staleFinish = h.engine.finishOutputAudio(identity);
|
||||
h.engine.setOutputMuted(true);
|
||||
await staleFinish;
|
||||
assert.equal(staleContext.sources.length, 0);
|
||||
assert.equal(staleContext.state, 'closed');
|
||||
assert.equal(h.completed.length, 0);
|
||||
h.engine.setOutputMuted(false);
|
||||
await h.engine.play(new Uint8Array([0, 0]), identity);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.deepEqual(Array.from(h.contexts[1].buffers[0].data), [0, 0]);
|
||||
h.contexts[1].sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves the legacy no-marker drain without retaining any samples', async () => {
|
||||
const h = fixture(44_100);
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
const identity = { epoch: 6, outputId: 1 };
|
||||
await h.engine.play(new Uint8Array([0, 64]), identity);
|
||||
const context = h.contexts[0];
|
||||
assert.equal(context.buffers.length, 1);
|
||||
assert.equal(context.buffers[0].rate, 24_000);
|
||||
assert.deepEqual(Array.from(context.buffers[0].data), [0.5]);
|
||||
context.sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(context.buffers.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('clears the new tail state when a mode switch interrupts playback', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(false);
|
||||
h.engine.setOutputEndMarkerMode(true);
|
||||
const identity = { epoch: 7, outputId: 1 };
|
||||
await h.engine.play(new Uint8Array(960), identity);
|
||||
const context = h.contexts[0];
|
||||
h.engine.setOutputEndMarkerMode(false);
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(h.completed.length, 0);
|
||||
await h.engine.finishOutputAudio(identity);
|
||||
assert.equal(context.buffers.length, 1);
|
||||
await h.engine.play(new Uint8Array([0, 0]), identity);
|
||||
assert.equal(h.contexts[1].buffers[0].rate, 24_000);
|
||||
h.contexts[1].sources[0].onended?.();
|
||||
assert.equal(h.completed.length, 1);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not acquire a microphone when capture starts muted', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, true, 1);
|
||||
assert.equal(h.constraints.length, 0);
|
||||
assert.equal(h.contexts.length, 0);
|
||||
assert.equal(h.levels.at(-1), 0);
|
||||
const ready = h.diagnostics.findLast(
|
||||
(entry) => entry.event === 'capture_ready',
|
||||
);
|
||||
assert.equal(ready?.details.epoch, 1);
|
||||
assert.equal(ready?.details.muted, true);
|
||||
assert.equal(ready?.details.capturing, false);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('releases muted capture and rebuilds it on unmute without clearing playback', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
await h.engine.play(new Uint8Array(4800), { epoch: 1, outputId: 1 });
|
||||
const captureContext = h.contexts[0];
|
||||
const outputContext = h.contexts[1];
|
||||
await h.engine.setCapture(true, true, 1);
|
||||
assert.equal(h.streams[0].track.stopped, true);
|
||||
assert.equal(captureContext.virtualStreams[0].track.stopped, true);
|
||||
assert.equal(captureContext.state, 'closed');
|
||||
assert.equal(outputContext.state, 'running');
|
||||
assert.equal(outputContext.sources[0].stopped, false);
|
||||
assert.equal(h.levels.at(-1), 0);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
assert.equal(h.streams.length, 2);
|
||||
assert.equal(h.streams[1].track.stopped, false);
|
||||
assert.equal(h.contexts.length, 3);
|
||||
h.worklets[1].frame();
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
1,
|
||||
);
|
||||
const ready = h.diagnostics.findLast(
|
||||
(entry) => entry.event === 'capture_ready',
|
||||
);
|
||||
assert.equal(ready?.details.capturing, true);
|
||||
assert.equal(ready?.details.contextSampleRate, 48_000);
|
||||
assert.equal(ready?.details.inputSampleRate, 48_000);
|
||||
assert.doesNotMatch(JSON.stringify(h.diagnostics), /mic-1|Microphone/);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores stale worklet messages after a same-epoch mute and unmute', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
const oldWorklet = h.worklets[0];
|
||||
await h.engine.setCapture(true, true, 1);
|
||||
const mutedLevelCount = h.levels.length;
|
||||
oldWorklet.frame(0.9);
|
||||
assert.equal(h.levels.length, mutedLevelCount);
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
0,
|
||||
);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
const levelCount = h.levels.length;
|
||||
const inputCount = h.ipc.filter(
|
||||
(entry) => entry.channel === 'live:audio:input',
|
||||
).length;
|
||||
oldWorklet.frame(0.9);
|
||||
assert.equal(h.levels.length, levelCount);
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
inputCount,
|
||||
);
|
||||
h.worklets.at(-1)?.frame(0.3);
|
||||
assert.equal(h.levels.at(-1), 0.3);
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
inputCount + 1,
|
||||
);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not reacquire muted input on device changes or selection', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, true, 1);
|
||||
h.mediaDevices.dispatchEvent(new Event('devicechange'));
|
||||
await h.engine.setInputDevice('mic-2');
|
||||
await h.engine.setCapture(true, true, 1);
|
||||
assert.equal(h.constraints.length, 0);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
assert.equal(h.constraints.length, 1);
|
||||
assert.equal(
|
||||
(h.constraints[0].audio as MediaTrackConstraints).deviceId &&
|
||||
(
|
||||
(h.constraints[0].audio as MediaTrackConstraints)
|
||||
.deviceId as ConstrainDOMStringParameters
|
||||
).exact,
|
||||
'mic-2',
|
||||
);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves the worklet and output clock when replacing an active input', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
await h.engine.play(new Uint8Array(4800), { epoch: 1, outputId: 1 });
|
||||
const worklet = h.worklets[0];
|
||||
const captureContext = h.contexts[0];
|
||||
const outputContext = h.contexts[1];
|
||||
h.mediaDevices.dispatchEvent(new Event('devicechange'));
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
assert.equal(h.streams.length, 2);
|
||||
assert.equal(h.streams[0].track.stopped, true);
|
||||
assert.equal(h.streams[1].track.stopped, false);
|
||||
assert.equal(h.worklets.length, 1);
|
||||
assert.equal(h.contexts.length, 2);
|
||||
assert.equal(captureContext.state, 'running');
|
||||
assert.equal(outputContext.state, 'running');
|
||||
assert.equal(outputContext.sources[0].stopped, false);
|
||||
worklet.frame();
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores capture callbacks after call end and disposal', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.engine.initialize(true);
|
||||
await h.engine.setCapture(true, false, 1);
|
||||
const worklet = h.worklets[0];
|
||||
await h.engine.setCapture(false, false, 1);
|
||||
const levelCount = h.levels.length;
|
||||
worklet.frame();
|
||||
assert.equal(h.levels.length, levelCount);
|
||||
await h.engine.dispose();
|
||||
worklet.frame();
|
||||
assert.equal(h.levels.length, levelCount);
|
||||
assert.equal(
|
||||
h.ipc.filter((entry) => entry.channel === 'live:audio:input').length,
|
||||
0,
|
||||
);
|
||||
} finally {
|
||||
await h.engine.dispose();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,10 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { scheduleOutputFrame } from '../../preload/audio-output-queue.ts';
|
||||
import {
|
||||
MAX_COMPLETED_OUTPUT_TOMBSTONES,
|
||||
OutputPlaybackTracker,
|
||||
scheduleOutputFrame,
|
||||
} from '../../preload/audio-output-queue.ts';
|
||||
|
||||
describe('Live Host output queue', () => {
|
||||
it('schedules frames while the resulting queue stays below ten seconds', () => {
|
||||
@@ -22,4 +26,134 @@ describe('Live Host output queue', () => {
|
||||
assert.equal(schedule.startAt, 4.01);
|
||||
assert.ok(Math.abs(schedule.endAt - 4.03) < Number.EPSILON * 4.03);
|
||||
});
|
||||
|
||||
it('keeps queued audio contiguous even when less than ten milliseconds remain', () => {
|
||||
assert.deepEqual(scheduleOutputFrame(0.505, 0.51, 0.5), {
|
||||
startAt: 0.51,
|
||||
endAt: 1.01,
|
||||
});
|
||||
assert.deepEqual(scheduleOutputFrame(0.505, 0.51, 0.5, 48_000), {
|
||||
startAt: 0.51,
|
||||
endAt: 1.01,
|
||||
});
|
||||
});
|
||||
|
||||
it('uses integer device-frame boundaries without cumulative rounding drift', () => {
|
||||
for (const rate of [16_000, 44_100, 48_000, 96_000]) {
|
||||
let cursor = 0;
|
||||
for (let index = 0; index < 10_000; index += 1) {
|
||||
const schedule = scheduleOutputFrame(0, cursor, 131 / rate, rate);
|
||||
if (index > 0) assert.equal(schedule.startAt, cursor);
|
||||
cursor = schedule.endAt;
|
||||
}
|
||||
assert.equal(cursor, (Math.ceil(0.01 * rate) + 1_310_000) / rate);
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for a marker across a temporary source gap', () => {
|
||||
const tracker = new OutputPlaybackTracker();
|
||||
tracker.setEndMarkerRequired(true);
|
||||
const identity = { epoch: 4, outputId: 11 };
|
||||
|
||||
const first = tracker.beginFrame(identity);
|
||||
assert(first);
|
||||
assert.equal(first.playbackStarted, true);
|
||||
assert.deepEqual(tracker.endFrame(first.output), { accepted: true });
|
||||
|
||||
const second = tracker.beginFrame(identity);
|
||||
assert(second);
|
||||
assert.equal(second.playbackStarted, false);
|
||||
assert.deepEqual(tracker.finish(identity), { accepted: true });
|
||||
assert.deepEqual(tracker.endFrame(second.output), {
|
||||
accepted: true,
|
||||
completed: identity,
|
||||
});
|
||||
assert.deepEqual(tracker.finish(identity), { accepted: false });
|
||||
});
|
||||
|
||||
it('completes immediately when the marker arrives after the drain', () => {
|
||||
const tracker = new OutputPlaybackTracker();
|
||||
tracker.setEndMarkerRequired(true);
|
||||
const identity = { epoch: 4, outputId: 12 };
|
||||
const frame = tracker.beginFrame(identity);
|
||||
assert(frame);
|
||||
|
||||
assert.deepEqual(tracker.endFrame(frame.output), { accepted: true });
|
||||
assert.deepEqual(tracker.finish(identity), {
|
||||
accepted: true,
|
||||
completed: identity,
|
||||
});
|
||||
});
|
||||
|
||||
it('tracks consecutive sealed outputs independently', () => {
|
||||
const tracker = new OutputPlaybackTracker();
|
||||
tracker.setEndMarkerRequired(true);
|
||||
const firstIdentity = { epoch: 4, outputId: 13 };
|
||||
const secondIdentity = { epoch: 4, outputId: 14 };
|
||||
const first = tracker.beginFrame(firstIdentity);
|
||||
assert(first);
|
||||
assert.deepEqual(tracker.finish(firstIdentity), { accepted: true });
|
||||
const second = tracker.beginFrame(secondIdentity);
|
||||
assert(second);
|
||||
assert.deepEqual(tracker.finish(secondIdentity), { accepted: true });
|
||||
|
||||
assert.deepEqual(tracker.endFrame(first.output), {
|
||||
accepted: true,
|
||||
completed: firstIdentity,
|
||||
});
|
||||
assert.deepEqual(tracker.endFrame(second.output), {
|
||||
accepted: true,
|
||||
completed: secondIdentity,
|
||||
});
|
||||
});
|
||||
|
||||
it('bounds completed tombstones while rejecting recent late traffic', () => {
|
||||
const tracker = new OutputPlaybackTracker();
|
||||
tracker.setEndMarkerRequired(true);
|
||||
for (
|
||||
let outputId = 1;
|
||||
outputId <= MAX_COMPLETED_OUTPUT_TOMBSTONES + 1;
|
||||
outputId += 1
|
||||
) {
|
||||
const identity = { epoch: 8, outputId };
|
||||
const frame = tracker.beginFrame(identity);
|
||||
assert(frame);
|
||||
assert.deepEqual(tracker.finish(identity), { accepted: true });
|
||||
assert.deepEqual(tracker.endFrame(frame.output), {
|
||||
accepted: true,
|
||||
completed: identity,
|
||||
});
|
||||
}
|
||||
|
||||
const recent = {
|
||||
epoch: 8,
|
||||
outputId: MAX_COMPLETED_OUTPUT_TOMBSTONES + 1,
|
||||
};
|
||||
assert.equal(tracker.beginFrame(recent), undefined);
|
||||
assert.deepEqual(tracker.finish(recent), { accepted: false });
|
||||
|
||||
const evicted = tracker.beginFrame({ epoch: 8, outputId: 1 });
|
||||
assert(evicted);
|
||||
assert.equal(evicted.playbackStarted, true);
|
||||
});
|
||||
|
||||
it('invalidates stale source endings on clear and preserves legacy drain', () => {
|
||||
const tracker = new OutputPlaybackTracker();
|
||||
tracker.setEndMarkerRequired(true);
|
||||
const staleIdentity = { epoch: 4, outputId: 15 };
|
||||
const stale = tracker.beginFrame(staleIdentity);
|
||||
assert(stale);
|
||||
tracker.clear();
|
||||
assert.deepEqual(tracker.endFrame(stale.output), { accepted: false });
|
||||
assert.deepEqual(tracker.finish(staleIdentity), { accepted: false });
|
||||
|
||||
tracker.setEndMarkerRequired(false);
|
||||
const legacyIdentity = { epoch: 5, outputId: 1 };
|
||||
const legacy = tracker.beginFrame(legacyIdentity);
|
||||
assert(legacy);
|
||||
assert.deepEqual(tracker.endFrame(legacy.output), {
|
||||
accepted: true,
|
||||
completed: legacyIdentity,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { StreamingOutputResampler } from '../../preload/audio-output-resampler.ts';
|
||||
|
||||
function join(parts: Float32Array[]): Float32Array {
|
||||
const output = new Float32Array(
|
||||
parts.reduce((length, part) => length + part.length, 0),
|
||||
);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
output.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function tone(frequency: number, length = 24_001): Float32Array {
|
||||
return Float32Array.from(
|
||||
{ length },
|
||||
(_, index) =>
|
||||
0.4 * Math.sin((2 * Math.PI * frequency * index) / 24_000 + 0.4),
|
||||
);
|
||||
}
|
||||
|
||||
function convert(
|
||||
input: Float32Array,
|
||||
outputRate: number,
|
||||
partition: number,
|
||||
): Float32Array {
|
||||
const resampler = new StreamingOutputResampler(24_000, outputRate);
|
||||
const parts: Float32Array[] = [];
|
||||
for (let offset = 0; offset < input.length; offset += partition) {
|
||||
parts.push(resampler.push(input.subarray(offset, offset + partition)));
|
||||
}
|
||||
parts.push(resampler.finish());
|
||||
assert.equal(resampler.finish().length, 0);
|
||||
assert.equal(resampler.push(input).length, 0);
|
||||
return join(parts);
|
||||
}
|
||||
|
||||
describe('Live Host streaming output resampler', () => {
|
||||
for (const rate of [8_000, 16_000, 24_000, 44_100, 48_000, 96_000]) {
|
||||
it(`preserves the entire waveform across arbitrary chunks at ${rate} Hz`, () => {
|
||||
const input = tone(431);
|
||||
const expected = convert(input, rate, input.length);
|
||||
assert.equal(expected.length, Math.round((input.length * rate) / 24_000));
|
||||
for (const size of [1, 7, 480, 512, 1024, 2400, 3072]) {
|
||||
assert.deepEqual(convert(input, rate, size), expected);
|
||||
}
|
||||
assert.ok(expected.every(Number.isFinite));
|
||||
});
|
||||
}
|
||||
|
||||
it('passes native-rate PCM through without filtering', () => {
|
||||
const input = Float32Array.of(-1, 0.25, 0.9999, 0, -0.125);
|
||||
assert.deepEqual(convert(input, 24_000, 1), input);
|
||||
});
|
||||
|
||||
it('keeps tiny chunks until the end marker without losing their tail', () => {
|
||||
const resampler = new StreamingOutputResampler(24_000, 44_100);
|
||||
assert.equal(resampler.push(Float32Array.of(0.4)).length, 0);
|
||||
assert.equal(resampler.push(Float32Array.of(0.4)).length, 0);
|
||||
const tail = resampler.finish();
|
||||
assert.equal(tail.length, 4);
|
||||
for (const sample of tail) assert.ok(Math.abs(sample - 0.4) < 0.000001);
|
||||
});
|
||||
|
||||
it('does not add gain or discontinuities to a constant stream', () => {
|
||||
const input = new Float32Array(1201).fill(0.4);
|
||||
for (const rate of [16_000, 44_100, 48_000]) {
|
||||
for (const sample of convert(input, rate, 17)) {
|
||||
assert.ok(Math.abs(sample - 0.4) < 0.000001);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('filters above-Nyquist energy when the output device runs at 16 kHz', () => {
|
||||
const rms = (samples: Float32Array) => {
|
||||
const middle = samples.subarray(200, samples.length - 200);
|
||||
return Math.sqrt(
|
||||
middle.reduce((sum, value) => sum + value * value, 0) / middle.length,
|
||||
);
|
||||
};
|
||||
const audible = rms(convert(tone(1000), 16_000, 512));
|
||||
const alias = rms(convert(tone(10_000), 16_000, 512));
|
||||
assert.ok(audible > 0.28 && audible < 0.285);
|
||||
assert.ok(alias / audible < 0.001);
|
||||
});
|
||||
|
||||
it('does not leak the previous output into a new resampler', () => {
|
||||
convert(new Float32Array(19).fill(1), 44_100, 7);
|
||||
assert.ok(
|
||||
convert(new Float32Array(19), 44_100, 7).every((value) => value === 0),
|
||||
);
|
||||
assert.equal(
|
||||
new StreamingOutputResampler(24_000, 44_100).finish().length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { describe, it } from 'node:test';
|
||||
import { liveText } from '@qwen-code/qwen-live/i18n';
|
||||
|
||||
const CAMERA_ENGINE = new URL(
|
||||
'../../preload/camera-engine.ts',
|
||||
import.meta.url,
|
||||
);
|
||||
const PRELOAD = new URL('../../preload/index.ts', import.meta.url);
|
||||
const RENDERER = new URL('../../renderer/live-view.ts', import.meta.url);
|
||||
const MAIN_PROCESS = new URL('../index.ts', import.meta.url);
|
||||
const BUILDER_CONFIG = new URL(
|
||||
'../../../electron-builder.yml',
|
||||
import.meta.url,
|
||||
);
|
||||
const ENTITLEMENTS = new URL(
|
||||
'../../../build/entitlements.mac.plist',
|
||||
import.meta.url,
|
||||
);
|
||||
const AFTER_PACK = new URL('../../../scripts/after-pack.cjs', import.meta.url);
|
||||
|
||||
describe('Live Host visual input architecture', () => {
|
||||
it('captures bounded camera JPEG frames without opening another microphone', async () => {
|
||||
const source = await readFile(CAMERA_ENGINE, 'utf8');
|
||||
|
||||
assert.match(source, /getUserMedia\(\{[\s\S]*audio: false/u);
|
||||
assert.match(source, /width: \{ ideal: requestedWidth \}/u);
|
||||
assert.match(source, /height: \{ ideal: requestedHeight \}/u);
|
||||
assert.match(source, /LIVE_JPEG_ATTEMPTS/u);
|
||||
assert.match(source, /SNAPSHOT_JPEG_QUALITIES/u);
|
||||
assert.match(source, /toBlob\(resolve, 'image\/jpeg'/u);
|
||||
assert.match(source, /MAX_INPUT_IMAGE_FRAME_BYTES/u);
|
||||
assert.match(source, /generation !== this\.generation\) return/u);
|
||||
});
|
||||
|
||||
it('keeps visual lifecycle and frame routing in the Host process', async () => {
|
||||
const source = await readFile(MAIN_PROCESS, 'utf8');
|
||||
|
||||
assert.match(source, /let visualInput: VisualInput \| undefined/u);
|
||||
assert.match(source, /stopLocalVisual\(\)/u);
|
||||
assert.match(source, /daemon\.sendVisualFrame/u);
|
||||
assert.match(source, /settings\.source !== 'screen'/u);
|
||||
assert.match(source, /settings\.mode !== 'live-feed'/u);
|
||||
assert.match(source, /live\.callId !== pending\.callId/u);
|
||||
assert.match(source, /daemon\.getEpoch\(\) !== pending\.epoch/u);
|
||||
assert.match(source, /pendingVisualSourceChange/u);
|
||||
assert.match(source, /applyPendingVisualSourceChange\(\)/u);
|
||||
assert.match(
|
||||
source,
|
||||
/if \(snapshot\.phase !== 'ready'\) \{[\s\S]{0,120}stopLocalVisual\(\)/u,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/if \(nextCamera !== 'granted'\) \{[\s\S]{0,160}failClosedForReadinessLoss\(\)/u,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/if \(visualInput\?\.source === 'camera'\) \{[\s\S]{0,80}failClosedForReadinessLoss\(\)/u,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/visualError = nextError;[\s\S]{0,80}appshotReadiness\.refresh\(\)/u,
|
||||
);
|
||||
assert.match(source, /camera_transport_rejected/u);
|
||||
assert.match(source, /generation !== visualGeneration/u);
|
||||
assert.match(source, /function dispatchNextCameraSnapshot\(\): void/u);
|
||||
assert.match(source, /pending\.sent = true/u);
|
||||
assert.match(
|
||||
source,
|
||||
/function requestCameraSnapshot[\s\S]*const timer = setTimeout[\s\S]*pendingCameraSnapshots\.set\(requestId, \{[\s\S]*timer,/u,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
/pending\.sent = true;\s*pending\.timer = setTimeout/u,
|
||||
);
|
||||
assert.match(source, /request\.persistAsset === false/u);
|
||||
assert.match(source, /visual_frame_sent[\s\S]*bytes: Buffer\.byteLength/u);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
/writeLiveDiagnostic\('visual_frame_sent',[\s\S]{0,400}image:/u,
|
||||
);
|
||||
});
|
||||
|
||||
it('reuses the private capture stream for an in-Host live preview', async () => {
|
||||
const [camera, preload, renderer] = await Promise.all([
|
||||
readFile(CAMERA_ENGINE, 'utf8'),
|
||||
readFile(PRELOAD, 'utf8'),
|
||||
readFile(RENDERER, 'utf8'),
|
||||
]);
|
||||
|
||||
assert.match(camera, /attachPreview\(\): void/u);
|
||||
assert.match(camera, /slot\.replaceChildren\(this\.video\)/u);
|
||||
assert.match(
|
||||
preload,
|
||||
/attachCameraPreview: \(\) => camera\.attachPreview\(\)/u,
|
||||
);
|
||||
assert.match(renderer, /slot\.dataset\.liveCameraPreview = ''/u);
|
||||
assert.match(renderer, /'ui\.cameraBadge'/u);
|
||||
assert.equal(
|
||||
liveText('en', 'ui.cameraBadge', { mode: 'Local preview' }),
|
||||
'Camera · Local preview',
|
||||
);
|
||||
assert.match(renderer, /shouldShowCameraPreview/u);
|
||||
assert.match(renderer, /'ui\.videoSource'/u);
|
||||
assert.equal(liveText('en', 'ui.videoSource'), 'Video Source');
|
||||
assert.match(renderer, /'ui\.onDemand'/u);
|
||||
assert.equal(liveText('en', 'ui.onDemand'), 'On Demand');
|
||||
assert.match(renderer, /'ui\.liveFeed'/u);
|
||||
assert.equal(liveText('en', 'ui.liveFeed'), 'Live Feed');
|
||||
assert.match(renderer, /SettingsPanel/u);
|
||||
});
|
||||
|
||||
it('declares the macOS camera purpose and hardened-runtime entitlement', async () => {
|
||||
const [builder, entitlements, afterPack] = await Promise.all([
|
||||
readFile(BUILDER_CONFIG, 'utf8'),
|
||||
readFile(ENTITLEMENTS, 'utf8'),
|
||||
readFile(AFTER_PACK, 'utf8'),
|
||||
]);
|
||||
|
||||
assert.match(builder, /NSCameraUsageDescription:/u);
|
||||
assert.match(builder, /resetAdHocDarwinSignature: true/u);
|
||||
assert.match(entitlements, /com\.apple\.security\.device\.camera/u);
|
||||
assert.doesNotMatch(afterPack, /NSCameraUsageDescription/u);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,614 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { HostCameraEngine } from '../../preload/camera-engine.ts';
|
||||
import {
|
||||
MAX_INPUT_IMAGE_FRAME_BYTES,
|
||||
isValidInputImageFrame,
|
||||
isValidCameraSnapshotAsset,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
function cameraEnvironment(
|
||||
options: {
|
||||
photoAvailable?: boolean;
|
||||
photoFailure?: boolean;
|
||||
nativeSizeAvailable?: boolean;
|
||||
automaticFrames?: boolean;
|
||||
takePhoto?: () => Promise<Blob>;
|
||||
restoreFailure?: boolean;
|
||||
negotiatedVideoSize?: { width: number; height: number };
|
||||
actualPhotoSize?: { width: number; height: number };
|
||||
} = {},
|
||||
) {
|
||||
const original = new Map<string, PropertyDescriptor | undefined>();
|
||||
const install = (key: string, value: unknown) => {
|
||||
original.set(key, Object.getOwnPropertyDescriptor(globalThis, key));
|
||||
Object.defineProperty(globalThis, key, { configurable: true, value });
|
||||
};
|
||||
const photoSettings: PhotoSettings[] = [];
|
||||
const appliedConstraints: MediaTrackConstraints[] = [];
|
||||
const encodes: Array<{ width: number; height: number }> = [];
|
||||
const errors: string[] = [];
|
||||
let photoCaptures = 0;
|
||||
let bitmapCloses = 0;
|
||||
let cameraWidth = 1280;
|
||||
let cameraHeight = 720;
|
||||
let stopped = false;
|
||||
const previewConstraints: MediaTrackConstraints = {
|
||||
width: { ideal: 1280 },
|
||||
height: { ideal: 720 },
|
||||
};
|
||||
const track = Object.assign(new EventTarget(), {
|
||||
getCapabilities: () =>
|
||||
options.nativeSizeAvailable === false
|
||||
? {}
|
||||
: {
|
||||
width: { min: 640, max: 3840 },
|
||||
height: { min: 480, max: 2160 },
|
||||
resizeMode: ['none', 'crop-and-scale'],
|
||||
},
|
||||
getConstraints: () => structuredClone(previewConstraints),
|
||||
getSettings: () => ({ width: cameraWidth, height: cameraHeight }),
|
||||
applyConstraints: async (constraints: MediaTrackConstraints) => {
|
||||
appliedConstraints.push(structuredClone(constraints));
|
||||
if (options.restoreFailure && appliedConstraints.length > 1) {
|
||||
throw new Error('restore_failed');
|
||||
}
|
||||
cameraWidth = (constraints.width as ConstrainULongRange).ideal ?? 1280;
|
||||
cameraHeight = (constraints.height as ConstrainULongRange).ideal ?? 720;
|
||||
if (appliedConstraints.length === 1 && options.negotiatedVideoSize) {
|
||||
cameraWidth = options.negotiatedVideoSize.width;
|
||||
cameraHeight = options.negotiatedVideoSize.height;
|
||||
}
|
||||
},
|
||||
stop: () => {
|
||||
stopped = true;
|
||||
track.dispatchEvent(new Event('ended'));
|
||||
},
|
||||
});
|
||||
const stream = {
|
||||
getTracks: () => [track],
|
||||
getVideoTracks: () => [track],
|
||||
} as unknown as MediaStream;
|
||||
let nextFrameId = 0;
|
||||
const frameCallbacks = new Map<number, VideoFrameRequestCallback>();
|
||||
const presentFrame = (width: number, height: number) => {
|
||||
video.videoWidth = width;
|
||||
video.videoHeight = height;
|
||||
for (const [id, callback] of [...frameCallbacks]) {
|
||||
frameCallbacks.delete(id);
|
||||
callback(0, { width, height } as VideoFrameCallbackMetadata);
|
||||
}
|
||||
};
|
||||
const video = Object.assign(new EventTarget(), {
|
||||
autoplay: false,
|
||||
muted: false,
|
||||
playsInline: false,
|
||||
srcObject: null as MediaStream | null,
|
||||
className: '',
|
||||
readyState: 2,
|
||||
videoWidth: 1280,
|
||||
videoHeight: 720,
|
||||
parentElement: null,
|
||||
play: async () => undefined,
|
||||
remove: () => undefined,
|
||||
setAttribute: () => undefined,
|
||||
requestVideoFrameCallback: (callback: VideoFrameRequestCallback) => {
|
||||
const id = ++nextFrameId;
|
||||
frameCallbacks.set(id, callback);
|
||||
if (options.automaticFrames !== false) {
|
||||
queueMicrotask(() => {
|
||||
if (frameCallbacks.has(id)) presentFrame(cameraWidth, cameraHeight);
|
||||
});
|
||||
}
|
||||
return id;
|
||||
},
|
||||
cancelVideoFrameCallback: (id: number) => frameCallbacks.delete(id),
|
||||
});
|
||||
const canvas = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext: () => ({ drawImage: () => undefined }),
|
||||
toBlob: (callback: (blob: Blob) => void) => {
|
||||
encodes.push({ width: canvas.width, height: canvas.height });
|
||||
const jpeg = Buffer.alloc(
|
||||
Math.max(4, Math.round((canvas.width * canvas.height) / 12)),
|
||||
);
|
||||
jpeg[0] = 0xff;
|
||||
jpeg[1] = 0xd8;
|
||||
jpeg[jpeg.length - 2] = 0xff;
|
||||
jpeg[jpeg.length - 1] = 0xd9;
|
||||
callback(new Blob([jpeg], { type: 'image/jpeg' }));
|
||||
},
|
||||
};
|
||||
install('navigator', {
|
||||
mediaDevices: { getUserMedia: async () => stream },
|
||||
});
|
||||
install('document', {
|
||||
createElement: (tag: string) => (tag === 'video' ? video : canvas),
|
||||
querySelector: () => null,
|
||||
});
|
||||
install('HTMLMediaElement', { HAVE_CURRENT_DATA: 2 });
|
||||
install(
|
||||
'FileReader',
|
||||
class {
|
||||
result: string | null = null;
|
||||
error: Error | null = null;
|
||||
onload: (() => void) | null = null;
|
||||
readAsDataURL(blob: Blob) {
|
||||
void blob.arrayBuffer().then((bytes) => {
|
||||
this.result = `data:image/jpeg;base64,${Buffer.from(bytes).toString('base64')}`;
|
||||
this.onload?.();
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
install(
|
||||
'ImageCapture',
|
||||
options.photoAvailable === false
|
||||
? undefined
|
||||
: class {
|
||||
getPhotoCapabilities() {
|
||||
return Promise.resolve({
|
||||
imageWidth: { max: 4032 },
|
||||
imageHeight: { max: 3024 },
|
||||
});
|
||||
}
|
||||
takePhoto(settings: PhotoSettings) {
|
||||
photoCaptures += 1;
|
||||
photoSettings.push(settings);
|
||||
if (options.photoFailure) {
|
||||
return Promise.reject(new Error('photo_not_supported'));
|
||||
}
|
||||
return options.takePhoto?.() ?? Promise.resolve(new Blob());
|
||||
}
|
||||
},
|
||||
);
|
||||
install('createImageBitmap', async (source: Blob | typeof video) => ({
|
||||
width:
|
||||
source instanceof Blob
|
||||
? (options.actualPhotoSize?.width ?? 4032)
|
||||
: source.videoWidth,
|
||||
height:
|
||||
source instanceof Blob
|
||||
? (options.actualPhotoSize?.height ?? 3024)
|
||||
: source.videoHeight,
|
||||
close: () => {
|
||||
bitmapCloses += 1;
|
||||
},
|
||||
}));
|
||||
const camera = new HostCameraEngine(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
(error) => errors.push(error),
|
||||
);
|
||||
return {
|
||||
camera,
|
||||
photoSettings,
|
||||
appliedConstraints,
|
||||
encodes,
|
||||
errors,
|
||||
video,
|
||||
presentFrame,
|
||||
photoCaptures: () => photoCaptures,
|
||||
bitmapCloses: () => bitmapCloses,
|
||||
stopped: () => stopped,
|
||||
start: () =>
|
||||
camera.setCapture(true, {
|
||||
epoch: 1,
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
cameraWidth: 1280,
|
||||
cameraHeight: 720,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
}),
|
||||
restore: () => {
|
||||
camera.dispose();
|
||||
for (const [key, descriptor] of original) {
|
||||
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
|
||||
else Reflect.deleteProperty(globalThis, key);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function flushCameraCallbacks(): Promise<void> {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
describe('HostCameraEngine', () => {
|
||||
it('rejects a native video snapshot negotiated below the advertised size and restores preview', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
photoAvailable: false,
|
||||
negotiatedVideoSize: { width: 1280, height: 720 },
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
await assert.rejects(
|
||||
environment.camera.captureSnapshot(),
|
||||
/camera_snapshot_resolution_unavailable/,
|
||||
);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
assert.equal(environment.appliedConstraints.length, 2);
|
||||
assert.equal(environment.video.videoWidth, 1280);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects an undersized native photo when native video fallback is also unavailable', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
actualPhotoSize: { width: 1280, height: 720 },
|
||||
nativeSizeAvailable: false,
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
await assert.rejects(
|
||||
environment.camera.captureSnapshot(),
|
||||
/camera_snapshot_resolution_unavailable/,
|
||||
);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
assert.equal(environment.bitmapCloses(), 1);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts full native dimensions rotated by 90 degrees', async () => {
|
||||
for (const options of [
|
||||
{ actualPhotoSize: { width: 3024, height: 4032 } },
|
||||
{
|
||||
photoAvailable: false,
|
||||
negotiatedVideoSize: { width: 2160, height: 3840 },
|
||||
},
|
||||
]) {
|
||||
const environment = cameraEnvironment(options);
|
||||
try {
|
||||
await environment.start();
|
||||
assert.ok((await environment.camera.captureSnapshot()).assetImage);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps explicitly bounded snapshots as aspect-preserving upper bounds', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
photoAvailable: false,
|
||||
negotiatedVideoSize: { width: 1600, height: 1200 },
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
await environment.camera.captureSnapshot({
|
||||
snapshotWidth: 1920,
|
||||
snapshotHeight: 1080,
|
||||
});
|
||||
assert.deepEqual(environment.encodes[0], { width: 1440, height: 1080 });
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
it('keeps a native still asset separately from its provider-size preview', async () => {
|
||||
const environment = cameraEnvironment();
|
||||
try {
|
||||
await environment.start();
|
||||
const result = await environment.camera.captureSnapshot();
|
||||
assert.equal(environment.photoCaptures(), 1);
|
||||
assert.deepEqual(environment.photoSettings, [
|
||||
{ imageWidth: 4032, imageHeight: 3024 },
|
||||
]);
|
||||
assert.deepEqual(environment.encodes[0], { width: 4032, height: 3024 });
|
||||
assert.ok(result.assetImage);
|
||||
assert.ok(isValidCameraSnapshotAsset(result.assetImage));
|
||||
assert.ok(
|
||||
Buffer.byteLength(result.assetImage, 'base64') >
|
||||
MAX_INPUT_IMAGE_FRAME_BYTES,
|
||||
);
|
||||
assert.ok(isValidInputImageFrame(result.image));
|
||||
assert.ok(result.width <= 1920 && result.height <= 1080);
|
||||
assert.equal(environment.bitmapCloses(), 1);
|
||||
assert.deepEqual(environment.appliedConstraints, []);
|
||||
assert.equal(environment.video.videoWidth, 1280);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('fits a still to snapshot bounds independently of the preview stream', async () => {
|
||||
const environment = cameraEnvironment();
|
||||
try {
|
||||
await environment.start();
|
||||
await environment.camera.captureSnapshot({
|
||||
snapshotWidth: 2560,
|
||||
snapshotHeight: 1440,
|
||||
});
|
||||
assert.deepEqual(environment.encodes[0], { width: 1920, height: 1440 });
|
||||
assert.equal(environment.video.videoWidth, 1280);
|
||||
assert.equal(environment.video.videoHeight, 720);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('samples private monitoring frames without taking photos or changing constraints', async () => {
|
||||
const environment = cameraEnvironment();
|
||||
try {
|
||||
await environment.start();
|
||||
const result = await environment.camera.captureSnapshot({
|
||||
persistAsset: false,
|
||||
snapshotWidth: 3840,
|
||||
snapshotHeight: 2160,
|
||||
});
|
||||
assert.equal(environment.photoCaptures(), 0);
|
||||
assert.deepEqual(environment.appliedConstraints, []);
|
||||
assert.equal(result.assetImage, undefined);
|
||||
assert.equal(result.width, 1280);
|
||||
assert.equal(result.height, 720);
|
||||
assert.ok(isValidInputImageFrame(result.image));
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to a native video frame and restores preview constraints', async () => {
|
||||
const environment = cameraEnvironment({ photoFailure: true });
|
||||
try {
|
||||
await environment.start();
|
||||
const result = await environment.camera.captureSnapshot();
|
||||
assert.deepEqual(environment.encodes[0], { width: 3840, height: 2160 });
|
||||
assert.ok(result.assetImage);
|
||||
assert.deepEqual(environment.appliedConstraints, [
|
||||
{
|
||||
width: { ideal: 3840 },
|
||||
height: { ideal: 2160 },
|
||||
resizeMode: 'none',
|
||||
},
|
||||
{ width: { ideal: 1280 }, height: { ideal: 720 } },
|
||||
]);
|
||||
assert.equal(environment.video.videoWidth, 1280);
|
||||
assert.equal(environment.video.videoHeight, 720);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for a fresh still frame and then a restored preview frame', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
photoAvailable: false,
|
||||
automaticFrames: false,
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
let completed = false;
|
||||
const capture = environment.camera.captureSnapshot().then((frame) => {
|
||||
completed = true;
|
||||
return frame;
|
||||
});
|
||||
await flushCameraCallbacks();
|
||||
environment.presentFrame(1280, 720);
|
||||
await flushCameraCallbacks();
|
||||
assert.equal(completed, false);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
environment.presentFrame(3840, 2160);
|
||||
await flushCameraCallbacks();
|
||||
assert.equal(environment.appliedConstraints.length, 2);
|
||||
assert.equal(completed, false);
|
||||
environment.presentFrame(1280, 720);
|
||||
await capture;
|
||||
assert.equal(completed, true);
|
||||
assert.deepEqual(environment.encodes[0], { width: 3840, height: 2160 });
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a native snapshot when neither photo nor native video size is available', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
photoAvailable: false,
|
||||
nativeSizeAvailable: false,
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
await assert.rejects(
|
||||
environment.camera.captureSnapshot(),
|
||||
/camera_snapshot_resolution_unavailable/u,
|
||||
);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
assert.deepEqual(environment.appliedConstraints, []);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('discards a photo that completes after the camera is stopped', async () => {
|
||||
let finishPhoto: ((photo: Blob) => void) | undefined;
|
||||
const environment = cameraEnvironment({
|
||||
takePhoto: () =>
|
||||
new Promise<Blob>((resolve) => {
|
||||
finishPhoto = resolve;
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
const capture = environment.camera.captureSnapshot();
|
||||
await flushCameraCallbacks();
|
||||
environment.camera.dispose();
|
||||
finishPhoto?.(new Blob());
|
||||
await assert.rejects(capture, /camera_not_ready/u);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
assert.deepEqual(environment.errors, []);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('stops capture if fallback cannot restore the preview', async () => {
|
||||
const environment = cameraEnvironment({
|
||||
photoAvailable: false,
|
||||
restoreFailure: true,
|
||||
});
|
||||
try {
|
||||
await environment.start();
|
||||
await assert.rejects(
|
||||
environment.camera.captureSnapshot(),
|
||||
/restore_failed/u,
|
||||
);
|
||||
assert.deepEqual(environment.errors, ['camera_preview_restore_failed']);
|
||||
assert.equal(environment.stopped(), true);
|
||||
assert.equal(environment.bitmapCloses(), 1);
|
||||
assert.deepEqual(environment.encodes, []);
|
||||
} finally {
|
||||
environment.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it('deduplicates identical settings while camera permission is pending', async () => {
|
||||
const originalNavigator = Object.getOwnPropertyDescriptor(
|
||||
globalThis,
|
||||
'navigator',
|
||||
);
|
||||
let rejectOpen: ((error: Error) => void) | undefined;
|
||||
let openCount = 0;
|
||||
let requestedConstraints: MediaStreamConstraints | undefined;
|
||||
Object.defineProperty(globalThis, 'navigator', {
|
||||
configurable: true,
|
||||
value: {
|
||||
mediaDevices: {
|
||||
getUserMedia: (constraints: MediaStreamConstraints) => {
|
||||
openCount += 1;
|
||||
requestedConstraints = constraints;
|
||||
return new Promise<MediaStream>((_resolve, reject) => {
|
||||
rejectOpen = reject;
|
||||
});
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const camera = new HostCameraEngine(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
const settings = {
|
||||
epoch: 1,
|
||||
mode: 'on-demand' as const,
|
||||
fps: 1,
|
||||
cameraWidth: 960,
|
||||
cameraHeight: 540,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
};
|
||||
|
||||
const first = camera.setCapture(true, settings);
|
||||
await camera.setCapture(true, settings);
|
||||
|
||||
assert.equal(openCount, 1);
|
||||
assert.deepEqual(requestedConstraints, {
|
||||
audio: false,
|
||||
video: {
|
||||
width: { ideal: 960 },
|
||||
height: { ideal: 540 },
|
||||
},
|
||||
});
|
||||
rejectOpen?.(new Error('permission_pending'));
|
||||
await assert.rejects(first, /permission_pending/u);
|
||||
camera.dispose();
|
||||
} finally {
|
||||
if (originalNavigator) {
|
||||
Object.defineProperty(globalThis, 'navigator', originalNavigator);
|
||||
} else {
|
||||
Reflect.deleteProperty(globalThis, 'navigator');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('signals ready only after the camera has a decodable frame', async () => {
|
||||
const originalNavigator = Object.getOwnPropertyDescriptor(
|
||||
globalThis,
|
||||
'navigator',
|
||||
);
|
||||
const originalDocument = Object.getOwnPropertyDescriptor(
|
||||
globalThis,
|
||||
'document',
|
||||
);
|
||||
const track = Object.assign(new EventTarget(), { stop: () => undefined });
|
||||
const stream = {
|
||||
getTracks: () => [track],
|
||||
getVideoTracks: () => [track],
|
||||
} as unknown as MediaStream;
|
||||
const video = Object.assign(new EventTarget(), {
|
||||
autoplay: false,
|
||||
muted: false,
|
||||
playsInline: false,
|
||||
srcObject: null as MediaStream | null,
|
||||
className: '',
|
||||
readyState: 0,
|
||||
videoWidth: 0,
|
||||
videoHeight: 0,
|
||||
parentElement: null,
|
||||
play: () => Promise.resolve(),
|
||||
remove: () => undefined,
|
||||
setAttribute: () => undefined,
|
||||
}) as unknown as HTMLVideoElement;
|
||||
const canvas = {} as HTMLCanvasElement;
|
||||
Object.defineProperty(globalThis, 'navigator', {
|
||||
configurable: true,
|
||||
value: {
|
||||
mediaDevices: { getUserMedia: () => Promise.resolve(stream) },
|
||||
},
|
||||
});
|
||||
Object.defineProperty(globalThis, 'document', {
|
||||
configurable: true,
|
||||
value: {
|
||||
createElement: (tag: string) => (tag === 'video' ? video : canvas),
|
||||
querySelector: () => null,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
let ready = false;
|
||||
const camera = new HostCameraEngine(
|
||||
() => undefined,
|
||||
() => {
|
||||
ready = true;
|
||||
},
|
||||
() => undefined,
|
||||
);
|
||||
const started = camera.setCapture(true, {
|
||||
epoch: 1,
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
cameraWidth: 1280,
|
||||
cameraHeight: 720,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
});
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
assert.equal(ready, false);
|
||||
|
||||
Object.assign(video, {
|
||||
readyState: 2,
|
||||
videoWidth: 1280,
|
||||
videoHeight: 720,
|
||||
});
|
||||
video.dispatchEvent(new Event('loadeddata'));
|
||||
await started;
|
||||
assert.equal(ready, true);
|
||||
camera.dispose();
|
||||
} finally {
|
||||
if (originalNavigator) {
|
||||
Object.defineProperty(globalThis, 'navigator', originalNavigator);
|
||||
} else {
|
||||
Reflect.deleteProperty(globalThis, 'navigator');
|
||||
}
|
||||
if (originalDocument) {
|
||||
Object.defineProperty(globalThis, 'document', originalDocument);
|
||||
} else {
|
||||
Reflect.deleteProperty(globalThis, 'document');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
unlinkSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import { liveMessage, type LiveMessageKey } from '@qwen-code/qwen-live/i18n';
|
||||
import type { HostPublicState } from '../../shared/host-api.ts';
|
||||
|
||||
const source = readFileSync(new URL('../index.ts', import.meta.url), 'utf8');
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = new Set(['isTrustedSender', 'registerIpc', 'publicState']);
|
||||
const functions = tree.statements.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) && node.name && names.has(node.name.text),
|
||||
);
|
||||
assert.equal(functions.length, names.size);
|
||||
const directories: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
type Handler = (...args: unknown[]) => unknown;
|
||||
const errorCode = (key: LiveMessageKey) => (error: unknown) =>
|
||||
Boolean(
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'message' in error &&
|
||||
error.message === liveMessage(key),
|
||||
);
|
||||
|
||||
function fixture() {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'qwen-live-open-config-'));
|
||||
directories.push(directory);
|
||||
const dataDir = join(directory, '自定义 Live data');
|
||||
mkdirSync(dataDir);
|
||||
const path = join(dataDir, 'config.json');
|
||||
const content = '{"language":"en"}\n';
|
||||
writeFileSync(path, content, { mode: 0o600 });
|
||||
const opened: string[] = [];
|
||||
const flags = {
|
||||
configPath: path as string | undefined,
|
||||
destroyed: false,
|
||||
openError: '' as string | Error,
|
||||
};
|
||||
const handlers = new Map<string, Handler>();
|
||||
const context = {
|
||||
ipcMain: {
|
||||
on: (channel: string, callback: Handler) =>
|
||||
handlers.set(channel, callback),
|
||||
handle: (channel: string, callback: Handler) =>
|
||||
handlers.set(channel, callback),
|
||||
},
|
||||
overlay: { isDestroyed: () => flags.destroyed, webContents: {} },
|
||||
rendererEventsEnabled: true,
|
||||
quitState: undefined as HostPublicState['quitState'],
|
||||
connection: { phase: 'ready' },
|
||||
daemon: { getConfigFilePath: () => flags.configPath },
|
||||
lstatSync,
|
||||
shell: {
|
||||
openPath: async (openedPath: string) => {
|
||||
opened.push(openedPath);
|
||||
if (flags.openError instanceof Error) throw flags.openError;
|
||||
return flags.openError;
|
||||
},
|
||||
},
|
||||
liveMessage,
|
||||
theme: 'system',
|
||||
resolvedTheme: () => 'light',
|
||||
language: 'en',
|
||||
overlayOffset: { x: 0, y: 0 },
|
||||
visualInput: undefined,
|
||||
visualError: undefined,
|
||||
visualReady: false,
|
||||
screenDisplays: [],
|
||||
screenDisplaysError: undefined,
|
||||
permissions: {},
|
||||
selfChecks: {},
|
||||
effectiveLiveStatus: () => ({
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
}),
|
||||
};
|
||||
const code = `${functions.map((node) => node.getText(tree)).join('\n')}\nregisterIpc(); ({ publicState });`;
|
||||
const controls = runInNewContext(
|
||||
ts.transpileModule(code, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText,
|
||||
context,
|
||||
) as { publicState: () => HostPublicState };
|
||||
const openConfig = handlers.get('live:open-config');
|
||||
assert(openConfig);
|
||||
return {
|
||||
path,
|
||||
content,
|
||||
dataDir,
|
||||
flags,
|
||||
context,
|
||||
controls,
|
||||
opened,
|
||||
open: (sender: unknown = context.overlay.webContents, ...args: unknown[]) =>
|
||||
openConfig({ sender }, ...args) as Promise<void>,
|
||||
};
|
||||
}
|
||||
|
||||
describe('native config-file opening', () => {
|
||||
it('opens only the active config and exposes availability, not the private path', async () => {
|
||||
const h = fixture();
|
||||
const state = h.controls.publicState();
|
||||
assert.equal(state.canOpenConfig, true);
|
||||
assert.equal(JSON.stringify(state).includes(h.path), false);
|
||||
await h.open(h.context.overlay.webContents, '/another/config.json');
|
||||
assert.deepEqual(h.opened, [h.path]);
|
||||
assert.equal(readFileSync(h.path, 'utf8'), h.content);
|
||||
assert.equal(lstatSync(h.path).mode & 0o777, 0o600);
|
||||
});
|
||||
|
||||
it('rejects foreign/stale renderers, reload, Quit and unavailable connections', async () => {
|
||||
const h = fixture();
|
||||
const unavailable = errorCode('host.config.unavailable');
|
||||
await assert.rejects(h.open({}), unavailable);
|
||||
const stale = h.context.overlay.webContents;
|
||||
h.context.overlay.webContents = {};
|
||||
await assert.rejects(h.open(stale), unavailable);
|
||||
h.flags.destroyed = true;
|
||||
await assert.rejects(h.open(), unavailable);
|
||||
h.flags.destroyed = false;
|
||||
h.context.rendererEventsEnabled = false;
|
||||
await assert.rejects(h.open(), unavailable);
|
||||
h.context.rendererEventsEnabled = true;
|
||||
for (const quitState of ['pending', 'failed'] as const) {
|
||||
h.context.quitState = quitState;
|
||||
assert.equal(h.controls.publicState().canOpenConfig, false);
|
||||
await assert.rejects(h.open(), unavailable);
|
||||
}
|
||||
h.context.quitState = undefined;
|
||||
h.context.connection.phase = 'disconnected';
|
||||
assert.equal(h.controls.publicState().canOpenConfig, false);
|
||||
await assert.rejects(h.open(), unavailable);
|
||||
h.context.connection.phase = 'ready';
|
||||
h.flags.configPath = undefined;
|
||||
assert.equal(h.controls.publicState().canOpenConfig, false);
|
||||
await assert.rejects(h.open(), unavailable);
|
||||
assert.deepEqual(h.opened, []);
|
||||
});
|
||||
|
||||
it('does not create missing config files or open directories and symlinks', async () => {
|
||||
const h = fixture();
|
||||
const inaccessible = errorCode('host.config.inaccessible');
|
||||
unlinkSync(h.path);
|
||||
await assert.rejects(h.open(), inaccessible);
|
||||
assert.deepEqual(readdirSync(h.dataDir), []);
|
||||
mkdirSync(h.path);
|
||||
await assert.rejects(h.open(), inaccessible);
|
||||
rmSync(h.path, { recursive: true });
|
||||
const target = join(h.dataDir, 'other.json');
|
||||
writeFileSync(target, h.content);
|
||||
symlinkSync(target, h.path);
|
||||
await assert.rejects(h.open(), inaccessible);
|
||||
assert.equal(readFileSync(target, 'utf8'), h.content);
|
||||
assert.deepEqual(h.opened, []);
|
||||
});
|
||||
|
||||
it('handles native opener error strings and rejections without exposing details, and allows retry', async () => {
|
||||
const h = fixture();
|
||||
for (const failure of [
|
||||
'private launch details',
|
||||
new Error('private launch details'),
|
||||
]) {
|
||||
h.flags.openError = failure;
|
||||
await assert.rejects(h.open(), errorCode('host.config.openFailed'));
|
||||
}
|
||||
h.flags.openError = '';
|
||||
await h.open();
|
||||
assert.deepEqual(h.opened, [h.path, h.path, h.path]);
|
||||
assert.equal(readFileSync(h.path, 'utf8'), h.content);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, it } from 'node:test';
|
||||
import { WebSocketServer } from 'ws';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
MAX_CONTROL_FRAME_BYTES,
|
||||
MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES,
|
||||
parseDaemonControlMessage,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
describe('Host incoming frame limits', () => {
|
||||
it('accepts control frames above the audio limit while retaining the binary audio limit', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'live-control-limit-'));
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
let connection: LiveDaemonConnection | undefined;
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
discovery,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'fixture-token',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'fixture_nonce_2026',
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const large = JSON.stringify({
|
||||
type: 'host.state',
|
||||
epoch: 1,
|
||||
memory: {
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
locked: false,
|
||||
libraryId: 'default',
|
||||
model: 'fixture',
|
||||
libraries: Array.from({ length: 2800 }, (_, index) => ({
|
||||
id: `library_${index}`,
|
||||
name: 'x'.repeat(80),
|
||||
})),
|
||||
},
|
||||
status: { v: 1, available: true, state: 'idle', shortcut: 'Command+E' },
|
||||
});
|
||||
assert(Buffer.byteLength(large) > MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES);
|
||||
assert(Buffer.byteLength(large) < MAX_CONTROL_FRAME_BYTES);
|
||||
assert(parseDaemonControlMessage(large));
|
||||
let memoryReceived = false;
|
||||
let resolveClose: (code: number) => void = () => {};
|
||||
const closed = new Promise<number>((resolve, reject) => {
|
||||
resolveClose = resolve;
|
||||
timeout = setTimeout(
|
||||
() => reject(new Error('Connection did not close')),
|
||||
3000,
|
||||
);
|
||||
});
|
||||
server.once('connection', (peer) => {
|
||||
peer.once('message', () => {
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
epoch: 1,
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'fixture_nonce_2026',
|
||||
heartbeatIntervalMs: 10000,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
}),
|
||||
);
|
||||
peer.send(large);
|
||||
peer.send(Buffer.alloc(MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES + 2));
|
||||
});
|
||||
peer.once('close', resolveClose);
|
||||
});
|
||||
connection = new LiveDaemonConnection(
|
||||
'fixture',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'denied',
|
||||
camera: 'denied',
|
||||
accessibility: 'denied',
|
||||
screenRecording: 'denied',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: false,
|
||||
audioOutput: false,
|
||||
globalShortcut: false,
|
||||
appshot: false,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => {
|
||||
if (snapshot.memory?.libraries.length === 2800)
|
||||
memoryReceived = true;
|
||||
},
|
||||
onOutputAudio: () => assert.fail('Oversized audio was accepted'),
|
||||
onOutputAudioFinished() {},
|
||||
onClearOutput() {},
|
||||
},
|
||||
discovery,
|
||||
);
|
||||
connection.start();
|
||||
assert.equal(await closed, 1009);
|
||||
assert.equal(memoryReceived, true);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
connection?.stop();
|
||||
for (const peer of server.clients) peer.terminate();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter, once } from 'node:events';
|
||||
import { liveText, liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
@@ -14,8 +16,10 @@ import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
MAX_CONTROL_FRAME_BYTES,
|
||||
MAX_SOCKET_BUFFERED_BYTES,
|
||||
encodeOutputAudioFrame,
|
||||
type HostAction,
|
||||
type HostControlMessage,
|
||||
type OutputAudioFrame,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
@@ -55,11 +59,13 @@ describe('LiveDaemonConnection', () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-live-connection-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const discoveryPath = join(directory, 'daemon.json');
|
||||
const configPath = join(directory, 'custom data', 'config.json');
|
||||
await writeFile(
|
||||
discoveryPath,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'private-token',
|
||||
configPath,
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
@@ -77,15 +83,18 @@ describe('LiveDaemonConnection', () => {
|
||||
},
|
||||
);
|
||||
const snapshots: string[] = [];
|
||||
const outputFrames: Uint8Array[] = [];
|
||||
const outputFrames: OutputAudioFrame[] = [];
|
||||
const outputEvents: string[] = [];
|
||||
const shortcuts: string[] = [];
|
||||
let captureCalls = 0;
|
||||
let visualCaptureCalls = 0;
|
||||
let visualCaptureSource: 'screen' | 'camera' = 'screen';
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -97,31 +106,51 @@ describe('LiveDaemonConnection', () => {
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => snapshots.push(snapshot.phase),
|
||||
onOutputAudio: (frame) => outputFrames.push(frame),
|
||||
onOutputAudio: (frame) => {
|
||||
outputFrames.push(frame);
|
||||
outputEvents.push(`audio:${frame.epoch}:${frame.outputId}`);
|
||||
},
|
||||
onOutputAudioFinished: (identity) => {
|
||||
outputEvents.push(`finished:${identity.epoch}:${identity.outputId}`);
|
||||
},
|
||||
onClearOutput: () => undefined,
|
||||
setShortcut: (shortcut) => {
|
||||
shortcuts.push(shortcut);
|
||||
return { success: true };
|
||||
},
|
||||
captureScreenContext: async () => {
|
||||
captureCalls += 1;
|
||||
if (captureCalls === 3) {
|
||||
captureVisual: async (request) => {
|
||||
visualCaptureCalls += 1;
|
||||
if (visualCaptureCalls === 3) {
|
||||
throw new Error('x'.repeat(100_000));
|
||||
}
|
||||
if (visualCaptureCalls === 4)
|
||||
throw new Error(liveMessage('host.error.visualUnavailable'));
|
||||
assert.deepEqual(request, {
|
||||
source: 'screen',
|
||||
snapshotWidth: 1920,
|
||||
snapshotHeight: 1080,
|
||||
...(visualCaptureCalls === 1 ? { persistAsset: false } : {}),
|
||||
});
|
||||
return {
|
||||
source: visualCaptureSource,
|
||||
image: Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64'),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
appName: 'Safari',
|
||||
windowTitle: 'LIVE_APP_A',
|
||||
accessibilityText:
|
||||
captureCalls === 1
|
||||
? 'AXWindow LIVE_APP_A'
|
||||
: '\u0001'.repeat(32_000),
|
||||
screenshotPath: '/private/tmp/qwen-live-appshot/test.png',
|
||||
accessibilityText: 'AXWindow LIVE_APP_A',
|
||||
...(request.persistAsset === false
|
||||
? {}
|
||||
: {
|
||||
screenshotPath: '/private/tmp/qwen-live-appshot/visual.png',
|
||||
}),
|
||||
};
|
||||
},
|
||||
},
|
||||
discoveryPath,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
connection.start();
|
||||
|
||||
const request = await requestPromise;
|
||||
@@ -132,6 +161,7 @@ describe('LiveDaemonConnection', () => {
|
||||
assert(peer);
|
||||
|
||||
const helloFrame = await nextMessage(peer);
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
assert.equal(helloFrame.isBinary, false);
|
||||
const hello = JSON.parse(
|
||||
helloFrame.data.toString('utf8'),
|
||||
@@ -139,6 +169,9 @@ describe('LiveDaemonConnection', () => {
|
||||
assert.equal(hello.type, 'host.hello');
|
||||
assert.equal(hello.protocolVersion, LIVE_PROTOCOL_VERSION);
|
||||
assert.equal(hello.bundleId, 'com.alibaba.qwen-code.live-host');
|
||||
assert.deepEqual(hello.capabilities, {
|
||||
outputAudioEndMarkerV1: true,
|
||||
});
|
||||
|
||||
const requiredActions: HostAction[] = [
|
||||
{ type: 'host.action', action: 'stop', epoch: 0 },
|
||||
@@ -171,6 +204,14 @@ describe('LiveDaemonConnection', () => {
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 1_000,
|
||||
epoch: 0,
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
visualInput: {
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
@@ -181,6 +222,190 @@ describe('LiveDaemonConnection', () => {
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(snapshots.at(-1), 'ready');
|
||||
assert.equal(connection.getConfigFilePath(), configPath);
|
||||
assert.equal('configPath' in connection.getSnapshot(), false);
|
||||
assert.deepEqual(connection.getSnapshot().capabilities, {
|
||||
outputAudioEndMarkerV1: true,
|
||||
});
|
||||
assert.deepEqual(connection.getSnapshot().visualInput, {
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
});
|
||||
|
||||
const image = Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64');
|
||||
assert.equal(connection.sendVisualFrame('camera', image, 1), false);
|
||||
const imageFramePromise = nextMessage(peer);
|
||||
assert.equal(connection.sendVisualFrame('camera', image, 0), true);
|
||||
const imageFrame = await imageFramePromise;
|
||||
assert.equal(imageFrame.isBinary, false);
|
||||
assert.deepEqual(JSON.parse(imageFrame.data.toString('utf8')), {
|
||||
type: 'host.visual_frame',
|
||||
epoch: 0,
|
||||
source: 'camera',
|
||||
image,
|
||||
});
|
||||
|
||||
const visualSourcePromise = nextMessage(peer);
|
||||
assert.equal(connection.sendVisualSettings({ source: 'screen' }, 0), true);
|
||||
const visualSource = await visualSourcePromise;
|
||||
assert.deepEqual(JSON.parse(visualSource.data.toString('utf8')), {
|
||||
type: 'host.visual_settings',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
mode: 'live-feed',
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
});
|
||||
const visualModePromise = nextMessage(peer);
|
||||
assert.equal(connection.sendVisualSettings({ mode: 'on-demand' }, 0), true);
|
||||
const visualMode = await visualModePromise;
|
||||
assert.deepEqual(JSON.parse(visualMode.data.toString('utf8')), {
|
||||
type: 'host.visual_settings',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
});
|
||||
assert.deepEqual(connection.getSnapshot().visualInput, {
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
});
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'visual-1',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
snapshotWidth: 1920,
|
||||
snapshotHeight: 1080,
|
||||
persistAsset: false,
|
||||
}),
|
||||
);
|
||||
const visualCaptureFrame = await nextMessage(peer);
|
||||
assert.deepEqual(JSON.parse(visualCaptureFrame.data.toString('utf8')), {
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: 'visual-1',
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
appName: 'Safari',
|
||||
windowTitle: 'LIVE_APP_A',
|
||||
accessibilityText: 'AXWindow LIVE_APP_A',
|
||||
});
|
||||
assert.equal(visualCaptureCalls, 1);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.state',
|
||||
epoch: 0,
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+Q',
|
||||
},
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.deepEqual(connection.getSnapshot().visualInput, {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
});
|
||||
assert.deepEqual(connection.getSnapshot().capabilities, {
|
||||
outputAudioEndMarkerV1: true,
|
||||
});
|
||||
|
||||
visualCaptureSource = 'camera';
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'visual-wrong-source',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
snapshotWidth: 1920,
|
||||
snapshotHeight: 1080,
|
||||
}),
|
||||
);
|
||||
const mismatchedVisualCapture = await nextMessage(peer);
|
||||
assert.deepEqual(
|
||||
JSON.parse(mismatchedVisualCapture.data.toString('utf8')),
|
||||
{
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: 'visual-wrong-source',
|
||||
success: false,
|
||||
error: liveText('en', 'host.error.visualWrongSource'),
|
||||
},
|
||||
);
|
||||
assert.equal(visualCaptureCalls, 2);
|
||||
|
||||
visualCaptureSource = 'screen';
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'visual-error',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
snapshotWidth: 1920,
|
||||
snapshotHeight: 1080,
|
||||
}),
|
||||
);
|
||||
const failedVisualCaptureFrame = await nextMessage(peer);
|
||||
const failedVisualCapture = JSON.parse(
|
||||
failedVisualCaptureFrame.data.toString('utf8'),
|
||||
) as { success: boolean; error: string };
|
||||
assert.equal(failedVisualCapture.success, false);
|
||||
assert.equal(failedVisualCapture.error.length, 1_024);
|
||||
assert.equal(
|
||||
Buffer.byteLength(
|
||||
failedVisualCaptureFrame.data.toString('utf8'),
|
||||
'utf8',
|
||||
) <= MAX_CONTROL_FRAME_BYTES,
|
||||
true,
|
||||
);
|
||||
assert.equal(visualCaptureCalls, 3);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'visual-localized-error',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
}),
|
||||
);
|
||||
const localizedVisualFrame = await nextMessage(peer);
|
||||
const localizedVisualResult = JSON.parse(
|
||||
localizedVisualFrame.data.toString('utf8'),
|
||||
);
|
||||
assert.equal(localizedVisualResult.error, 'Visual capture is unavailable.');
|
||||
assert.equal(localizedVisualResult.error.includes('qwen-live-ui:'), false);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
@@ -198,65 +423,6 @@ describe('LiveDaemonConnection', () => {
|
||||
});
|
||||
assert.deepEqual(shortcuts, ['Command+E']);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_screen_context',
|
||||
requestId: 'capture-1',
|
||||
epoch: 0,
|
||||
}),
|
||||
);
|
||||
const captureFrame = await nextMessage(peer);
|
||||
assert.deepEqual(JSON.parse(captureFrame.data.toString('utf8')), {
|
||||
type: 'host.screen_context_result',
|
||||
requestId: 'capture-1',
|
||||
success: true,
|
||||
appName: 'Safari',
|
||||
windowTitle: 'LIVE_APP_A',
|
||||
accessibilityText: 'AXWindow LIVE_APP_A',
|
||||
screenshotPath: '/private/tmp/qwen-live-appshot/test.png',
|
||||
});
|
||||
assert.equal(captureCalls, 1);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_screen_context',
|
||||
requestId: 'capture-2',
|
||||
epoch: 0,
|
||||
}),
|
||||
);
|
||||
const boundedCaptureFrame = await nextMessage(peer);
|
||||
const boundedCapture = JSON.parse(
|
||||
boundedCaptureFrame.data.toString('utf8'),
|
||||
) as { accessibilityText: string; success: boolean };
|
||||
assert.equal(boundedCapture.success, true);
|
||||
assert.equal(
|
||||
Buffer.byteLength(boundedCaptureFrame.data.toString('utf8'), 'utf8') <=
|
||||
MAX_CONTROL_FRAME_BYTES,
|
||||
true,
|
||||
);
|
||||
assert.equal(boundedCapture.accessibilityText.length < 32_000, true);
|
||||
assert.equal(captureCalls, 2);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_screen_context',
|
||||
requestId: 'capture-3',
|
||||
epoch: 0,
|
||||
}),
|
||||
);
|
||||
const failedCaptureFrame = await nextMessage(peer);
|
||||
const failedCapture = JSON.parse(
|
||||
failedCaptureFrame.data.toString('utf8'),
|
||||
) as { success: boolean; error: string };
|
||||
assert.equal(failedCapture.success, false);
|
||||
assert.equal(failedCapture.error.length, 1_024);
|
||||
assert.equal(
|
||||
Buffer.byteLength(failedCaptureFrame.data.toString('utf8'), 'utf8') <=
|
||||
MAX_CONTROL_FRAME_BYTES,
|
||||
true,
|
||||
);
|
||||
assert.equal(captureCalls, 3);
|
||||
|
||||
peer.send(JSON.stringify({ type: 'host.ping', pingId: 'ping-1' }));
|
||||
const pongFrame = await nextMessage(peer);
|
||||
assert.deepEqual(JSON.parse(pongFrame.data.toString('utf8')), {
|
||||
@@ -275,9 +441,155 @@ describe('LiveDaemonConnection', () => {
|
||||
Buffer.alloc(640),
|
||||
);
|
||||
|
||||
peer.send(Buffer.alloc(1_920), { binary: true });
|
||||
const outputFrame = encodeOutputAudioFrame(0, 23, Buffer.alloc(1_920));
|
||||
assert(outputFrame);
|
||||
peer.send(outputFrame, { binary: true });
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(outputFrames.at(-1)?.byteLength, 1_920);
|
||||
assert.equal(outputFrames.at(-1)?.epoch, 0);
|
||||
assert.equal(outputFrames.at(-1)?.outputId, 23);
|
||||
assert.equal(outputFrames.at(-1)?.audio.byteLength, 1_920);
|
||||
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.output_audio_finished',
|
||||
epoch: 0,
|
||||
outputId: 23,
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.deepEqual(outputEvents, ['audio:0:23', 'finished:0:23']);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.output_audio_finished',
|
||||
epoch: 1,
|
||||
outputId: 24,
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.deepEqual(outputEvents, ['audio:0:23', 'finished:0:23']);
|
||||
|
||||
const startedPromise = nextMessage(peer);
|
||||
assert.equal(connection.sendPlaybackStarted(0, 23), true);
|
||||
const started = await startedPromise;
|
||||
assert.equal(started.isBinary, false);
|
||||
assert.deepEqual(JSON.parse(started.data.toString('utf8')), {
|
||||
type: 'host.playback_started',
|
||||
epoch: 0,
|
||||
outputId: 23,
|
||||
});
|
||||
|
||||
const completedPromise = nextMessage(peer);
|
||||
assert.equal(connection.sendPlaybackCompleted(0, 23), true);
|
||||
const completed = await completedPromise;
|
||||
assert.equal(completed.isBinary, false);
|
||||
assert.deepEqual(JSON.parse(completed.data.toString('utf8')), {
|
||||
type: 'host.playback_completed',
|
||||
epoch: 0,
|
||||
outputId: 23,
|
||||
});
|
||||
connection.stop();
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
});
|
||||
|
||||
it('withholds configuration authority on nonce mismatch, disconnect and Quit', async () => {
|
||||
for (const outcome of ['mismatch', 'disconnect', 'quit']) {
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
cleanup.push(() => {
|
||||
for (const client of server.clients) client.terminate();
|
||||
server.close();
|
||||
});
|
||||
await once(server, 'listening');
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const directory = await mkdtemp(join(tmpdir(), 'live-config-connect-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const discoveryPath = join(directory, 'daemon.json');
|
||||
const configPath = join(directory, 'config.json');
|
||||
await writeFile(
|
||||
discoveryPath,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'fixture-private-token',
|
||||
configPath,
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const changes = new EventEmitter();
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: () => changes.emit('snapshot'),
|
||||
onOutputAudio: () => undefined,
|
||||
onOutputAudioFinished: () => undefined,
|
||||
onClearOutput: () => undefined,
|
||||
},
|
||||
discoveryPath,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
const waitForPhase = async (phase: string) => {
|
||||
const signal = AbortSignal.timeout(3_000);
|
||||
while (connection.getSnapshot().phase !== phase)
|
||||
await once(changes, 'snapshot', { signal });
|
||||
};
|
||||
const peerReady = new Promise<WebSocket>((resolve, reject) => {
|
||||
server.once('connection', (peer) => {
|
||||
void nextMessage(peer).then(() => resolve(peer), reject);
|
||||
});
|
||||
});
|
||||
connection.start();
|
||||
const peer = await peerReady;
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce:
|
||||
outcome === 'mismatch' ? 'wrong_nonce_0001' : 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 10_000,
|
||||
epoch: 0,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (outcome === 'mismatch') {
|
||||
await waitForPhase('error');
|
||||
assert.equal(connection.getSnapshot().error, 'daemon_identity');
|
||||
} else {
|
||||
await waitForPhase('ready');
|
||||
assert.equal(connection.getConfigFilePath(), configPath);
|
||||
if (outcome === 'quit') {
|
||||
const quitting = connection.requestQuit();
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
await quitting;
|
||||
} else {
|
||||
peer.close();
|
||||
await waitForPhase('disconnected');
|
||||
}
|
||||
}
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
connection.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps retrying the same discovery identity slowly after the fast budget', async () => {
|
||||
@@ -307,6 +619,7 @@ describe('LiveDaemonConnection', () => {
|
||||
);
|
||||
|
||||
let connectionCount = 0;
|
||||
let readyPeer: WebSocket | undefined;
|
||||
server.on('connection', async (socket) => {
|
||||
connectionCount += 1;
|
||||
await nextMessage(socket);
|
||||
@@ -314,6 +627,7 @@ describe('LiveDaemonConnection', () => {
|
||||
socket.close(1012, 'retry');
|
||||
return;
|
||||
}
|
||||
readyPeer = socket;
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
@@ -332,12 +646,14 @@ describe('LiveDaemonConnection', () => {
|
||||
});
|
||||
|
||||
const errors: Array<string | undefined> = [];
|
||||
const finishedOutputs: string[] = [];
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -350,6 +666,9 @@ describe('LiveDaemonConnection', () => {
|
||||
}),
|
||||
onSnapshot: (snapshot) => errors.push(snapshot.error),
|
||||
onOutputAudio: () => undefined,
|
||||
onOutputAudioFinished: (identity) => {
|
||||
finishedOutputs.push(`${identity.epoch}:${identity.outputId}`);
|
||||
},
|
||||
onClearOutput: () => undefined,
|
||||
},
|
||||
discoveryPath,
|
||||
@@ -376,5 +695,16 @@ describe('LiveDaemonConnection', () => {
|
||||
assert.equal(connectionCount, 3);
|
||||
assert(errors.includes('daemon_reconnect_exhausted'));
|
||||
assert.equal(connection.getSnapshot().phase, 'ready');
|
||||
assert.equal(connection.getConfigFilePath(), undefined);
|
||||
assert(readyPeer);
|
||||
readyPeer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.output_audio_finished',
|
||||
epoch: 0,
|
||||
outputId: 1,
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.deepEqual(finishedOutputs, []);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { isLiveHostDiagnosticsEnabled } from '../../shared/diagnostics.ts';
|
||||
|
||||
describe('Live Host diagnostics', () => {
|
||||
it('supports the public Host flag and the private renderer flag', () => {
|
||||
assert.equal(
|
||||
isLiveHostDiagnosticsEnabled(['electron', '.', '--live-debug'], {}),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isLiveHostDiagnosticsEnabled(
|
||||
['electron-helper', '--qwen-live-debug'],
|
||||
{},
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('supports the diagnostics environment variable', () => {
|
||||
assert.equal(
|
||||
isLiveHostDiagnosticsEnabled([], { QWEN_LIVE_DIAGNOSTICS: '1' }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not use Electron's reserved --debug argument", () => {
|
||||
assert.equal(
|
||||
isLiveHostDiagnosticsEnabled(['electron', '.', '--debug'], {}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
@@ -22,7 +22,10 @@ afterEach(async () => {
|
||||
);
|
||||
});
|
||||
|
||||
async function discoveryFile(mode = 0o600): Promise<string> {
|
||||
async function discoveryFile(
|
||||
mode = 0o600,
|
||||
configPath?: unknown,
|
||||
): Promise<string> {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-live-discovery-'));
|
||||
temporaryDirectories.push(directory);
|
||||
const path = join(directory, 'daemon.json');
|
||||
@@ -34,6 +37,7 @@ async function discoveryFile(mode = 0o600): Promise<string> {
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
...(configPath !== undefined ? { configPath } : {}),
|
||||
}),
|
||||
{ mode },
|
||||
);
|
||||
@@ -48,9 +52,69 @@ describe('Live daemon discovery', () => {
|
||||
if (result.kind === 'ready') {
|
||||
assert.equal(result.record.protocolVersion, LIVE_PROTOCOL_VERSION);
|
||||
assert.equal(result.record.token, 'secret-not-logged');
|
||||
assert.equal(result.record.configPath, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts a custom absolute config.json path, including spaces and Unicode', async () => {
|
||||
const configPath = join(tmpdir(), 'Live config 中文', 'config.json');
|
||||
const result = await readDiscoveryFile(
|
||||
await discoveryFile(0o600, configPath),
|
||||
);
|
||||
assert.equal(result.kind, 'ready');
|
||||
if (result.kind === 'ready')
|
||||
assert.equal(result.record.configPath, configPath);
|
||||
});
|
||||
|
||||
it('rejects malformed, relative, non-config and unbounded config paths', async () => {
|
||||
for (const configPath of [
|
||||
null,
|
||||
1,
|
||||
true,
|
||||
{},
|
||||
[],
|
||||
'',
|
||||
'relative/config.json',
|
||||
'~/config.json',
|
||||
'file:///tmp/config.json',
|
||||
join(tmpdir(), 'config.toml'),
|
||||
join(tmpdir(), 'config.json.exe'),
|
||||
join(tmpdir(), 'bad\0', 'config.json'),
|
||||
join(tmpdir(), 'x'.repeat(4_096), 'config.json'),
|
||||
]) {
|
||||
assert.deepEqual(
|
||||
await readDiscoveryFile(await discoveryFile(0o600, configPath)),
|
||||
{ kind: 'invalid', reason: 'discovery_shape' },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('notifies discovery consumers when only the configuration path changes', async () => {
|
||||
const firstPath = join(tmpdir(), 'live-one', 'config.json');
|
||||
const nextPath = join(tmpdir(), 'live-two', 'config.json');
|
||||
const path = await discoveryFile(0o600, firstPath);
|
||||
const record = JSON.parse(await readFile(path, 'utf8')) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const observed: DiscoveryResult[] = [];
|
||||
const monitor = new DiscoveryMonitor(path, (result) =>
|
||||
observed.push(result),
|
||||
);
|
||||
|
||||
await monitor.poll();
|
||||
await writeFile(path, JSON.stringify({ ...record, configPath: nextPath }));
|
||||
await monitor.poll();
|
||||
await monitor.poll();
|
||||
|
||||
assert.equal(observed.length, 2);
|
||||
const [first, next] = observed;
|
||||
assert(first?.kind === 'ready' && next?.kind === 'ready');
|
||||
assert.notEqual(first.signature, next.signature);
|
||||
assert.equal(first.record.configPath, firstPath);
|
||||
assert.equal(next.record.configPath, nextPath);
|
||||
});
|
||||
|
||||
it('rejects group-readable discovery records', async () => {
|
||||
assert.deepEqual(await readDiscoveryFile(await discoveryFile(0o640)), {
|
||||
kind: 'invalid',
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
type HostControlMessage,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
afterEach(async () => {
|
||||
for (const task of cleanup.splice(0).reverse()) await task();
|
||||
});
|
||||
|
||||
function nextMessage(socket: WebSocket): Promise<HostControlMessage> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Missing fixture frame')),
|
||||
3_000,
|
||||
);
|
||||
socket.once('message', (data) => {
|
||||
clearTimeout(timer);
|
||||
resolve(JSON.parse(data.toString()) as HostControlMessage);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function fixture(displayCaptureV1 = true) {
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
cleanup.push(() => {
|
||||
for (const socket of server.clients) socket.terminate();
|
||||
server.close();
|
||||
});
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
const address = server.address();
|
||||
if (typeof address !== 'object' || !address)
|
||||
throw new Error('Missing server address');
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-live-display-wire-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const discoveryPath = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
discoveryPath,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'fixture-token',
|
||||
instanceNonce: 'fixture-display-daemon',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
let captureRequest: unknown;
|
||||
let resultDisplayId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64');
|
||||
const connection = new LiveDaemonConnection(
|
||||
'fixture',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: () => undefined,
|
||||
onOutputAudio: () => undefined,
|
||||
onOutputAudioFinished: () => undefined,
|
||||
onClearOutput: () => undefined,
|
||||
captureVisual: async (request) => {
|
||||
captureRequest = request;
|
||||
return {
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
displayId: resultDisplayId,
|
||||
image: jpeg,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
};
|
||||
},
|
||||
},
|
||||
discoveryPath,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
const connected = new Promise<{
|
||||
socket: WebSocket;
|
||||
hello: Promise<HostControlMessage>;
|
||||
}>((resolve) => {
|
||||
server.once('connection', (socket) =>
|
||||
resolve({ socket, hello: nextMessage(socket) }),
|
||||
);
|
||||
});
|
||||
connection.start();
|
||||
const { socket, hello } = await connected;
|
||||
const helloMessage = await hello;
|
||||
assert.equal(helloMessage.type, 'host.hello');
|
||||
assert('displayCaptureV1' in helloMessage);
|
||||
assert.equal(helloMessage.displayCaptureV1, true);
|
||||
const status = {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
};
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'fixture-display-daemon',
|
||||
heartbeatIntervalMs: 30_000,
|
||||
epoch: 0,
|
||||
...(displayCaptureV1 ? { displayCaptureV1: true } : {}),
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
screenDisplayId: 'primary',
|
||||
},
|
||||
status,
|
||||
}),
|
||||
);
|
||||
const deadline = Date.now() + 3_000;
|
||||
while (connection.getSnapshot().phase !== 'ready') {
|
||||
if (Date.now() > deadline) throw new Error('Missing fixture welcome');
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
return {
|
||||
connection,
|
||||
socket,
|
||||
jpeg,
|
||||
status,
|
||||
captureRequest: () => captureRequest,
|
||||
setResultDisplay: (id: string) => {
|
||||
resultDisplayId = id;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('display capture connection', () => {
|
||||
it('retains rejected visual selection errors across state updates and drops the rejected pending UUID before retry', async () => {
|
||||
const { connection, socket, status } = await fixture();
|
||||
const displayId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
const selected = nextMessage(socket);
|
||||
assert.equal(
|
||||
connection.sendVisualSettings({ screenDisplayId: displayId }, 0),
|
||||
true,
|
||||
);
|
||||
await selected;
|
||||
const previousState = {
|
||||
type: 'host.state',
|
||||
epoch: 0,
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
screenDisplayId: 'primary',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status,
|
||||
};
|
||||
const error =
|
||||
'Could not save the selected display. The previous selection is unchanged.';
|
||||
const processed = nextMessage(socket);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.error',
|
||||
code: 'invalid_message',
|
||||
message: error,
|
||||
}),
|
||||
);
|
||||
socket.send(JSON.stringify(previousState));
|
||||
socket.send(
|
||||
JSON.stringify({ type: 'host.ping', pingId: 'rejected-selection' }),
|
||||
);
|
||||
assert.deepEqual(await processed, {
|
||||
type: 'host.pong',
|
||||
pingId: 'rejected-selection',
|
||||
});
|
||||
assert.equal(connection.getSnapshot().visualSettingsError, error);
|
||||
assert.equal(
|
||||
connection.getSnapshot().visualInput?.screenDisplayId,
|
||||
'primary',
|
||||
);
|
||||
|
||||
const repeated = nextMessage(socket);
|
||||
socket.send(JSON.stringify(previousState));
|
||||
socket.send(JSON.stringify({ type: 'host.ping', pingId: 'later-state' }));
|
||||
await repeated;
|
||||
assert.equal(connection.getSnapshot().visualSettingsError, error);
|
||||
|
||||
const retry = nextMessage(socket);
|
||||
assert.equal(connection.sendVisualSettings({ mode: 'live-feed' }, 0), true);
|
||||
assert.equal(connection.getSnapshot().visualSettingsError, undefined);
|
||||
const retried = await retry;
|
||||
assert.equal(retried.type, 'host.visual_settings');
|
||||
assert('screenDisplayId' in retried);
|
||||
assert.equal(retried.screenDisplayId, 'primary');
|
||||
|
||||
const rejectedAgain = nextMessage(socket);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.error',
|
||||
code: 'invalid_message',
|
||||
message: error,
|
||||
}),
|
||||
);
|
||||
socket.send(JSON.stringify(previousState));
|
||||
socket.send(
|
||||
JSON.stringify({ type: 'host.ping', pingId: 'rejected-again' }),
|
||||
);
|
||||
await rejectedAgain;
|
||||
assert.equal(connection.getSnapshot().visualSettingsError, error);
|
||||
connection.stop();
|
||||
assert.equal(connection.getSnapshot().visualSettingsError, undefined);
|
||||
});
|
||||
|
||||
it('negotiates display support, retains pending selection across mode changes, and binds captured frames to a UUID', async () => {
|
||||
const { connection, socket, jpeg, captureRequest, setResultDisplay } =
|
||||
await fixture();
|
||||
const displayId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
assert.equal(connection.getSnapshot().displayCaptureV1, true);
|
||||
const select = nextMessage(socket);
|
||||
assert.equal(
|
||||
connection.sendVisualSettings(
|
||||
{ screenDisplayId: displayId.toUpperCase() },
|
||||
0,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal((await select).type, 'host.visual_settings');
|
||||
const mode = nextMessage(socket);
|
||||
assert.equal(connection.sendVisualSettings({ mode: 'live-feed' }, 0), true);
|
||||
assert.deepEqual(await mode, {
|
||||
type: 'host.visual_settings',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
mode: 'live-feed',
|
||||
screenDisplayId: displayId,
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
});
|
||||
assert.equal(connection.sendVisualFrame('screen', jpeg, 0), false);
|
||||
const frame = nextMessage(socket);
|
||||
assert.equal(
|
||||
connection.sendVisualFrame('screen', jpeg, 0, displayId),
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(await frame, {
|
||||
type: 'host.visual_frame',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
displayId,
|
||||
image: jpeg,
|
||||
});
|
||||
const result = nextMessage(socket);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'capture-1',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: displayId,
|
||||
persistAsset: false,
|
||||
}),
|
||||
);
|
||||
assert.equal((await result).type, 'host.visual_capture_result');
|
||||
assert.deepEqual(captureRequest(), {
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: displayId,
|
||||
persistAsset: false,
|
||||
});
|
||||
setResultDisplay('11111111-2222-3333-4444-555555555555');
|
||||
const mismatch = nextMessage(socket);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'capture-2',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: displayId,
|
||||
persistAsset: false,
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(await mismatch, {
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: 'capture-2',
|
||||
success: false,
|
||||
error: 'The captured display does not match the selection.',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not send full-display settings/frames or route scoped captures against an old daemon', async () => {
|
||||
const { connection, socket, jpeg, captureRequest } = await fixture(false);
|
||||
const displayId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
assert.equal(
|
||||
connection.sendVisualSettings({ screenDisplayId: displayId }, 0),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
connection.sendVisualFrame('screen', jpeg, 0, displayId),
|
||||
false,
|
||||
);
|
||||
const result = nextMessage(socket);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'capture-1',
|
||||
epoch: 0,
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: displayId,
|
||||
persistAsset: false,
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(await result, {
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: 'capture-1',
|
||||
success: false,
|
||||
error: 'Update Live Host to enable full-display capture.',
|
||||
});
|
||||
assert.equal(captureRequest(), undefined);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,319 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import { liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import * as policy from '../live-state-policy.ts';
|
||||
import { isScreenDisplayId } from '../../shared/protocol.ts';
|
||||
|
||||
const DISPLAY = '11223344-5566-7788-99aa-bbccddeeff00';
|
||||
const OTHER = '11223344-5566-7788-99aa-bbccddeeff11';
|
||||
const image = Uint8Array.of(1, 2, 3);
|
||||
|
||||
function fixture() {
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
readFileSync(new URL('../index.ts', import.meta.url), 'utf8'),
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = [
|
||||
'captureOnDemandVisual',
|
||||
'captureScreenFeed',
|
||||
'sameVisualInput',
|
||||
'hostReadinessBlocker',
|
||||
'visualSourceReady',
|
||||
'refreshScreenDisplays',
|
||||
'registerIpc',
|
||||
];
|
||||
const declarations = tree.statements.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) &&
|
||||
node.name &&
|
||||
names.includes(node.name.text),
|
||||
);
|
||||
assert.equal(declarations.length, names.length);
|
||||
const methods = ts.transpileModule(
|
||||
declarations.map((node) => node.getText(tree)).join('\n') +
|
||||
'\n({' +
|
||||
names.join(',') +
|
||||
'});',
|
||||
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } },
|
||||
).outputText;
|
||||
const calls: Array<{ kind: string; value?: unknown }> = [];
|
||||
const diagnostics: Array<{
|
||||
event: string;
|
||||
details: Record<string, unknown>;
|
||||
}> = [];
|
||||
const handlers = new Map<string, (...args: unknown[]) => unknown>();
|
||||
const context = {
|
||||
Buffer,
|
||||
createHash,
|
||||
diagnosticsEnabled: true,
|
||||
...policy,
|
||||
liveMessage,
|
||||
isScreenDisplayId,
|
||||
daemon: {
|
||||
getEpoch: () => 1,
|
||||
sendVisualFrame: (...args: unknown[]) => {
|
||||
calls.push({ kind: 'frame', value: args });
|
||||
return true;
|
||||
},
|
||||
sendVisualSettings: (...args: unknown[]) => {
|
||||
calls.push({ kind: 'settings', value: args });
|
||||
return true;
|
||||
},
|
||||
},
|
||||
appshotCapture: {
|
||||
captureFrame: async () => {
|
||||
calls.push({ kind: 'window' });
|
||||
return {
|
||||
screenshot: image,
|
||||
appName: 'Fixture app',
|
||||
accessibilityText: 'AX fixture',
|
||||
};
|
||||
},
|
||||
captureDisplayFrame: async (id: string) => {
|
||||
calls.push({ kind: 'display', value: id });
|
||||
return { screenshot: image, displayId: DISPLAY };
|
||||
},
|
||||
listDisplays: () => [
|
||||
{
|
||||
id: DISPLAY,
|
||||
name: 'Fixture display',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
primary: true,
|
||||
},
|
||||
],
|
||||
storePng: async () => {
|
||||
calls.push({ kind: 'asset' });
|
||||
return '/fixture/image.png';
|
||||
},
|
||||
},
|
||||
ipcMain: {
|
||||
on: (name: string, fn: (...args: unknown[]) => unknown) =>
|
||||
handlers.set(name, fn),
|
||||
handle: (name: string, fn: (...args: unknown[]) => unknown) =>
|
||||
handlers.set(name, fn),
|
||||
},
|
||||
isTrustedSender: () => true,
|
||||
rendererEventsEnabled: true,
|
||||
quitState: undefined,
|
||||
connection: { phase: 'ready', displayCaptureV1: true },
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
screenDisplayId: DISPLAY,
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
visualGeneration: 0,
|
||||
screenFeedGeneration: 0,
|
||||
screenFeedInFlight: false,
|
||||
visualCallId: 'call',
|
||||
visualReady: false,
|
||||
visualError: undefined,
|
||||
screenDisplays: [] as Array<{ id: string }>,
|
||||
screenDisplaysError: undefined,
|
||||
live: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
callId: 'call',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
permissions: {
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
appshot: true,
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
},
|
||||
appshotReadiness: { refresh() {}, requestPermission() {} },
|
||||
isHostReady: () => true,
|
||||
encodeScreenFrame: () => ({
|
||||
image: 'fixture-jpeg',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
}),
|
||||
writeLiveDiagnostic: (event: string, details: Record<string, unknown>) =>
|
||||
diagnostics.push({ event, details }),
|
||||
publishState: () => {},
|
||||
};
|
||||
const api = runInNewContext(methods, context) as {
|
||||
captureOnDemandVisual: (value: object) => Promise<Record<string, unknown>>;
|
||||
captureScreenFeed: (generation: number) => Promise<void>;
|
||||
sameVisualInput: (a: object, b: object) => boolean;
|
||||
hostReadinessBlocker: () => string | undefined;
|
||||
registerIpc: () => void;
|
||||
};
|
||||
return { api, context, calls, handlers, diagnostics };
|
||||
}
|
||||
|
||||
describe('selected display capture routing', () => {
|
||||
it('keeps foreground Appshot and its asset/AX separate from private monitor display capture', async () => {
|
||||
const f = fixture();
|
||||
const appshot = await f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
persistAsset: true,
|
||||
});
|
||||
assert.deepEqual(
|
||||
f.calls.map((call) => call.kind),
|
||||
['window', 'asset'],
|
||||
);
|
||||
assert.equal(appshot.accessibilityText, 'AX fixture');
|
||||
assert.equal(appshot.screenshotPath, '/fixture/image.png');
|
||||
f.calls.length = 0;
|
||||
f.context.permissions.accessibility = 'denied';
|
||||
f.context.selfChecks.appshot = false;
|
||||
const monitor = await f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: DISPLAY,
|
||||
persistAsset: false,
|
||||
});
|
||||
assert.deepEqual(f.calls, [{ kind: 'display', value: DISPLAY }]);
|
||||
assert.equal(monitor.displayId, DISPLAY);
|
||||
assert.equal(monitor.screenScope, 'display');
|
||||
assert.equal(monitor.accessibilityText, undefined);
|
||||
assert.equal(monitor.screenshotPath, undefined);
|
||||
});
|
||||
|
||||
it('feeds the complete selected display with its identity and no foreground-window call', async () => {
|
||||
const f = fixture();
|
||||
f.context.visualInput.mode = 'live-feed';
|
||||
await f.api.captureScreenFeed(0);
|
||||
assert.equal(f.calls[0]?.kind, 'display');
|
||||
assert.deepEqual(f.calls[1], {
|
||||
kind: 'frame',
|
||||
value: ['screen', 'fixture-jpeg', 1, DISPLAY],
|
||||
});
|
||||
assert.equal(
|
||||
f.calls.some((call) => call.kind === 'window'),
|
||||
false,
|
||||
);
|
||||
assert.equal(f.context.visualReady, true);
|
||||
});
|
||||
|
||||
it('correlates debug snapshot/feed bytes without logging images or accessibility contents', async () => {
|
||||
const f = fixture();
|
||||
await f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: DISPLAY,
|
||||
persistAsset: false,
|
||||
});
|
||||
f.context.visualInput.mode = 'live-feed';
|
||||
await f.api.captureScreenFeed(0);
|
||||
assert.deepEqual(
|
||||
f.diagnostics.map((log) => log.event),
|
||||
['visual_snapshot_captured', 'visual_frame_sent'],
|
||||
);
|
||||
const expectedHash = createHash('sha256')
|
||||
.update(Buffer.from('fixture-jpeg', 'base64'))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
for (const { details } of f.diagnostics) {
|
||||
assert.equal(details.frameHash, expectedHash);
|
||||
assert.equal(details.displayId, DISPLAY);
|
||||
assert.equal(details.width, 1280);
|
||||
assert.equal(details.height, 720);
|
||||
assert.equal(details.bytes, Buffer.byteLength('fixture-jpeg', 'base64'));
|
||||
assert.equal(details.image, undefined);
|
||||
assert.equal(details.accessibilityText, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not calculate snapshot or feed fingerprints when debug is off', async () => {
|
||||
const f = fixture();
|
||||
f.context.diagnosticsEnabled = false;
|
||||
f.context.createHash = () => {
|
||||
throw new Error('Unexpected fingerprint without diagnostics');
|
||||
};
|
||||
await f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: DISPLAY,
|
||||
persistAsset: false,
|
||||
});
|
||||
assert.equal(f.diagnostics.length, 0);
|
||||
f.context.visualInput.mode = 'live-feed';
|
||||
await f.api.captureScreenFeed(0);
|
||||
assert.equal(f.context.visualReady, true);
|
||||
assert.equal(f.diagnostics[0]?.details.frameHash, undefined);
|
||||
});
|
||||
|
||||
it('discards a monitor capture after display/topology generation changes', async () => {
|
||||
const f = fixture();
|
||||
let finish!: () => void;
|
||||
f.context.appshotCapture.captureDisplayFrame = () =>
|
||||
new Promise((resolve) => {
|
||||
finish = () => resolve({ screenshot: image, displayId: DISPLAY });
|
||||
});
|
||||
const capture = f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: DISPLAY,
|
||||
persistAsset: false,
|
||||
});
|
||||
f.context.visualGeneration++;
|
||||
finish();
|
||||
await assert.rejects(capture, /stale_visual_capture/);
|
||||
assert.equal(f.calls.length, 0);
|
||||
});
|
||||
|
||||
it('rejects stale target selection before acquiring any display frame', async () => {
|
||||
const f = fixture();
|
||||
await assert.rejects(
|
||||
f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: OTHER,
|
||||
persistAsset: false,
|
||||
}),
|
||||
);
|
||||
assert.equal(f.calls.length, 0);
|
||||
assert.equal(
|
||||
f.api.sameVisualInput(f.context.visualInput, {
|
||||
...f.context.visualInput,
|
||||
screenDisplayId: OTHER,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('requires AX for the original window tool, but not for Screen Live Feed', () => {
|
||||
const f = fixture();
|
||||
f.context.permissions.accessibility = 'denied';
|
||||
f.context.selfChecks.appshot = false;
|
||||
assert.equal(f.api.hostReadinessBlocker(), 'accessibility_permission');
|
||||
f.context.visualInput.mode = 'live-feed';
|
||||
assert.equal(f.api.hostReadinessBlocker(), undefined);
|
||||
f.context.permissions.screenRecording = 'denied';
|
||||
assert.equal(f.api.hostReadinessBlocker(), 'screen_recording_permission');
|
||||
});
|
||||
|
||||
it('forwards only trusted supported and connected display selections', () => {
|
||||
const f = fixture();
|
||||
f.api.registerIpc();
|
||||
const setDisplay = f.handlers.get('live:set-screen-display')!;
|
||||
setDisplay({}, DISPLAY.toUpperCase());
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(f.calls)), [
|
||||
{ kind: 'settings', value: [{ screenDisplayId: DISPLAY }, 1] },
|
||||
]);
|
||||
assert.throws(() => setDisplay({}, OTHER));
|
||||
assert.throws(() => setDisplay({}, 'window'));
|
||||
f.context.isTrustedSender = () => false;
|
||||
assert.throws(() => setDisplay({}, 'primary'));
|
||||
assert.equal(f.calls.length, 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,278 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import * as positions from '../overlay-position.ts';
|
||||
import * as policy from '../live-state-policy.ts';
|
||||
import { OVERLAY_GEOMETRY } from '../../shared/overlay-geometry.ts';
|
||||
|
||||
function fixture() {
|
||||
const source = readFileSync(new URL('../index.ts', import.meta.url), 'utf8');
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = new Set([
|
||||
'handleDisplayChange',
|
||||
'clampOverlayToDisplays',
|
||||
'overlayWorkArea',
|
||||
'positionOverlay',
|
||||
'dragOverlay',
|
||||
'syncPointerInteractivity',
|
||||
'captureOnDemandVisual',
|
||||
]);
|
||||
const declarations = tree.statements.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) && node.name && names.has(node.name.text),
|
||||
);
|
||||
assert.equal(declarations.length, names.size);
|
||||
const registrations = source.match(
|
||||
/^ screen\.on\('display-(?:added|removed|metrics-changed)',[\s\S]*?^ \}\);/gm,
|
||||
);
|
||||
assert.equal(registrations?.length, 3);
|
||||
const area = { x: 0, y: 25, width: 1440, height: 875 };
|
||||
const bounds = { x: 900, y: 200, width: 384, height: 480 };
|
||||
const display = {
|
||||
id: 1,
|
||||
bounds: { x: 0, y: 0, width: 1440, height: 900 },
|
||||
workArea: area,
|
||||
scaleFactor: 2,
|
||||
rotation: 0,
|
||||
};
|
||||
const handlers = new Map<string, (...args: unknown[]) => void>();
|
||||
const moves: Array<{ x: number; y: number }> = [];
|
||||
const saved: Array<{ x: number; y: number }> = [];
|
||||
const dragging: boolean[] = [];
|
||||
const diagnostics: Array<Record<string, unknown>> = [];
|
||||
const counters = { capture: 0, restart: 0, refresh: 0, publish: 0 };
|
||||
let finishCapture: (() => void) | undefined;
|
||||
const context = {
|
||||
...positions,
|
||||
...policy,
|
||||
OVERLAY_GEOMETRY,
|
||||
diagnosticsEnabled: false,
|
||||
screen: {
|
||||
on: (name: string, handler: (...args: unknown[]) => void) =>
|
||||
handlers.set(name, handler),
|
||||
getDisplayNearestPoint: () => display,
|
||||
},
|
||||
overlay: {
|
||||
isDestroyed: () => false,
|
||||
getBounds: () => ({ ...bounds }),
|
||||
setPosition: (x: number, y: number) => {
|
||||
Object.assign(bounds, { x, y });
|
||||
moves.push({ x, y });
|
||||
},
|
||||
setIgnoreMouseEvents: () => {},
|
||||
},
|
||||
desiredOverlayPosition: { x: bounds.x, y: bounds.y },
|
||||
hasCustomOverlayPosition: true,
|
||||
overlayLayout: 'orb',
|
||||
overlayOffset: { x: 0, y: 0 },
|
||||
overlayDrag: undefined,
|
||||
settingsOpen: false,
|
||||
pointerInteractive: false,
|
||||
pointerOverInteractive: false,
|
||||
subagents: {
|
||||
displaysChanged: () => {},
|
||||
setDragging: (value: boolean) => dragging.push(value),
|
||||
},
|
||||
appshotCapture: {
|
||||
captureDisplayFrame: () => {
|
||||
counters.capture++;
|
||||
return new Promise((resolve) => {
|
||||
finishCapture = () =>
|
||||
resolve({
|
||||
screenshot: Uint8Array.of(1),
|
||||
displayId: 'fixture-display',
|
||||
});
|
||||
});
|
||||
},
|
||||
},
|
||||
refreshScreenDisplays: () => counters.refresh++,
|
||||
stopScreenFeed: () => counters.restart++,
|
||||
syncVisualCapture: () => {},
|
||||
publishState: () => counters.publish++,
|
||||
persistOverlayPosition: (): void => {
|
||||
saved.push({ ...context.desiredOverlayPosition });
|
||||
},
|
||||
sendRendererCommand: () => {},
|
||||
writeLiveDiagnostic: (event: string, details: object) =>
|
||||
diagnostics.push({ event, ...details }),
|
||||
daemon: { getEpoch: () => 1 },
|
||||
appshotReadiness: { refresh: () => {} },
|
||||
permissions: { screenRecording: 'granted' },
|
||||
selfChecks: { appshot: true },
|
||||
visualGeneration: 1,
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
screenDisplayId: 'primary',
|
||||
},
|
||||
live: { state: 'listening', callId: 'fixture', available: true },
|
||||
isHostReady: () => true,
|
||||
encodeScreenFrame: () => ({ image: 'fixture', width: 1280, height: 720 }),
|
||||
liveMessage: (key: string) => key,
|
||||
};
|
||||
const code = ts.transpileModule(
|
||||
declarations.map((node) => node.getText(tree)).join('\n') +
|
||||
'\n' +
|
||||
registrations?.join('\n') +
|
||||
'\n({ captureOnDemandVisual, dragOverlay });',
|
||||
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } },
|
||||
).outputText;
|
||||
const api = runInNewContext(code, context) as {
|
||||
captureOnDemandVisual(request: object): Promise<unknown>;
|
||||
dragOverlay(phase: 'start' | 'move' | 'end', x: number, y: number): void;
|
||||
};
|
||||
return {
|
||||
api,
|
||||
context,
|
||||
display,
|
||||
area,
|
||||
bounds,
|
||||
counters,
|
||||
moves,
|
||||
saved,
|
||||
dragging,
|
||||
diagnostics,
|
||||
finish: () => finishCapture?.(),
|
||||
event: (
|
||||
name: string,
|
||||
changedMetrics?: string[],
|
||||
changedDisplay = display,
|
||||
) => handlers.get(name)?.({}, changedDisplay, changedMetrics),
|
||||
};
|
||||
}
|
||||
|
||||
describe('native display event isolation', () => {
|
||||
it('preserves a pending monitor frame and drag through non-geometric metrics', async () => {
|
||||
const f = fixture();
|
||||
const frame = f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: 'primary',
|
||||
persistAsset: false,
|
||||
});
|
||||
f.api.dragOverlay('start', 1000, 500);
|
||||
f.api.dragOverlay('move', 1010, 510);
|
||||
const before = { ...f.bounds };
|
||||
f.event('display-metrics-changed', ['colorSpace']);
|
||||
f.finish();
|
||||
await frame;
|
||||
f.api.dragOverlay('move', 1060, 560);
|
||||
assert.equal(f.bounds.x, before.x + 50);
|
||||
assert.equal(f.bounds.y, before.y + 50);
|
||||
assert.equal(f.context.visualGeneration, 1);
|
||||
assert.deepEqual(f.counters, {
|
||||
capture: 1,
|
||||
restart: 0,
|
||||
refresh: 0,
|
||||
publish: 0,
|
||||
});
|
||||
assert.deepEqual(f.saved, []);
|
||||
assert.deepEqual(f.dragging, [true]);
|
||||
assert.equal(
|
||||
f.diagnostics.find((log) => log.event === 'native_display_changed')
|
||||
?.geometryChanged,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores empty and unknown metrics without moving a stationary default orb', () => {
|
||||
const f = fixture();
|
||||
f.context.hasCustomOverlayPosition = false;
|
||||
const before = { ...f.bounds };
|
||||
f.event('display-metrics-changed', []);
|
||||
f.event('display-metrics-changed', ['colorSpace', 'unknown']);
|
||||
assert.deepEqual(f.bounds, before);
|
||||
assert.deepEqual(f.moves, []);
|
||||
assert.deepEqual(f.dragging, []);
|
||||
assert.equal(f.context.visualGeneration, 1);
|
||||
});
|
||||
|
||||
for (const metric of ['bounds', 'workArea', 'scaleFactor', 'rotation']) {
|
||||
it(`invalidates frames conservatively for another display's ${metric} change without interrupting the orb drag`, async () => {
|
||||
const f = fixture();
|
||||
const frame = f.api.captureOnDemandVisual({
|
||||
source: 'screen',
|
||||
screenScope: 'display',
|
||||
screenDisplayId: 'primary',
|
||||
persistAsset: false,
|
||||
});
|
||||
const rejection = assert.rejects(frame, /stale_visual_capture/);
|
||||
f.api.dragOverlay('start', 1000, 500);
|
||||
f.api.dragOverlay('move', 1010, 510);
|
||||
const before = { ...f.bounds };
|
||||
f.event('display-metrics-changed', ['colorSpace', metric], {
|
||||
...f.display,
|
||||
id: 2,
|
||||
bounds: { ...f.display.bounds, x: 1440 },
|
||||
workArea: { ...f.area, x: 1440 },
|
||||
});
|
||||
f.finish();
|
||||
await rejection;
|
||||
f.api.dragOverlay('move', 1060, 560);
|
||||
assert.equal(f.bounds.x, before.x + 50);
|
||||
assert.equal(f.bounds.y, before.y + 50);
|
||||
assert.equal(f.context.visualGeneration, 2);
|
||||
assert.equal(f.counters.restart, 1);
|
||||
assert.deepEqual(f.saved, []);
|
||||
assert.deepEqual(f.dragging, [true]);
|
||||
});
|
||||
}
|
||||
|
||||
for (const event of ['display-added', 'display-removed']) {
|
||||
it(`${event} invalidates capture and preserves an already reachable default position`, () => {
|
||||
const f = fixture();
|
||||
f.context.hasCustomOverlayPosition = false;
|
||||
const before = { ...f.bounds };
|
||||
f.event(event);
|
||||
assert.equal(f.context.visualGeneration, 2);
|
||||
assert.equal(f.counters.refresh, 1);
|
||||
assert.equal(f.counters.restart, 1);
|
||||
assert.deepEqual(f.bounds, before);
|
||||
assert.deepEqual(f.moves, []);
|
||||
assert.deepEqual(f.dragging, []);
|
||||
});
|
||||
}
|
||||
|
||||
it('clamps onto the remaining display after removal without overwriting the saved desired location', () => {
|
||||
const f = fixture();
|
||||
f.api.dragOverlay('start', 1000, 500);
|
||||
f.api.dragOverlay('move', 1010, 510);
|
||||
const desired = { ...f.context.desiredOverlayPosition };
|
||||
f.area.width = 800;
|
||||
f.event('display-removed', undefined, { ...f.display, id: 2 });
|
||||
const corrected = { ...f.bounds };
|
||||
f.api.dragOverlay('move', 1060, 560);
|
||||
assert.deepEqual(f.bounds, corrected);
|
||||
assert.equal(
|
||||
corrected.x +
|
||||
OVERLAY_GEOMETRY.bounds.orb.x +
|
||||
OVERLAY_GEOMETRY.bounds.orb.width,
|
||||
f.area.width,
|
||||
);
|
||||
assert.deepEqual(f.context.desiredOverlayPosition, desired);
|
||||
assert.deepEqual(f.saved, [desired]);
|
||||
assert.deepEqual(f.dragging, [true, false]);
|
||||
const log = f.diagnostics.at(-1);
|
||||
assert.equal(log?.event, 'overlay_position');
|
||||
assert.equal(log?.reason, 'display-removed');
|
||||
assert.deepEqual(log?.after, corrected);
|
||||
});
|
||||
|
||||
it('clamps the compensated logical position without resetting its macOS offset', () => {
|
||||
const f = fixture();
|
||||
f.bounds.y = 25;
|
||||
f.context.overlayOffset.y = -130;
|
||||
const before = { ...f.bounds };
|
||||
f.event('display-metrics-changed', ['bounds']);
|
||||
assert.deepEqual(f.bounds, before);
|
||||
assert.equal(f.context.overlayOffset.y, -130);
|
||||
assert.deepEqual(f.moves, []);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { describe, it } from 'node:test';
|
||||
import {
|
||||
LiveGlobalShortcut,
|
||||
@@ -76,7 +77,7 @@ describe('LiveGlobalShortcut', () => {
|
||||
assert.deepEqual(state, {
|
||||
accelerator: 'Command+Shift+E',
|
||||
healthy: false,
|
||||
error: 'That shortcut is already in use.',
|
||||
error: liveMessage('host.error.shortcutInUse'),
|
||||
});
|
||||
assert.deepEqual(value.unregistered, []);
|
||||
value.callbacks.get('Command+E')?.();
|
||||
@@ -93,7 +94,7 @@ describe('LiveGlobalShortcut', () => {
|
||||
{
|
||||
accelerator: 'Command+E',
|
||||
healthy: false,
|
||||
error: 'That shortcut is already in use.',
|
||||
error: liveMessage('host.error.shortcutInUse'),
|
||||
},
|
||||
]);
|
||||
failed.shortcut.replace('Command+E');
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { displayLiveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
encodeHostControlMessage,
|
||||
parseDaemonControlMessage,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanups: Array<() => void | Promise<void>> = [];
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0).reverse()) await cleanup();
|
||||
});
|
||||
const status = {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
shortcut: 'Command+E',
|
||||
};
|
||||
function receive(peer: WebSocket): Promise<Record<string, unknown>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Missing language action')),
|
||||
3000,
|
||||
);
|
||||
peer.once('message', (raw) => {
|
||||
clearTimeout(timer);
|
||||
resolve(JSON.parse(String(raw)));
|
||||
});
|
||||
});
|
||||
}
|
||||
async function fixture(supports = true) {
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
cleanups.push(() => {
|
||||
for (const socket of server.clients) socket.terminate();
|
||||
server.close();
|
||||
});
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const directory = await mkdtemp(join(tmpdir(), 'live-language-host-'));
|
||||
cleanups.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const path = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
path,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'fixture',
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
let ready!: () => void;
|
||||
const readiness = new Promise<void>((resolve) => {
|
||||
ready = resolve;
|
||||
});
|
||||
const peerReady = new Promise<WebSocket>((resolve) =>
|
||||
server.once('connection', async (peer) => {
|
||||
await receive(peer);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 10000,
|
||||
epoch: 3,
|
||||
status,
|
||||
...(supports ? { uiLanguageV1: { language: 'en' } } : {}),
|
||||
}),
|
||||
);
|
||||
resolve(peer);
|
||||
}),
|
||||
);
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
appshot: true,
|
||||
globalShortcut: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => {
|
||||
if (snapshot.phase === 'ready') ready();
|
||||
},
|
||||
onOutputAudio: () => {},
|
||||
onOutputAudioFinished: () => {},
|
||||
onClearOutput: () => {},
|
||||
},
|
||||
path,
|
||||
);
|
||||
cleanups.push(() => connection.stop());
|
||||
connection.start();
|
||||
const peer = await peerReady;
|
||||
await readiness;
|
||||
return { connection, peer };
|
||||
}
|
||||
const localized = (expression: RegExp) => (error: Error) =>
|
||||
expression.test(displayLiveMessage('en', error.message));
|
||||
|
||||
describe('Host language protocol', () => {
|
||||
it('validates locale and correlated result shapes without accepting coercion', () => {
|
||||
const valid = {
|
||||
type: 'host.state',
|
||||
epoch: 3,
|
||||
status,
|
||||
uiLanguageV1: { language: 'zh-CN' },
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(valid)), valid);
|
||||
for (const language of [null, 'zh', 'fr', 1, true]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...valid, uiLanguageV1: { language } }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({
|
||||
type: 'host.language_action',
|
||||
requestId: 'r',
|
||||
epoch: 3,
|
||||
language,
|
||||
} as never),
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.language_result',
|
||||
requestId: 'r',
|
||||
ok: true,
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('changes language only on matching acknowledgement and rejects duplicate requests', async () => {
|
||||
const { connection, peer } = await fixture();
|
||||
const frame = receive(peer);
|
||||
const result = connection.requestLanguage('zh-CN');
|
||||
assert.equal(connection.getSnapshot().uiLanguageV1?.language, 'en');
|
||||
await assert.rejects(
|
||||
connection.requestLanguage('en'),
|
||||
localized(/already in progress/),
|
||||
);
|
||||
const request = await frame;
|
||||
assert.equal(request['epoch'], 3);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.language_result',
|
||||
requestId: 'not-this-request',
|
||||
ok: true,
|
||||
uiLanguageV1: { language: 'zh-CN' },
|
||||
}),
|
||||
);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.language_result',
|
||||
requestId: request['requestId'],
|
||||
ok: true,
|
||||
uiLanguageV1: { language: 'zh-CN' },
|
||||
}),
|
||||
);
|
||||
assert.equal(await result, 'zh-CN');
|
||||
assert.equal(connection.getSnapshot().uiLanguageV1?.language, 'zh-CN');
|
||||
});
|
||||
|
||||
it('keeps confirmed language after failed saves and rejects mismatched success', async () => {
|
||||
const { connection, peer } = await fixture();
|
||||
for (const wrongSuccess of [false, true]) {
|
||||
const frame = receive(peer);
|
||||
const result = connection.requestLanguage('zh-CN');
|
||||
const rejected = assert.rejects(result);
|
||||
const request = await frame;
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.language_result',
|
||||
requestId: request['requestId'],
|
||||
ok: wrongSuccess,
|
||||
error: 'write refused',
|
||||
uiLanguageV1: { language: 'en' },
|
||||
}),
|
||||
);
|
||||
await rejected;
|
||||
assert.equal(connection.getSnapshot().uiLanguageV1?.language, 'en');
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects changes on legacy connections and pending changes on disconnect', async () => {
|
||||
const legacy = await fixture(false);
|
||||
await assert.rejects(
|
||||
legacy.connection.requestLanguage('zh-CN'),
|
||||
localized(/unavailable/),
|
||||
);
|
||||
const { connection, peer } = await fixture();
|
||||
const result = connection.requestLanguage('zh-CN');
|
||||
const rejected = assert.rejects(result, localized(/disconnected/));
|
||||
peer.close();
|
||||
await rejected;
|
||||
});
|
||||
|
||||
it('fences a pending language change when the call epoch changes', async () => {
|
||||
const { connection, peer } = await fixture();
|
||||
const frame = receive(peer);
|
||||
const result = connection.requestLanguage('zh-CN');
|
||||
const rejected = assert.rejects(result, localized(/call changed/));
|
||||
const request = await frame;
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.state',
|
||||
epoch: 4,
|
||||
status,
|
||||
uiLanguageV1: { language: 'en' },
|
||||
}),
|
||||
);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.language_result',
|
||||
requestId: request['requestId'],
|
||||
ok: true,
|
||||
uiLanguageV1: { language: 'zh-CN' },
|
||||
}),
|
||||
);
|
||||
await rejected;
|
||||
assert.equal(connection.getSnapshot().uiLanguageV1?.language, 'en');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { it } from 'node:test';
|
||||
import { readHostLanguage, saveHostLanguage } from '../language-store.ts';
|
||||
|
||||
it('persists the local locale privately and falls back for absent or invalid cache', () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'live-locale-cache-'));
|
||||
try {
|
||||
const path = join(directory, 'language.json');
|
||||
assert.equal(readHostLanguage(path), 'en');
|
||||
saveHostLanguage(path, 'zh-CN');
|
||||
assert.equal(readHostLanguage(path), 'zh-CN');
|
||||
assert.equal(statSync(path).mode & 0o777, 0o600);
|
||||
assert.throws(() => saveHostLanguage(path, 'fr' as never));
|
||||
assert.equal(readHostLanguage(path), 'zh-CN');
|
||||
assert.deepEqual(readdirSync(directory), ['language.json']);
|
||||
writeFileSync(path, '{');
|
||||
assert.equal(readHostLanguage(path), 'en');
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -2,11 +2,15 @@ import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import type { LiveStatus } from '../../shared/protocol.ts';
|
||||
import {
|
||||
canChangeLiveVisualInput,
|
||||
canToggleLive,
|
||||
isActiveLiveCall,
|
||||
projectLiveStatusForCapture,
|
||||
shouldCaptureLiveAudio,
|
||||
shouldCaptureLiveVisual,
|
||||
shouldRenderSetup,
|
||||
shouldRequestVisualSourceChange,
|
||||
shouldShowCameraPreview,
|
||||
shouldStopLiveOnToggle,
|
||||
} from '../live-state-policy.ts';
|
||||
|
||||
@@ -53,6 +57,105 @@ describe('Live Host state policy', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('activates visual capture only for a configured active call', () => {
|
||||
assert.equal(
|
||||
shouldCaptureLiveVisual(
|
||||
{ state: 'listening', callId: 'call-1' },
|
||||
{ source: 'screen', mode: 'on-demand' },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCaptureLiveVisual(
|
||||
{ state: 'idle', callId: undefined },
|
||||
{ source: 'camera', mode: 'live-feed' },
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCaptureLiveVisual(
|
||||
{ state: 'stopping', callId: 'call-1' },
|
||||
{ source: 'camera', mode: 'live-feed' },
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCaptureLiveVisual(
|
||||
{ state: 'listening', callId: 'call-1' },
|
||||
undefined,
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('shows a selected camera preview while the usable orb is visible', () => {
|
||||
const camera = { source: 'camera' as const };
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('idle', true), camera, true),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('listening', true), camera, true),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('stopping', true), camera, true),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('idle', true), { source: 'screen' }, true),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('idle', true), camera, false),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldShowCameraPreview(status('unavailable', false), camera, true),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('allows visual settings to escape an unavailable source', () => {
|
||||
const visualInput = { source: 'screen', mode: 'on-demand' };
|
||||
assert.equal(
|
||||
canChangeLiveVisualInput(
|
||||
{ state: 'unavailable', callId: undefined },
|
||||
visualInput,
|
||||
true,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
canChangeLiveVisualInput(
|
||||
{ state: 'stopping', callId: 'call-1' },
|
||||
visualInput,
|
||||
true,
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
canChangeLiveVisualInput(
|
||||
{ state: 'idle', callId: undefined },
|
||||
visualInput,
|
||||
false,
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('honors a rapid source rollback while the first selection is pending', () => {
|
||||
assert.equal(shouldRequestVisualSourceChange('screen', 'screen'), false);
|
||||
assert.equal(
|
||||
shouldRequestVisualSourceChange('screen', 'screen', 'camera'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldRequestVisualSourceChange('screen', 'camera', 'camera'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps the surface loading until microphone capture is ready', () => {
|
||||
const listening = {
|
||||
...status('listening', true),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,213 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { displayLiveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
type MemoryState,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
afterEach(async () => {
|
||||
for (const task of cleanup.splice(0).reverse()) await task();
|
||||
});
|
||||
|
||||
const memory: MemoryState = {
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
libraryId: 'default',
|
||||
model: 'qwen3.7-plus',
|
||||
libraries: [{ id: 'default', name: 'Default' }],
|
||||
locked: false,
|
||||
};
|
||||
|
||||
function nextMessage(peer: WebSocket): Promise<Record<string, unknown>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Missing memory action')),
|
||||
3_000,
|
||||
);
|
||||
peer.once('message', (data) => {
|
||||
clearTimeout(timer);
|
||||
resolve(JSON.parse(data.toString()) as Record<string, unknown>);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function connectedMemory(locked = false) {
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
cleanup.push(() => {
|
||||
for (const peer of server.clients) peer.terminate();
|
||||
server.close();
|
||||
});
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-live-memory-host-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
discovery,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'memory-token',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
let onReady: () => void = () => undefined;
|
||||
const ready = new Promise<void>((resolve) => {
|
||||
onReady = resolve;
|
||||
});
|
||||
const peerPromise = new Promise<WebSocket>((resolve) => {
|
||||
server.once('connection', async (peer) => {
|
||||
await nextMessage(peer);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 10_000,
|
||||
epoch: 3,
|
||||
memory: { ...memory, locked },
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: locked ? 'listening' : 'idle',
|
||||
shortcut: 'Command+Q',
|
||||
},
|
||||
}),
|
||||
);
|
||||
resolve(peer);
|
||||
});
|
||||
});
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => {
|
||||
if (snapshot.phase === 'ready') onReady();
|
||||
},
|
||||
onOutputAudio: () => undefined,
|
||||
onOutputAudioFinished: () => undefined,
|
||||
onClearOutput: () => undefined,
|
||||
},
|
||||
discovery,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
connection.start();
|
||||
const peer = await peerPromise;
|
||||
await ready;
|
||||
return { peer, connection };
|
||||
}
|
||||
|
||||
describe('memory requests over the Host connection', () => {
|
||||
it('waits for a matching authoritative reply and prevents duplicate changes', async () => {
|
||||
const { peer, connection } = await connectedMemory();
|
||||
const frame = nextMessage(peer);
|
||||
const result = connection.requestMemoryAction({
|
||||
action: 'set_enabled',
|
||||
enabled: false,
|
||||
});
|
||||
assert.equal(connection.getSnapshot().memory?.enabled, true);
|
||||
await assert.rejects(
|
||||
connection.requestMemoryAction({ action: 'create', name: 'Work' }),
|
||||
(error: Error) =>
|
||||
/already in progress/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
const request = await frame;
|
||||
assert.equal(request.type, 'host.memory_action');
|
||||
assert.equal(request.action, 'set_enabled');
|
||||
assert.equal(request.epoch, 3);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.memory_result',
|
||||
requestId: 'unrelated',
|
||||
ok: true,
|
||||
memory: { ...memory, enabled: false },
|
||||
}),
|
||||
);
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.memory_result',
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
memory: { ...memory, enabled: false },
|
||||
}),
|
||||
);
|
||||
assert.equal((await result).enabled, false);
|
||||
assert.equal(connection.getSnapshot().memory?.enabled, false);
|
||||
});
|
||||
|
||||
it('surfaces rejected changes without altering the current library', async () => {
|
||||
const { peer, connection } = await connectedMemory();
|
||||
const frame = nextMessage(peer);
|
||||
const result = connection.requestMemoryAction({
|
||||
action: 'create',
|
||||
name: 'Work',
|
||||
});
|
||||
const rejection = assert.rejects(result, /Disk is full/);
|
||||
const request = await frame;
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.memory_result',
|
||||
requestId: request.requestId,
|
||||
ok: false,
|
||||
error: 'Disk is full',
|
||||
}),
|
||||
);
|
||||
await rejection;
|
||||
assert.equal(connection.getSnapshot().memory?.libraryId, 'default');
|
||||
});
|
||||
|
||||
it('locks library/model changes during calls but allows rename and immediately rejects disconnects', async () => {
|
||||
const { peer, connection } = await connectedMemory(true);
|
||||
for (const action of [
|
||||
{ action: 'select', libraryId: 'work' },
|
||||
{ action: 'create', name: 'Work' },
|
||||
{ action: 'set_model', model: 'other-model' },
|
||||
] as const) {
|
||||
await assert.rejects(
|
||||
connection.requestMemoryAction(action),
|
||||
(error: Error) =>
|
||||
/End the current call/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
}
|
||||
const frame = nextMessage(peer);
|
||||
const result = connection.requestMemoryAction({
|
||||
action: 'rename',
|
||||
libraryId: 'default',
|
||||
name: 'Personal',
|
||||
});
|
||||
const rejection = assert.rejects(result, (error: Error) =>
|
||||
/disconnected/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
assert.equal((await frame).action, 'rename');
|
||||
peer.close();
|
||||
await rejection;
|
||||
await assert.rejects(
|
||||
connection.requestMemoryAction({ action: 'set_enabled', enabled: false }),
|
||||
(error: Error) =>
|
||||
/disconnected/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { MemoryPanel } from '../../renderer/memory-panel.ts';
|
||||
import type { HostPublicState } from '../../shared/host-api.ts';
|
||||
import type { MemoryAction, MemoryState } from '../../shared/protocol.ts';
|
||||
|
||||
const memory: MemoryState = {
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
libraryId: 'default',
|
||||
model: 'qwen3.7-plus',
|
||||
libraries: [{ id: 'default', name: 'Default' }],
|
||||
locked: false,
|
||||
};
|
||||
const state: HostPublicState = {
|
||||
connection: 'ready',
|
||||
memory,
|
||||
live: { v: 1, available: true, state: 'idle', shortcut: 'Command+E' },
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
visualReady: true,
|
||||
};
|
||||
const cleanup: Array<() => void> = [];
|
||||
afterEach(() => {
|
||||
for (const task of cleanup.splice(0).reverse()) task();
|
||||
});
|
||||
|
||||
function setup(
|
||||
handler: (action: MemoryAction) => Promise<MemoryState> = async () => memory,
|
||||
) {
|
||||
const dom = new JSDOM(
|
||||
'<!doctype html><html><body><main id="app"></main></body></html>',
|
||||
);
|
||||
const previous = Object.getOwnPropertyDescriptor(globalThis, 'document');
|
||||
Object.defineProperty(globalThis, 'document', {
|
||||
configurable: true,
|
||||
value: dom.window.document,
|
||||
});
|
||||
cleanup.push(() => {
|
||||
dom.window.close();
|
||||
if (previous) Object.defineProperty(globalThis, 'document', previous);
|
||||
else Reflect.deleteProperty(globalThis, 'document');
|
||||
});
|
||||
const calls: MemoryAction[] = [];
|
||||
const panel = new MemoryPanel({
|
||||
memoryAction: async (action) => {
|
||||
calls.push(action);
|
||||
return handler(action);
|
||||
},
|
||||
});
|
||||
dom.window.document.body.append(panel.element);
|
||||
panel.update(state);
|
||||
const button = (text: string) => {
|
||||
const result = Array.from(panel.element.querySelectorAll('button')).find(
|
||||
(item) => item.textContent === text,
|
||||
);
|
||||
assert(result, `Missing ${text} button`);
|
||||
return result;
|
||||
};
|
||||
const input = (selector: string) => {
|
||||
const result = panel.element.querySelector<HTMLInputElement>(selector);
|
||||
assert(result, `Missing ${selector} input`);
|
||||
return result;
|
||||
};
|
||||
const change = (target: HTMLInputElement, value: string) => {
|
||||
target.value = value;
|
||||
target.dispatchEvent(new dom.window.Event('input', { bubbles: true }));
|
||||
};
|
||||
return { dom, panel, calls, button, input, change };
|
||||
}
|
||||
|
||||
const settled = () => new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
describe('memory settings panel', () => {
|
||||
it('embeds in Settings and retains drafts while disconnected controls are disabled', () => {
|
||||
const { panel, button, input, change } = setup();
|
||||
assert.equal(panel.element.getAttribute('role'), 'group');
|
||||
assert.equal(panel.element.hasAttribute('aria-modal'), false);
|
||||
button('Rename').click();
|
||||
change(input('#memory-library-name'), 'Unsaved name');
|
||||
panel.update({ ...state, connection: 'disconnected', memory: undefined });
|
||||
assert.equal(panel.element.hidden, true);
|
||||
assert.equal(button('Save').disabled, true);
|
||||
panel.update({ ...state, memory: undefined });
|
||||
assert.equal(panel.element.hidden, true);
|
||||
panel.update(state);
|
||||
assert.equal(panel.element.hidden, false);
|
||||
assert.equal(input('#memory-library-name').value, 'Unsaved name');
|
||||
assert.equal(button('Save').disabled, false);
|
||||
});
|
||||
|
||||
it('preserves an in-progress rename and focus across call-state and orb redraws', () => {
|
||||
const { dom, panel, button, input, change } = setup();
|
||||
button('Rename').click();
|
||||
const name = input('#memory-library-name');
|
||||
change(name, 'My personal memory');
|
||||
name.setSelectionRange(3, 8);
|
||||
for (const callState of ['thinking', 'speaking', 'listening'] as const) {
|
||||
dom.window.document
|
||||
.querySelector('#app')
|
||||
?.replaceChildren(dom.window.document.createElement('div'));
|
||||
panel.update({
|
||||
...state,
|
||||
memory: { ...memory, locked: true },
|
||||
live: { ...state.live, state: callState, caption: 'new caption' },
|
||||
});
|
||||
assert.equal(name.value, 'My personal memory');
|
||||
assert.equal(dom.window.document.activeElement, name);
|
||||
assert.equal(name.selectionStart, 3);
|
||||
assert.equal(name.selectionEnd, 8);
|
||||
assert.equal(name.disabled, false);
|
||||
}
|
||||
assert.equal(button('New').disabled, true);
|
||||
assert.equal(button('Rename').disabled, false);
|
||||
assert.equal(input('[aria-label="Consolidation model"]').disabled, true);
|
||||
assert.match(panel.element.textContent ?? '', /End the current call/);
|
||||
});
|
||||
|
||||
it('shows only the acknowledged toggle and disables duplicate actions while saving', async () => {
|
||||
let complete: (next: MemoryState) => void = () => undefined;
|
||||
const { dom, panel, calls, button } = setup(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
complete = resolve;
|
||||
}),
|
||||
);
|
||||
const enabled = panel.element.querySelector<HTMLInputElement>(
|
||||
'input[type="checkbox"]',
|
||||
);
|
||||
assert(enabled);
|
||||
enabled.checked = false;
|
||||
enabled.dispatchEvent(new dom.window.Event('change', { bubbles: true }));
|
||||
assert.deepEqual(calls, [{ action: 'set_enabled', enabled: false }]);
|
||||
assert.equal(enabled.checked, true);
|
||||
assert.equal(enabled.disabled, true);
|
||||
assert.equal(button('New').disabled, true);
|
||||
complete({ ...memory, enabled: false });
|
||||
await settled();
|
||||
assert.equal(enabled.checked, false);
|
||||
assert.equal(enabled.disabled, false);
|
||||
});
|
||||
|
||||
it('selects a new library from the daemon result and renames by stable id', async () => {
|
||||
const created = {
|
||||
...memory,
|
||||
libraryId: 'lib_work',
|
||||
libraries: [...memory.libraries, { id: 'lib_work', name: 'Work' }],
|
||||
};
|
||||
const { panel, calls, button, input, change } = setup(async (action) => {
|
||||
if (action.action === 'create') return created;
|
||||
return {
|
||||
...created,
|
||||
libraries: [memory.libraries[0]!, { id: 'lib_work', name: 'Projects' }],
|
||||
};
|
||||
});
|
||||
button('New').click();
|
||||
change(input('#memory-library-name'), 'Work');
|
||||
button('Save').click();
|
||||
await settled();
|
||||
assert.deepEqual(calls[0], { action: 'create', name: 'Work' });
|
||||
assert.equal(panel.element.querySelector('select')?.value, 'lib_work');
|
||||
button('Rename').click();
|
||||
assert.equal(panel.element.querySelector('select')?.disabled, true);
|
||||
change(input('#memory-library-name'), 'Projects');
|
||||
button('Save').click();
|
||||
await settled();
|
||||
assert.deepEqual(calls[1], {
|
||||
action: 'rename',
|
||||
libraryId: 'lib_work',
|
||||
name: 'Projects',
|
||||
});
|
||||
assert.equal(
|
||||
panel.element.querySelector('select')?.selectedOptions[0]?.textContent,
|
||||
'Projects',
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves model drafts on state updates, saves explicitly, and retains failed name edits', async () => {
|
||||
const { panel, calls, button, input, change } = setup(async (action) => {
|
||||
if (action.action === 'set_model')
|
||||
return { ...memory, model: action.model };
|
||||
throw new Error('Could not save library name');
|
||||
});
|
||||
const model = input('[aria-label="Consolidation model"]');
|
||||
change(model, 'custom-model');
|
||||
panel.update({ ...state, live: { ...state.live, caption: 'updated' } });
|
||||
assert.equal(model.value, 'custom-model');
|
||||
assert.equal(calls.length, 0);
|
||||
button('Save model').click();
|
||||
await settled();
|
||||
assert.deepEqual(calls[0], { action: 'set_model', model: 'custom-model' });
|
||||
assert.equal(model.value, 'custom-model');
|
||||
button('Rename').click();
|
||||
change(input('#memory-library-name'), 'Unsaved name');
|
||||
button('Save').click();
|
||||
await settled();
|
||||
assert.equal(input('#memory-library-name').value, 'Unsaved name');
|
||||
assert.match(
|
||||
panel.element.querySelector('[role="status"]')?.textContent ?? '',
|
||||
/Could not save/,
|
||||
);
|
||||
panel.update({ ...state, connection: 'disconnected', memory: undefined });
|
||||
assert.equal(button('Save').disabled, true);
|
||||
assert.match(panel.element.textContent ?? '', /Connect to Qwen Live/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import type {
|
||||
LiveMemoryAction,
|
||||
LiveMemoryResult,
|
||||
LiveMemoryState,
|
||||
} from '../../../../qwen-live/src/host/types.ts';
|
||||
import {
|
||||
encodeHostControlMessage,
|
||||
parseDaemonControlMessage,
|
||||
parseMemoryAction,
|
||||
parseMemoryState,
|
||||
type MemoryAction,
|
||||
type MemoryState,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const memory: MemoryState = {
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
libraryId: 'default',
|
||||
model: 'qwen3.7-plus',
|
||||
libraries: [{ id: 'default', name: 'Default' }],
|
||||
locked: false,
|
||||
};
|
||||
const daemonMemory: LiveMemoryState = memory;
|
||||
const actions: LiveMemoryAction[] = [
|
||||
{ action: 'set_enabled', enabled: false },
|
||||
{ action: 'set_visual_enabled', enabled: true },
|
||||
{ action: 'select', libraryId: 'work' },
|
||||
{ action: 'create', name: 'Work' },
|
||||
{ action: 'rename', libraryId: 'default', name: 'Personal' },
|
||||
{ action: 'set_model', model: 'qwen3.7-plus' },
|
||||
];
|
||||
|
||||
describe('memory protocol', () => {
|
||||
it('round-trips all management actions and authoritative state', () => {
|
||||
for (const action of actions) {
|
||||
const hostAction: MemoryAction = action;
|
||||
assert.deepEqual(parseMemoryAction(action), action);
|
||||
assert.deepEqual(
|
||||
JSON.parse(
|
||||
encodeHostControlMessage({
|
||||
type: 'host.memory_action',
|
||||
requestId: 'request-1',
|
||||
epoch: 3,
|
||||
...hostAction,
|
||||
}),
|
||||
),
|
||||
{
|
||||
type: 'host.memory_action',
|
||||
requestId: 'request-1',
|
||||
epoch: 3,
|
||||
...action,
|
||||
},
|
||||
);
|
||||
}
|
||||
const result: LiveMemoryResult = {
|
||||
type: 'host.memory_result',
|
||||
requestId: 'request-1',
|
||||
ok: true,
|
||||
memory: daemonMemory,
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(result)), result);
|
||||
assert.deepEqual(parseMemoryState(memory), memory);
|
||||
});
|
||||
|
||||
it('rejects unsafe or malformed actions, names, states, and incomplete results', () => {
|
||||
for (const action of [
|
||||
{ action: 'select', libraryId: '../other' },
|
||||
{ action: 'create', name: ' ' },
|
||||
{ action: 'create', name: 'x'.repeat(81) },
|
||||
{ action: 'rename', libraryId: 'default', name: 'bad\u0000name' },
|
||||
{ action: 'set_enabled', enabled: 'true' },
|
||||
{ action: 'set_model', model: '' },
|
||||
{ action: 'remove', libraryId: 'default' },
|
||||
]) {
|
||||
assert.equal(parseMemoryAction(action), undefined);
|
||||
}
|
||||
assert.deepEqual(parseMemoryAction({ action: 'create', name: ' 工作 ' }), {
|
||||
action: 'create',
|
||||
name: '工作',
|
||||
});
|
||||
assert.equal(parseMemoryState({ ...memory, locked: 'yes' }), undefined);
|
||||
assert.equal(
|
||||
parseMemoryState({
|
||||
...memory,
|
||||
libraries: [
|
||||
{ id: 'default', name: 'One' },
|
||||
{ id: 'default', name: 'Two' },
|
||||
],
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.memory_result',
|
||||
requestId: 'request-1',
|
||||
ok: true,
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.memory_result',
|
||||
requestId: 'request-1',
|
||||
ok: false,
|
||||
error: '',
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps memory optional for qwen serve and validates it when supplied', () => {
|
||||
const state = {
|
||||
type: 'host.state',
|
||||
epoch: 0,
|
||||
status: { v: 1, available: true, state: 'idle', shortcut: 'Command+Q' },
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(state)), state);
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(JSON.stringify({ ...state, memory })),
|
||||
{ ...state, memory },
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...state, memory: { enabled: true } }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,836 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import * as positions from '../overlay-position.ts';
|
||||
import type { HostPublicState } from '../../shared/host-api.ts';
|
||||
import { OVERLAY_GEOMETRY } from '../../shared/overlay-geometry.ts';
|
||||
import { StartupInteraction } from '../startup-interaction.ts';
|
||||
import {
|
||||
displayLiveMessage,
|
||||
isLiveLanguage,
|
||||
liveMessage,
|
||||
} from '@qwen-code/qwen-live/i18n';
|
||||
import { canToggleLive, shouldStopLiveOnToggle } from '../live-state-policy.ts';
|
||||
|
||||
type Callback = (...args: unknown[]) => unknown;
|
||||
|
||||
function fixture(minimumNativeY?: number, diagnosticsEnabled = false) {
|
||||
const source = readFileSync(new URL('../index.ts', import.meta.url), 'utf8');
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = new Set([
|
||||
'showOverlay',
|
||||
'stopLive',
|
||||
'createOverlay',
|
||||
'isTrustedSender',
|
||||
'registerIpc',
|
||||
'syncPointerInteractivity',
|
||||
'dragOverlay',
|
||||
'persistOverlayPosition',
|
||||
'clampOverlayToDisplays',
|
||||
'resetOverlayInteraction',
|
||||
'dismissSettings',
|
||||
'quitHost',
|
||||
'publicState',
|
||||
'resolvedTheme',
|
||||
'activateNativeServices',
|
||||
'deactivateNativeServices',
|
||||
'beginMediaPermissionMonitor',
|
||||
'scheduleReadinessReconnect',
|
||||
'microphonePermission',
|
||||
'cameraPermission',
|
||||
'requestCameraPermission',
|
||||
'applyOverlayPosition',
|
||||
'overlayWorkArea',
|
||||
'setOverlayLayout',
|
||||
'subagentsAnchor',
|
||||
'subagentsHoverRegions',
|
||||
'maybeStartStartupInteraction',
|
||||
'toggleLive',
|
||||
'newConversation',
|
||||
'positionOverlay',
|
||||
'sendRequiredPlaybackReceipt',
|
||||
]);
|
||||
const functions = tree.statements
|
||||
.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) &&
|
||||
node.name &&
|
||||
names.has(node.name.text),
|
||||
)
|
||||
.map((node) => node.getText(tree))
|
||||
.join('\n');
|
||||
const ipc = new Map<string, Callback>();
|
||||
const area = { x: 0, y: 23, width: 1280, height: 777 };
|
||||
const saved: Array<{ x: number; y: number }> = [];
|
||||
const stored = {
|
||||
position: undefined as { x: number; y: number } | undefined,
|
||||
};
|
||||
let hidden = 0;
|
||||
let quitCalls = 0;
|
||||
let quitResolves: (() => void) | undefined;
|
||||
let quitRejects: ((error: Error) => void) | undefined;
|
||||
const commands: string[] = [];
|
||||
const diagnostics: Array<{ event: string; details: object }> = [];
|
||||
const states: HostPublicState[] = [];
|
||||
const offsets: Array<{ x: number; y: number }> = [];
|
||||
const actions: string[] = [];
|
||||
const playback: Array<{ kind: string; epoch: number; outputId: number }> = [];
|
||||
const flags = { hostReady: false };
|
||||
const resources = { audio: false, camera: false, shortcut: false };
|
||||
const timers = new Set<object>();
|
||||
const grants = new Map<string, (granted: boolean) => void>();
|
||||
const timer = () => {
|
||||
const value = { unref() {} };
|
||||
timers.add(value);
|
||||
return value;
|
||||
};
|
||||
class Window {
|
||||
readonly events = new Map<string, Callback>();
|
||||
readonly webContents = {
|
||||
on: (name: string, callback: Callback) => this.events.set(name, callback),
|
||||
setWindowOpenHandler: () => {},
|
||||
isDestroyed: () => false,
|
||||
};
|
||||
readonly ignored: boolean[] = [];
|
||||
readonly moves: Array<{ x: number; y: number }> = [];
|
||||
constructor(
|
||||
readonly options: {
|
||||
x?: number;
|
||||
y?: number;
|
||||
width: number;
|
||||
height: number;
|
||||
},
|
||||
) {}
|
||||
isDestroyed() {
|
||||
return false;
|
||||
}
|
||||
getBounds() {
|
||||
return {
|
||||
x: this.options.x ?? 40,
|
||||
y: this.options.y ?? 80,
|
||||
width: this.options.width,
|
||||
height: this.options.height,
|
||||
};
|
||||
}
|
||||
setPosition(x: number, y: number) {
|
||||
this.options.x = x;
|
||||
this.options.y =
|
||||
minimumNativeY === undefined ? y : Math.max(minimumNativeY, y);
|
||||
this.moves.push({ x, y });
|
||||
}
|
||||
setAlwaysOnTop() {}
|
||||
setVisibleOnAllWorkspaces() {}
|
||||
setIgnoreMouseEvents(value: boolean) {
|
||||
this.ignored.push(value);
|
||||
}
|
||||
showInactive() {}
|
||||
hide() {
|
||||
hidden++;
|
||||
}
|
||||
on(name: string, callback: Callback) {
|
||||
this.events.set(name, callback);
|
||||
}
|
||||
once() {}
|
||||
loadFile() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
const context = {
|
||||
isLiveLanguage,
|
||||
liveMessage,
|
||||
language: 'en',
|
||||
screenDisplays: [],
|
||||
screenDisplaysError: undefined,
|
||||
appshotCapture: undefined,
|
||||
refreshScreenDisplays: () => {},
|
||||
theme: 'system',
|
||||
nativeTheme: { shouldUseDarkColors: true },
|
||||
subagents: undefined,
|
||||
BrowserWindow: Window,
|
||||
screen: {
|
||||
getCursorScreenPoint: () => ({ x: 400, y: 400 }),
|
||||
getDisplayNearestPoint: () => ({ workArea: area }),
|
||||
getDisplayMatching: () => ({ workArea: area }),
|
||||
},
|
||||
ipcMain: {
|
||||
on: (name: string, fn: Callback) => ipc.set(name, fn),
|
||||
handle: (name: string, fn: Callback) => ipc.set(name, fn),
|
||||
},
|
||||
app: {
|
||||
getPath: () => '/fixture',
|
||||
quit: () => {
|
||||
quitCalls++;
|
||||
},
|
||||
},
|
||||
daemon: {
|
||||
getConfigFilePath: () => undefined,
|
||||
getEpoch: () => 1,
|
||||
sendPlaybackStarted: (epoch: number, outputId: number) => {
|
||||
playback.push({ kind: 'started', epoch, outputId });
|
||||
return true;
|
||||
},
|
||||
sendPlaybackCompleted: (epoch: number, outputId: number) => {
|
||||
playback.push({ kind: 'completed', epoch, outputId });
|
||||
return true;
|
||||
},
|
||||
requestQuit: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
quitResolves = resolve;
|
||||
quitRejects = reject;
|
||||
}),
|
||||
},
|
||||
connection: { phase: 'ready' },
|
||||
OVERLAY_GEOMETRY,
|
||||
StartupInteraction,
|
||||
canToggleLive,
|
||||
shouldStopLiveOnToggle,
|
||||
isHostReady: () => flags.hostReady,
|
||||
setInterval: timer,
|
||||
setTimeout: timer,
|
||||
clearInterval: (value: object) => timers.delete(value),
|
||||
clearTimeout: (value: object) => timers.delete(value),
|
||||
systemPreferences: {
|
||||
getMediaAccessStatus: () => 'granted',
|
||||
askForMediaAccess: (permission: string) =>
|
||||
new Promise<boolean>((resolve) => grants.set(permission, resolve)),
|
||||
},
|
||||
shortcut: {
|
||||
stop: () => {
|
||||
resources.shortcut = false;
|
||||
},
|
||||
},
|
||||
appshotReadiness: { start() {}, stop() {} },
|
||||
failClosedForReadinessLoss: () => {},
|
||||
failRequiredDaemonMessage: () => {
|
||||
throw new Error('Unexpected failed playback receipt');
|
||||
},
|
||||
shell: {
|
||||
openExternal: async () => {
|
||||
throw new Error('Unexpected external settings');
|
||||
},
|
||||
},
|
||||
join,
|
||||
__dirname: '/fixture',
|
||||
...positions,
|
||||
readOverlayPosition: () => stored.position,
|
||||
saveOverlayPosition: (_path: string, position: { x: number; y: number }) =>
|
||||
saved.push({ x: position.x, y: position.y }),
|
||||
writeLiveDiagnostic: (event: string, details: object) =>
|
||||
diagnostics.push({ event, details }),
|
||||
sendRendererCommand: (channel: string, value?: unknown) => {
|
||||
commands.push(channel);
|
||||
if (channel === 'live:overlay-offset') {
|
||||
const point = value as { x: number; y: number };
|
||||
offsets.push({ x: point.x, y: point.y });
|
||||
}
|
||||
if (channel === 'live:audio:initialize') resources.audio = true;
|
||||
if (channel === 'live:audio:deactivate') resources.audio = false;
|
||||
if (channel === 'live:camera:deactivate') resources.camera = false;
|
||||
},
|
||||
stopLocalVisual: () => {
|
||||
resources.camera = false;
|
||||
},
|
||||
stopLocalAudio: () => {},
|
||||
closeHostAudioCapture: () => {},
|
||||
closeHostInputCapture: () => {},
|
||||
sendRequiredAction: (action: { action: string }) => {
|
||||
actions.push(action.action);
|
||||
return true;
|
||||
},
|
||||
overlayRecovery: { markReady: () => {}, handleFailure: () => {} },
|
||||
isRecoverableOverlayLoadFailure: () => false,
|
||||
syncOutputAudioEndMarkerMode: () => {},
|
||||
publishState: (): void => {
|
||||
states.push(controls.publicState());
|
||||
controls.maybeStart();
|
||||
},
|
||||
effectiveLiveStatus: () => ({
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
}),
|
||||
syncVisualCapture: () => {},
|
||||
};
|
||||
const script = `let overlay;
|
||||
let overlayReady = true, rendererEventsEnabled = true, pointerInteractive = false;
|
||||
let pointerOverInteractive = false, settingsOpen = false, overlayDrag, desiredOverlayPosition;
|
||||
let hasCustomOverlayPosition = false;
|
||||
let overlayLayout = 'setup';
|
||||
let overlayOffset = { x: 0, y: 0 };
|
||||
let nativeServicesActive = false, nativeServiceGeneration = 0, liveStartPending = false, quitting = false;
|
||||
let audioTransportFailed = false, captureReadyEpoch, pendingVisualSourceChange;
|
||||
let visualSourceChangeGeneration = 0, readinessReconnectTimer, readinessReconnectReason, mediaPermissionTimer;
|
||||
let quitApproved = false, quitOperation, quitState;
|
||||
let visualInput, visualError, visualReady = false;
|
||||
const permissions = {}, selfChecks = {};
|
||||
const startupInteraction = new StartupInteraction();
|
||||
let live = { v: 1, available: true, state: 'idle', shortcut: 'Command+E' };
|
||||
const OVERLAY_WIDTH = 384, OVERLAY_HEIGHT = 480, diagnosticsEnabled = ${diagnosticsEnabled}, READINESS_RECONNECT_DEBOUNCE_MS = 2500;
|
||||
${functions}
|
||||
registerIpc();
|
||||
({ create: (ready = true) => { overlay = createOverlay(); if (ready) overlay.events.get('did-finish-load')?.(); return overlay; }, showOverlay, stopLive,
|
||||
clamp: () => clampOverlayToDisplays(), quitHost: () => quitHost(), publicState,
|
||||
activate: activateNativeServices, maybeStart: maybeStartStartupInteraction, subagentsAnchor, subagentsHoverRegions });`;
|
||||
const controls = runInNewContext(
|
||||
ts.transpileModule(script, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText,
|
||||
context,
|
||||
) as {
|
||||
create: (ready?: boolean) => Window;
|
||||
showOverlay: () => void;
|
||||
stopLive: () => void;
|
||||
clamp: () => void;
|
||||
quitHost: () => Promise<void>;
|
||||
publicState: () => HostPublicState;
|
||||
activate: () => void;
|
||||
maybeStart: () => void;
|
||||
subagentsAnchor: () =>
|
||||
| { x: number; y: number; width: number; height: number }
|
||||
| undefined;
|
||||
subagentsHoverRegions: () => Array<{
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}>;
|
||||
};
|
||||
return {
|
||||
controls,
|
||||
ipc,
|
||||
stored,
|
||||
saved,
|
||||
area,
|
||||
hidden: () => hidden,
|
||||
quitCalls: () => quitCalls,
|
||||
finishQuit: () => quitResolves?.(),
|
||||
failQuit: () => quitRejects?.(new Error('private transport error')),
|
||||
commands,
|
||||
diagnostics,
|
||||
states,
|
||||
offsets,
|
||||
actions,
|
||||
playback,
|
||||
flags,
|
||||
connect: () => {
|
||||
context.connection = { phase: 'ready' };
|
||||
},
|
||||
resources,
|
||||
timers,
|
||||
grants,
|
||||
disconnect: () => {
|
||||
context.connection = { phase: 'disconnected' };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('native overlay interaction', () => {
|
||||
it('logs current native window movement only in debug mode without repositioning it', () => {
|
||||
const quiet = fixture();
|
||||
const quietWindow = quiet.controls.create();
|
||||
quiet.diagnostics.length = 0;
|
||||
quietWindow.events.get('move')?.();
|
||||
assert.equal(quiet.diagnostics.length, 0);
|
||||
|
||||
const host = fixture(undefined, true);
|
||||
const previous = host.controls.create();
|
||||
const window = host.controls.create();
|
||||
host.diagnostics.length = 0;
|
||||
const moves = window.moves.length;
|
||||
previous.events.get('move')?.();
|
||||
assert.equal(host.diagnostics.length, 0);
|
||||
window.events.get('move')?.();
|
||||
assert.equal(window.moves.length, moves);
|
||||
assert.equal(host.diagnostics.length, 1);
|
||||
const log = host.diagnostics[0];
|
||||
assert.equal(log?.event, 'overlay_native_moved');
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(log?.details)), {
|
||||
bounds: window.getBounds(),
|
||||
offset: { x: 0, y: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts playback receipts only from the current ready renderer with valid epoch and output identity', () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create(false);
|
||||
const event = { sender: window.webContents };
|
||||
const receipt = { epoch: 1, outputId: 7 };
|
||||
for (const channel of [
|
||||
'live:audio:playback-started',
|
||||
'live:audio:playback-completed',
|
||||
]) {
|
||||
const handle = host.ipc.get(channel)!;
|
||||
handle(event, receipt);
|
||||
assert.equal(host.playback.length, 0);
|
||||
}
|
||||
window.events.get('did-finish-load')?.();
|
||||
for (const channel of [
|
||||
'live:audio:playback-started',
|
||||
'live:audio:playback-completed',
|
||||
]) {
|
||||
const handle = host.ipc.get(channel)!;
|
||||
handle({ sender: {} }, receipt);
|
||||
for (const invalid of [
|
||||
null,
|
||||
1,
|
||||
{ epoch: 0, outputId: 7 },
|
||||
{ epoch: 1, outputId: -1 },
|
||||
{ epoch: 1, outputId: 1.5 },
|
||||
{ epoch: 1 },
|
||||
])
|
||||
handle(event, invalid);
|
||||
}
|
||||
assert.equal(host.playback.length, 0);
|
||||
host.ipc.get('live:audio:playback-started')!(event, receipt);
|
||||
host.ipc.get('live:audio:playback-completed')!(event, receipt);
|
||||
assert.deepEqual(host.playback, [
|
||||
{ kind: 'started', epoch: 1, outputId: 7 },
|
||||
{ kind: 'completed', epoch: 1, outputId: 7 },
|
||||
]);
|
||||
host.controls.create();
|
||||
host.ipc.get('live:audio:playback-started')!(event, receipt);
|
||||
assert.equal(host.playback.length, 2);
|
||||
});
|
||||
|
||||
it('places both first-run layouts at bottom-right with margins and retains a later drag', () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
const event = { sender: window.webContents };
|
||||
const layout = host.ipc.get('live:overlay-layout')!;
|
||||
const margins = (visible: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}) => {
|
||||
const bounds = window.getBounds();
|
||||
return {
|
||||
right:
|
||||
host.area.x + host.area.width - bounds.x - visible.x - visible.width,
|
||||
bottom:
|
||||
host.area.y +
|
||||
host.area.height -
|
||||
bounds.y -
|
||||
visible.y -
|
||||
visible.height,
|
||||
};
|
||||
};
|
||||
assert.deepEqual(margins(OVERLAY_GEOMETRY.bounds.setup), {
|
||||
right: 20,
|
||||
bottom: 20,
|
||||
});
|
||||
layout(event, 'orb');
|
||||
assert.deepEqual(margins(OVERLAY_GEOMETRY.bounds.orb), {
|
||||
right: 20,
|
||||
bottom: 20,
|
||||
});
|
||||
const anchor = host.controls.subagentsAnchor();
|
||||
assert(anchor);
|
||||
assert.equal(anchor.width, OVERLAY_GEOMETRY.bounds.orb.width);
|
||||
assert(anchor.x <= window.getBounds().x + OVERLAY_GEOMETRY.status.x);
|
||||
assert(
|
||||
anchor.x + anchor.width >=
|
||||
window.getBounds().x +
|
||||
OVERLAY_GEOMETRY.status.x +
|
||||
OVERLAY_GEOMETRY.status.width,
|
||||
);
|
||||
assert(
|
||||
anchor.y + anchor.height >=
|
||||
window.getBounds().y +
|
||||
OVERLAY_GEOMETRY.status.y +
|
||||
OVERLAY_GEOMETRY.status.height,
|
||||
);
|
||||
assert.equal(host.controls.subagentsHoverRegions().length, 3);
|
||||
const blank = {
|
||||
x: window.getBounds().x + 190,
|
||||
y: window.getBounds().y + 220,
|
||||
};
|
||||
assert(
|
||||
!host.controls
|
||||
.subagentsHoverRegions()
|
||||
.some(
|
||||
(r) =>
|
||||
blank.x >= r.x &&
|
||||
blank.x <= r.x + r.width &&
|
||||
blank.y >= r.y &&
|
||||
blank.y <= r.y + r.height,
|
||||
),
|
||||
);
|
||||
const drag = host.ipc.get('live:drag-overlay')!;
|
||||
drag(event, 'start', 1000, 700);
|
||||
drag(event, 'end', 900, 650);
|
||||
const dragged = window.getBounds();
|
||||
host.ipc.get('live:settings-open')!(event, true);
|
||||
host.ipc.get('live:settings-open')!(event, false);
|
||||
assert.deepEqual(window.getBounds(), dragged);
|
||||
const replacement = host.controls.create();
|
||||
layout({ sender: replacement.webContents }, 'orb');
|
||||
assert.deepEqual(replacement.getBounds(), dragged);
|
||||
});
|
||||
|
||||
it('does not reposition the window on show/state refresh and does not hide on stop', () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
const initial = window.getBounds();
|
||||
host.controls.showOverlay();
|
||||
host.controls.showOverlay();
|
||||
assert.deepEqual(window.getBounds(), initial);
|
||||
assert.equal(window.moves.length, 0);
|
||||
host.controls.stopLive();
|
||||
assert.equal(host.hidden(), 0);
|
||||
assert.equal(window.options.height, 480);
|
||||
});
|
||||
|
||||
it('restores the dragged bottom-right position after native re-show clamps the canvas', () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
const event = { sender: window.webContents };
|
||||
const layout = host.ipc.get('live:overlay-layout')!;
|
||||
layout(event, 'orb');
|
||||
const drag = host.ipc.get('live:drag-overlay')!;
|
||||
drag(event, 'start', 1000, 700);
|
||||
drag(event, 'end', 1100, 800);
|
||||
const desired = { x: 956, y: 326 };
|
||||
const before = window.getBounds();
|
||||
assert.deepEqual({ x: before.x, y: before.y }, desired);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
const moves = window.moves.length;
|
||||
host.controls.showOverlay();
|
||||
assert.equal(window.moves.length, moves);
|
||||
|
||||
window.hide();
|
||||
window.showInactive = () => {
|
||||
window.options.x = 896;
|
||||
window.options.y = 320;
|
||||
};
|
||||
host.controls.showOverlay();
|
||||
assert.deepEqual(window.getBounds(), before);
|
||||
assert.equal(window.moves.length, moves + 1);
|
||||
assert.deepEqual(window.moves.at(-1), desired);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.x, 0);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, 0);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
const replacement = host.controls.create();
|
||||
layout({ sender: replacement.webContents }, 'orb');
|
||||
assert.deepEqual(replacement.getBounds(), before);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
});
|
||||
|
||||
it('preserves an existing native offset and saved drag position when re-show clamps another edge', () => {
|
||||
const host = fixture(33);
|
||||
host.area.y = 33;
|
||||
host.stored.position = { x: 956, y: -97 };
|
||||
const window = host.controls.create();
|
||||
const event = { sender: window.webContents };
|
||||
const layout = host.ipc.get('live:overlay-layout')!;
|
||||
layout(event, 'orb');
|
||||
const drag = host.ipc.get('live:drag-overlay')!;
|
||||
drag(event, 'start', 292, 255);
|
||||
drag(event, 'end', 292, 265);
|
||||
const desired = { x: 956, y: -87 };
|
||||
const before = window.getBounds();
|
||||
assert.equal(before.y, 33);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
const moves = window.moves.length;
|
||||
const offsets = host.offsets.length;
|
||||
host.controls.showOverlay();
|
||||
assert.equal(window.moves.length, moves);
|
||||
assert.equal(host.offsets.length, offsets);
|
||||
|
||||
window.hide();
|
||||
window.showInactive = () => {
|
||||
window.options.x = 896;
|
||||
};
|
||||
host.controls.showOverlay();
|
||||
assert.deepEqual(window.getBounds(), before);
|
||||
assert.equal(window.moves.length, moves + 1);
|
||||
assert.deepEqual(window.moves.at(-1), desired);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.x, 0);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
const replacement = host.controls.create();
|
||||
layout({ sender: replacement.webContents }, 'orb');
|
||||
assert.deepEqual(replacement.getBounds(), before);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.deepEqual(host.saved, [desired]);
|
||||
});
|
||||
|
||||
it('resets the pointer cache before accepting events from a replacement window', () => {
|
||||
const host = fixture();
|
||||
const first = host.controls.create();
|
||||
const pointer = host.ipc.get('live:pointer-interactivity');
|
||||
assert(pointer);
|
||||
pointer({ sender: first.webContents }, true);
|
||||
const replacement = host.controls.create();
|
||||
pointer({ sender: first.webContents }, true);
|
||||
assert.equal(replacement.ignored.at(-1), true);
|
||||
pointer({ sender: replacement.webContents }, true);
|
||||
assert.equal(replacement.ignored.at(-1), false);
|
||||
});
|
||||
|
||||
it('restores a saved position and accepts only bounded drag messages from its renderer', () => {
|
||||
const host = fixture();
|
||||
host.stored.position = { x: 100, y: 200 };
|
||||
const window = host.controls.create();
|
||||
assert.equal(window.getBounds().x, 100);
|
||||
const drag = host.ipc.get('live:drag-overlay');
|
||||
assert(drag);
|
||||
drag({ sender: {} }, 'start', 300, 300);
|
||||
drag({ sender: window.webContents }, 'move', 350, 350);
|
||||
assert.equal(window.moves.length, 0);
|
||||
drag({ sender: window.webContents }, 'start', NaN, 300);
|
||||
drag({ sender: window.webContents }, 'move', 350, 350);
|
||||
assert.equal(window.moves.length, 0);
|
||||
drag({ sender: window.webContents }, 'start', 300, 300);
|
||||
drag({ sender: window.webContents }, 'move', 350, 350);
|
||||
drag({ sender: window.webContents }, 'end', 350, 350);
|
||||
assert.equal(window.getBounds().x, 150);
|
||||
assert.equal(window.getBounds().y, 250);
|
||||
assert.deepEqual(host.saved.at(-1), { x: 150, y: 250 });
|
||||
host.area.width = 400;
|
||||
host.controls.clamp();
|
||||
assert.equal(window.getBounds().x, 16);
|
||||
});
|
||||
|
||||
it('waits for the single graceful daemon quit before exiting Host', async () => {
|
||||
const host = fixture();
|
||||
host.controls.create();
|
||||
const first = host.controls.quitHost();
|
||||
const second = host.controls.quitHost();
|
||||
assert.equal(first, second);
|
||||
assert.equal(host.quitCalls(), 0);
|
||||
host.finishQuit();
|
||||
await first;
|
||||
assert.equal(host.quitCalls(), 1);
|
||||
});
|
||||
|
||||
it('keeps Host alive after a failed daemon quit and allows a safe retry', async () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
const quit = host.ipc.get('live:quit');
|
||||
assert(quit);
|
||||
assert.throws(
|
||||
() => quit({ sender: {} }),
|
||||
(error: Error) =>
|
||||
/Untrusted/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
const failed = host.controls.quitHost();
|
||||
host.failQuit();
|
||||
await assert.rejects(failed, (error: Error) =>
|
||||
/Could not shut down Qwen Live/.test(
|
||||
displayLiveMessage('en', error.message),
|
||||
),
|
||||
);
|
||||
assert.equal(host.quitCalls(), 0);
|
||||
const retry = quit({ sender: window.webContents });
|
||||
host.finishQuit();
|
||||
await retry;
|
||||
assert.equal(host.quitCalls(), 1);
|
||||
});
|
||||
|
||||
it('publishes native Quit pending and failed state independently of daemon connection snapshots', async () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
host.states.length = 0;
|
||||
assert.equal(host.controls.publicState().quitState, undefined);
|
||||
const first = host.controls.quitHost();
|
||||
assert.equal(host.states.at(-1)?.quitState, 'pending');
|
||||
host.disconnect();
|
||||
assert.equal(host.controls.publicState().quitState, 'pending');
|
||||
host.failQuit();
|
||||
await assert.rejects(first, (error: Error) =>
|
||||
/Could not shut down/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
assert.equal(host.states.at(-1)?.quitState, 'failed');
|
||||
assert.equal(host.quitCalls(), 0);
|
||||
const retry = host.ipc.get('live:quit')?.({ sender: window.webContents });
|
||||
assert.equal(host.states.at(-1)?.quitState, 'pending');
|
||||
host.finishQuit();
|
||||
await retry;
|
||||
assert.deepEqual(
|
||||
host.states.map((state) => state.quitState),
|
||||
['pending', 'failed', 'pending'],
|
||||
);
|
||||
assert.equal(host.quitCalls(), 1);
|
||||
});
|
||||
|
||||
it('stops local media, permissions polling and shortcuts before waiting for daemon Quit and does not restart on failure or retry', async () => {
|
||||
const host = fixture();
|
||||
host.controls.create();
|
||||
host.controls.activate();
|
||||
host.resources.camera = host.resources.shortcut = true;
|
||||
assert.equal(host.resources.audio, true);
|
||||
assert.equal(host.timers.size, 1);
|
||||
const first = host.controls.quitHost();
|
||||
assert.deepEqual(host.resources, {
|
||||
audio: false,
|
||||
camera: false,
|
||||
shortcut: false,
|
||||
});
|
||||
assert.equal(host.timers.size, 0);
|
||||
assert.equal(host.controls.quitHost(), first);
|
||||
host.controls.activate();
|
||||
assert.equal(host.resources.audio, false);
|
||||
host.failQuit();
|
||||
await assert.rejects(first, (error: Error) =>
|
||||
/Could not shut down/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
host.controls.activate();
|
||||
assert.equal(host.resources.audio, false);
|
||||
const retry = host.controls.quitHost();
|
||||
host.finishQuit();
|
||||
await retry;
|
||||
assert.equal(
|
||||
host.commands.filter((value) => value === 'live:audio:initialize').length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it('discards late microphone and camera grants across pending and failed Quit', async () => {
|
||||
const host = fixture();
|
||||
const window = host.controls.create();
|
||||
host.controls.activate();
|
||||
const request = host.ipc.get('live:request-permission');
|
||||
assert(request);
|
||||
const microphone = request({ sender: window.webContents }, 'microphone');
|
||||
const camera = request({ sender: window.webContents }, 'camera');
|
||||
const quit = host.controls.quitHost();
|
||||
const before = host.commands.length;
|
||||
host.grants.get('microphone')?.(true);
|
||||
await microphone;
|
||||
host.failQuit();
|
||||
await assert.rejects(quit, (error: Error) =>
|
||||
/Could not shut down/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
host.grants.get('camera')?.(true);
|
||||
await camera;
|
||||
assert.equal(
|
||||
host.commands.slice(before).includes('live:audio:initialize'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
host.controls.publicState().permissions.camera,
|
||||
'not_determined',
|
||||
);
|
||||
assert.equal(host.resources.audio, false);
|
||||
assert.equal(host.timers.size, 0);
|
||||
});
|
||||
|
||||
it('dismisses settings on current-window blur without accepting stale-window blur', () => {
|
||||
const host = fixture();
|
||||
const previous = host.controls.create();
|
||||
const window = host.controls.create();
|
||||
host.commands.length = 0;
|
||||
const settings = host.ipc.get('live:settings-open');
|
||||
assert(settings);
|
||||
settings({ sender: window.webContents }, true);
|
||||
assert.equal(window.ignored.at(-1), false);
|
||||
previous.events.get('blur')?.();
|
||||
assert.deepEqual(host.commands, []);
|
||||
window.events.get('blur')?.();
|
||||
assert.deepEqual(host.commands, ['live:settings-dismiss']);
|
||||
assert.equal(window.ignored.at(-1), true);
|
||||
});
|
||||
|
||||
it('uses compact bounds at edges and restores desired position after settings and preview clamps', () => {
|
||||
const host = fixture();
|
||||
host.stored.position = { x: 956, y: -100 };
|
||||
const window = host.controls.create();
|
||||
const event = { sender: window.webContents };
|
||||
const layout = host.ipc.get('live:overlay-layout')!;
|
||||
const settings = host.ipc.get('live:settings-open')!;
|
||||
assert.equal(window.getBounds().x, 896);
|
||||
layout(event, 'orb');
|
||||
assert.equal(window.getBounds().x, 956);
|
||||
assert.equal(window.getBounds().y, -100);
|
||||
settings(event, true);
|
||||
assert.equal(window.getBounds().x, 896);
|
||||
assert.equal(window.getBounds().y, 23);
|
||||
settings(event, false);
|
||||
assert.equal(window.getBounds().x, 956);
|
||||
assert.equal(window.getBounds().y, -100);
|
||||
layout(event, 'orb-preview');
|
||||
assert.equal(window.getBounds().y, 1);
|
||||
layout(event, 'orb');
|
||||
assert.equal(window.getBounds().y, -100);
|
||||
assert.deepEqual(host.saved, []);
|
||||
const recreated = host.controls.create();
|
||||
layout({ sender: recreated.webContents }, 'orb');
|
||||
assert.equal(recreated.getBounds().y, -100);
|
||||
assert.equal(recreated.getBounds().x, 956);
|
||||
});
|
||||
|
||||
it('accepts an initial layout before renderer ready and rejects malformed or foreign layout changes', () => {
|
||||
const host = fixture();
|
||||
host.stored.position = { x: 956, y: -100 };
|
||||
const window = host.controls.create(false);
|
||||
const layout = host.ipc.get('live:overlay-layout')!;
|
||||
layout({ sender: window.webContents }, 'orb');
|
||||
assert.equal(window.getBounds().x, 896);
|
||||
window.events.get('did-finish-load')?.();
|
||||
assert.equal(window.getBounds().x, 956);
|
||||
layout({ sender: {} }, 'setup');
|
||||
layout({ sender: window.webContents }, 'unexpected');
|
||||
assert.equal(window.getBounds().x, 956);
|
||||
assert.deepEqual(host.saved, []);
|
||||
});
|
||||
|
||||
it('starts once only after readiness, and explicit stop prevents restart after reconnect', () => {
|
||||
const host = fixture();
|
||||
host.disconnect();
|
||||
host.controls.create();
|
||||
host.flags.hostReady = true;
|
||||
host.controls.maybeStart();
|
||||
assert.deepEqual(host.actions, []);
|
||||
host.connect();
|
||||
host.flags.hostReady = false;
|
||||
host.controls.maybeStart();
|
||||
assert.deepEqual(host.actions, []);
|
||||
host.flags.hostReady = true;
|
||||
host.controls.maybeStart();
|
||||
host.controls.maybeStart();
|
||||
assert.deepEqual(host.actions, ['toggle']);
|
||||
host.controls.stopLive();
|
||||
host.disconnect();
|
||||
host.connect();
|
||||
host.controls.maybeStart();
|
||||
assert.deepEqual(host.actions, ['toggle', 'stop']);
|
||||
});
|
||||
|
||||
it('compensates macOS top clamping and keeps drag deltas, settings and reload on the same logical position', () => {
|
||||
const host = fixture(33);
|
||||
host.area.y = 33;
|
||||
host.stored.position = { x: 100, y: -97 };
|
||||
const window = host.controls.create();
|
||||
const event = { sender: window.webContents };
|
||||
host.ipc.get('live:overlay-layout')!(event, 'orb');
|
||||
assert.equal(window.getBounds().y, 33);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -130);
|
||||
const drag = host.ipc.get('live:drag-overlay')!;
|
||||
drag(event, 'start', 292, 255);
|
||||
drag(event, 'move', 292, 265);
|
||||
drag(event, 'end', 292, 265);
|
||||
assert.equal(window.getBounds().y, 33);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.equal(host.saved.at(-1)?.y, -87);
|
||||
host.ipc.get('live:settings-open')!(event, true);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, 0);
|
||||
host.ipc.get('live:settings-open')!(event, false);
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.equal(host.saved.at(-1)?.y, -87);
|
||||
window.events.get('did-start-loading')?.();
|
||||
window.events.get('did-finish-load')?.();
|
||||
assert.equal(host.controls.publicState().overlayOffset?.y, -120);
|
||||
assert.equal(host.offsets.at(-1)?.y, -120);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
|
||||
describe('preload overlay pointer routing', () => {
|
||||
it('admits drag-only regions and recalculates after DOM changes without pointer movement', () => {
|
||||
const source = readFileSync(
|
||||
new URL('../../preload/index.ts', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const script = source.slice(source.indexOf('let lastPointerInteractive'));
|
||||
const listeners = new Map<string, (event: unknown) => void>();
|
||||
const frames: Array<() => void> = [];
|
||||
const messages: boolean[] = [];
|
||||
let overDragRegion = true;
|
||||
let changed = () => {};
|
||||
const context = {
|
||||
window: {
|
||||
addEventListener: (type: string, listener: (event: unknown) => void) =>
|
||||
listeners.set(type, listener),
|
||||
},
|
||||
document: {
|
||||
elementFromPoint: () =>
|
||||
overDragRegion
|
||||
? {
|
||||
closest: (selector: string) =>
|
||||
selector.includes('[data-live-drag]') ? {} : null,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
requestAnimationFrame: (callback: () => void) => frames.push(callback),
|
||||
ipcRenderer: {
|
||||
send: (_channel: string, interactive: boolean) =>
|
||||
messages.push(interactive),
|
||||
},
|
||||
MutationObserver: class {
|
||||
constructor(callback: () => void) {
|
||||
changed = callback;
|
||||
}
|
||||
observe() {}
|
||||
disconnect() {}
|
||||
},
|
||||
camera: { dispose: () => {} },
|
||||
audio: { dispose: () => Promise.resolve() },
|
||||
};
|
||||
runInNewContext(
|
||||
ts.transpileModule(script, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText,
|
||||
context,
|
||||
);
|
||||
listeners.get('mousemove')?.({ clientX: 20, clientY: 30 });
|
||||
frames.shift()?.();
|
||||
assert.deepEqual(messages, [true]);
|
||||
overDragRegion = false;
|
||||
changed();
|
||||
frames.shift()?.();
|
||||
assert.deepEqual(messages, [true, false]);
|
||||
overDragRegion = true;
|
||||
changed();
|
||||
frames.shift()?.();
|
||||
listeners.get('blur')?.({});
|
||||
frames.shift()?.();
|
||||
assert.deepEqual(messages, [true, false, true, false]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import {
|
||||
readOverlayPosition,
|
||||
saveOverlayPosition,
|
||||
} from '../overlay-position-store.ts';
|
||||
|
||||
const directories: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true });
|
||||
});
|
||||
function fixture(): string {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'live-overlay-position-'));
|
||||
directories.push(directory);
|
||||
return join(directory, 'overlay-position.json');
|
||||
}
|
||||
|
||||
describe('overlay position persistence', () => {
|
||||
it('falls back for absent, corrupt and non-finite positions', () => {
|
||||
const path = fixture();
|
||||
assert.equal(readOverlayPosition(path), undefined);
|
||||
for (const contents of ['{', '{}', '{"x":"1","y":2}', '{"x":1e30,"y":2}']) {
|
||||
writeFileSync(path, contents);
|
||||
assert.equal(readOverlayPosition(path), undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it('atomically replaces and restores a private rounded position without temporary files', () => {
|
||||
const path = fixture();
|
||||
saveOverlayPosition(path, { x: -800.4, y: 240.8 });
|
||||
assert.deepEqual(readOverlayPosition(path), { x: -800, y: 241 });
|
||||
saveOverlayPosition(path, { x: 420, y: 100 });
|
||||
assert.deepEqual(readOverlayPosition(path), { x: 420, y: 100 });
|
||||
assert.equal(statSync(path).mode & 0o777, 0o600);
|
||||
assert.deepEqual(readdirSync(directories[0]!), ['overlay-position.json']);
|
||||
});
|
||||
|
||||
it('rejects malformed writes before replacing a saved position', () => {
|
||||
const path = fixture();
|
||||
saveOverlayPosition(path, { x: 20, y: 30 });
|
||||
assert.throws(() => saveOverlayPosition(path, { x: Infinity, y: 50 }));
|
||||
assert.deepEqual(readOverlayPosition(path), { x: 20, y: 30 });
|
||||
});
|
||||
});
|
||||
@@ -1,14 +1,14 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { overlayPosition } from '../overlay-position.ts';
|
||||
import { clampOverlayPosition, overlayPosition } from '../overlay-position.ts';
|
||||
import { OVERLAY_GEOMETRY } from '../../shared/overlay-geometry.ts';
|
||||
|
||||
describe('overlayPosition', () => {
|
||||
it('anchors to the bottom right of the selected display work area', () => {
|
||||
assert.deepEqual(
|
||||
overlayPosition(
|
||||
{ x: -1_920, y: 23, width: 1_920, height: 1_057 },
|
||||
420,
|
||||
300,
|
||||
{ x: 0, y: 0, width: 420, height: 300 },
|
||||
),
|
||||
{ x: -440, y: 760 },
|
||||
);
|
||||
@@ -16,8 +16,69 @@ describe('overlayPosition', () => {
|
||||
|
||||
it('does not place a window before a tiny work area origin', () => {
|
||||
assert.deepEqual(
|
||||
overlayPosition({ x: 100, y: 200, width: 200, height: 100 }, 420, 300),
|
||||
overlayPosition(
|
||||
{ x: 100, y: 200, width: 200, height: 100 },
|
||||
{ x: 0, y: 0, width: 420, height: 300 },
|
||||
),
|
||||
{ x: 100, y: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves a remembered position within a negative-coordinate display', () => {
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: -900, y: 300 },
|
||||
{ x: -1920, y: 23, width: 1920, height: 1057 },
|
||||
{ x: 0, y: 0, width: 384, height: 480 },
|
||||
),
|
||||
{ x: -900, y: 300 },
|
||||
);
|
||||
});
|
||||
|
||||
it('clamps a removed display position into the remaining work area', () => {
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: -900, y: 900 },
|
||||
{ x: 0, y: 23, width: 1280, height: 777 },
|
||||
{ x: 0, y: 0, width: 384, height: 480 },
|
||||
),
|
||||
{ x: 0, y: 320 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: 900, y: 900 },
|
||||
{ x: 100, y: 200, width: 200, height: 100 },
|
||||
{ x: 0, y: 0, width: 384, height: 480 },
|
||||
),
|
||||
{ x: 100, y: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
it('allows transparent canvas outside the screen while protecting compact content', () => {
|
||||
const area = { x: 0, y: 23, width: 1280, height: 777 };
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: 1200, y: 900 },
|
||||
area,
|
||||
OVERLAY_GEOMETRY.bounds.orb,
|
||||
),
|
||||
{ x: 956, y: 326 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: -300, y: -300 },
|
||||
area,
|
||||
OVERLAY_GEOMETRY.bounds.orb,
|
||||
),
|
||||
{ x: -60, y: -107 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
clampOverlayPosition(
|
||||
{ x: -300, y: -300 },
|
||||
area,
|
||||
OVERLAY_GEOMETRY.bounds['orb-preview'],
|
||||
),
|
||||
{ x: -60, y: 1 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
import { OVERLAY_GEOMETRY } from '../../shared/overlay-geometry.ts';
|
||||
|
||||
const css = readFileSync(
|
||||
new URL('../../renderer/style.css', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const rule = (selector: string): string => {
|
||||
const escaped = selector.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const match = css.match(new RegExp(`${escaped}\\s*\\{([^}]+)\\}`));
|
||||
assert(match, `Missing style rule: ${selector}`);
|
||||
return match[1]!;
|
||||
};
|
||||
|
||||
describe('orb presentation geometry', () => {
|
||||
it('keeps hover controls above the animation and status below, with both inside edge bounds', () => {
|
||||
const { toolbar, orbMotion, status, bounds, caption, previewWithCaption } =
|
||||
OVERLAY_GEOMETRY;
|
||||
assert(toolbar.y + toolbar.height <= orbMotion.y);
|
||||
assert(status.y >= orbMotion.y + orbMotion.height + 4);
|
||||
assert(previewWithCaption.y + previewWithCaption.height < caption.y);
|
||||
for (const envelope of [bounds.orb, bounds['orb-preview']]) {
|
||||
for (const rect of [toolbar, orbMotion, status]) {
|
||||
assert(rect.x >= envelope.x && rect.y >= envelope.y);
|
||||
assert(rect.x + rect.width <= envelope.x + envelope.width);
|
||||
assert(rect.y + rect.height <= envelope.y + envelope.height);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('uses a translucent status surface and a round settings button', () => {
|
||||
assert.match(
|
||||
rule('.voice-status'),
|
||||
/background:\s*var\(--live-status-bg\)/,
|
||||
);
|
||||
assert.match(
|
||||
rule('.voice-controls .settings-control'),
|
||||
/border-radius:\s*50%/,
|
||||
);
|
||||
});
|
||||
|
||||
it('lets the full status receive hover and keeps stopping visually idle', () => {
|
||||
assert.match(rule('.voice-status-primary'), /pointer-events:\s*auto/);
|
||||
assert.match(
|
||||
css,
|
||||
/\.voice-orb\.idle \.orb-core,\s*\.voice-orb\.stopping \.orb-core,/,
|
||||
);
|
||||
});
|
||||
|
||||
it('overrides state-specific orb animations and input scaling under reduced motion', () => {
|
||||
const reduced = css.slice(
|
||||
css.indexOf('@media (prefers-reduced-motion: reduce)'),
|
||||
);
|
||||
assert.match(
|
||||
reduced,
|
||||
/\.voice-surface \.voice-orb \.orb-core,\s*\.voice-surface \.voice-orb \.orb-core::after\s*\{\s*animation:\s*none;\s*transform:\s*none;/,
|
||||
);
|
||||
});
|
||||
|
||||
it('fits persistent mute indicators below the primary status without increasing the orb bounds', () => {
|
||||
assert.match(rule('.voice-status'), /flex-direction:\s*column/);
|
||||
assert.match(rule('.voice-status-audio'), /line-height:\s*11px/);
|
||||
assert.match(rule('.voice-status-audio'), /flex-shrink:\s*0/);
|
||||
assert.match(
|
||||
rule('.voice-status.has-audio-status .voice-status-primary'),
|
||||
/max-height:\s*14px/,
|
||||
);
|
||||
assert.match(
|
||||
rule('.voice-status.has-audio-status .voice-status-primary'),
|
||||
/text-overflow:\s*ellipsis/,
|
||||
);
|
||||
assert.doesNotMatch(rule('.permission-link'), /position:\s*absolute/);
|
||||
assert(14 + 11 + 4 <= OVERLAY_GEOMETRY.status.height);
|
||||
});
|
||||
|
||||
it('reserves and paints the Settings scrollbar without waiting for hover', () => {
|
||||
assert.match(rule('.settings-body'), /overflow-y:\s*scroll/);
|
||||
assert.match(rule('.settings-body'), /scrollbar-gutter:\s*stable/);
|
||||
assert.match(rule('.settings-body::-webkit-scrollbar'), /width:\s*8px/);
|
||||
assert.match(
|
||||
rule('.settings-body::-webkit-scrollbar-track'),
|
||||
/background:/,
|
||||
);
|
||||
assert.match(
|
||||
rule('.settings-body::-webkit-scrollbar-thumb'),
|
||||
/background:/,
|
||||
);
|
||||
});
|
||||
|
||||
it('constrains translated or long Settings content without widening the panel', () => {
|
||||
assert.match(
|
||||
rule('.settings-layer'),
|
||||
/grid-template-columns:\s*minmax\(0, 1fr\)/,
|
||||
);
|
||||
assert.match(
|
||||
rule('.settings-layer'),
|
||||
/grid-template-rows:\s*minmax\(0, 1fr\)/,
|
||||
);
|
||||
assert.match(rule('.settings-panel'), /min-width:\s*0/);
|
||||
assert.match(rule('.settings-body'), /min-width:\s*0/);
|
||||
assert.match(rule('.settings-panel > header'), /flex-shrink:\s*0/);
|
||||
});
|
||||
|
||||
it('fits the listening gain and animated outline into the reserved motion envelope', () => {
|
||||
assert.match(
|
||||
rule('.voice-orb.listening .orb-core::after'),
|
||||
/inset:\s*-1px/,
|
||||
);
|
||||
assert(
|
||||
(OVERLAY_GEOMETRY.orb.width + 2) * 1.3 * 1.04 <
|
||||
OVERLAY_GEOMETRY.orbMotion.width,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -12,14 +12,26 @@ import {
|
||||
LIVE_HOST_BUNDLE_ID,
|
||||
MAX_CONTROL_FRAME_BYTES,
|
||||
MAX_INPUT_AUDIO_FRAME_BYTES,
|
||||
MAX_INPUT_IMAGE_FRAME_BYTES,
|
||||
MAX_CAPTURE_ASSET_BYTES,
|
||||
INPUT_AUDIO_EPOCH_BYTES,
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES,
|
||||
OUTPUT_AUDIO_EPOCH_BYTES,
|
||||
OUTPUT_AUDIO_HEADER_BYTES,
|
||||
OUTPUT_AUDIO_ID_BYTES,
|
||||
decodeOutputAudioFrame,
|
||||
encodeInputAudioFrame,
|
||||
encodeOutputAudioFrame,
|
||||
fitRealtimeVisualDimensions,
|
||||
MAX_OUTPUT_AUDIO_FRAME_BYTES,
|
||||
encodeHostControlMessage,
|
||||
isValidInputAudioFrame,
|
||||
isValidInputImageFrame,
|
||||
isValidCameraSnapshotAsset,
|
||||
isValidOutputAudioFrame,
|
||||
parseDaemonControlMessage,
|
||||
isScreenDisplayId,
|
||||
type DaemonControlMessage as HostDaemonMessage,
|
||||
type HostAction,
|
||||
type HostHello,
|
||||
@@ -59,7 +71,7 @@ type DaemonMuteAction = Extract<DaemonHostAction, { action: 'mute' }>;
|
||||
|
||||
const PROTOCOL_TYPE_PARITY: {
|
||||
helloAssignable: HostHello extends DaemonHostHello ? true : false;
|
||||
helloKeys: HasSameKeys<HostHello, DaemonHostHello>;
|
||||
helloKeys: HasSameKeys<Omit<HostHello, 'subagentsV1'>, DaemonHostHello>;
|
||||
permissionKeys: HasSameKeys<HostPermissions, DaemonHostHello['permissions']>;
|
||||
permissionStates: IsEqual<
|
||||
HostPermissions[keyof HostPermissions],
|
||||
@@ -87,7 +99,7 @@ const PROTOCOL_TYPE_PARITY: {
|
||||
NonNullable<HostVisibleDaemonStatus['host']>
|
||||
>;
|
||||
daemonMessageNames: IsEqual<
|
||||
MessageType<HostDaemonMessage>,
|
||||
Exclude<MessageType<HostDaemonMessage>, 'host.subagents'>,
|
||||
MessageType<DaemonMessage>
|
||||
>;
|
||||
} = {
|
||||
@@ -113,12 +125,71 @@ const DAEMON_PROTOCOL_TYPES_URL = new URL(
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
const ELECTRON_BUILDER_CONFIG_URL = new URL(
|
||||
'../../../electron-builder.yml',
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
const QWEN_LIVE_PROTOCOL_TYPES_URL = new URL(
|
||||
'../../../../qwen-live/src/host/types.ts',
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
describe('Live Host protocol', () => {
|
||||
it('keeps display-scope requests explicit and rejects invalid identities or camera/display combinations', () => {
|
||||
const displayId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
const base = {
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'display-1',
|
||||
epoch: 1,
|
||||
source: 'screen',
|
||||
persistAsset: false,
|
||||
screenScope: 'display',
|
||||
screenDisplayId: displayId.toUpperCase(),
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(base)), {
|
||||
...base,
|
||||
screenDisplayId: displayId,
|
||||
});
|
||||
for (const change of [
|
||||
{ screenScope: 'window' },
|
||||
{ source: 'camera' },
|
||||
{ screenDisplayId: 'screen-2' },
|
||||
{ screenDisplayId: displayId + '\n' },
|
||||
])
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(JSON.stringify({ ...base, ...change })),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(isScreenDisplayId('primary'), true);
|
||||
assert.equal(isScreenDisplayId(displayId.toUpperCase()), true);
|
||||
assert.equal(isScreenDisplayId(displayId + '\n'), false);
|
||||
});
|
||||
|
||||
it('encodes resolved display identity and refuses a primary token or partial display metadata as pixels', () => {
|
||||
const frame = {
|
||||
type: 'host.visual_frame' as const,
|
||||
epoch: 1,
|
||||
source: 'screen' as const,
|
||||
screenScope: 'display' as const,
|
||||
displayId: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
|
||||
image: Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64'),
|
||||
};
|
||||
assert.deepEqual(JSON.parse(encodeHostControlMessage(frame)), frame);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({ ...frame, displayId: 'primary' }),
|
||||
);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({ ...frame, source: 'camera' }),
|
||||
);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({ ...frame, screenScope: undefined }),
|
||||
);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({ ...frame, displayId: undefined }),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps the qwen-live daemon contract byte-identical to the cli copy', async () => {
|
||||
// PROTOCOL_TYPE_PARITY type-checks against the cli copy only; the
|
||||
// standalone qwen-live daemon validates and emits against its own copy.
|
||||
@@ -131,10 +202,10 @@ describe('Live Host protocol', () => {
|
||||
});
|
||||
|
||||
it('stays synchronized with the daemon protocol contract', async () => {
|
||||
const source = await readFile(
|
||||
fileURLToPath(DAEMON_PROTOCOL_TYPES_URL),
|
||||
'utf8',
|
||||
);
|
||||
const [source, builderConfig] = await Promise.all([
|
||||
readFile(fileURLToPath(DAEMON_PROTOCOL_TYPES_URL), 'utf8'),
|
||||
readFile(fileURLToPath(ELECTRON_BUILDER_CONFIG_URL), 'utf8'),
|
||||
]);
|
||||
const daemonVersion = Number(
|
||||
source.match(/LIVE_HOST_PROTOCOL_VERSION = (\d+)/u)?.[1],
|
||||
);
|
||||
@@ -142,9 +213,13 @@ describe('Live Host protocol', () => {
|
||||
/LIVE_HOST_BUNDLE_ID = '([^']+)'/u,
|
||||
)?.[1];
|
||||
|
||||
assert.equal(LIVE_PROTOCOL_VERSION, 7);
|
||||
assert.equal(LIVE_PROTOCOL_VERSION, 9);
|
||||
assert.equal(daemonVersion, LIVE_PROTOCOL_VERSION);
|
||||
assert.equal(daemonBundleId, LIVE_HOST_BUNDLE_ID);
|
||||
assert.equal(
|
||||
Number(builderConfig.match(/QwenLiveProtocolVersion: (\d+)/u)?.[1]),
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
);
|
||||
assert.equal(Object.values(PROTOCOL_TYPE_PARITY).every(Boolean), true);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
@@ -159,8 +234,10 @@ describe('Live Host protocol', () => {
|
||||
hostVersion: '0.0.6',
|
||||
bundleId: LIVE_HOST_BUNDLE_ID,
|
||||
instanceNonce: 'host-instance-nonce',
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'denied',
|
||||
screenRecording: 'not_determined',
|
||||
},
|
||||
@@ -279,6 +356,16 @@ describe('Live Host protocol', () => {
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 50,
|
||||
epoch: 2,
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
cameraWidth: 1280,
|
||||
cameraHeight: 720,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
@@ -293,6 +380,16 @@ describe('Live Host protocol', () => {
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 1_000,
|
||||
epoch: 2,
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
cameraWidth: 1280,
|
||||
cameraHeight: 720,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
@@ -303,6 +400,86 @@ describe('Live Host protocol', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('strictly negotiates output audio end markers and validates identities', () => {
|
||||
const welcome = {
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 1_000,
|
||||
epoch: 2,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+Q',
|
||||
},
|
||||
};
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...welcome, daemonShutdownV1: true }),
|
||||
),
|
||||
{ ...welcome, daemonShutdownV1: true },
|
||||
);
|
||||
for (const daemonShutdownV1 of [false, 'true', 1, null]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...welcome, daemonShutdownV1 }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
}),
|
||||
),
|
||||
{
|
||||
...welcome,
|
||||
capabilities: { outputAudioEndMarkerV1: true },
|
||||
},
|
||||
);
|
||||
for (const capabilities of [
|
||||
{},
|
||||
{ outputAudioEndMarkerV1: false },
|
||||
{ outputAudioEndMarkerV1: true, unknown: true },
|
||||
null,
|
||||
]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(JSON.stringify({ ...welcome, capabilities })),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.output_audio_finished',
|
||||
epoch: 2,
|
||||
outputId: 17,
|
||||
}),
|
||||
),
|
||||
{ type: 'host.output_audio_finished', epoch: 2, outputId: 17 },
|
||||
);
|
||||
for (const identity of [
|
||||
{ epoch: -1, outputId: 17 },
|
||||
{ epoch: 2, outputId: -1 },
|
||||
{ epoch: 2.5, outputId: 17 },
|
||||
{ epoch: 2, outputId: Number.MAX_SAFE_INTEGER + 1 },
|
||||
]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.output_audio_finished',
|
||||
...identity,
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects invalid state and oversized control frames', () => {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
@@ -325,6 +502,177 @@ describe('Live Host protocol', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('requires bounded visual resolutions as complete pairs', () => {
|
||||
const welcome = {
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 1_000,
|
||||
epoch: 0,
|
||||
visualInput: {
|
||||
source: 'screen',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+Q',
|
||||
},
|
||||
};
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
visualInput: {
|
||||
...welcome.visualInput,
|
||||
cameraWidth: 1280,
|
||||
},
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
const cameraSettings = {
|
||||
...welcome.visualInput,
|
||||
cameraSnapshotWidth: 3840,
|
||||
cameraSnapshotHeight: 2160,
|
||||
snapshotWidth: 2560,
|
||||
snapshotHeight: 1440,
|
||||
};
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...welcome, visualInput: cameraSettings }),
|
||||
),
|
||||
{ ...welcome, visualInput: cameraSettings },
|
||||
);
|
||||
for (const override of [
|
||||
{ cameraSnapshotHeight: undefined },
|
||||
{ cameraSnapshotWidth: 8000 },
|
||||
{ cameraSnapshotHeight: 100 },
|
||||
]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
visualInput: { ...cameraSettings, ...override },
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
visualInput: {
|
||||
...welcome.visualInput,
|
||||
snapshotWidth: 1280,
|
||||
},
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
visualInput: {
|
||||
...welcome.visualInput,
|
||||
liveWidth: 4096,
|
||||
},
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('validates visual settings messages', () => {
|
||||
assert.equal(
|
||||
encodeHostControlMessage({
|
||||
type: 'host.visual_settings',
|
||||
epoch: 3,
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'host.visual_settings',
|
||||
epoch: 3,
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
}),
|
||||
);
|
||||
assert.throws(() =>
|
||||
encodeHostControlMessage({
|
||||
type: 'host.visual_settings',
|
||||
epoch: 3,
|
||||
source: 'camera',
|
||||
mode: 'invalid' as 'live-feed',
|
||||
permissions: {
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
appshot: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('parses optional visual capture persistence without coercion', () => {
|
||||
const base = {
|
||||
type: 'host.capture_visual',
|
||||
requestId: 'visual-1',
|
||||
epoch: 3,
|
||||
source: 'screen',
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(base)), base);
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...base, persistAsset: false }),
|
||||
),
|
||||
{ ...base, persistAsset: false },
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...base, persistAsset: 'false' }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('allows larger local camera assets without relaxing provider image limits', () => {
|
||||
const jpeg = Buffer.alloc(MAX_INPUT_IMAGE_FRAME_BYTES + 1);
|
||||
jpeg[0] = 0xff;
|
||||
jpeg[1] = 0xd8;
|
||||
jpeg[jpeg.length - 2] = 0xff;
|
||||
jpeg[jpeg.length - 1] = 0xd9;
|
||||
const encoded = jpeg.toString('base64');
|
||||
assert.equal(isValidCameraSnapshotAsset(encoded), true);
|
||||
assert.equal(isValidInputImageFrame(encoded), false);
|
||||
assert.equal(isValidCameraSnapshotAsset(`${encoded}\n`), false);
|
||||
assert.equal(isValidCameraSnapshotAsset('not an image'), false);
|
||||
assert.equal(
|
||||
isValidCameraSnapshotAsset(
|
||||
'A'.repeat(Math.ceil(MAX_CAPTURE_ASSET_BYTES / 3) * 4 + 4),
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('bounds audio frames and requires complete PCM16 samples', () => {
|
||||
assert.equal(isValidInputAudioFrame(new Uint8Array(640)), true);
|
||||
assert.equal(isValidInputAudioFrame(new Uint8Array(641)), false);
|
||||
@@ -339,6 +687,148 @@ describe('Live Host protocol', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('frames output PCM with a bounded epoch and output identity', () => {
|
||||
const pcm16 = new Uint8Array([1, 0, 2, 0]);
|
||||
const encoded = encodeOutputAudioFrame(42, 7, pcm16);
|
||||
assert(encoded);
|
||||
assert.equal(OUTPUT_AUDIO_EPOCH_BYTES, 8);
|
||||
assert.equal(OUTPUT_AUDIO_ID_BYTES, 8);
|
||||
assert.equal(OUTPUT_AUDIO_HEADER_BYTES, 16);
|
||||
assert.equal(encoded.byteLength, OUTPUT_AUDIO_HEADER_BYTES + pcm16.length);
|
||||
const header = new DataView(
|
||||
encoded.buffer,
|
||||
encoded.byteOffset,
|
||||
encoded.byteLength,
|
||||
);
|
||||
assert.equal(header.getBigUint64(0, false), 42n);
|
||||
assert.equal(header.getBigUint64(OUTPUT_AUDIO_EPOCH_BYTES, false), 7n);
|
||||
assert.deepEqual(decodeOutputAudioFrame(encoded), {
|
||||
epoch: 42,
|
||||
outputId: 7,
|
||||
audio: pcm16,
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects invalid or oversized output audio wire frames', () => {
|
||||
const pcm16 = new Uint8Array([1, 0]);
|
||||
assert.equal(encodeOutputAudioFrame(-1, 1, pcm16), undefined);
|
||||
assert.equal(encodeOutputAudioFrame(1, -1, pcm16), undefined);
|
||||
assert.equal(
|
||||
encodeOutputAudioFrame(Number.MAX_SAFE_INTEGER + 1, 1, pcm16),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
encodeOutputAudioFrame(1, Number.MAX_SAFE_INTEGER + 1, pcm16),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(encodeOutputAudioFrame(1, 1, new Uint8Array(1)), undefined);
|
||||
assert.equal(
|
||||
decodeOutputAudioFrame(new Uint8Array(OUTPUT_AUDIO_HEADER_BYTES)),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
decodeOutputAudioFrame(
|
||||
new Uint8Array(MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES + 1),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
const maximum = encodeOutputAudioFrame(
|
||||
1,
|
||||
1,
|
||||
new Uint8Array(MAX_OUTPUT_AUDIO_FRAME_BYTES),
|
||||
);
|
||||
assert(maximum);
|
||||
assert.equal(maximum.byteLength, MAX_OUTPUT_AUDIO_WIRE_FRAME_BYTES);
|
||||
assert.equal(
|
||||
decodeOutputAudioFrame(maximum)?.audio.byteLength,
|
||||
MAX_OUTPUT_AUDIO_FRAME_BYTES,
|
||||
);
|
||||
|
||||
const unsafeEpoch = new Uint8Array(OUTPUT_AUDIO_HEADER_BYTES + 2);
|
||||
new DataView(unsafeEpoch.buffer).setBigUint64(
|
||||
0,
|
||||
BigInt(Number.MAX_SAFE_INTEGER) + 1n,
|
||||
false,
|
||||
);
|
||||
assert.equal(decodeOutputAudioFrame(unsafeEpoch), undefined);
|
||||
|
||||
const unsafeOutputId = new Uint8Array(OUTPUT_AUDIO_HEADER_BYTES + 2);
|
||||
new DataView(unsafeOutputId.buffer).setBigUint64(
|
||||
OUTPUT_AUDIO_EPOCH_BYTES,
|
||||
BigInt(Number.MAX_SAFE_INTEGER) + 1n,
|
||||
false,
|
||||
);
|
||||
assert.equal(decodeOutputAudioFrame(unsafeOutputId), undefined);
|
||||
});
|
||||
|
||||
it('encodes only bounded JPEG visual frames', () => {
|
||||
const image = Buffer.from([0xff, 0xd8, 0xff, 0xd9]).toString('base64');
|
||||
assert.equal(isValidInputImageFrame(image), true);
|
||||
assert.deepEqual(
|
||||
JSON.parse(
|
||||
encodeHostControlMessage({
|
||||
type: 'host.visual_frame',
|
||||
epoch: 7,
|
||||
source: 'camera',
|
||||
image,
|
||||
}),
|
||||
),
|
||||
{ type: 'host.visual_frame', epoch: 7, source: 'camera', image },
|
||||
);
|
||||
assert.equal(
|
||||
isValidInputImageFrame(Buffer.from('not a jpeg').toString('base64')),
|
||||
false,
|
||||
);
|
||||
|
||||
const oversized = Buffer.alloc(MAX_INPUT_IMAGE_FRAME_BYTES + 1);
|
||||
oversized[0] = 0xff;
|
||||
oversized[1] = 0xd8;
|
||||
oversized[oversized.length - 2] = 0xff;
|
||||
oversized[oversized.length - 1] = 0xd9;
|
||||
assert.equal(isValidInputImageFrame(oversized.toString('base64')), false);
|
||||
});
|
||||
|
||||
it('admits a visual capture with every text and image field at its limit', () => {
|
||||
const jpeg = Buffer.alloc(MAX_INPUT_IMAGE_FRAME_BYTES);
|
||||
jpeg[0] = 0xff;
|
||||
jpeg[1] = 0xd8;
|
||||
jpeg[jpeg.length - 2] = 0xff;
|
||||
jpeg[jpeg.length - 1] = 0xd9;
|
||||
const encoded = encodeHostControlMessage({
|
||||
type: 'host.visual_capture_result',
|
||||
requestId: '\ud800'.repeat(128),
|
||||
success: true,
|
||||
source: 'screen',
|
||||
image: jpeg.toString('base64'),
|
||||
width: Number.MAX_SAFE_INTEGER,
|
||||
height: Number.MAX_SAFE_INTEGER,
|
||||
appName: '\ud800'.repeat(512),
|
||||
windowTitle: '\ud800'.repeat(2_048),
|
||||
accessibilityText: '\ud800'.repeat(32_000),
|
||||
screenshotPath: '\ud800'.repeat(4_096),
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
Buffer.byteLength(encoded, 'utf8') <= MAX_CONTROL_FRAME_BYTES,
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('fits every Omni-bound image within 1080p without upscaling', () => {
|
||||
assert.deepEqual(fitRealtimeVisualDimensions(3840, 2160), {
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
});
|
||||
assert.deepEqual(fitRealtimeVisualDimensions(2560, 1600, 1280, 720), {
|
||||
width: 1152,
|
||||
height: 720,
|
||||
});
|
||||
assert.deepEqual(fitRealtimeVisualDimensions(640, 480), {
|
||||
width: 640,
|
||||
height: 480,
|
||||
});
|
||||
});
|
||||
|
||||
it('binds input PCM to a safe call epoch', () => {
|
||||
const pcm16 = new Uint8Array([1, 0, 2, 0]);
|
||||
const encoded = encodeInputAudioFrame(42, pcm16);
|
||||
@@ -366,6 +856,7 @@ describe('Live Host protocol', () => {
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
@@ -407,24 +898,36 @@ describe('Live Host protocol', () => {
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
const state = {
|
||||
type: 'host.state',
|
||||
epoch: 1,
|
||||
visualInput: {
|
||||
source: 'camera',
|
||||
mode: 'live-feed',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: '',
|
||||
},
|
||||
};
|
||||
assert.deepEqual(parseDaemonControlMessage(JSON.stringify(state)), state);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.state',
|
||||
epoch: 1,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: '',
|
||||
},
|
||||
...state,
|
||||
visualInput: { ...state.visualInput, fps: 0 },
|
||||
}),
|
||||
)?.type,
|
||||
'host.state',
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('requires a shortcut and rejects the removed session-window message', () => {
|
||||
it('rejects removed protocol messages', () => {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
@@ -448,6 +951,16 @@ describe('Live Host protocol', () => {
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({
|
||||
type: 'host.capture_screen_context',
|
||||
requestId: 'capture-1',
|
||||
epoch: 1,
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not retain the removed install URL field', () => {
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { displayLiveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import {
|
||||
createServer,
|
||||
type IncomingMessage,
|
||||
type ServerResponse,
|
||||
} from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it, mock } from 'node:test';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import { LIVE_PROTOCOL_VERSION } from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
afterEach(async () => {
|
||||
mock.restoreAll();
|
||||
for (const task of cleanup.splice(0).reverse()) await task();
|
||||
});
|
||||
|
||||
const callbacks = {
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted' as const,
|
||||
camera: 'granted' as const,
|
||||
accessibility: 'granted' as const,
|
||||
screenRecording: 'granted' as const,
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: () => undefined,
|
||||
onOutputAudio: () => undefined,
|
||||
onOutputAudioFinished: () => undefined,
|
||||
onClearOutput: () => undefined,
|
||||
};
|
||||
|
||||
async function fixture(
|
||||
options: {
|
||||
standalone?: boolean;
|
||||
token?: string | null;
|
||||
welcomeNonce?: string;
|
||||
handleQuit?: (request: IncomingMessage, response: ServerResponse) => void;
|
||||
} = {},
|
||||
) {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-live-quit-host-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const nonce = 'authenticated_instance_01';
|
||||
const requests: IncomingMessage[] = [];
|
||||
const server = createServer((request, response) => {
|
||||
requests.push(request);
|
||||
if (options.handleQuit) options.handleQuit(request, response);
|
||||
else
|
||||
response
|
||||
.writeHead(200, { 'content-type': 'application/json' })
|
||||
.end(JSON.stringify({ stopped: true, instanceNonce: nonce }));
|
||||
});
|
||||
const peers = new WebSocketServer({ server });
|
||||
cleanup.push(async () => {
|
||||
for (const peer of peers.clients) peer.terminate();
|
||||
peers.close();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
const record = {
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
...(options.token !== null ? { token: options.token ?? 'quit-token' } : {}),
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: nonce,
|
||||
};
|
||||
await writeFile(discovery, JSON.stringify(record), { mode: 0o600 });
|
||||
let ready!: () => void;
|
||||
const waiting = new Promise<void>((resolve) => {
|
||||
ready = resolve;
|
||||
});
|
||||
let handshakes = 0;
|
||||
const peerPromise = new Promise<WebSocket>((resolve) => {
|
||||
peers.on('connection', (peer) => {
|
||||
handshakes++;
|
||||
peer.once('message', () => {
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: options.welcomeNonce ?? nonce,
|
||||
...(options.standalone !== false ? { daemonShutdownV1: true } : {}),
|
||||
heartbeatIntervalMs: 10_000,
|
||||
epoch: 7,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
}),
|
||||
);
|
||||
resolve(peer);
|
||||
});
|
||||
});
|
||||
});
|
||||
const snapshots: string[] = [];
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
...callbacks,
|
||||
onSnapshot: (snapshot) => {
|
||||
snapshots.push(snapshot.phase);
|
||||
if (snapshot.phase === 'ready' || snapshot.phase === 'error') ready();
|
||||
},
|
||||
},
|
||||
discovery,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
connection.start();
|
||||
const peer = await peerPromise;
|
||||
await waiting;
|
||||
return {
|
||||
connection,
|
||||
peer,
|
||||
requests,
|
||||
record,
|
||||
discovery,
|
||||
snapshots,
|
||||
handshakes: () => handshakes,
|
||||
};
|
||||
}
|
||||
|
||||
describe('Host Quit ownership and acknowledgement', () => {
|
||||
it('closes only Host when there is no authenticated connection', async () => {
|
||||
const connection = new LiveDaemonConnection('0.0.6', callbacks);
|
||||
await connection.requestQuit();
|
||||
connection.stop();
|
||||
const wrongInstance = await fixture({
|
||||
welcomeNonce: 'not_the_authenticated_instance',
|
||||
});
|
||||
await wrongInstance.connection.requestQuit();
|
||||
assert.equal(wrongInstance.requests.length, 0);
|
||||
});
|
||||
|
||||
it('waits for the instance-specific shutdown receipt and deduplicates Quit', async () => {
|
||||
let reply!: () => void;
|
||||
let received!: () => void;
|
||||
const requestArrived = new Promise<void>((resolve) => {
|
||||
received = resolve;
|
||||
});
|
||||
const value = await fixture({
|
||||
handleQuit: (_request, response) => {
|
||||
reply = () =>
|
||||
response.writeHead(200, { 'content-type': 'application/json' }).end(
|
||||
JSON.stringify({
|
||||
stopped: true,
|
||||
instanceNonce: value.record.instanceNonce,
|
||||
}),
|
||||
);
|
||||
received();
|
||||
},
|
||||
});
|
||||
const before = [...value.snapshots];
|
||||
let completed = false;
|
||||
const first = value.connection.requestQuit();
|
||||
const second = value.connection.requestQuit();
|
||||
assert.equal(first, second);
|
||||
void first.then(() => {
|
||||
completed = true;
|
||||
});
|
||||
await requestArrived;
|
||||
assert.equal(completed, false);
|
||||
assert.equal(value.requests[0]?.method, 'POST');
|
||||
assert.equal(value.requests[0]?.url, '/live/quit');
|
||||
assert.equal(value.requests[0]?.headers.authorization, 'Bearer quit-token');
|
||||
assert.equal(
|
||||
value.requests[0]?.headers['x-qwen-live-nonce'],
|
||||
value.record.instanceNonce,
|
||||
);
|
||||
value.peer.close(1001, 'Daemon is shutting down');
|
||||
value.connection.reconnectNow();
|
||||
value.connection.forceReconnectNow();
|
||||
reply();
|
||||
await first;
|
||||
assert.equal(value.handshakes(), 1);
|
||||
assert.deepEqual(value.snapshots, before);
|
||||
});
|
||||
|
||||
it('sends only a Live stop action to a shared WebShell daemon', async () => {
|
||||
const value = await fixture({ standalone: false });
|
||||
const action = new Promise<unknown>((resolve) =>
|
||||
value.peer.once('message', (data) => resolve(JSON.parse(String(data)))),
|
||||
);
|
||||
await value.connection.requestQuit();
|
||||
assert.deepEqual(await action, {
|
||||
type: 'host.action',
|
||||
action: 'stop',
|
||||
epoch: 7,
|
||||
});
|
||||
assert.equal(value.requests.length, 0);
|
||||
});
|
||||
|
||||
it('rejects a shutdown capability without bearer credentials', async () => {
|
||||
const value = await fixture({ token: null });
|
||||
await assert.rejects(value.connection.requestQuit(), (error: Error) =>
|
||||
/not confirmed/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
assert.equal(value.requests.length, 0);
|
||||
});
|
||||
|
||||
it('retries the same authenticated target after cleanup failure and disconnect', async () => {
|
||||
let attempts = 0;
|
||||
const value = await fixture({
|
||||
handleQuit: (_request, response) => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
value.peer.close(1001, 'Daemon cleanup failed');
|
||||
response.writeHead(500).end('failure');
|
||||
} else
|
||||
response.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
stopped: true,
|
||||
instanceNonce: value.record.instanceNonce,
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit(), (error: Error) =>
|
||||
/not confirmed/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
await writeFile(
|
||||
value.discovery,
|
||||
JSON.stringify({
|
||||
...value.record,
|
||||
url: 'http://127.0.0.1:1',
|
||||
instanceNonce: 'different_instance_0001',
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
value.connection.reconnectNow();
|
||||
await value.connection.requestQuit();
|
||||
assert.equal(attempts, 2);
|
||||
assert.equal(value.handshakes(), 1);
|
||||
assert(
|
||||
value.requests.every(
|
||||
(request) =>
|
||||
request.headers['x-qwen-live-nonce'] === value.record.instanceNonce,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects mismatched receipts, redirects and lost responses', async () => {
|
||||
for (const mode of ['nonce', 'redirect', 'lost'] as const) {
|
||||
const value = await fixture({
|
||||
handleQuit: (request, response) => {
|
||||
if (mode === 'lost') request.socket.destroy();
|
||||
else if (mode === 'redirect')
|
||||
response.writeHead(302, { location: 'http://127.0.0.1:1' }).end();
|
||||
else
|
||||
response.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
stopped: true,
|
||||
instanceNonce: 'wrong_instance_0001',
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit(), (error: Error) =>
|
||||
/not confirmed/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
assert.equal(value.requests.length, 1);
|
||||
}
|
||||
});
|
||||
|
||||
it(
|
||||
'does not report a timed-out shutdown as success',
|
||||
{ timeout: 2_000 },
|
||||
async () => {
|
||||
const controller = new AbortController();
|
||||
const value = await fixture({
|
||||
handleQuit: () => {
|
||||
setImmediate(() =>
|
||||
controller.abort(new DOMException('Timed out', 'TimeoutError')),
|
||||
);
|
||||
},
|
||||
});
|
||||
mock.method(AbortSignal, 'timeout', () => controller.signal);
|
||||
await assert.rejects(value.connection.requestQuit(), (error: Error) =>
|
||||
/not confirmed/.test(displayLiveMessage('en', error.message)),
|
||||
);
|
||||
assert.equal(value.requests.length, 1);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
|
||||
const workflow = readFileSync(
|
||||
new URL('../../../../../.github/workflows/live-host.yml', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const entitlements = readFileSync(
|
||||
new URL('../../../build/entitlements.mac.plist', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const packaging = readFileSync(
|
||||
new URL('../../../electron-builder.yml', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const assertion = workflow.match(/\| node -e '([\s\S]*?)\n\s*'/)?.[1];
|
||||
assert(assertion, 'The exact workflow entitlement assertion must be found');
|
||||
const values = Object.fromEntries(
|
||||
[...entitlements.matchAll(/<key>([^<]+)<\/key>\s*<(true|false)\/>/g)].map(
|
||||
([, key, value]) => [key, value === 'true'],
|
||||
),
|
||||
);
|
||||
|
||||
function checkEntitlements(value: Record<string, boolean>) {
|
||||
return spawnSync(process.execPath, ['-e', assertion!], {
|
||||
input: JSON.stringify(value),
|
||||
encoding: 'utf8',
|
||||
});
|
||||
}
|
||||
|
||||
describe('Host camera packaging contract review regression', () => {
|
||||
it('accepts the source entitlement plist in the exact CI guard', () => {
|
||||
const result = checkEntitlements(values);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
it('still rejects an unrelated extra entitlement', () => {
|
||||
const result = checkEntitlements({
|
||||
...values,
|
||||
'com.apple.security.device.bluetooth': true,
|
||||
});
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /Unexpected Live Host entitlements/);
|
||||
});
|
||||
|
||||
it('rejects disabled or missing camera entitlement', () => {
|
||||
const withoutCamera = { ...values };
|
||||
delete withoutCamera['com.apple.security.device.camera'];
|
||||
for (const value of [
|
||||
withoutCamera,
|
||||
{ ...values, 'com.apple.security.device.camera': false },
|
||||
]) {
|
||||
assert.equal(checkEntitlements(value).status, 1);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not forbid the explicitly packaged camera usage description', () => {
|
||||
assert.match(packaging, /NSCameraUsageDescription: ['"][^'"]+['"]/);
|
||||
const unused = workflow.match(/for unused_permission in ([^;]+); do/)?.[1];
|
||||
assert(unused, 'The workflow unused-permission guard must be found');
|
||||
assert.equal(
|
||||
unused.split(/\s+/).includes('NSCameraUsageDescription'),
|
||||
false,
|
||||
);
|
||||
assert.match(workflow, /Print :NSCameraUsageDescription/);
|
||||
assert.match(workflow, /\$\{camera_usage\/\/\[\[:space:\]\]\/\}/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,443 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { createServer, type ServerResponse } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it, mock } from 'node:test';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { WebSocket, WebSocketServer } from 'ws';
|
||||
import { liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import {
|
||||
LiveDaemonConnection,
|
||||
type ConnectionSnapshot,
|
||||
} from '../daemon-connection.ts';
|
||||
import { BoundedReconnectPolicy } from '../reconnect-policy.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
MAX_SOCKET_BUFFERED_BYTES,
|
||||
encodeOutputAudioFrame,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
afterEach(async () => {
|
||||
mock.restoreAll();
|
||||
for (const task of cleanup.splice(0).reverse()) await task();
|
||||
});
|
||||
|
||||
async function waitFor(check: () => boolean, description: string) {
|
||||
for (let attempt = 0; attempt < 150; attempt++) {
|
||||
if (check()) return;
|
||||
await delay(10);
|
||||
}
|
||||
assert(check(), description);
|
||||
}
|
||||
|
||||
async function fixture(
|
||||
options: {
|
||||
standalone?: boolean;
|
||||
welcomeNonce?: string;
|
||||
protocolVersion?: number;
|
||||
handleQuit?: (response: ServerResponse, attempt: number) => void;
|
||||
} = {},
|
||||
) {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'qwen-host-review-quit-'));
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
const nonce = 'review_authenticated_instance_01';
|
||||
const requests: Array<{ url?: string; nonce?: string | string[] }> = [];
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === '/health') {
|
||||
response.writeHead(200).end('alive');
|
||||
return;
|
||||
}
|
||||
requests.push({
|
||||
url: request.url,
|
||||
nonce: request.headers['x-qwen-live-nonce'],
|
||||
});
|
||||
if (options.handleQuit) options.handleQuit(response, requests.length);
|
||||
else
|
||||
response
|
||||
.writeHead(200)
|
||||
.end(JSON.stringify({ stopped: true, instanceNonce: nonce }));
|
||||
});
|
||||
const peers = new WebSocketServer({ server });
|
||||
const stopHttp = async () => {
|
||||
for (const peer of peers.clients) peer.terminate();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
};
|
||||
cleanup.push(async () => {
|
||||
peers.close();
|
||||
await stopHttp();
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const record = {
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'review-fixture-token',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: nonce,
|
||||
};
|
||||
await writeFile(discovery, JSON.stringify(record), { mode: 0o600 });
|
||||
let handshakes = 0;
|
||||
let firstPeer: WebSocket | undefined;
|
||||
peers.on('connection', (peer) => {
|
||||
handshakes++;
|
||||
firstPeer ??= peer;
|
||||
peer.once('message', () => {
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: options.protocolVersion ?? LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: options.welcomeNonce ?? nonce,
|
||||
...(options.standalone !== false ? { daemonShutdownV1: true } : {}),
|
||||
heartbeatIntervalMs: 10_000,
|
||||
epoch: 7,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
const snapshots: ConnectionSnapshot[] = [];
|
||||
let outputFrames = 0;
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => snapshots.push(snapshot),
|
||||
onOutputAudio: () => {
|
||||
outputFrames++;
|
||||
},
|
||||
onOutputAudioFinished: () => undefined,
|
||||
onClearOutput: () => undefined,
|
||||
},
|
||||
discovery,
|
||||
{ policy: new BoundedReconnectPolicy([5, 5, 5], 0) },
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
connection.start();
|
||||
await waitFor(
|
||||
() =>
|
||||
snapshots.some((snapshot) =>
|
||||
['ready', 'error', 'incompatible'].includes(snapshot.phase),
|
||||
),
|
||||
'Host handshake settled',
|
||||
);
|
||||
assert(firstPeer);
|
||||
return {
|
||||
connection,
|
||||
peer: firstPeer,
|
||||
requests,
|
||||
snapshots,
|
||||
record,
|
||||
discovery,
|
||||
server,
|
||||
stopHttp,
|
||||
handshakes: () => handshakes,
|
||||
outputFrames: () => outputFrames,
|
||||
};
|
||||
}
|
||||
|
||||
describe('Host Quit review recovery regressions', () => {
|
||||
for (const invalid of ['nonce', 'version'] as const) {
|
||||
it(`preserves the ${invalid} rejection after its WebSocket closes`, async () => {
|
||||
const value = await fixture(
|
||||
invalid === 'nonce'
|
||||
? { welcomeNonce: 'unverified_instance_0001' }
|
||||
: { protocolVersion: LIVE_PROTOCOL_VERSION - 1 },
|
||||
);
|
||||
await waitFor(
|
||||
() => value.peer.readyState === WebSocket.CLOSED,
|
||||
'Rejected peer closes',
|
||||
);
|
||||
await delay(20);
|
||||
assert.equal(
|
||||
value.connection.getSnapshot().phase,
|
||||
invalid === 'nonce' ? 'error' : 'incompatible',
|
||||
);
|
||||
assert.equal(
|
||||
value.connection.getSnapshot().error,
|
||||
invalid === 'nonce' ? 'daemon_identity' : 'host_version',
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
it('does not retain a ready snapshot after failed shared stop and socket close', async () => {
|
||||
const value = await fixture({ standalone: false });
|
||||
const internals = value.connection as unknown as { socket: WebSocket };
|
||||
Object.defineProperty(internals.socket, 'bufferedAmount', {
|
||||
configurable: true,
|
||||
value: MAX_SOCKET_BUFFERED_BYTES + 1,
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
assert.deepEqual(value.connection.getSnapshot(), {
|
||||
phase: 'error',
|
||||
error: liveMessage('host.error.quitUnconfirmed'),
|
||||
});
|
||||
const closed = once(value.peer, 'close');
|
||||
value.peer.close(1001, 'fixture disconnection after failed stop');
|
||||
await closed;
|
||||
await delay(20);
|
||||
assert.notEqual(
|
||||
value.connection.getSnapshot().phase,
|
||||
'ready',
|
||||
`After closed peer: ${JSON.stringify(value.connection.getSnapshot())}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('latches a failed shared Quit against late state, errors and media until explicit retry', async () => {
|
||||
const value = await fixture({ standalone: false });
|
||||
const internals = value.connection as unknown as { socket: WebSocket };
|
||||
Object.defineProperty(internals.socket, 'bufferedAmount', {
|
||||
configurable: true,
|
||||
value: MAX_SOCKET_BUFFERED_BYTES + 1,
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
const failed = value.connection.getSnapshot();
|
||||
assert.equal(failed.phase, 'error');
|
||||
Reflect.deleteProperty(internals.socket, 'bufferedAmount');
|
||||
const before = [...value.snapshots];
|
||||
for (const message of [
|
||||
{
|
||||
type: 'host.state',
|
||||
epoch: 8,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
},
|
||||
{ type: 'host.error', code: 'provider_config' },
|
||||
]) {
|
||||
value.peer.send(JSON.stringify(message));
|
||||
await delay(20);
|
||||
assert.deepEqual(value.connection.getSnapshot(), failed);
|
||||
}
|
||||
const output = encodeOutputAudioFrame(7, 1, Buffer.alloc(8));
|
||||
assert(output);
|
||||
value.peer.send(output, { binary: true });
|
||||
await delay(20);
|
||||
assert.equal(value.outputFrames(), 0);
|
||||
assert.equal(value.connection.getEpoch(), 7);
|
||||
assert.equal(value.connection.sendAudio(Buffer.alloc(8), 7), false);
|
||||
assert.equal(
|
||||
value.connection.sendAction({
|
||||
type: 'host.action',
|
||||
action: 'toggle',
|
||||
epoch: 7,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
value.connection.start();
|
||||
value.connection.reconnectNow();
|
||||
value.connection.forceReconnectNow();
|
||||
assert.equal(value.handshakes(), 1);
|
||||
assert.deepEqual(value.snapshots, before);
|
||||
const action = once(value.peer, 'message');
|
||||
await value.connection.requestQuit();
|
||||
assert.deepEqual(JSON.parse(String((await action)[0])), {
|
||||
type: 'host.action',
|
||||
action: 'stop',
|
||||
epoch: 7,
|
||||
});
|
||||
});
|
||||
|
||||
it('still sends a stop frame when retrying the same shared connection', async () => {
|
||||
const value = await fixture({ standalone: false });
|
||||
const internals = value.connection as unknown as { socket: WebSocket };
|
||||
Object.defineProperty(internals.socket, 'bufferedAmount', {
|
||||
configurable: true,
|
||||
value: MAX_SOCKET_BUFFERED_BYTES + 1,
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
Reflect.deleteProperty(internals.socket, 'bufferedAmount');
|
||||
const messages: unknown[] = [];
|
||||
value.peer.once('message', (data) =>
|
||||
messages.push(JSON.parse(String(data))),
|
||||
);
|
||||
await value.connection.requestQuit();
|
||||
await waitFor(
|
||||
() => messages.length === 1,
|
||||
'Retry must not resolve without sending the shared stop frame',
|
||||
);
|
||||
assert.deepEqual(messages, [
|
||||
{ type: 'host.action', action: 'stop', epoch: 7 },
|
||||
]);
|
||||
assert.equal(value.requests.length, 0);
|
||||
assert.equal(value.handshakes(), 1);
|
||||
});
|
||||
|
||||
it('requests authenticated standalone shutdown during a reconnect window', async () => {
|
||||
const value = await fixture();
|
||||
value.connection.forceReconnectNow();
|
||||
assert.equal(value.connection.getSnapshot().phase, 'connecting');
|
||||
await value.connection.requestQuit();
|
||||
assert.deepEqual(value.requests, [
|
||||
{ url: '/live/quit', nonce: value.record.instanceNonce },
|
||||
]);
|
||||
});
|
||||
|
||||
it('revokes a cached shutdown target when discovery changes before authentication', async () => {
|
||||
const value = await fixture();
|
||||
await writeFile(
|
||||
value.discovery,
|
||||
JSON.stringify({
|
||||
...value.record,
|
||||
instanceNonce: 'replacement_unverified_instance_01',
|
||||
}),
|
||||
);
|
||||
await waitFor(
|
||||
() => value.connection.getSnapshot().error === 'daemon_identity',
|
||||
'Replacement must fail authentication',
|
||||
);
|
||||
await delay(20);
|
||||
assert.equal(value.connection.getSnapshot().error, 'daemon_identity');
|
||||
await value.connection.requestQuit();
|
||||
assert.deepEqual(value.requests, []);
|
||||
});
|
||||
|
||||
for (const failure of [404, 410, 'reset'] as const) {
|
||||
it(`does not treat ${failure} as evidence a still-live daemon exited`, async () => {
|
||||
const value = await fixture({
|
||||
handleQuit: (response, attempt) => {
|
||||
if (attempt === 1) {
|
||||
if (failure === 'reset') response.destroy();
|
||||
else response.writeHead(failure).end('not confirmed');
|
||||
} else
|
||||
response.writeHead(200).end(
|
||||
JSON.stringify({
|
||||
stopped: true,
|
||||
instanceNonce: value.record.instanceNonce,
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
assert.equal(value.server.listening, true);
|
||||
assert.equal(
|
||||
await (await fetch(`${value.record.url}/health`)).text(),
|
||||
'alive',
|
||||
);
|
||||
value.connection.reconnectNow();
|
||||
value.connection.forceReconnectNow();
|
||||
await value.connection.requestQuit();
|
||||
assert.equal(value.requests.length, 2);
|
||||
assert(
|
||||
value.requests.every(
|
||||
(request) => request.nonce === value.record.instanceNonce,
|
||||
),
|
||||
);
|
||||
assert.equal(value.handshakes(), 1);
|
||||
});
|
||||
}
|
||||
|
||||
for (const failure of [404, 410, 500, 'reset'] as const) {
|
||||
it(`still attempts authenticated Quit with an absent PID when the live listener returns ${failure}`, async () => {
|
||||
const value = await fixture({
|
||||
handleQuit: (response) => {
|
||||
if (failure === 'reset') response.destroy();
|
||||
else response.writeHead(failure).end('not confirmed');
|
||||
},
|
||||
});
|
||||
const probe = mock.method(process, 'kill', () => {
|
||||
throw Object.assign(new Error('fixture process gone'), {
|
||||
code: 'ESRCH',
|
||||
});
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
assert.equal(value.requests.length, 2);
|
||||
assert(
|
||||
value.requests.every(
|
||||
(request) => request.nonce === value.record.instanceNonce,
|
||||
),
|
||||
);
|
||||
assert.equal(probe.mock.callCount(), 0);
|
||||
assert.equal(
|
||||
await (await fetch(`${value.record.url}/health`)).text(),
|
||||
'alive',
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
it('keeps a failed quit unconfirmed when the fixture listener actually disappears', async () => {
|
||||
const value = await fixture({
|
||||
handleQuit: (response) => response.writeHead(500).end('cleanup failed'),
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
await value.stopHttp();
|
||||
assert.equal(value.server.listening, false);
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
assert.equal(value.requests.length, 1);
|
||||
});
|
||||
|
||||
it('permits retry to finish only after a refused HTTP attempt and an absent original authenticated PID', async () => {
|
||||
const value = await fixture({
|
||||
handleQuit: (response) => response.writeHead(500).end('cleanup failed'),
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
await value.stopHttp();
|
||||
const fetchProbe = mock.method(globalThis, 'fetch');
|
||||
const probe = mock.method(
|
||||
process,
|
||||
'kill',
|
||||
(pid: number, signal?: number | string) => {
|
||||
assert.equal(pid, value.record.pid);
|
||||
assert.equal(signal, 0);
|
||||
throw Object.assign(new Error('fixture process gone'), {
|
||||
code: 'ESRCH',
|
||||
});
|
||||
},
|
||||
);
|
||||
await value.connection.requestQuit();
|
||||
assert.equal(fetchProbe.mock.callCount(), 1);
|
||||
assert.equal(
|
||||
String(fetchProbe.mock.calls[0]?.arguments[0]),
|
||||
`${value.record.url}/live/quit`,
|
||||
);
|
||||
assert.equal(probe.mock.callCount(), 1);
|
||||
assert.equal(value.requests.length, 1);
|
||||
});
|
||||
|
||||
for (const code of ['EPERM', 'EACCES', 'UNKNOWN']) {
|
||||
it(`does not mistake a ${code} process probe for shutdown proof`, async () => {
|
||||
const value = await fixture({
|
||||
handleQuit: (response) => response.writeHead(500).end('cleanup failed'),
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
await value.stopHttp();
|
||||
mock.method(process, 'kill', (pid: number, signal?: number | string) => {
|
||||
assert.equal(pid, value.record.pid);
|
||||
assert.equal(signal, 0);
|
||||
throw Object.assign(new Error('fixture ambiguous process state'), {
|
||||
code,
|
||||
});
|
||||
});
|
||||
await assert.rejects(value.connection.requestQuit());
|
||||
assert.equal(value.requests.length, 1);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,261 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import { liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import {
|
||||
canChangeLiveVisualInput,
|
||||
shouldRequestVisualSourceChange,
|
||||
} from '../live-state-policy.ts';
|
||||
|
||||
type Callback = (...args: unknown[]) => unknown;
|
||||
|
||||
function fixture() {
|
||||
const source = readFileSync(new URL('../index.ts', import.meta.url), 'utf8');
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = new Set([
|
||||
'registerIpc',
|
||||
'applyPendingVisualSourceChange',
|
||||
'visualSourceReady',
|
||||
'scheduleReadinessReconnect',
|
||||
'cancelReadinessReconnect',
|
||||
'beginMediaPermissionMonitor',
|
||||
'microphonePermission',
|
||||
'cameraPermission',
|
||||
'requestCameraPermission',
|
||||
]);
|
||||
const declarations = tree.statements
|
||||
.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) &&
|
||||
node.name &&
|
||||
names.has(node.name.text),
|
||||
)
|
||||
.map((node) => node.getText(tree));
|
||||
assert.equal(
|
||||
declarations.length,
|
||||
names.size,
|
||||
'Every real function must be found',
|
||||
);
|
||||
const ipc = new Map<string, Callback>();
|
||||
const calls: Array<{ update: object; epoch: number }> = [];
|
||||
const diagnostics: Array<{ event: string; details: object }> = [];
|
||||
const flags = { sent: true };
|
||||
let reconnects = 0;
|
||||
let now = 0;
|
||||
type Timer = {
|
||||
at: number;
|
||||
repeat?: number;
|
||||
callback: () => void;
|
||||
unref: () => void;
|
||||
};
|
||||
const timers = new Set<Timer>();
|
||||
const timer = (
|
||||
callback: () => void,
|
||||
milliseconds: number,
|
||||
repeat?: number,
|
||||
) => {
|
||||
const value = { at: now + milliseconds, repeat, callback, unref() {} };
|
||||
timers.add(value);
|
||||
return value;
|
||||
};
|
||||
const context = {
|
||||
liveMessage,
|
||||
canChangeLiveVisualInput,
|
||||
shouldRequestVisualSourceChange,
|
||||
isTrustedSender: () => true,
|
||||
ipcMain: {
|
||||
on: (name: string, callback: Callback) => ipc.set(name, callback),
|
||||
handle: (name: string, callback: Callback) => ipc.set(name, callback),
|
||||
},
|
||||
connection: { phase: 'ready' },
|
||||
daemon: {
|
||||
getEpoch: () => 7,
|
||||
sendVisualSettings: (update: object, epoch: number) => {
|
||||
calls.push({ update: JSON.parse(JSON.stringify(update)), epoch });
|
||||
return flags.sent;
|
||||
},
|
||||
reconnectNow: () => {
|
||||
reconnects++;
|
||||
},
|
||||
},
|
||||
writeLiveDiagnostic: (event: string, details: object) =>
|
||||
diagnostics.push({ event, details: { ...details } }),
|
||||
systemPreferences: {
|
||||
getMediaAccessStatus: () => 'granted',
|
||||
askForMediaAccess: () => {
|
||||
throw new Error('Media prompts are forbidden in this probe');
|
||||
},
|
||||
},
|
||||
appshotReadiness: { refresh() {} },
|
||||
setTimeout: timer,
|
||||
clearTimeout: (value: Timer) => timers.delete(value),
|
||||
setInterval: (callback: () => void, milliseconds: number) =>
|
||||
timer(callback, milliseconds, milliseconds),
|
||||
clearInterval: (value: Timer) => timers.delete(value),
|
||||
publishState: () => undefined,
|
||||
sendRendererCommand: () => undefined,
|
||||
syncVisualCapture: () => undefined,
|
||||
failClosedForReadinessLoss: () => {
|
||||
throw new Error('Unexpected readiness loss');
|
||||
},
|
||||
};
|
||||
const script = `
|
||||
let nativeServicesActive = true, nativeServiceGeneration = 1, quitState;
|
||||
let readinessReconnectTimer, readinessReconnectReason, mediaPermissionTimer;
|
||||
let pendingVisualSourceChange, visualSourceChangeGeneration = 0;
|
||||
const READINESS_RECONNECT_DEBOUNCE_MS = 2500;
|
||||
const permissions = { microphone: 'denied', camera: 'granted', accessibility: 'granted', screenRecording: 'granted' };
|
||||
const selfChecks = { audioInput: false, audioOutput: true, globalShortcut: true, appshot: true };
|
||||
const live = { v: 1, available: true, state: 'listening', callId: 'fixture-call', shortcut: 'Command+E' };
|
||||
const visualInput = { source: 'screen', mode: 'live-feed' };
|
||||
${declarations.join('\n')}
|
||||
registerIpc();
|
||||
({ beginMediaPermissionMonitor, scheduleReadinessReconnect,
|
||||
pending: () => Boolean(readinessReconnectTimer),
|
||||
microphone: () => permissions.microphone,
|
||||
mode: () => visualInput.mode });
|
||||
`;
|
||||
const controls = runInNewContext(
|
||||
ts.transpileModule(script, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText,
|
||||
context,
|
||||
) as {
|
||||
beginMediaPermissionMonitor: () => void;
|
||||
scheduleReadinessReconnect: (reason?: 'readiness' | 'visual') => void;
|
||||
pending: () => boolean;
|
||||
microphone: () => string;
|
||||
mode: () => string;
|
||||
};
|
||||
return {
|
||||
controls,
|
||||
flags,
|
||||
calls,
|
||||
diagnostics,
|
||||
reconnects: () => reconnects,
|
||||
invoke: (name: string, value: unknown) => {
|
||||
const handler = ipc.get(name);
|
||||
assert(handler);
|
||||
return handler({}, value);
|
||||
},
|
||||
advance: (milliseconds: number) => {
|
||||
const end = now + milliseconds;
|
||||
for (;;) {
|
||||
const next = [...timers]
|
||||
.filter((value) => value.at <= end)
|
||||
.sort((a, b) => a.at - b.at)[0];
|
||||
if (!next) break;
|
||||
now = next.at;
|
||||
if (next.repeat) next.at += next.repeat;
|
||||
else timers.delete(next);
|
||||
next.callback();
|
||||
}
|
||||
now = end;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('Host visual readiness review regressions', () => {
|
||||
it('performs a pending microphone readiness reconnect without a visual change', () => {
|
||||
const value = fixture();
|
||||
value.controls.beginMediaPermissionMonitor();
|
||||
value.advance(2_000);
|
||||
assert.equal(value.controls.microphone(), 'granted');
|
||||
assert.equal(value.controls.pending(), true);
|
||||
value.advance(2_501);
|
||||
assert.equal(value.reconnects(), 1);
|
||||
});
|
||||
|
||||
it('keeps microphone recovery scheduled when already-granted Camera is selected', async () => {
|
||||
const value = fixture();
|
||||
value.controls.beginMediaPermissionMonitor();
|
||||
value.advance(2_000);
|
||||
assert.equal(value.controls.microphone(), 'granted');
|
||||
assert.equal(value.controls.pending(), true);
|
||||
await value.invoke('live:set-visual-source', 'camera');
|
||||
assert.deepEqual(value.calls, [{ update: { source: 'camera' }, epoch: 7 }]);
|
||||
value.advance(2_501);
|
||||
assert.equal(
|
||||
value.reconnects(),
|
||||
1,
|
||||
'Camera source send must not discard microphone recovery',
|
||||
);
|
||||
});
|
||||
|
||||
it('cancels a purely visual reconnect after its source update was sent', async () => {
|
||||
const value = fixture();
|
||||
value.controls.scheduleReadinessReconnect('visual');
|
||||
await value.invoke('live:set-visual-source', 'camera');
|
||||
value.advance(2_501);
|
||||
assert.equal(value.reconnects(), 0);
|
||||
assert.equal(value.controls.pending(), false);
|
||||
value.controls.scheduleReadinessReconnect();
|
||||
value.advance(2_501);
|
||||
assert.equal(value.reconnects(), 1);
|
||||
});
|
||||
|
||||
for (const reasons of [
|
||||
['readiness', 'visual'],
|
||||
['visual', 'readiness', 'visual'],
|
||||
] as const) {
|
||||
it(`does not downgrade the full readiness requirement: ${reasons.join(', ')}`, async () => {
|
||||
const value = fixture();
|
||||
for (const reason of reasons)
|
||||
value.controls.scheduleReadinessReconnect(reason);
|
||||
await value.invoke('live:set-visual-source', 'camera');
|
||||
value.advance(2_501);
|
||||
assert.equal(value.reconnects(), 1);
|
||||
});
|
||||
}
|
||||
|
||||
it('surfaces a failed visual-mode send to the IPC caller', async () => {
|
||||
const value = fixture();
|
||||
value.flags.sent = false;
|
||||
await assert.rejects(async () =>
|
||||
value.invoke('live:set-visual-mode', 'on-demand'),
|
||||
);
|
||||
assert.deepEqual(value.calls, [
|
||||
{ update: { mode: 'on-demand' }, epoch: 7 },
|
||||
]);
|
||||
assert.equal(value.controls.mode(), 'live-feed');
|
||||
});
|
||||
|
||||
it('records a diagnostic for failed visual-mode transport', async () => {
|
||||
const value = fixture();
|
||||
value.flags.sent = false;
|
||||
try {
|
||||
await value.invoke('live:set-visual-mode', 'on-demand');
|
||||
} catch {
|
||||
// The failure is allowed to reject; this case independently checks diagnostics.
|
||||
}
|
||||
assert.deepEqual(value.calls, [
|
||||
{ update: { mode: 'on-demand' }, epoch: 7 },
|
||||
]);
|
||||
assert.deepEqual(value.diagnostics, [
|
||||
{
|
||||
event: 'visual_mode_rejected',
|
||||
details: { epoch: 7, mode: 'on-demand' },
|
||||
},
|
||||
]);
|
||||
assert.equal(value.controls.mode(), 'live-feed');
|
||||
});
|
||||
|
||||
it('accepts a successful visual-mode send without claiming unacknowledged state', async () => {
|
||||
const value = fixture();
|
||||
assert.equal(
|
||||
await value.invoke('live:set-visual-mode', 'on-demand'),
|
||||
undefined,
|
||||
);
|
||||
assert.deepEqual(value.calls, [
|
||||
{ update: { mode: 'on-demand' }, epoch: 7 },
|
||||
]);
|
||||
assert.equal(value.controls.mode(), 'live-feed');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import {
|
||||
StartupInteraction,
|
||||
type StartupInteractionState,
|
||||
} from '../startup-interaction.ts';
|
||||
|
||||
const ready: StartupInteractionState = {
|
||||
connectionReady: true,
|
||||
rendererReady: true,
|
||||
hostReady: true,
|
||||
startPending: false,
|
||||
live: { available: true, state: 'idle' },
|
||||
};
|
||||
|
||||
describe('Host startup interaction', () => {
|
||||
it('waits for every startup readiness gate before consuming the intention', () => {
|
||||
for (const key of [
|
||||
'connectionReady',
|
||||
'rendererReady',
|
||||
'hostReady',
|
||||
] as const) {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(startup.shouldStart({ ...ready, [key]: false }), false);
|
||||
assert.equal(startup.shouldStart({ ...ready, [key]: false }), false);
|
||||
assert.equal(startup.shouldStart(ready), true);
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for provider availability and idle state', () => {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(
|
||||
startup.shouldStart({
|
||||
...ready,
|
||||
live: { available: false, state: 'idle' },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
startup.shouldStart({
|
||||
...ready,
|
||||
live: { available: false, state: 'unavailable' },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
startup.shouldStart({
|
||||
...ready,
|
||||
live: { available: true, state: 'error' },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(startup.shouldStart(ready), true);
|
||||
});
|
||||
|
||||
it('treats an existing call as fulfilled intent even before renderer readiness', () => {
|
||||
for (const state of [
|
||||
'starting',
|
||||
'listening',
|
||||
'thinking',
|
||||
'speaking',
|
||||
'stopping',
|
||||
] as const) {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(
|
||||
startup.shouldStart({
|
||||
...ready,
|
||||
rendererReady: false,
|
||||
live: { available: true, state },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not duplicate an already pending start', () => {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(startup.shouldStart({ ...ready, startPending: true }), false);
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
});
|
||||
|
||||
it('lets explicit user actions cancel startup before it becomes ready', () => {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(
|
||||
startup.shouldStart({ ...ready, connectionReady: false }),
|
||||
false,
|
||||
);
|
||||
startup.cancel();
|
||||
startup.cancel();
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
assert.equal(
|
||||
startup.shouldStart({ ...ready, rendererReady: false }),
|
||||
false,
|
||||
);
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
});
|
||||
|
||||
it('consumes before dispatch so failure, reconnect and a later idle state never retry', () => {
|
||||
const startup = new StartupInteraction();
|
||||
assert.equal(startup.shouldStart(ready), true);
|
||||
assert.equal(
|
||||
startup.shouldStart({
|
||||
...ready,
|
||||
live: { available: false, state: 'error' },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
startup.shouldStart({ ...ready, connectionReady: false }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
startup.shouldStart({ ...ready, rendererReady: false }),
|
||||
false,
|
||||
);
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
startup.cancel();
|
||||
assert.equal(startup.shouldStart(ready), false);
|
||||
});
|
||||
|
||||
it('gives each new Host process its own single startup intention', () => {
|
||||
const previous = new StartupInteraction();
|
||||
previous.cancel();
|
||||
assert.equal(previous.shouldStart(ready), false);
|
||||
const restarted = new StartupInteraction();
|
||||
assert.equal(restarted.shouldStart(ready), true);
|
||||
assert.equal(restarted.shouldStart(ready), false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { it } from 'node:test';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
parseDaemonControlMessage,
|
||||
} from '../../shared/protocol.ts';
|
||||
import type { SubagentsSnapshot } from '@qwen-code/qwen-live/subagents';
|
||||
|
||||
const snapshot = (revision: number): SubagentsSnapshot => ({
|
||||
revision,
|
||||
counts: {
|
||||
running: 0,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
},
|
||||
tasks: [],
|
||||
omitted: 0,
|
||||
});
|
||||
it('publishes standalone task revisions without republishing media state and ignores stale updates', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'live-subagent-connection-'));
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
let connection: LiveDaemonConnection | undefined;
|
||||
try {
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
discovery,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'fixture',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: 'abcdefghijklmnop',
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
let ready!: () => void;
|
||||
const readyPromise = new Promise<void>((resolve) => {
|
||||
ready = resolve;
|
||||
});
|
||||
let changed!: () => void;
|
||||
const changedPromise = new Promise<void>((resolve) => {
|
||||
changed = resolve;
|
||||
});
|
||||
let stateCount = 0;
|
||||
const updates: SubagentsSnapshot[] = [];
|
||||
const peerPromise = new Promise<WebSocket>((resolve) =>
|
||||
server.once('connection', (peer) => {
|
||||
peer.once('message', () => {
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: 'abcdefghijklmnop',
|
||||
heartbeatIntervalMs: 10000,
|
||||
epoch: 0,
|
||||
status: {
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'idle',
|
||||
shortcut: 'Command+E',
|
||||
},
|
||||
subagentsV1: snapshot(0),
|
||||
}),
|
||||
);
|
||||
resolve(peer);
|
||||
});
|
||||
}),
|
||||
);
|
||||
connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'denied',
|
||||
camera: 'denied',
|
||||
accessibility: 'denied',
|
||||
screenRecording: 'denied',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: false,
|
||||
audioOutput: false,
|
||||
appshot: false,
|
||||
globalShortcut: false,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (state) => {
|
||||
stateCount++;
|
||||
if (state.phase === 'ready') ready();
|
||||
},
|
||||
onSubagents: (value) => {
|
||||
updates.push(value);
|
||||
changed();
|
||||
},
|
||||
onOutputAudio: () => {},
|
||||
onOutputAudioFinished: () => {},
|
||||
onClearOutput: () => {},
|
||||
},
|
||||
discovery,
|
||||
);
|
||||
connection.start();
|
||||
const peer = await peerPromise;
|
||||
await readyPromise;
|
||||
const before = stateCount;
|
||||
peer.send(
|
||||
JSON.stringify({ type: 'host.subagents', subagentsV1: snapshot(1) }),
|
||||
);
|
||||
await changedPromise;
|
||||
assert.equal(stateCount, before);
|
||||
assert.equal(updates.length, 1);
|
||||
assert.equal(connection.getSnapshot().subagentsV1?.revision, 1);
|
||||
peer.send(
|
||||
JSON.stringify({ type: 'host.subagents', subagentsV1: snapshot(0) }),
|
||||
);
|
||||
const barrier = new Promise<void>((resolve) =>
|
||||
peer.once('message', () => resolve()),
|
||||
);
|
||||
peer.send(JSON.stringify({ type: 'host.ping', pingId: 'barrier' }));
|
||||
await barrier;
|
||||
assert.equal(updates.length, 1);
|
||||
assert.equal(stateCount, before);
|
||||
} finally {
|
||||
connection?.stop();
|
||||
for (const peer of server.clients) peer.terminate();
|
||||
server.close();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('validates standalone task messages and rejects unbounded or malformed snapshots', () => {
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ type: 'host.subagents', subagentsV1: snapshot(1) }),
|
||||
),
|
||||
{ type: 'host.subagents', subagentsV1: snapshot(1) },
|
||||
);
|
||||
for (const bad of [
|
||||
{ ...snapshot(1), revision: -1 },
|
||||
{ ...snapshot(1), counts: { running: '2' } },
|
||||
{ ...snapshot(1), tasks: [{ id: 'unsafe' }] },
|
||||
{ ...snapshot(1), extra: 'x'.repeat(260 * 1024) },
|
||||
]) {
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ type: 'host.subagents', subagentsV1: bad }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,303 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import {
|
||||
createServer,
|
||||
type IncomingMessage,
|
||||
type ServerResponse,
|
||||
} from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { WebSocketServer } from 'ws';
|
||||
import { MAX_SUBAGENTS_CONTROL_BYTES } from '@qwen-code/qwen-live/subagents';
|
||||
import { LiveDaemonConnection } from '../daemon-connection.ts';
|
||||
import {
|
||||
LIVE_PROTOCOL_VERSION,
|
||||
parseDaemonControlMessage,
|
||||
} from '../../shared/protocol.ts';
|
||||
|
||||
const cleanup: Array<() => Promise<void> | void> = [];
|
||||
afterEach(async () => {
|
||||
for (const task of cleanup.splice(0).reverse()) await task();
|
||||
});
|
||||
|
||||
const nonce = 'subagents_fixture_instance';
|
||||
const page = {
|
||||
type: 'page',
|
||||
page: {
|
||||
offset: 0,
|
||||
total: 0,
|
||||
snapshot: {
|
||||
revision: 0,
|
||||
counts: {
|
||||
running: 0,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
},
|
||||
tasks: [],
|
||||
omitted: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
const welcome = {
|
||||
type: 'host.welcome',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
daemonInstanceNonce: nonce,
|
||||
heartbeatIntervalMs: 10_000,
|
||||
epoch: 0,
|
||||
status: { v: 1, available: true, state: 'idle', shortcut: 'Command+E' },
|
||||
};
|
||||
|
||||
async function fixture(
|
||||
options: {
|
||||
capability?: boolean;
|
||||
handleRequest?: (
|
||||
request: IncomingMessage,
|
||||
response: ServerResponse,
|
||||
) => void;
|
||||
} = {},
|
||||
) {
|
||||
const directory = await mkdtemp(
|
||||
join(tmpdir(), 'qwen-live-subagent-controls-'),
|
||||
);
|
||||
cleanup.push(() => rm(directory, { recursive: true, force: true }));
|
||||
const requests: IncomingMessage[] = [];
|
||||
const server = createServer((request, response) => {
|
||||
requests.push(request);
|
||||
if (options.handleRequest) options.handleRequest(request, response);
|
||||
else
|
||||
response
|
||||
.writeHead(200, { 'content-type': 'application/json' })
|
||||
.end(JSON.stringify(page));
|
||||
});
|
||||
const peers = new WebSocketServer({ server });
|
||||
cleanup.push(async () => {
|
||||
for (const peer of peers.clients) peer.terminate();
|
||||
peers.close();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === 'object');
|
||||
const discovery = join(directory, 'daemon.json');
|
||||
await writeFile(
|
||||
discovery,
|
||||
JSON.stringify({
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
token: 'management-token',
|
||||
protocolVersion: LIVE_PROTOCOL_VERSION,
|
||||
pid: process.pid,
|
||||
instanceNonce: nonce,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
peers.on('connection', (peer) =>
|
||||
peer.once('message', () => {
|
||||
peer.send(
|
||||
JSON.stringify({
|
||||
...welcome,
|
||||
...(options.capability !== false ? { subagentsControlV1: true } : {}),
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
let ready!: () => void;
|
||||
const waiting = new Promise<void>((resolve) => {
|
||||
ready = resolve;
|
||||
});
|
||||
const connection = new LiveDaemonConnection(
|
||||
'0.0.6',
|
||||
{
|
||||
getReadiness: () => ({
|
||||
permissions: {
|
||||
microphone: 'denied',
|
||||
camera: 'denied',
|
||||
accessibility: 'denied',
|
||||
screenRecording: 'denied',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: false,
|
||||
audioOutput: false,
|
||||
appshot: false,
|
||||
globalShortcut: false,
|
||||
},
|
||||
}),
|
||||
onSnapshot: (snapshot) => {
|
||||
if (snapshot.phase === 'ready') ready();
|
||||
},
|
||||
onOutputAudio: () => {},
|
||||
onOutputAudioFinished: () => {},
|
||||
onClearOutput: () => {},
|
||||
},
|
||||
discovery,
|
||||
);
|
||||
cleanup.push(() => connection.stop());
|
||||
connection.start();
|
||||
await waiting;
|
||||
return { connection, requests };
|
||||
}
|
||||
|
||||
describe('standalone subagent management transport', () => {
|
||||
it('uses nonce-authenticated daemon management while the call is idle', async () => {
|
||||
const value = await fixture();
|
||||
assert.equal(value.connection.getSnapshot().subagentsControlV1, true);
|
||||
assert.deepEqual(
|
||||
await value.connection.requestSubagents({ action: 'list' }, nonce),
|
||||
page,
|
||||
);
|
||||
assert.equal(value.requests.length, 1);
|
||||
assert.equal(value.requests[0]?.method, 'POST');
|
||||
assert.equal(value.requests[0]?.url, '/live/subagents');
|
||||
assert.equal(
|
||||
value.requests[0]?.headers.authorization,
|
||||
'Bearer management-token',
|
||||
);
|
||||
assert.equal(value.requests[0]?.headers['x-qwen-live-nonce'], nonce);
|
||||
});
|
||||
|
||||
it('rejects old renderer instance IDs and unsupported daemons before HTTP dispatch', async () => {
|
||||
const value = await fixture();
|
||||
assert.deepEqual(
|
||||
await value.connection.requestSubagents(
|
||||
{ action: 'stop', taskId: 'harness:job_1' },
|
||||
'old_instance',
|
||||
),
|
||||
{ type: 'error', code: 'stale_instance' },
|
||||
);
|
||||
assert.equal(value.requests.length, 0);
|
||||
const legacy = await fixture({ capability: false });
|
||||
assert.deepEqual(
|
||||
await legacy.connection.requestSubagents({ action: 'list' }, nonce),
|
||||
{ type: 'error', code: 'unsupported' },
|
||||
);
|
||||
assert.equal(legacy.requests.length, 0);
|
||||
});
|
||||
|
||||
it('drops in-flight results when the authenticated connection changes', async () => {
|
||||
let received!: () => void;
|
||||
const arrived = new Promise<void>((resolve) => {
|
||||
received = resolve;
|
||||
});
|
||||
let reply!: () => void;
|
||||
const value = await fixture({
|
||||
handleRequest: (_request, response) => {
|
||||
reply = () => response.writeHead(200).end(JSON.stringify(page));
|
||||
received();
|
||||
},
|
||||
});
|
||||
const pending = value.connection.requestSubagents(
|
||||
{ action: 'list' },
|
||||
nonce,
|
||||
);
|
||||
await arrived;
|
||||
value.connection.stop();
|
||||
reply();
|
||||
assert.deepEqual(await pending, { type: 'error', code: 'stale_instance' });
|
||||
});
|
||||
|
||||
it('rejects oversized streamed responses and invalid outcomes', async () => {
|
||||
for (const body of [
|
||||
JSON.stringify({ type: 'outcome', outcome: 'stopped' }),
|
||||
'x'.repeat(MAX_SUBAGENTS_CONTROL_BYTES + 1),
|
||||
]) {
|
||||
const value = await fixture({
|
||||
handleRequest: (_request, response) => {
|
||||
response.writeHead(200, { 'transfer-encoding': 'chunked' });
|
||||
response.write(body.slice(0, body.length / 2));
|
||||
response.end(body.slice(body.length / 2));
|
||||
},
|
||||
});
|
||||
assert.deepEqual(
|
||||
await value.connection.requestSubagents({ action: 'list' }, nonce),
|
||||
{ type: 'error', code: 'action_failed' },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves owned errors and distinguishes nonce rejection', async () => {
|
||||
const denied = await fixture({
|
||||
handleRequest: (_request, response) =>
|
||||
response
|
||||
.writeHead(200)
|
||||
.end(
|
||||
JSON.stringify({ type: 'error', code: 'permission_unavailable' }),
|
||||
),
|
||||
});
|
||||
assert.deepEqual(
|
||||
await denied.connection.requestSubagents(
|
||||
{ action: 'permission', requestHandle: 'req_1', decision: 'allow' },
|
||||
nonce,
|
||||
),
|
||||
{ type: 'error', code: 'permission_unavailable' },
|
||||
);
|
||||
const stale = await fixture({
|
||||
handleRequest: (_request, response) => response.writeHead(409).end(),
|
||||
});
|
||||
assert.deepEqual(
|
||||
await stale.connection.requestSubagents({ action: 'list' }, nonce),
|
||||
{ type: 'error', code: 'stale_instance' },
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a well-formed result belonging to another task or permission decision', async () => {
|
||||
const wrongTask = await fixture({
|
||||
handleRequest: (_request, response) =>
|
||||
response
|
||||
.writeHead(200)
|
||||
.end(
|
||||
JSON.stringify({
|
||||
type: 'outcome',
|
||||
outcome: 'stopped',
|
||||
taskId: 'harness:job_2',
|
||||
}),
|
||||
),
|
||||
});
|
||||
assert.deepEqual(
|
||||
await wrongTask.connection.requestSubagents(
|
||||
{ action: 'stop', taskId: 'harness:job_1' },
|
||||
nonce,
|
||||
),
|
||||
{ type: 'error', code: 'action_failed' },
|
||||
);
|
||||
const wrongDecision = await fixture({
|
||||
handleRequest: (_request, response) =>
|
||||
response
|
||||
.writeHead(200)
|
||||
.end(
|
||||
JSON.stringify({
|
||||
type: 'outcome',
|
||||
outcome: 'denied',
|
||||
requestHandle: 'req_1',
|
||||
}),
|
||||
),
|
||||
});
|
||||
assert.deepEqual(
|
||||
await wrongDecision.connection.requestSubagents(
|
||||
{ action: 'permission', requestHandle: 'req_1', decision: 'allow' },
|
||||
nonce,
|
||||
),
|
||||
{ type: 'error', code: 'action_failed' },
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts an optional true capability but rejects malformed capability values', () => {
|
||||
assert(parseDaemonControlMessage(JSON.stringify(welcome)));
|
||||
assert.deepEqual(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...welcome, subagentsControlV1: true }),
|
||||
)?.type,
|
||||
'host.welcome',
|
||||
);
|
||||
for (const subagentsControlV1 of [false, 'true', 1])
|
||||
assert.equal(
|
||||
parseDaemonControlMessage(
|
||||
JSON.stringify({ ...welcome, subagentsControlV1 }),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import {
|
||||
fitSubagentsBounds,
|
||||
subagentsSidecarBounds,
|
||||
} from '../subagents-position.ts';
|
||||
import { OVERLAY_GEOMETRY } from '../../shared/overlay-geometry.ts';
|
||||
import { overlayPosition } from '../overlay-position.ts';
|
||||
|
||||
const primary = { x: 0, y: 30, width: 2048, height: 1028 };
|
||||
const right = { x: 2048, y: 33, width: 1728, height: 1084 };
|
||||
const sizes = [
|
||||
{ width: 132, height: 62 },
|
||||
{ width: 330, height: 430 },
|
||||
{ width: 330, height: 430 },
|
||||
];
|
||||
describe('Subagents floating panel geometry', () => {
|
||||
it('keeps expanded panels clear of the complete dock including its wider status bar', () => {
|
||||
const area = { x: 0, y: 33, width: 1728, height: 990 };
|
||||
const visible = OVERLAY_GEOMETRY.bounds.orb;
|
||||
const origin = overlayPosition(area, visible);
|
||||
const anchor = {
|
||||
x: origin.x + visible.x,
|
||||
y: origin.y + visible.y,
|
||||
width: visible.width,
|
||||
height: visible.height,
|
||||
};
|
||||
for (const size of sizes) {
|
||||
const result = subagentsSidecarBounds(anchor, size, area);
|
||||
assert(result.bounds.x + result.bounds.width + 12 <= anchor.x);
|
||||
assert(result.bounds.y >= area.y);
|
||||
assert(result.bounds.y + result.bounds.height <= area.y + area.height);
|
||||
}
|
||||
});
|
||||
|
||||
it('changes an undersized summary side on expansion instead of clamping into the dock', () => {
|
||||
const area = { x: 0, y: 0, width: 1000, height: 800 };
|
||||
const anchor = { x: 200, y: 400, width: 264, height: 344 };
|
||||
const summary = subagentsSidecarBounds(anchor, sizes[0]!, area);
|
||||
assert.equal(summary.side, 'left');
|
||||
const expanded = subagentsSidecarBounds(
|
||||
anchor,
|
||||
sizes[1]!,
|
||||
area,
|
||||
summary.side,
|
||||
);
|
||||
assert.equal(expanded.side, 'right');
|
||||
assert(expanded.bounds.x >= anchor.x + anchor.width + 12);
|
||||
});
|
||||
|
||||
it('fits summary, list and detail at every edge without modifying the orb anchor', () => {
|
||||
for (const area of [
|
||||
primary,
|
||||
right,
|
||||
{ x: -1920, y: -200, width: 1920, height: 1080 },
|
||||
]) {
|
||||
for (const point of [
|
||||
{ x: area.x, y: area.y },
|
||||
{ x: area.x + area.width - 156, y: area.y },
|
||||
{ x: area.x, y: area.y + area.height - 156 },
|
||||
{ x: area.x + area.width - 156, y: area.y + area.height - 156 },
|
||||
]) {
|
||||
const anchor = { ...point, width: 156, height: 156 };
|
||||
const before = { ...anchor };
|
||||
for (const size of sizes) {
|
||||
const { bounds } = subagentsSidecarBounds(anchor, size, area);
|
||||
assert(bounds.x >= area.x && bounds.y >= area.y);
|
||||
assert(bounds.x + bounds.width <= area.x + area.width);
|
||||
assert(bounds.y + bounds.height <= area.y + area.height);
|
||||
assert.deepEqual(anchor, before);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
it('selects the inward side and retains it while expanding or receiving updates', () => {
|
||||
const anchor = { x: 1810, y: 850, width: 156, height: 156 };
|
||||
const summary = subagentsSidecarBounds(anchor, sizes[0]!, primary);
|
||||
assert.equal(summary.side, 'left');
|
||||
const list = subagentsSidecarBounds(
|
||||
anchor,
|
||||
sizes[1]!,
|
||||
primary,
|
||||
summary.side,
|
||||
);
|
||||
assert.equal(list.side, 'left');
|
||||
assert(list.bounds.x + list.bounds.width < anchor.x);
|
||||
assert.deepEqual(
|
||||
subagentsSidecarBounds(anchor, sizes[1]!, primary, list.side),
|
||||
list,
|
||||
);
|
||||
assert.equal(
|
||||
subagentsSidecarBounds(
|
||||
{ x: 8, y: 200, width: 156, height: 156 },
|
||||
sizes[0]!,
|
||||
primary,
|
||||
).side,
|
||||
'right',
|
||||
);
|
||||
});
|
||||
it('shrinks task windows for small work areas and clamps removed-display preferences', () => {
|
||||
const tiny = { x: 100, y: 50, width: 300, height: 220 };
|
||||
const fitted = fitSubagentsBounds({ x: 3000, y: -900 }, sizes[2]!, tiny);
|
||||
assert.deepEqual(fitted, { x: 108, y: 58, width: 284, height: 204 });
|
||||
const restored = fitSubagentsBounds(
|
||||
{ x: 2500, y: 600 },
|
||||
sizes[2]!,
|
||||
primary,
|
||||
);
|
||||
assert(restored.x + restored.width <= 2048);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,935 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { liveMessage, liveText } from '@qwen-code/qwen-live/i18n';
|
||||
import type {
|
||||
SubagentTask,
|
||||
SubagentsControlResult,
|
||||
SubagentsSnapshot,
|
||||
} from '@qwen-code/qwen-live/subagents';
|
||||
import { SubagentsView } from '../../renderer/subagents-view.ts';
|
||||
import type {
|
||||
SubagentsWindowApi,
|
||||
SubagentsWindowState,
|
||||
} from '../../shared/subagents-api.ts';
|
||||
|
||||
const cleanup: Array<() => void> = [];
|
||||
afterEach(() => {
|
||||
for (const run of cleanup.splice(0).reverse()) run();
|
||||
});
|
||||
const settled = () => new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
function task(overrides: Partial<SubagentTask> = {}): SubagentTask {
|
||||
return {
|
||||
id: 'task-1',
|
||||
kind: 'harness',
|
||||
title: 'Check build',
|
||||
status: 'running',
|
||||
createdAt: 1_788_790_000_000,
|
||||
updatedAt: 1_788_790_001_000,
|
||||
request: 'Run focused tests',
|
||||
activity: 'Running the tests',
|
||||
output: 'Build output',
|
||||
events: [{ at: 1_788_790_001_000, kind: 'tool', text: 'npm test' }],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function snapshot(tasks = [task()]): SubagentsSnapshot {
|
||||
return {
|
||||
revision: 1,
|
||||
counts: {
|
||||
running: 3,
|
||||
completed: 7,
|
||||
needsAttention: 2,
|
||||
failed: 1,
|
||||
cancelled: 1,
|
||||
interrupted: 1,
|
||||
},
|
||||
tasks,
|
||||
omitted: 4,
|
||||
};
|
||||
}
|
||||
|
||||
function setup(
|
||||
overrides: Partial<SubagentsWindowApi> = {},
|
||||
initial: Partial<SubagentsWindowState> = {},
|
||||
) {
|
||||
const dom = new JSDOM('<!doctype html><main id="app"></main>');
|
||||
const previous = Object.getOwnPropertyDescriptor(globalThis, 'document');
|
||||
Object.defineProperty(globalThis, 'document', {
|
||||
configurable: true,
|
||||
value: dom.window.document,
|
||||
});
|
||||
cleanup.push(() => {
|
||||
dom.window.close();
|
||||
if (previous) Object.defineProperty(globalThis, 'document', previous);
|
||||
else Reflect.deleteProperty(globalThis, 'document');
|
||||
});
|
||||
const app = dom.window.document.querySelector<HTMLElement>('#app')!;
|
||||
const calls: unknown[][] = [];
|
||||
const state: SubagentsWindowState = {
|
||||
language: 'en',
|
||||
connected: true,
|
||||
mode: 'summary',
|
||||
snapshot: snapshot(),
|
||||
...initial,
|
||||
};
|
||||
const api: SubagentsWindowApi = {
|
||||
getState: async () => state,
|
||||
onState: () => () => {},
|
||||
setHover: (value) => calls.push(['hover', value]),
|
||||
setKeyboardHeld: (value) => calls.push(['keyboard', value]),
|
||||
back: async () => {
|
||||
calls.push(['back']);
|
||||
},
|
||||
expand: async () => {
|
||||
calls.push(['expand']);
|
||||
},
|
||||
close: () => calls.push(['close']),
|
||||
openDetail: async (id) => {
|
||||
calls.push(['detail', id]);
|
||||
},
|
||||
control: async (instanceId, request) => {
|
||||
calls.push(['control', instanceId, request]);
|
||||
return {
|
||||
type: 'outcome',
|
||||
outcome: 'stopping',
|
||||
...(request.action === 'stop' ? { taskId: request.taskId } : {}),
|
||||
};
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
const view = new SubagentsView(app, api);
|
||||
cleanup.push(() => view.dispose());
|
||||
view.update(state);
|
||||
const get = <T extends HTMLElement = HTMLElement>(selector: string): T => {
|
||||
const node = app.querySelector<T>(selector);
|
||||
assert(node, `Missing ${selector}`);
|
||||
return node;
|
||||
};
|
||||
const update = (next: Partial<SubagentsWindowState>) => {
|
||||
Object.assign(state, next);
|
||||
view.update({ ...state });
|
||||
};
|
||||
return { dom, app, calls, view, get, update, state };
|
||||
}
|
||||
|
||||
describe('Subagents read-only surfaces', () => {
|
||||
it('marks an unassigned backend approval without inventing an active task', () => {
|
||||
const value = snapshot([]);
|
||||
value.counts = {
|
||||
running: 0,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
};
|
||||
value.pendingUnassignedPermissions = 1;
|
||||
const h = setup({}, { snapshot: value });
|
||||
assert.equal(h.get('.subagents-summary-waiting').hidden, false);
|
||||
assert.match(
|
||||
h.get('.subagents-summary').getAttribute('aria-label') ?? '',
|
||||
/0 running.*1 waiting/,
|
||||
);
|
||||
h.update({ mode: 'list' });
|
||||
assert.equal(
|
||||
h.get('.subagents-panel [data-count="needsAttention"]').textContent,
|
||||
'1',
|
||||
);
|
||||
});
|
||||
|
||||
it('stops the exact selected task outside the summary page and distinguishes requested from confirmed', async () => {
|
||||
const selected = task({ id: 'harness:40', canStop: true });
|
||||
const h = setup(
|
||||
{},
|
||||
{
|
||||
mode: 'detail',
|
||||
selectedId: selected.id,
|
||||
instanceId: 'daemon-one',
|
||||
controlsAvailable: true,
|
||||
page: { snapshot: snapshot(), offset: 0, total: 40, selected },
|
||||
},
|
||||
);
|
||||
const stop = h.get<HTMLButtonElement>('.subagent-identity .subagents-stop');
|
||||
assert.equal(stop.hidden, false);
|
||||
stop.click();
|
||||
stop.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls, [
|
||||
['control', 'daemon-one', { action: 'stop', taskId: 'harness:40' }],
|
||||
]);
|
||||
assert.equal(
|
||||
h.get('.subagent-identity .subagent-status').textContent,
|
||||
'Running',
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagents-feedback').textContent ?? '',
|
||||
/Waiting for the backend/,
|
||||
);
|
||||
h.update({
|
||||
page: {
|
||||
...h.state.page!,
|
||||
selected: { ...selected, canStop: false, stopReason: 'stopping' },
|
||||
},
|
||||
});
|
||||
assert.equal(stop.disabled, true);
|
||||
assert.equal(stop.textContent, 'Stopping…');
|
||||
h.update({
|
||||
language: 'zh-CN',
|
||||
page: {
|
||||
...h.state.page!,
|
||||
selected: {
|
||||
...selected,
|
||||
canStop: false,
|
||||
stopReason: 'ended',
|
||||
status: 'cancelled',
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(stop.hidden, true);
|
||||
assert.equal(h.get('.subagents-feedback').textContent, '任务已停止。');
|
||||
assert.equal(
|
||||
h.get('.subagent-identity .subagent-status').textContent,
|
||||
'已取消',
|
||||
);
|
||||
h.get<HTMLButtonElement>('.subagents-close').click();
|
||||
assert.equal(h.calls.filter(([call]) => call === 'control').length, 1);
|
||||
});
|
||||
|
||||
it('renders real pending decisions with explicit scopes, keeps focus, and sends the request handle', async () => {
|
||||
const selected = task({
|
||||
status: 'waiting',
|
||||
canStop: true,
|
||||
permissions: [
|
||||
{
|
||||
requestHandle: 'req_12',
|
||||
title: '<script>Write file outside project</script>',
|
||||
choices: [
|
||||
{ decision: 'allow', scope: 'always' },
|
||||
{ decision: 'deny', scope: 'once' },
|
||||
],
|
||||
},
|
||||
],
|
||||
permissionsOmitted: 3,
|
||||
});
|
||||
const h = setup(
|
||||
{
|
||||
control: async (instance, request) => {
|
||||
h.calls.push(['control', instance, request]);
|
||||
return {
|
||||
type: 'outcome',
|
||||
outcome: 'allowed',
|
||||
requestHandle: 'req_12',
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
mode: 'detail',
|
||||
instanceId: 'daemon-one',
|
||||
controlsAvailable: true,
|
||||
selectedId: selected.id,
|
||||
page: { snapshot: snapshot([selected]), offset: 0, total: 1, selected },
|
||||
},
|
||||
);
|
||||
const allow = h.get<HTMLButtonElement>('[data-decision="allow"]');
|
||||
assert.equal(allow.textContent, 'Always allow');
|
||||
assert.equal(h.get('[data-decision="deny"]').textContent, 'Deny once');
|
||||
assert.equal(h.app.querySelector('script'), null);
|
||||
allow.focus();
|
||||
h.update({ language: 'zh-CN' });
|
||||
assert.equal(h.get('[data-decision="allow"]'), allow);
|
||||
assert.equal(document.activeElement, allow);
|
||||
assert.equal(allow.textContent, '始终允许');
|
||||
assert.match(h.get('.subagents-more-permissions').textContent ?? '', /3/);
|
||||
allow.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), [
|
||||
'control',
|
||||
'daemon-one',
|
||||
{ action: 'permission', requestHandle: 'req_12', decision: 'allow' },
|
||||
]);
|
||||
assert.equal(
|
||||
h.get('.subagents-feedback').textContent,
|
||||
liveText('zh-CN', 'subagents.outcome.allowed'),
|
||||
);
|
||||
h.update({ connected: false });
|
||||
assert.equal(allow.disabled, true);
|
||||
});
|
||||
|
||||
it('shows unassigned approvals separately and pages retained tasks instead of hiding active details', async () => {
|
||||
const entries = [task({ id: 'harness:33', canStop: true })];
|
||||
const h = setup(
|
||||
{},
|
||||
{
|
||||
mode: 'list',
|
||||
instanceId: 'daemon-one',
|
||||
controlsAvailable: true,
|
||||
page: {
|
||||
snapshot: snapshot(entries),
|
||||
offset: 32,
|
||||
total: 34,
|
||||
unassignedPermissions: [
|
||||
{
|
||||
requestHandle: 'req_2',
|
||||
title: 'Unassigned write',
|
||||
choices: [{ decision: 'deny', scope: 'once' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagent-unassigned').textContent ?? '',
|
||||
/Task identity unconfirmed/,
|
||||
);
|
||||
assert.equal(h.app.querySelector('[data-task-id="task-1"]'), null);
|
||||
assert.equal(h.get('.subagents-page-label').textContent, '33–33 of 34');
|
||||
const denial = h.get<HTMLButtonElement>(
|
||||
'.subagent-unassigned [data-decision="deny"]',
|
||||
);
|
||||
denial.focus();
|
||||
h.update({ language: 'zh-CN' });
|
||||
assert.equal(h.get('.subagent-unassigned [data-decision="deny"]'), denial);
|
||||
assert.equal(document.activeElement, denial);
|
||||
const next = h.get<HTMLButtonElement>(
|
||||
'.subagents-pagination button:nth-of-type(2)',
|
||||
);
|
||||
next.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), [
|
||||
'control',
|
||||
'daemon-one',
|
||||
{ action: 'list', offset: 33 },
|
||||
]);
|
||||
assert.equal(h.get('.subagents-retention').hidden, true);
|
||||
});
|
||||
|
||||
it('does not apply late action feedback to a replacement daemon or permanently lock controls after closing', async () => {
|
||||
let finish: (result: SubagentsControlResult) => void = () => {};
|
||||
const selected = task({ canStop: true });
|
||||
const h = setup(
|
||||
{
|
||||
control: () =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
},
|
||||
{
|
||||
mode: 'detail',
|
||||
instanceId: 'one',
|
||||
controlsAvailable: true,
|
||||
selectedId: selected.id,
|
||||
page: { snapshot: snapshot([selected]), offset: 0, total: 1, selected },
|
||||
},
|
||||
);
|
||||
h.get<HTMLButtonElement>('.subagent-identity .subagents-stop').click();
|
||||
h.update({ instanceId: 'two' });
|
||||
finish({ type: 'outcome', outcome: 'stopped' });
|
||||
await settled();
|
||||
assert.equal(h.get('.subagents-feedback').hidden, true);
|
||||
const stop = h.get<HTMLButtonElement>('.subagent-identity .subagents-stop');
|
||||
assert.equal(stop.disabled, false);
|
||||
stop.click();
|
||||
h.get<HTMLButtonElement>('.subagents-close').click();
|
||||
finish({ type: 'error', code: 'action_failed' });
|
||||
await settled();
|
||||
h.update({ mode: 'summary' });
|
||||
assert.equal(
|
||||
h.get<HTMLButtonElement>('.subagents-summary').disabled,
|
||||
false,
|
||||
);
|
||||
assert.equal(h.get('.subagents-error').hidden, true);
|
||||
});
|
||||
|
||||
it('shows authoritative paired-language counts rather than counting retained tasks', async () => {
|
||||
const h = setup();
|
||||
const summary = h.get<HTMLButtonElement>('.subagents-summary');
|
||||
assert.match(summary.textContent ?? '', /Subagents/);
|
||||
assert.equal(
|
||||
summary.querySelector('[data-count="running"]')?.textContent,
|
||||
'3',
|
||||
);
|
||||
assert.equal(
|
||||
summary.querySelector('[data-count="completed"]')?.textContent,
|
||||
'7',
|
||||
);
|
||||
assert.equal(summary.querySelector('[data-count="needsAttention"]'), null);
|
||||
assert.equal(h.get('.subagents-summary-waiting').hidden, false);
|
||||
assert.equal(
|
||||
h.get('.subagents-summary-waiting').title,
|
||||
liveText('en', 'subagents.summaryWaiting', { count: 2 }),
|
||||
);
|
||||
assert.equal(
|
||||
summary.querySelector('.subagents-bot')?.getAttribute('aria-hidden'),
|
||||
'true',
|
||||
);
|
||||
assert.equal(summary.querySelector('.subagents-arrow'), null);
|
||||
assert.doesNotMatch(
|
||||
summary.textContent ?? '',
|
||||
/Running|Completed|Needs you/,
|
||||
);
|
||||
assert.equal(
|
||||
summary.getAttribute('aria-label'),
|
||||
'View subagents: 3 running, 7 completed, 2 waiting for your input.',
|
||||
);
|
||||
const running = summary.querySelector('[data-count="running"]');
|
||||
const completed = summary.querySelector('[data-count="completed"]');
|
||||
summary.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls, [['expand']]);
|
||||
assert.equal(h.get('.subagents-panel').hidden, true);
|
||||
h.update({ language: 'zh-CN' });
|
||||
assert.equal(h.get('.subagents-summary'), summary);
|
||||
assert.equal(summary.querySelector('[data-count="running"]'), running);
|
||||
assert.equal(summary.querySelector('[data-count="completed"]'), completed);
|
||||
assert.match(summary.textContent ?? '', /子智能体/);
|
||||
assert.doesNotMatch(summary.textContent ?? '', /进行中|已完成|需关注/);
|
||||
assert.equal(
|
||||
summary.getAttribute('aria-label'),
|
||||
liveText('zh-CN', 'subagents.summaryLabel', {
|
||||
running: 3,
|
||||
completed: 7,
|
||||
waiting: 2,
|
||||
}),
|
||||
);
|
||||
assert.equal(summary.title, summary.getAttribute('aria-label'));
|
||||
assert.equal(
|
||||
summary.getAttribute('aria-label'),
|
||||
'查看子智能体：3 项进行中，7 项已完成，2 项等待你处理。',
|
||||
);
|
||||
assert.equal(h.get('.subagents-summary .running').title, '进行中: 3');
|
||||
assert.equal(h.get('.subagents-summary .completed').title, '已完成: 7');
|
||||
});
|
||||
|
||||
it('pulses only for connected active summary counts and hides the waiting marker at zero', () => {
|
||||
const empty = snapshot([]);
|
||||
empty.counts = {
|
||||
running: 0,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
};
|
||||
const h = setup({}, { snapshot: empty });
|
||||
const summary = h.get('.subagents-summary');
|
||||
const point = h.get('.subagents-count-symbol.running');
|
||||
assert.equal(summary.classList.contains('running-active'), false);
|
||||
assert.equal(h.get('.subagents-summary-waiting').hidden, true);
|
||||
h.update({
|
||||
snapshot: {
|
||||
...empty,
|
||||
counts: { ...empty.counts, running: 2, needsAttention: 1 },
|
||||
},
|
||||
});
|
||||
assert.equal(summary.classList.contains('running-active'), true);
|
||||
assert.equal(h.get('.subagents-summary-waiting').hidden, false);
|
||||
h.update({ connected: false });
|
||||
assert.equal(summary.classList.contains('running-active'), false);
|
||||
assert.match(summary.title, /Disconnected/);
|
||||
h.update({ connected: true, mode: 'list' });
|
||||
assert.equal(summary.classList.contains('running-active'), false);
|
||||
h.update({ mode: 'summary' });
|
||||
assert.equal(summary.classList.contains('running-active'), true);
|
||||
h.update({ snapshot: empty });
|
||||
assert.equal(summary.classList.contains('running-active'), false);
|
||||
assert.equal(h.get('.subagents-summary-waiting').hidden, true);
|
||||
assert.equal(h.get('.subagents-count-symbol.running'), point);
|
||||
});
|
||||
|
||||
it('keeps large summary numbers compact while titles, accessibility and list counts stay exact', () => {
|
||||
const value = snapshot([]);
|
||||
value.counts = {
|
||||
...value.counts,
|
||||
running: 1_234_567,
|
||||
completed: 9_876_543,
|
||||
needsAttention: 456,
|
||||
};
|
||||
const h = setup({}, { snapshot: value });
|
||||
const running = h.get('.subagents-summary [data-count="running"]');
|
||||
const completed = h.get('.subagents-summary [data-count="completed"]');
|
||||
assert.equal(running.textContent, '999+');
|
||||
assert.equal(completed.textContent, '999+');
|
||||
assert.equal(running.title, 'Running: 1234567');
|
||||
assert.equal(completed.title, 'Completed: 9876543');
|
||||
assert.equal(
|
||||
h.get('.subagents-summary').getAttribute('aria-label'),
|
||||
liveText('en', 'subagents.summaryLabel', {
|
||||
running: 1_234_567,
|
||||
completed: 9_876_543,
|
||||
waiting: 456,
|
||||
}),
|
||||
);
|
||||
h.update({ mode: 'list', language: 'zh-CN' });
|
||||
assert.equal(
|
||||
h.get('.subagents-panel [data-count="running"]').textContent,
|
||||
'1234567',
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.subagents-panel [data-count="completed"]').textContent,
|
||||
'9876543',
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.subagents-panel [data-count="needsAttention"]').textContent,
|
||||
'456',
|
||||
);
|
||||
});
|
||||
|
||||
it('uses a gentle 1.4 second opacity pulse and disables it for reduced motion', async () => {
|
||||
const css = await readFile(
|
||||
new URL('../../renderer/subagents.css', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(
|
||||
css,
|
||||
/\.subagents-summary\.running-active \.subagents-count-symbol\.running\s*\{\s*animation: subagents-running-pulse 1\.4s ease-in-out infinite;/,
|
||||
);
|
||||
assert.match(
|
||||
css,
|
||||
/@keyframes subagents-running-pulse\s*\{[\s\S]*?opacity: 0\.45;[\s\S]*?opacity: 1;/,
|
||||
);
|
||||
assert.match(
|
||||
css,
|
||||
/@media \(prefers-reduced-motion: reduce\)\s*\{\s*\.subagents-summary\.running-active \.subagents-count-symbol\.running\s*\{\s*animation: none;/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
css,
|
||||
/^\.subagents-count-symbol\.running\s*\{[^}]*animation:/m,
|
||||
);
|
||||
assert.match(
|
||||
css,
|
||||
/\.subagents-summary-waiting\s*\{[^}]*position: absolute;/,
|
||||
);
|
||||
});
|
||||
|
||||
it('reports pointer hover separately from keyboard focus and closes with ordered releases', () => {
|
||||
const h = setup();
|
||||
const summary = h.get<HTMLButtonElement>('.subagents-summary');
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerenter'));
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerleave'));
|
||||
summary.focus();
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerleave'));
|
||||
summary.blur();
|
||||
assert.deepEqual(h.calls, [
|
||||
['hover', true],
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
]);
|
||||
document.dispatchEvent(
|
||||
new h.dom.window.KeyboardEvent('keydown', { key: 'Tab' }),
|
||||
);
|
||||
summary.focus();
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', true],
|
||||
]);
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerleave'));
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', true],
|
||||
]);
|
||||
summary.dispatchEvent(
|
||||
new h.dom.window.Event('pointerdown', { bubbles: true }),
|
||||
);
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
]);
|
||||
document.dispatchEvent(
|
||||
new h.dom.window.KeyboardEvent('keydown', { key: 'Escape' }),
|
||||
);
|
||||
assert.deepEqual(h.calls.slice(-3), [
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['close'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not retain hidden-list keyboard hover after native blur and summary reopen', () => {
|
||||
const h = setup({}, { mode: 'list' });
|
||||
document.dispatchEvent(
|
||||
new h.dom.window.KeyboardEvent('keydown', { key: 'Tab' }),
|
||||
);
|
||||
h.get<HTMLButtonElement>('.subagents-close').focus();
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', true],
|
||||
]);
|
||||
h.dom.window.dispatchEvent(new h.dom.window.Event('blur'));
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
]);
|
||||
h.update({ mode: 'summary' });
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerenter'));
|
||||
h.app.dispatchEvent(new h.dom.window.Event('pointerleave'));
|
||||
assert.deepEqual(h.calls.slice(-2), [
|
||||
['hover', false],
|
||||
['keyboard', false],
|
||||
]);
|
||||
});
|
||||
|
||||
it('switches the same panel from summary through list and detail and uses Back without closing', async () => {
|
||||
const h = setup();
|
||||
const summary = h.get<HTMLButtonElement>('.subagents-summary');
|
||||
const panel = h.get('.subagents-panel');
|
||||
const list = h.get('.subagents-list');
|
||||
const row = h.get<HTMLButtonElement>('[data-task-id="task-1"]');
|
||||
const back = h.get<HTMLButtonElement>('.subagents-back');
|
||||
summary.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), ['expand']);
|
||||
assert.equal(panel.hidden, true);
|
||||
h.update({ mode: 'list' });
|
||||
assert.equal(summary.hidden, true);
|
||||
assert.equal(panel.hidden, false);
|
||||
assert.equal(back.hidden, true);
|
||||
list.scrollTop = 41;
|
||||
row.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), ['detail', 'task-1']);
|
||||
h.update({ mode: 'detail', selectedId: 'task-1' });
|
||||
assert.equal(h.get('.subagents-panel'), panel);
|
||||
assert.equal(back.hidden, false);
|
||||
assert.equal(list.hidden, true);
|
||||
assert.equal(h.get('.subagent-detail-body').hidden, false);
|
||||
back.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), ['back']);
|
||||
assert.equal(h.state.mode, 'detail');
|
||||
h.update({ mode: 'list', selectedId: undefined });
|
||||
assert.equal(h.get('.subagents-panel'), panel);
|
||||
assert.equal(h.get('.subagents-list'), list);
|
||||
assert.equal(h.get('[data-task-id="task-1"]'), row);
|
||||
assert.equal(list.scrollTop, 41);
|
||||
assert.equal(back.hidden, true);
|
||||
assert.equal(
|
||||
h.calls.some(([call]) => call === 'close'),
|
||||
false,
|
||||
);
|
||||
h.get<HTMLButtonElement>('.subagents-close').click();
|
||||
assert.deepEqual(h.calls.slice(-3), [
|
||||
['keyboard', false],
|
||||
['hover', false],
|
||||
['close'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('allows Back from disconnected details while remote task-opening actions remain disabled', async () => {
|
||||
const h = setup(
|
||||
{},
|
||||
{
|
||||
mode: 'detail',
|
||||
selectedId: 'task-1',
|
||||
connected: false,
|
||||
},
|
||||
);
|
||||
h.get<HTMLButtonElement>('.subagents-back').click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls, [['back']]);
|
||||
h.update({ mode: 'list', selectedId: undefined });
|
||||
const row = h.get<HTMLButtonElement>('[data-task-id="task-1"]');
|
||||
assert.equal(row.disabled, true);
|
||||
row.click();
|
||||
h.update({ mode: 'summary' });
|
||||
const summary = h.get<HTMLButtonElement>('.subagents-summary');
|
||||
assert.equal(summary.disabled, true);
|
||||
summary.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls, [['back']]);
|
||||
});
|
||||
|
||||
it('keeps task rows, click identity, focus and list scroll stable across updates', async () => {
|
||||
const first = task();
|
||||
const second = task({
|
||||
id: 'task-2',
|
||||
title: 'Read docs',
|
||||
status: 'completed',
|
||||
});
|
||||
const h = setup({}, { mode: 'list', snapshot: snapshot([first, second]) });
|
||||
const list = h.get('.subagents-list');
|
||||
const row = h.get<HTMLButtonElement>('[data-task-id="task-1"]');
|
||||
list.scrollTop = 53;
|
||||
row.focus();
|
||||
h.update({
|
||||
language: 'zh-CN',
|
||||
snapshot: {
|
||||
...snapshot([
|
||||
second,
|
||||
{ ...first, status: 'waiting', activity: 'Need your input' },
|
||||
]),
|
||||
revision: 2,
|
||||
},
|
||||
});
|
||||
assert.equal(h.get('[data-task-id="task-1"]'), row);
|
||||
assert.equal(document.activeElement, row);
|
||||
assert.equal(list.scrollTop, 53);
|
||||
assert.equal(list.firstElementChild?.firstElementChild, row);
|
||||
assert.match(row.textContent ?? '', /等待输入/);
|
||||
assert.match(row.textContent ?? '', /Need your input/);
|
||||
row.click();
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), ['detail', first.id]);
|
||||
assert.match(h.get('.subagents-other-counts').textContent ?? '', /1 失败/);
|
||||
assert.match(h.get('.subagents-retention').textContent ?? '', /4/);
|
||||
h.update({ snapshot: snapshot([second]) });
|
||||
assert.equal(h.app.querySelector('[data-task-id="task-1"]'), null);
|
||||
});
|
||||
|
||||
it('renders public title, request and output as text, not markup, while localizing owned activity', () => {
|
||||
const payload =
|
||||
'<img src="https://example.com/x" onerror="alert(1)"><script>bad()</script>';
|
||||
const entry = task({
|
||||
title: payload,
|
||||
request: payload,
|
||||
output: payload.repeat(80),
|
||||
activity: liveMessage('subagents.reconnecting'),
|
||||
events: [
|
||||
{ at: 1, kind: 'message', text: payload },
|
||||
{ at: 2, kind: 'status', text: liveMessage('subagents.callEnded') },
|
||||
],
|
||||
outputTruncated: true,
|
||||
});
|
||||
const h = setup(
|
||||
{},
|
||||
{ mode: 'detail', selectedId: entry.id, snapshot: snapshot([entry]) },
|
||||
);
|
||||
const output = h.get('.subagent-output');
|
||||
h.update({ language: 'zh-CN' });
|
||||
assert.equal(h.get('.subagent-title').textContent, payload);
|
||||
assert.equal(h.get('.subagent-request').textContent, payload);
|
||||
assert.equal(output.textContent, payload.repeat(80));
|
||||
assert.equal(h.app.querySelector('img, script, a'), null);
|
||||
assert.equal(
|
||||
h.get('.subagent-latest').textContent,
|
||||
liveText('zh-CN', 'subagents.reconnecting'),
|
||||
);
|
||||
assert.match(h.get('.subagent-events').textContent ?? '', /语音通话已结束/);
|
||||
assert.equal(h.get('.subagent-truncated').hidden, false);
|
||||
assert.equal(h.get('.subagent-output'), output);
|
||||
assert.equal(
|
||||
h.app.querySelector('button[data-cancel], button[data-approve]'),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps notification delivery distinct from task completion and preserves exact statuses', () => {
|
||||
const entry = task({
|
||||
kind: 'proactive',
|
||||
status: 'monitoring',
|
||||
triggerCount: 3,
|
||||
pendingNotifications: 2,
|
||||
notification: 'delivered',
|
||||
remainingSec: 4.2,
|
||||
});
|
||||
const h = setup(
|
||||
{},
|
||||
{ mode: 'detail', selectedId: entry.id, snapshot: snapshot([entry]) },
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.subagent-identity .subagent-status').textContent,
|
||||
'Monitoring',
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagent-notifications').textContent ?? '',
|
||||
/Triggers: 3/,
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagent-notifications').textContent ?? '',
|
||||
/Pending announcements: 2/,
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagent-notifications').textContent ?? '',
|
||||
/Announcement delivered/,
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagent-notifications').textContent ?? '',
|
||||
/Remaining: 5s/,
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.subagent-section:last-child h2').textContent,
|
||||
'Public output',
|
||||
);
|
||||
for (const status of [
|
||||
'completed',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'interrupted',
|
||||
] as const) {
|
||||
h.update({ snapshot: snapshot([{ ...entry, status }]) });
|
||||
assert.equal(
|
||||
h.get('.subagent-identity .subagent-status').textContent,
|
||||
liveText('en', `subagents.${status}`),
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.subagent-section:last-child h2').textContent,
|
||||
status === 'completed' ? 'Result' : 'Public output',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('follows output and activity only from the bottom and retains content and focus on disconnect', () => {
|
||||
let entry = task();
|
||||
const h = setup(
|
||||
{},
|
||||
{ mode: 'detail', selectedId: entry.id, snapshot: snapshot([entry]) },
|
||||
);
|
||||
const output = h.get('.subagent-output');
|
||||
const events = h.get('.subagent-events');
|
||||
const body = h.get('.subagent-detail-body');
|
||||
let height = 500;
|
||||
for (const node of [output, events, body]) {
|
||||
Object.defineProperty(node, 'scrollHeight', {
|
||||
configurable: true,
|
||||
get: () => height,
|
||||
});
|
||||
Object.defineProperty(node, 'clientHeight', {
|
||||
configurable: true,
|
||||
value: 100,
|
||||
});
|
||||
node.scrollTop = 80;
|
||||
}
|
||||
output.focus();
|
||||
entry = {
|
||||
...entry,
|
||||
output: 'more output',
|
||||
events: [
|
||||
...entry.events,
|
||||
{ at: 2, kind: 'message', text: 'more activity' },
|
||||
],
|
||||
};
|
||||
h.update({ snapshot: snapshot([entry]) });
|
||||
for (const node of [output, events, body]) assert.equal(node.scrollTop, 80);
|
||||
assert.equal(document.activeElement, output);
|
||||
for (const node of [output, events, body]) node.scrollTop = 400;
|
||||
entry = {
|
||||
...entry,
|
||||
output: 'even more output',
|
||||
updatedAt: entry.updatedAt + 1000,
|
||||
};
|
||||
h.update({ snapshot: snapshot([entry]) });
|
||||
for (const node of [output, events, body])
|
||||
assert.equal(node.scrollTop, height);
|
||||
height = 600;
|
||||
h.update({ connected: false });
|
||||
assert.equal(h.get('.subagents-notice').hidden, false);
|
||||
assert.match(h.get('.subagents-notice').textContent ?? '', /Disconnected/);
|
||||
assert.equal(output.textContent, 'even more output');
|
||||
assert.equal(document.activeElement, output);
|
||||
h.update({
|
||||
connected: true,
|
||||
selectedId: undefined,
|
||||
snapshot: snapshot([]),
|
||||
});
|
||||
assert.equal(h.get('.subagent-detail-body').hidden, true);
|
||||
assert.match(h.get('.subagents-empty').textContent ?? '', /no longer/);
|
||||
});
|
||||
|
||||
it('handles empty, unsupported and missing-task states without inventing entries', () => {
|
||||
const h = setup({}, { mode: 'list', snapshot: snapshot([]) });
|
||||
assert.equal(h.get('.subagents-empty').hidden, false);
|
||||
assert.match(
|
||||
h.get('.subagents-empty').textContent ?? '',
|
||||
/No task details/,
|
||||
);
|
||||
assert.match(
|
||||
h.get('.subagents-retention').textContent ?? '',
|
||||
/4 other tasks/,
|
||||
);
|
||||
h.update({
|
||||
snapshot: {
|
||||
...snapshot([]),
|
||||
omitted: 0,
|
||||
counts: {
|
||||
running: 0,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.match(
|
||||
h.get('.subagents-empty').textContent ?? '',
|
||||
/No subagent tasks/,
|
||||
);
|
||||
assert.equal(h.app.querySelector('[data-task-id]'), null);
|
||||
h.update({ snapshot: undefined });
|
||||
assert.match(h.get('.subagents-empty').textContent ?? '', /unavailable/);
|
||||
h.update({ mode: 'summary' });
|
||||
assert.equal(h.get<HTMLButtonElement>('.subagents-summary').disabled, true);
|
||||
h.update({ mode: 'detail', selectedId: 'unknown' });
|
||||
assert.match(h.get('.subagents-empty').textContent ?? '', /no longer/);
|
||||
h.get<HTMLButtonElement>('.subagents-close').click();
|
||||
assert.deepEqual(h.calls.at(-1), ['close']);
|
||||
});
|
||||
|
||||
it('shows failed opens, suppresses duplicate actions, and does not reopen after close', async () => {
|
||||
let rejectOpen: (error: Error) => void = () => {};
|
||||
let openings = 0;
|
||||
const h = setup({
|
||||
expand: () => {
|
||||
openings++;
|
||||
return new Promise<void>((_resolve, reject) => {
|
||||
rejectOpen = reject;
|
||||
});
|
||||
},
|
||||
});
|
||||
const summary = h.get<HTMLButtonElement>('.subagents-summary');
|
||||
summary.click();
|
||||
summary.click();
|
||||
assert.equal(openings, 1);
|
||||
rejectOpen(new Error(liveMessage('subagents.openFailed')));
|
||||
await settled();
|
||||
assert.equal(summary.disabled, false);
|
||||
assert.equal(h.get('.subagents-summary-error').hidden, false);
|
||||
h.update({ language: 'zh-CN' });
|
||||
assert.equal(
|
||||
h.get('.subagents-summary-error').textContent,
|
||||
liveText('zh-CN', 'subagents.openFailed'),
|
||||
);
|
||||
summary.click();
|
||||
document.dispatchEvent(
|
||||
new h.dom.window.KeyboardEvent('keydown', { key: 'Escape' }),
|
||||
);
|
||||
rejectOpen(new Error('late failure'));
|
||||
await settled();
|
||||
assert.deepEqual(h.calls.at(-1), ['close']);
|
||||
assert.equal(h.app.textContent?.includes('late failure'), false);
|
||||
});
|
||||
|
||||
it('isolates media and scripts and makes expanded-panel headers draggable without dragging controls', async () => {
|
||||
const [html, css] = await Promise.all([
|
||||
readFile(
|
||||
new URL('../../renderer/subagents.html', import.meta.url),
|
||||
'utf8',
|
||||
),
|
||||
readFile(
|
||||
new URL('../../renderer/subagents.css', import.meta.url),
|
||||
'utf8',
|
||||
),
|
||||
]);
|
||||
assert.match(html, /connect-src 'none'; media-src 'none'/);
|
||||
assert.match(html, /script-src 'self'/);
|
||||
assert.match(css, /\.subagents-header\s*\{\s*-webkit-app-region: drag;/);
|
||||
assert.match(css, /button\s*\{[^}]*-webkit-app-region: no-drag;/);
|
||||
assert.match(css, /\.subagent-output\s*\{[^}]*overflow-y: auto;/);
|
||||
assert.match(css, /\.subagent-task-title\s*\{[^}]*height: 36px;/);
|
||||
assert.match(css, /\.subagent-task-activity\s*\{[^}]*height: 34px;/);
|
||||
assert.match(
|
||||
css,
|
||||
/\.subagent-task > \.subagent-status\s*\{[^}]*max-height: 36px;/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,626 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import { describe, it } from 'node:test';
|
||||
import ts from 'typescript';
|
||||
import * as geometry from '../subagents-position.ts';
|
||||
import {
|
||||
parseSubagentsControlRequest,
|
||||
type SubagentsControlRequest,
|
||||
type SubagentsControlResult,
|
||||
type SubagentsSnapshot,
|
||||
} from '@qwen-code/qwen-live/subagents';
|
||||
import type { SubagentsWindowState } from '../../shared/subagents-api.ts';
|
||||
|
||||
const snapshot: SubagentsSnapshot = {
|
||||
revision: 1,
|
||||
counts: {
|
||||
running: 1,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
},
|
||||
omitted: 0,
|
||||
tasks: [
|
||||
{
|
||||
id: 'harness:1',
|
||||
kind: 'harness',
|
||||
title: 'Task',
|
||||
status: 'running',
|
||||
request: 'User request',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
activity: 'Running tests',
|
||||
output: 'output',
|
||||
events: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
function fixture(
|
||||
hoverRegions?: Array<{ x: number; y: number; width: number; height: number }>,
|
||||
requestControl?: (
|
||||
request: SubagentsControlRequest,
|
||||
instanceId: string,
|
||||
) => Promise<SubagentsControlResult>,
|
||||
) {
|
||||
type Handler = (...args: unknown[]) => unknown;
|
||||
const handlers = new Map<string, Handler>();
|
||||
const windows: Window[] = [];
|
||||
const events: Array<{ action: string; window?: Window }> = [];
|
||||
let now = 10_000;
|
||||
let cursor = { x: 1800, y: 800 };
|
||||
const anchor = { x: 1750, y: 780, width: 156, height: 156 };
|
||||
let workArea = { x: 0, y: 30, width: 2048, height: 1028 };
|
||||
type Timer = { at: number; callback: () => void; unref: () => void };
|
||||
const timers = new Set<Timer>();
|
||||
class Window {
|
||||
visible = false;
|
||||
destroyed = false;
|
||||
loading = true;
|
||||
focused = 0;
|
||||
bounds: { x: number; y: number; width: number; height: number };
|
||||
events = new Map<string, Handler>();
|
||||
readonly moves: unknown[] = [];
|
||||
readonly sent: unknown[] = [];
|
||||
readonly backgrounds: string[] = [];
|
||||
webContents = {
|
||||
isDestroyed: () => this.destroyed,
|
||||
isLoadingMainFrame: () => this.loading,
|
||||
send: (channel: string, state: unknown) => {
|
||||
assert.equal(channel, 'live:subagents:state');
|
||||
this.sent.push(state);
|
||||
events.push({ action: 'publish', window: this });
|
||||
},
|
||||
setWindowOpenHandler: () => {},
|
||||
on: (name: string, fn: Handler) => this.events.set(name, fn),
|
||||
};
|
||||
constructor(readonly options: Record<string, unknown>) {
|
||||
this.bounds = {
|
||||
x: Number(options.x),
|
||||
y: Number(options.y),
|
||||
width: Number(options.width),
|
||||
height: Number(options.height),
|
||||
};
|
||||
windows.push(this);
|
||||
events.push({ action: 'create', window: this });
|
||||
}
|
||||
setAlwaysOnTop() {}
|
||||
setVisibleOnAllWorkspaces() {}
|
||||
setBackgroundColor(color: string) {
|
||||
this.backgrounds.push(color);
|
||||
}
|
||||
on(name: string, fn: Handler) {
|
||||
this.events.set(name, fn);
|
||||
}
|
||||
loadFile() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
isDestroyed() {
|
||||
return this.destroyed;
|
||||
}
|
||||
isVisible() {
|
||||
return this.visible;
|
||||
}
|
||||
getBounds() {
|
||||
return { ...this.bounds };
|
||||
}
|
||||
setBounds(bounds: Window['bounds']) {
|
||||
this.bounds = { ...bounds };
|
||||
this.moves.push(bounds);
|
||||
events.push({ action: 'place', window: this });
|
||||
}
|
||||
show() {
|
||||
this.visible = true;
|
||||
events.push({ action: 'show', window: this });
|
||||
}
|
||||
showInactive() {
|
||||
this.visible = true;
|
||||
events.push({ action: 'showInactive', window: this });
|
||||
}
|
||||
focus() {
|
||||
this.focused++;
|
||||
events.push({ action: 'focus', window: this });
|
||||
}
|
||||
hide() {
|
||||
this.visible = false;
|
||||
events.push({ action: 'hide', window: this });
|
||||
}
|
||||
close() {
|
||||
this.events.get('close')?.();
|
||||
this.destroy();
|
||||
}
|
||||
destroy() {
|
||||
this.destroyed = true;
|
||||
this.visible = false;
|
||||
events.push({ action: 'destroy', window: this });
|
||||
this.events.get('closed')?.();
|
||||
}
|
||||
ready() {
|
||||
this.loading = false;
|
||||
this.events.get('did-finish-load')?.();
|
||||
}
|
||||
}
|
||||
const tree = ts.createSourceFile(
|
||||
'subagents-windows.ts',
|
||||
readFileSync(new URL('../subagents-windows.ts', import.meta.url), 'utf8'),
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const declaration = tree.statements.find(
|
||||
(node) =>
|
||||
ts.isClassDeclaration(node) && node.name?.text === 'SubagentsWindows',
|
||||
);
|
||||
assert(declaration);
|
||||
const source = ts.transpileModule(
|
||||
declaration.getText(tree).replace('export class', 'class') +
|
||||
'\nSubagentsWindows;',
|
||||
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } },
|
||||
).outputText;
|
||||
const Controller = runInNewContext(source, {
|
||||
BrowserWindow: Window,
|
||||
ipcMain: {
|
||||
handle: (name: string, fn: Handler) => handlers.set(name, fn),
|
||||
on: (name: string, fn: Handler) => handlers.set(name, fn),
|
||||
removeHandler: (name: string) => handlers.delete(name),
|
||||
removeAllListeners: (name: string) => handlers.delete(name),
|
||||
},
|
||||
screen: {
|
||||
getDisplayMatching: () => ({ workArea }),
|
||||
getPrimaryDisplay: () => ({ workArea }),
|
||||
getCursorScreenPoint: () => ({ ...cursor }),
|
||||
},
|
||||
...geometry,
|
||||
parseSubagentsControlRequest,
|
||||
join: (...values: string[]) => values.join('/'),
|
||||
Date: class extends Date {
|
||||
static override now() {
|
||||
return now;
|
||||
}
|
||||
},
|
||||
setTimeout: (callback: () => void, delay: number) => {
|
||||
const timer = { at: now + delay, callback, unref: () => {} };
|
||||
timers.add(timer);
|
||||
return timer;
|
||||
},
|
||||
clearTimeout: (timer: Timer) => timers.delete(timer),
|
||||
}) as new (options: unknown) => {
|
||||
update: (
|
||||
lang: string,
|
||||
connected: boolean,
|
||||
snapshot?: SubagentsSnapshot,
|
||||
instanceId?: string,
|
||||
controlsAvailable?: boolean,
|
||||
) => void;
|
||||
setOrbHovered: (value: boolean) => void;
|
||||
setOrbKeyboardHeld: (value: boolean) => void;
|
||||
setDragging: (value: boolean) => void;
|
||||
setBlocked: (value: boolean) => void;
|
||||
setTheme: (theme: string, appearance: string) => void;
|
||||
dispose: () => void;
|
||||
displaysChanged: () => void;
|
||||
dismissPeek: () => void;
|
||||
};
|
||||
const controller = new Controller({
|
||||
baseDirectory: '/fixture',
|
||||
requestControl,
|
||||
anchor: () => anchor,
|
||||
...(hoverRegions ? { hoverRegions: () => hoverRegions } : {}),
|
||||
});
|
||||
const invoke = (
|
||||
name: string,
|
||||
window: Window | undefined,
|
||||
...args: unknown[]
|
||||
) => handlers.get(name)?.({ sender: window?.webContents ?? {} }, ...args);
|
||||
return {
|
||||
controller,
|
||||
windows,
|
||||
invoke,
|
||||
events,
|
||||
anchor,
|
||||
timers,
|
||||
state: (window: Window) =>
|
||||
invoke('live:subagents:get-state', window) as SubagentsWindowState,
|
||||
setCursor: (point: typeof cursor) => {
|
||||
cursor = point;
|
||||
},
|
||||
setWorkArea: (area: typeof workArea) => {
|
||||
workArea = area;
|
||||
},
|
||||
advance: (milliseconds: number) => {
|
||||
const end = now + milliseconds;
|
||||
for (;;) {
|
||||
const next = [...timers].sort((a, b) => a.at - b.at)[0];
|
||||
if (!next || next.at > end) break;
|
||||
timers.delete(next);
|
||||
now = next.at;
|
||||
next.callback();
|
||||
}
|
||||
now = end;
|
||||
},
|
||||
};
|
||||
}
|
||||
describe('Subagents native lifecycle', () => {
|
||||
it('coalesces page refreshes, preserves geometry and rejects stale or foreign mutations', async () => {
|
||||
const requests: Array<{
|
||||
request: SubagentsControlRequest;
|
||||
instance: string;
|
||||
resolve: (result: SubagentsControlResult) => void;
|
||||
}> = [];
|
||||
const f = fixture(
|
||||
undefined,
|
||||
(request, instance) =>
|
||||
new Promise((resolve) => {
|
||||
requests.push({ request, instance, resolve });
|
||||
}),
|
||||
);
|
||||
const settle = () => new Promise<void>((resolve) => setImmediate(resolve));
|
||||
f.controller.update('en', true, snapshot, 'one', true);
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
assert.equal(requests.length, 0);
|
||||
f.invoke('live:subagents:expand', window);
|
||||
assert.equal(requests.length, 1);
|
||||
window.setBounds({ x: 240, y: 150, width: 330, height: 430 });
|
||||
const moves = window.moves.length;
|
||||
for (let revision = 2; revision <= 20; revision++)
|
||||
f.controller.update('en', true, { ...snapshot, revision }, 'one', true);
|
||||
assert.equal(requests.length, 1);
|
||||
requests[0]!.resolve({
|
||||
type: 'page',
|
||||
page: { snapshot, offset: 0, total: 40 },
|
||||
});
|
||||
await settle();
|
||||
assert.equal(requests.length, 2);
|
||||
requests[1]!.resolve({
|
||||
type: 'page',
|
||||
page: { snapshot: { ...snapshot, revision: 20 }, offset: 0, total: 40 },
|
||||
});
|
||||
await settle();
|
||||
assert.equal(f.state(window).page?.total, 40);
|
||||
assert.equal(window.moves.length, moves);
|
||||
const serialize = (value: unknown) =>
|
||||
JSON.parse(JSON.stringify(value)) as unknown;
|
||||
assert.deepEqual(
|
||||
serialize(
|
||||
await f.invoke('live:subagents:control', undefined, 'one', {
|
||||
action: 'stop',
|
||||
taskId: 'harness:1',
|
||||
}),
|
||||
),
|
||||
{ type: 'error', code: 'invalid_request' },
|
||||
);
|
||||
assert.deepEqual(
|
||||
serialize(
|
||||
await f.invoke('live:subagents:control', window, 'old', {
|
||||
action: 'stop',
|
||||
taskId: 'harness:1',
|
||||
}),
|
||||
),
|
||||
{ type: 'error', code: 'stale_instance' },
|
||||
);
|
||||
assert.equal(requests.length, 2);
|
||||
const stopping = f.invoke('live:subagents:control', window, 'one', {
|
||||
action: 'stop',
|
||||
taskId: 'harness:1',
|
||||
});
|
||||
assert.deepEqual(serialize(requests[2]!.request), {
|
||||
action: 'stop',
|
||||
taskId: 'harness:1',
|
||||
});
|
||||
f.controller.update('en', true, snapshot, 'two', true);
|
||||
requests[2]!.resolve({ type: 'outcome', outcome: 'stopping' });
|
||||
assert.deepEqual(serialize(await stopping), {
|
||||
type: 'error',
|
||||
code: 'stale_instance',
|
||||
});
|
||||
assert.equal(f.state(window).page, undefined);
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.dispose();
|
||||
});
|
||||
|
||||
it('discards a page reply after close and keeps controls usable while the voice call is inactive', async () => {
|
||||
let resolvePage: (result: SubagentsControlResult) => void = () => {};
|
||||
const calls: SubagentsControlRequest[] = [];
|
||||
const f = fixture(undefined, async (request) => {
|
||||
calls.push(request);
|
||||
if (request.action === 'list')
|
||||
return await new Promise((resolve) => {
|
||||
resolvePage = resolve;
|
||||
});
|
||||
return { type: 'outcome', outcome: 'denied' };
|
||||
});
|
||||
f.controller.update('en', true, snapshot, 'one', true);
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
f.invoke('live:subagents:expand', window);
|
||||
const result = await f.invoke('live:subagents:control', window, 'one', {
|
||||
action: 'permission',
|
||||
requestHandle: 'req_1',
|
||||
decision: 'deny',
|
||||
});
|
||||
assert.equal((result as SubagentsControlResult).type, 'outcome');
|
||||
f.invoke('live:subagents:close', window);
|
||||
resolvePage({ type: 'page', page: { snapshot, offset: 0, total: 1 } });
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(f.state(window).page, undefined);
|
||||
assert.equal(window.visible, false);
|
||||
assert.equal(calls.length, 2);
|
||||
f.controller.dispose();
|
||||
});
|
||||
|
||||
it('does not treat transparent placement padding as a hovered control', () => {
|
||||
const f = fixture([{ x: 1800, y: 800, width: 20, height: 20 }]);
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
f.advance(2_000);
|
||||
assert(window.visible);
|
||||
f.setCursor({ x: 1760, y: 790 });
|
||||
f.advance(1_100);
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('does not create surfaces for legacy state and never shows after a delayed load is blocked', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true);
|
||||
f.controller.setOrbHovered(true);
|
||||
assert.equal(f.windows.length, 0);
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const summary = f.windows[0]!;
|
||||
f.controller.setBlocked(true);
|
||||
summary.ready();
|
||||
assert.equal(summary.visible, false);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('uses native cursor containment and a 1000 ms backstop even when pointer-leave is lost', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const summary = f.windows[0]!;
|
||||
summary.ready();
|
||||
assert(summary.visible);
|
||||
f.advance(2_000);
|
||||
assert(summary.visible);
|
||||
f.setCursor({ x: summary.bounds.x + 20, y: summary.bounds.y + 20 });
|
||||
f.invoke('live:subagents:hover', summary, true);
|
||||
f.controller.setOrbHovered(false);
|
||||
f.advance(2_000);
|
||||
assert(summary.visible);
|
||||
f.setCursor({ x: 10, y: 10 });
|
||||
f.advance(100);
|
||||
f.advance(999);
|
||||
assert(summary.visible);
|
||||
f.advance(1);
|
||||
assert.equal(summary.visible, false);
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('resets the outside grace on native re-entry and distinguishes keyboard hold from hover', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const summary = f.windows[0]!;
|
||||
summary.ready();
|
||||
f.setCursor({ x: 10, y: 10 });
|
||||
f.advance(900);
|
||||
f.setCursor({ x: f.anchor.x + 10, y: f.anchor.y + 10 });
|
||||
f.advance(100);
|
||||
f.setCursor({ x: 10, y: 10 });
|
||||
f.advance(1_000);
|
||||
assert(summary.visible);
|
||||
f.invoke('live:subagents:keyboard', summary, true);
|
||||
f.controller.setOrbKeyboardHeld(false);
|
||||
f.advance(3_000);
|
||||
assert(summary.visible);
|
||||
summary.events.get('blur')?.();
|
||||
f.advance(1_100);
|
||||
assert.equal(summary.visible, false);
|
||||
f.controller.setOrbKeyboardHeld(true);
|
||||
assert(summary.visible);
|
||||
f.advance(3_000);
|
||||
assert(summary.visible);
|
||||
f.controller.setOrbKeyboardHeld(false);
|
||||
f.advance(1_100);
|
||||
assert.equal(summary.visible, false);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('reuses one borderless window for summary, pinned list, detail and back without moving the orb', () => {
|
||||
const f = fixture();
|
||||
const originalAnchor = { ...f.anchor };
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
assert.equal(window.options.frame, false);
|
||||
assert.equal(window.options.resizable, false);
|
||||
assert.equal(window.bounds.width, 132);
|
||||
assert.equal(window.bounds.height, 62);
|
||||
const start = f.events.length;
|
||||
f.invoke('live:subagents:expand', window);
|
||||
assert.equal(f.state(window).mode, 'list');
|
||||
assert.equal(window.bounds.width, 330);
|
||||
assert.deepEqual(
|
||||
f.events.slice(start).map((event) => event.action),
|
||||
['place', 'publish', 'show', 'focus'],
|
||||
);
|
||||
f.invoke('live:subagents:detail', window, 'harness:1');
|
||||
assert.equal(f.windows.length, 1);
|
||||
assert.equal(f.state(window).mode, 'detail');
|
||||
assert.equal(f.state(window).selectedId, 'harness:1');
|
||||
assert.equal(window.bounds.width, 330);
|
||||
assert.equal(window.bounds.height, 430);
|
||||
assert(window.bounds.x + window.bounds.width < f.anchor.x);
|
||||
window.setBounds({ x: 240, y: 150, width: 330, height: 430 });
|
||||
const moves = window.moves.length;
|
||||
f.controller.update('zh-CN', true, { ...snapshot, revision: 2 }, 'one');
|
||||
assert.equal(window.moves.length, moves);
|
||||
f.invoke('live:subagents:back', window);
|
||||
assert.equal(f.state(window).mode, 'list');
|
||||
assert.equal(f.state(window).selectedId, undefined);
|
||||
assert.equal(window.bounds.width, 330);
|
||||
assert.equal(window.bounds.x, 240);
|
||||
assert.equal(window.bounds.y, 150);
|
||||
assert.equal(f.windows.length, 1);
|
||||
assert.deepEqual(f.anchor, originalAnchor);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('keeps list and detail pinned through blur, outside cursor, drag, blocked state and disconnect until close', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
for (const mode of ['list', 'detail'] as const) {
|
||||
if (mode === 'list') f.invoke('live:subagents:expand', window);
|
||||
else f.invoke('live:subagents:detail', window, 'harness:1');
|
||||
f.controller.setOrbHovered(false);
|
||||
f.invoke('live:subagents:hover', window, false);
|
||||
window.events.get('blur')?.();
|
||||
f.setCursor({ x: 10, y: 10 });
|
||||
f.controller.setDragging(true);
|
||||
f.controller.setBlocked(true);
|
||||
f.controller.dismissPeek();
|
||||
f.controller.update('en', false);
|
||||
f.advance(5_000);
|
||||
assert(window.visible);
|
||||
assert.equal(f.state(window).mode, mode);
|
||||
assert.equal(f.state(window).connected, false);
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.controller.setDragging(false);
|
||||
f.controller.setBlocked(false);
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
}
|
||||
f.invoke('live:subagents:close', window);
|
||||
assert.equal(window.visible, false);
|
||||
assert.equal(window.destroyed, false);
|
||||
assert.equal(f.state(window).mode, 'summary');
|
||||
assert.equal(f.state(window).selectedId, undefined);
|
||||
f.controller.update('en', true, { ...snapshot, revision: 3 }, 'one');
|
||||
f.advance(5_000);
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.setOrbHovered(true);
|
||||
assert(window.visible);
|
||||
assert.equal(window.bounds.width, 132);
|
||||
assert.equal(f.windows.length, 1);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('hides only the unpinned peek for orb drag or display changes and clamps a pinned window', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
f.controller.setDragging(true);
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.setDragging(false);
|
||||
f.advance(2_000);
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.setOrbHovered(true);
|
||||
f.controller.displaysChanged();
|
||||
assert.equal(window.visible, false);
|
||||
f.controller.setOrbHovered(true);
|
||||
f.invoke('live:subagents:expand', window);
|
||||
f.setWorkArea({ x: 0, y: 0, width: 800, height: 600 });
|
||||
f.controller.displaysChanged();
|
||||
assert(window.visible);
|
||||
assert(window.bounds.x + window.bounds.width <= 800);
|
||||
assert(window.bounds.y + window.bounds.height <= 600);
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('native Escape closes the pinned panel without destroying its reusable window', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const summary = f.windows[0]!;
|
||||
summary.ready();
|
||||
f.invoke('live:subagents:detail', summary, 'harness:1');
|
||||
let prevented = false;
|
||||
summary.events.get('before-input-event')?.(
|
||||
{
|
||||
preventDefault: () => {
|
||||
prevented = true;
|
||||
},
|
||||
},
|
||||
{ type: 'keyDown', key: 'Escape' },
|
||||
);
|
||||
assert(prevented);
|
||||
assert.equal(summary.visible, false);
|
||||
assert.equal(summary.destroyed, false);
|
||||
f.controller.setOrbHovered(true);
|
||||
assert(summary.visible);
|
||||
assert.equal(f.state(summary).mode, 'summary');
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('updates theme in every mode without moving, resizing or focusing the surface', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const window = f.windows[0]!;
|
||||
window.ready();
|
||||
for (const mode of ['summary', 'list', 'detail'] as const) {
|
||||
if (mode === 'list') f.invoke('live:subagents:expand', window);
|
||||
if (mode === 'detail')
|
||||
f.invoke('live:subagents:detail', window, 'harness:1');
|
||||
const bounds = { ...window.bounds };
|
||||
const start = f.events.length;
|
||||
f.controller.setTheme('system', 'light');
|
||||
assert.equal(f.state(window).resolvedTheme, 'light');
|
||||
assert.equal(window.backgrounds.at(-1), '#f7f7fc');
|
||||
f.controller.setTheme('dark', 'dark');
|
||||
assert.equal(f.state(window).theme, 'dark');
|
||||
assert.equal(f.state(window).resolvedTheme, 'dark');
|
||||
assert.equal(window.backgrounds.at(-1), '#1b1b29');
|
||||
assert.deepEqual(
|
||||
f.events.slice(start).map((event) => event.action),
|
||||
['publish', 'publish'],
|
||||
);
|
||||
assert.deepEqual(window.bounds, bounds);
|
||||
}
|
||||
f.controller.dispose();
|
||||
});
|
||||
it('rejects foreign IPC, invalid ids and clears detail selection when daemon identity changes', () => {
|
||||
const f = fixture();
|
||||
f.controller.update('en', true, snapshot, 'one');
|
||||
f.controller.setOrbHovered(true);
|
||||
const summary = f.windows[0]!;
|
||||
summary.ready();
|
||||
assert.throws(() =>
|
||||
f.invoke('live:subagents:detail', undefined, 'harness:1'),
|
||||
);
|
||||
f.invoke('live:subagents:detail', summary, 'missing');
|
||||
f.invoke('live:subagents:detail', summary, 'x'.repeat(129));
|
||||
f.invoke('live:subagents:expand', undefined);
|
||||
assert.equal(f.state(summary).mode, 'summary');
|
||||
assert.throws(() => f.invoke('live:subagents:get-state', undefined));
|
||||
assert.equal(f.windows.length, 1);
|
||||
f.invoke('live:subagents:detail', summary, 'harness:1');
|
||||
assert.equal(f.windows.length, 1);
|
||||
f.controller.update('en', false);
|
||||
assert.equal(
|
||||
(summary.sent.at(-1) as { connected: boolean }).connected,
|
||||
false,
|
||||
);
|
||||
assert.equal(f.state(summary).selectedId, 'harness:1');
|
||||
f.invoke('live:subagents:close', undefined);
|
||||
assert(summary.visible);
|
||||
f.controller.update('en', true, snapshot, 'two');
|
||||
assert.equal(
|
||||
(summary.sent.at(-1) as { selectedId?: string }).selectedId,
|
||||
undefined,
|
||||
);
|
||||
assert.equal(f.state(summary).mode, 'summary');
|
||||
assert.equal(summary.visible, false);
|
||||
f.controller.dispose();
|
||||
assert(f.windows.every((window) => window.destroyed));
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.controller.setOrbHovered(true);
|
||||
assert.equal(f.windows.length, 1);
|
||||
assert.equal(f.invoke('live:subagents:get-state', summary), undefined);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,363 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import ts from 'typescript';
|
||||
import { liveMessage, type LiveMessageKey } from '@qwen-code/qwen-live/i18n';
|
||||
import type { HostPublicState } from '../../shared/host-api.ts';
|
||||
import {
|
||||
isLiveTheme,
|
||||
type LiveTheme,
|
||||
type ResolvedTheme,
|
||||
} from '../../shared/theme.ts';
|
||||
import { readHostTheme, saveHostTheme } from '../theme-store.ts';
|
||||
|
||||
const source = readFileSync(new URL('../index.ts', import.meta.url), 'utf8');
|
||||
const tree = ts.createSourceFile(
|
||||
'index.ts',
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
);
|
||||
const names = new Set([
|
||||
'resolvedTheme',
|
||||
'publicState',
|
||||
'publishState',
|
||||
'registerIpc',
|
||||
'isTrustedSender',
|
||||
]);
|
||||
const functions = tree.statements.filter(
|
||||
(node) =>
|
||||
ts.isFunctionDeclaration(node) && node.name && names.has(node.name.text),
|
||||
);
|
||||
assert.equal(functions.length, names.size);
|
||||
let initialization = '';
|
||||
function findThemeStartup(node: ts.Node): void {
|
||||
if (
|
||||
ts.isCallExpression(node) &&
|
||||
ts.isPropertyAccessExpression(node.expression) &&
|
||||
node.expression.name.text === 'then' &&
|
||||
node.expression.expression.getText(tree) === 'app.whenReady()'
|
||||
) {
|
||||
const callback = node.arguments[0];
|
||||
assert(
|
||||
callback && ts.isArrowFunction(callback) && ts.isBlock(callback.body),
|
||||
);
|
||||
const statements = callback.body.statements;
|
||||
const languageIndex = statements.findIndex((statement) =>
|
||||
statement.getText(tree).includes('readHostLanguage'),
|
||||
);
|
||||
assert(languageIndex > 0);
|
||||
initialization = statements
|
||||
.slice(0, languageIndex)
|
||||
.map((statement) => statement.getText(tree))
|
||||
.join('\n');
|
||||
}
|
||||
ts.forEachChild(node, findThemeStartup);
|
||||
}
|
||||
findThemeStartup(tree);
|
||||
assert.match(initialization, /readHostTheme/);
|
||||
assert.match(initialization, /nativeTheme\.themeSource/);
|
||||
assert.match(initialization, /nativeTheme\.on/);
|
||||
|
||||
const directories: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
type Handler = (...args: unknown[]) => unknown;
|
||||
const errorCode = (key: LiveMessageKey) => (error: unknown) =>
|
||||
Boolean(
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'message' in error &&
|
||||
error.message === liveMessage(key),
|
||||
);
|
||||
|
||||
function fixture(saved?: LiveTheme, systemDark = false) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'live-theme-native-'));
|
||||
directories.push(directory);
|
||||
const path = join(directory, 'theme.json');
|
||||
if (saved) saveHostTheme(path, saved);
|
||||
const handlers = new Map<string, Handler>();
|
||||
const states: HostPublicState[] = [];
|
||||
const subagentThemes: Array<[LiveTheme, ResolvedTheme]> = [];
|
||||
const subagentUpdates: unknown[][] = [];
|
||||
const writes: LiveTheme[] = [];
|
||||
const flags = {
|
||||
failSave: false,
|
||||
destroyed: false,
|
||||
webContentsDestroyed: false,
|
||||
};
|
||||
class NativeTheme extends EventEmitter {
|
||||
private source: LiveTheme = 'system';
|
||||
systemDark = systemDark;
|
||||
get themeSource() {
|
||||
return this.source;
|
||||
}
|
||||
set themeSource(value: LiveTheme) {
|
||||
this.source = value;
|
||||
}
|
||||
get shouldUseDarkColors() {
|
||||
return this.source === 'system'
|
||||
? this.systemDark
|
||||
: this.source === 'dark';
|
||||
}
|
||||
systemAppearance(dark: boolean) {
|
||||
this.systemDark = dark;
|
||||
this.emit('updated');
|
||||
}
|
||||
}
|
||||
const nativeTheme = new NativeTheme();
|
||||
const overlay = {
|
||||
isDestroyed: () => flags.destroyed,
|
||||
webContents: {
|
||||
isDestroyed: () => flags.webContentsDestroyed,
|
||||
send: (channel: string, state: HostPublicState) => {
|
||||
assert.equal(channel, 'live:state');
|
||||
states.push(JSON.parse(JSON.stringify(state)) as HostPublicState);
|
||||
},
|
||||
},
|
||||
};
|
||||
const context = {
|
||||
app: {
|
||||
getPath: (name: string) => {
|
||||
assert.equal(name, 'userData');
|
||||
return directory;
|
||||
},
|
||||
},
|
||||
join,
|
||||
readHostTheme,
|
||||
saveHostTheme: (savePath: string, theme: LiveTheme) => {
|
||||
assert.equal(savePath, path);
|
||||
if (flags.failSave) throw new Error('private-filesystem-detail');
|
||||
saveHostTheme(savePath, theme);
|
||||
writes.push(theme);
|
||||
},
|
||||
isLiveTheme,
|
||||
liveMessage,
|
||||
nativeTheme,
|
||||
overlay,
|
||||
daemon: { getConfigFilePath: () => undefined },
|
||||
overlayReady: true,
|
||||
rendererEventsEnabled: true,
|
||||
screenDisplays: [],
|
||||
screenDisplaysError: undefined,
|
||||
theme: 'system',
|
||||
language: 'zh-CN',
|
||||
quitApproved: false,
|
||||
quitState: undefined as HostPublicState['quitState'],
|
||||
overlayOffset: { x: 0, y: 0 },
|
||||
connection: {
|
||||
phase: 'ready',
|
||||
instanceId: 'one',
|
||||
memory: { enabled: true },
|
||||
},
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
screenRecording: 'granted',
|
||||
accessibility: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
visualInput: {
|
||||
source: 'camera',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
visualReady: true,
|
||||
visualError: undefined,
|
||||
effectiveLiveStatus: () => ({
|
||||
v: 1,
|
||||
available: true,
|
||||
state: 'listening',
|
||||
shortcut: 'Command+E',
|
||||
callId: 'call-1',
|
||||
}),
|
||||
subagents: {
|
||||
setTheme: (theme: LiveTheme, appearance: ResolvedTheme) =>
|
||||
subagentThemes.push([theme, appearance]),
|
||||
update: (...args: unknown[]) => subagentUpdates.push(args),
|
||||
},
|
||||
rebuildTrayMenu: () => {},
|
||||
maybeStartStartupInteraction: () => {},
|
||||
ipcMain: {
|
||||
on: (channel: string, callback: Handler) =>
|
||||
handlers.set(channel, callback),
|
||||
handle: (channel: string, callback: Handler) =>
|
||||
handlers.set(channel, callback),
|
||||
},
|
||||
};
|
||||
const code = `${functions.map((node) => node.getText(tree)).join('\n')}
|
||||
${initialization}
|
||||
registerIpc();
|
||||
({ publishState, publicState });`;
|
||||
const controls = runInNewContext(
|
||||
ts.transpileModule(code, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText,
|
||||
context,
|
||||
) as { publishState: () => void; publicState: () => HostPublicState };
|
||||
const setTheme = handlers.get('live:set-theme');
|
||||
assert(setTheme);
|
||||
return {
|
||||
context,
|
||||
controls,
|
||||
states,
|
||||
subagentThemes,
|
||||
subagentUpdates,
|
||||
nativeTheme,
|
||||
flags,
|
||||
writes,
|
||||
path,
|
||||
setTheme: (theme: unknown, sender: unknown = overlay.webContents) =>
|
||||
setTheme({ sender }, theme),
|
||||
};
|
||||
}
|
||||
|
||||
describe('native theme ownership and broadcast', () => {
|
||||
it('loads the saved preference before publishing and defaults to current system appearance', () => {
|
||||
for (const saved of [undefined, 'system', 'light', 'dark'] as const) {
|
||||
const h = fixture(saved, true);
|
||||
assert.equal(h.nativeTheme.listenerCount('updated'), 1);
|
||||
h.controls.publishState();
|
||||
const theme = saved ?? 'system';
|
||||
const appearance = saved === 'light' ? 'light' : 'dark';
|
||||
assert.equal(h.nativeTheme.themeSource, theme);
|
||||
assert.equal(h.states.at(-1)?.theme, theme);
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, appearance);
|
||||
assert.deepEqual(h.subagentThemes.at(-1), [theme, appearance]);
|
||||
assert.deepEqual(h.writes, []);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects unknown values, foreign or stale renderers, and requests during reload or Quit', () => {
|
||||
const h = fixture('dark');
|
||||
for (const value of [undefined, null, 'auto', 'Dark', 1, true, {}, []])
|
||||
assert.throws(() => h.setTheme(value), errorCode('host.theme.invalid'));
|
||||
assert.throws(
|
||||
() => h.setTheme('light', {}),
|
||||
errorCode('host.theme.invalid'),
|
||||
);
|
||||
const stale = h.context.overlay.webContents;
|
||||
h.context.overlay = { ...h.context.overlay, webContents: { ...stale } };
|
||||
assert.throws(
|
||||
() => h.setTheme('light', stale),
|
||||
errorCode('host.theme.invalid'),
|
||||
);
|
||||
const current = h.context.overlay.webContents;
|
||||
h.context.rendererEventsEnabled = false;
|
||||
assert.throws(
|
||||
() => h.setTheme('light', current),
|
||||
errorCode('host.theme.invalid'),
|
||||
);
|
||||
h.context.rendererEventsEnabled = true;
|
||||
h.flags.destroyed = true;
|
||||
assert.throws(
|
||||
() => h.setTheme('light', current),
|
||||
errorCode('host.theme.invalid'),
|
||||
);
|
||||
h.flags.destroyed = false;
|
||||
for (const quitState of ['pending', 'failed'] as const) {
|
||||
h.context.quitState = quitState;
|
||||
assert.throws(
|
||||
() => h.setTheme('light', current),
|
||||
errorCode('host.theme.unavailable'),
|
||||
);
|
||||
}
|
||||
assert.equal(readHostTheme(h.path), 'dark');
|
||||
assert.equal(h.nativeTheme.themeSource, 'dark');
|
||||
assert.deepEqual(h.writes, []);
|
||||
assert.equal(h.states.length, 0);
|
||||
assert.deepEqual(h.subagentThemes, []);
|
||||
});
|
||||
|
||||
it('publishes a saved preference to both surfaces without changing language, media or connection state', () => {
|
||||
const h = fixture('dark');
|
||||
const before = JSON.parse(
|
||||
JSON.stringify(h.controls.publicState()),
|
||||
) as HostPublicState;
|
||||
h.setTheme('light');
|
||||
assert.deepEqual(h.writes, ['light']);
|
||||
assert.equal(readHostTheme(h.path), 'light');
|
||||
assert.equal(h.nativeTheme.themeSource, 'light');
|
||||
assert.deepEqual(h.states.at(-1), {
|
||||
...before,
|
||||
theme: 'light',
|
||||
resolvedTheme: 'light',
|
||||
});
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['light', 'light']);
|
||||
assert.deepEqual(h.subagentUpdates.at(-1), [
|
||||
'zh-CN',
|
||||
true,
|
||||
undefined,
|
||||
'one',
|
||||
false,
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps all current state and the old file intact when saving fails', () => {
|
||||
const h = fixture('dark');
|
||||
h.flags.failSave = true;
|
||||
const before = JSON.stringify(h.controls.publicState());
|
||||
assert.throws(
|
||||
() => h.setTheme('light'),
|
||||
errorCode('host.theme.saveFailed'),
|
||||
);
|
||||
assert.equal(JSON.stringify(h.controls.publicState()), before);
|
||||
assert.equal(readHostTheme(h.path), 'dark');
|
||||
assert.equal(h.nativeTheme.themeSource, 'dark');
|
||||
assert.deepEqual(h.states, []);
|
||||
assert.deepEqual(h.subagentThemes, []);
|
||||
});
|
||||
|
||||
it('follows system updates in System mode and preserves explicit Light or Dark choices', () => {
|
||||
const h = fixture('system', false);
|
||||
h.nativeTheme.systemAppearance(true);
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'dark');
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['system', 'dark']);
|
||||
h.nativeTheme.systemAppearance(false);
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'light');
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['system', 'light']);
|
||||
assert.deepEqual(h.writes, []);
|
||||
h.setTheme('dark');
|
||||
h.nativeTheme.systemAppearance(false);
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'dark');
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['dark', 'dark']);
|
||||
h.setTheme('light');
|
||||
h.nativeTheme.systemAppearance(true);
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'light');
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['light', 'light']);
|
||||
h.setTheme('system');
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'dark');
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['system', 'dark']);
|
||||
assert.deepEqual(h.writes, ['dark', 'light', 'system']);
|
||||
});
|
||||
|
||||
it('updates Subagents while the main renderer reloads and stays quiet after approved Quit', () => {
|
||||
const h = fixture();
|
||||
h.context.overlayReady = false;
|
||||
h.nativeTheme.systemAppearance(true);
|
||||
assert.equal(h.states.length, 0);
|
||||
assert.deepEqual(h.subagentThemes.at(-1), ['system', 'dark']);
|
||||
assert.equal(h.controls.publicState().resolvedTheme, 'dark');
|
||||
h.context.overlayReady = true;
|
||||
h.controls.publishState();
|
||||
assert.equal(h.states.at(-1)?.resolvedTheme, 'dark');
|
||||
h.context.quitApproved = true;
|
||||
h.states.length = h.subagentThemes.length = 0;
|
||||
h.nativeTheme.systemAppearance(false);
|
||||
assert.deepEqual(h.states, []);
|
||||
assert.deepEqual(h.subagentThemes, []);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,424 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { liveMessage, liveText } from '@qwen-code/qwen-live/i18n';
|
||||
import { LiveView } from '../../renderer/live-view.ts';
|
||||
import { SubagentsView } from '../../renderer/subagents-view.ts';
|
||||
import { applyTheme } from '../../renderer/theme.ts';
|
||||
import type { HostPublicState, LiveHostApi } from '../../shared/host-api.ts';
|
||||
import type { SubagentsWindowState } from '../../shared/subagents-api.ts';
|
||||
import type { LiveTheme } from '../../shared/theme.ts';
|
||||
|
||||
const cleanup: Array<() => void> = [];
|
||||
afterEach(() => {
|
||||
for (const dispose of cleanup.splice(0).reverse()) dispose();
|
||||
});
|
||||
const settled = () => new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
function documentRoot() {
|
||||
const dom = new JSDOM('<!doctype html><main id="app"></main>');
|
||||
const previous = Object.getOwnPropertyDescriptor(globalThis, 'document');
|
||||
Object.defineProperty(globalThis, 'document', {
|
||||
configurable: true,
|
||||
value: dom.window.document,
|
||||
});
|
||||
cleanup.push(() => {
|
||||
dom.window.close();
|
||||
if (previous) Object.defineProperty(globalThis, 'document', previous);
|
||||
else Reflect.deleteProperty(globalThis, 'document');
|
||||
});
|
||||
return {
|
||||
dom,
|
||||
app: dom.window.document.querySelector<HTMLElement>('#app')!,
|
||||
};
|
||||
}
|
||||
|
||||
function host(overrides: Partial<LiveHostApi> = {}) {
|
||||
const { dom, app } = documentRoot();
|
||||
const state: HostPublicState = {
|
||||
language: 'en',
|
||||
connection: 'ready',
|
||||
live: { v: 1, available: true, state: 'idle', shortcut: 'Command+E' },
|
||||
permissions: {
|
||||
microphone: 'granted',
|
||||
camera: 'granted',
|
||||
accessibility: 'granted',
|
||||
screenRecording: 'granted',
|
||||
},
|
||||
selfChecks: {
|
||||
audioInput: true,
|
||||
audioOutput: true,
|
||||
globalShortcut: true,
|
||||
appshot: true,
|
||||
},
|
||||
visualInput: {
|
||||
source: 'camera',
|
||||
mode: 'on-demand',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
},
|
||||
memory: {
|
||||
enabled: true,
|
||||
visualEnabled: false,
|
||||
libraryId: 'default',
|
||||
model: 'qwen3.7-plus',
|
||||
libraries: [{ id: 'default', name: 'Default' }],
|
||||
locked: false,
|
||||
},
|
||||
visualReady: true,
|
||||
};
|
||||
const themes: LiveTheme[] = [];
|
||||
let previews = 0;
|
||||
const api: LiveHostApi = {
|
||||
toggle: async () => {},
|
||||
stop: async () => {},
|
||||
quit: async () => {},
|
||||
newConversation: async () => {},
|
||||
setInputMuted: async () => {},
|
||||
setOutputMuted: async () => {},
|
||||
setVisualSource: async () => {},
|
||||
setVisualMode: async () => {},
|
||||
setScreenDisplay: async () => {},
|
||||
memoryAction: async () => {
|
||||
throw new Error('Theme updates must not change memory');
|
||||
},
|
||||
setLanguage: async () => {
|
||||
throw new Error('Theme updates must not change language');
|
||||
},
|
||||
setTheme: async (theme) => {
|
||||
themes.push(theme);
|
||||
},
|
||||
setSettingsOpen: async () => {},
|
||||
openConfig: async () => {},
|
||||
setOverlayLayout: () => {},
|
||||
onSettingsDismiss: () => () => {},
|
||||
onOverlayOffset: () => () => {},
|
||||
dragOverlay: () => {},
|
||||
attachCameraPreview: () => {
|
||||
previews++;
|
||||
},
|
||||
requestPermission: async () => {},
|
||||
listInputDevices: async () => [],
|
||||
setInputDevice: async () => {},
|
||||
openWebShellForPermission: async () => {},
|
||||
getState: async () => state,
|
||||
onInputLevel: () => () => {},
|
||||
onState: () => () => {},
|
||||
...overrides,
|
||||
};
|
||||
const view = new LiveView(app, api);
|
||||
cleanup.push(() => view.dispose());
|
||||
view.update(state);
|
||||
const get = <T extends HTMLElement = HTMLElement>(selector: string): T => {
|
||||
const node = app.querySelector<T>(selector);
|
||||
assert(node, `Missing ${selector}`);
|
||||
return node;
|
||||
};
|
||||
const update = (next: Partial<HostPublicState>) => {
|
||||
Object.assign(state, next);
|
||||
view.update({ ...state });
|
||||
};
|
||||
return { dom, app, get, update, themes, previews: () => previews };
|
||||
}
|
||||
|
||||
describe('Live Host theme settings', () => {
|
||||
it('offers a persistent display choice without changing Camera or init and retains unavailable selections', async () => {
|
||||
const selections: string[] = [];
|
||||
const h = host({
|
||||
setScreenDisplay: async (id) => {
|
||||
selections.push(id);
|
||||
},
|
||||
});
|
||||
const id = '11223344-5566-7788-99aa-bbccddeeff00';
|
||||
const settings = {
|
||||
source: 'screen',
|
||||
mode: 'live-feed',
|
||||
fps: 1,
|
||||
liveWidth: 1280,
|
||||
liveHeight: 720,
|
||||
screenDisplayId: 'primary',
|
||||
} as const;
|
||||
h.update({
|
||||
visualInput: settings,
|
||||
canSelectScreenDisplay: true,
|
||||
screenDisplays: [
|
||||
{
|
||||
id,
|
||||
name: 'Studio Display',
|
||||
width: 5120,
|
||||
height: 2880,
|
||||
primary: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.get<HTMLButtonElement>('.settings-control').click();
|
||||
await settled();
|
||||
const display = h.get<HTMLSelectElement>('select[aria-label="Display"]');
|
||||
assert.equal(display.value, 'primary');
|
||||
assert.equal(display.options[0]?.textContent, 'Primary display');
|
||||
assert.match(display.options[1]?.textContent ?? '', /Studio Display.*5120/);
|
||||
display.value = id;
|
||||
display.dispatchEvent(new h.dom.window.Event('change', { bubbles: true }));
|
||||
await settled();
|
||||
assert.deepEqual(selections, [id]);
|
||||
assert.equal(display.value, 'primary');
|
||||
h.update({
|
||||
visualInput: { ...settings, screenDisplayId: id },
|
||||
language: 'zh-CN',
|
||||
});
|
||||
assert.equal(display.value, id);
|
||||
assert.equal(display.getAttribute('aria-label'), '显示器');
|
||||
h.update({ screenDisplays: [] });
|
||||
h.update({ visualSettingsError: liveMessage('runtime.displaySaveFailed') });
|
||||
assert.match(
|
||||
h.get('.settings-status').textContent ?? '',
|
||||
/无法保存显示器选择/,
|
||||
);
|
||||
assert.equal(display.value, id);
|
||||
assert.match(display.selectedOptions[0]?.textContent ?? '', /不可用/);
|
||||
h.update({ visualInput: { ...settings, source: 'camera' } });
|
||||
assert.equal(display.closest<HTMLElement>('.settings-field')?.hidden, true);
|
||||
});
|
||||
|
||||
it('places Theme after Language, defaults to System and sends each preference', async () => {
|
||||
const h = host();
|
||||
h.get<HTMLButtonElement>('.settings-control').click();
|
||||
await settled();
|
||||
const fields = h.app.querySelectorAll('.settings-body > .settings-field');
|
||||
assert.equal(
|
||||
fields[fields.length - 2]?.firstChild?.textContent,
|
||||
'Language',
|
||||
);
|
||||
assert.equal(fields[fields.length - 1]?.firstChild?.textContent, 'Theme');
|
||||
assert.equal(
|
||||
h.get('[data-theme="system"]').getAttribute('aria-pressed'),
|
||||
'true',
|
||||
);
|
||||
for (const theme of ['light', 'dark', 'system'] as const) {
|
||||
h.get<HTMLButtonElement>(`[data-theme="${theme}"]`).click();
|
||||
await settled();
|
||||
}
|
||||
assert.deepEqual(h.themes, ['light', 'dark', 'system']);
|
||||
assert.equal(
|
||||
h.get('[data-theme="system"]').getAttribute('aria-pressed'),
|
||||
'true',
|
||||
);
|
||||
assert.equal(h.dom.window.document.documentElement.dataset.theme, 'dark');
|
||||
h.update({ theme: 'system', resolvedTheme: 'light', language: 'zh-CN' });
|
||||
assert.equal(h.get('[data-theme="system"]').textContent, '跟随系统');
|
||||
assert.equal(h.get('[data-theme="light"]').textContent, '白天模式');
|
||||
assert.equal(h.get('[data-theme="dark"]').textContent, '黑暗模式');
|
||||
assert.equal(
|
||||
h.get('[data-theme="system"]').getAttribute('aria-pressed'),
|
||||
'true',
|
||||
);
|
||||
assert.equal(h.get('[data-language="en"]').textContent, 'English');
|
||||
assert.equal(h.get('[data-language="zh-CN"]').textContent, '简体中文');
|
||||
});
|
||||
|
||||
it('keeps the saved preference selected when saving fails and restores controls', async () => {
|
||||
const h = host({
|
||||
setTheme: async () => {
|
||||
throw new Error(liveMessage('host.theme.saveFailed'));
|
||||
},
|
||||
});
|
||||
h.update({ theme: 'dark', resolvedTheme: 'dark', language: 'zh-CN' });
|
||||
h.get<HTMLButtonElement>('.settings-control').click();
|
||||
await settled();
|
||||
const light = h.get<HTMLButtonElement>('[data-theme="light"]');
|
||||
light.click();
|
||||
assert.equal(light.disabled, true);
|
||||
await settled();
|
||||
assert.equal(light.disabled, false);
|
||||
assert.equal(
|
||||
h.get('[data-theme="dark"]').getAttribute('aria-pressed'),
|
||||
'true',
|
||||
);
|
||||
assert.equal(
|
||||
h.get('.settings-status.error').textContent,
|
||||
liveText('zh-CN', 'host.theme.saveFailed'),
|
||||
);
|
||||
assert.equal(h.dom.window.document.documentElement.dataset.theme, 'dark');
|
||||
});
|
||||
|
||||
it('applies the same resolved appearance to every surface without replacing media or task nodes', () => {
|
||||
const h = host();
|
||||
const orb = h.get('.voice-orb');
|
||||
const slot = h.get('.camera-preview-slot');
|
||||
const video = h.dom.window.document.createElement('video');
|
||||
slot.append(video);
|
||||
const model = h.get<HTMLInputElement>('.memory-model-form input');
|
||||
model.value = 'Keep my model draft';
|
||||
model.dispatchEvent(new h.dom.window.Event('input', { bubbles: true }));
|
||||
const { dom, app } = documentRoot();
|
||||
const view = new SubagentsView(app, {
|
||||
getState: async () => ({ language: 'en', connected: true, mode: 'list' }),
|
||||
onState: () => () => {},
|
||||
setHover: () => {},
|
||||
back: async () => {},
|
||||
expand: async () => {},
|
||||
close: () => {},
|
||||
openDetail: async () => {},
|
||||
control: async () => ({ type: 'error', code: 'unsupported' }),
|
||||
});
|
||||
cleanup.push(() => view.dispose());
|
||||
for (const mode of ['summary', 'list', 'detail'] as const) {
|
||||
for (const resolvedTheme of ['light', 'dark'] as const) {
|
||||
h.update({ theme: 'system', resolvedTheme });
|
||||
const state: SubagentsWindowState = {
|
||||
language: 'en',
|
||||
connected: true,
|
||||
mode,
|
||||
theme: 'system',
|
||||
resolvedTheme,
|
||||
selectedId: 'theme-task',
|
||||
snapshot: {
|
||||
revision: 1,
|
||||
counts: {
|
||||
running: 1,
|
||||
completed: 0,
|
||||
needsAttention: 0,
|
||||
failed: 0,
|
||||
cancelled: 0,
|
||||
interrupted: 0,
|
||||
},
|
||||
omitted: 0,
|
||||
tasks: [
|
||||
{
|
||||
id: 'theme-task',
|
||||
kind: 'harness',
|
||||
title: 'Keep task nodes',
|
||||
status: 'running',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
request: 'Original request',
|
||||
output: 'Original output',
|
||||
activity: 'Still running',
|
||||
events: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
view.update(state);
|
||||
const children = Array.from(app.children);
|
||||
const task = app.querySelector('.subagent-task');
|
||||
const output = app.querySelector('.subagent-output');
|
||||
view.update({
|
||||
...state,
|
||||
resolvedTheme: resolvedTheme === 'light' ? 'dark' : 'light',
|
||||
});
|
||||
view.update(state);
|
||||
assert.equal(
|
||||
dom.window.document.documentElement.dataset.theme,
|
||||
resolvedTheme,
|
||||
);
|
||||
assert.equal(
|
||||
h.dom.window.document.documentElement.dataset.theme,
|
||||
resolvedTheme,
|
||||
);
|
||||
assert.deepEqual(Array.from(app.children), children);
|
||||
assert.equal(app.querySelector('.subagent-task'), task);
|
||||
assert.equal(app.querySelector('.subagent-output'), output);
|
||||
assert.equal(h.get('.voice-orb'), orb);
|
||||
assert.equal(h.get('.camera-preview-slot').firstChild, video);
|
||||
assert.equal(model.value, 'Keep my model draft');
|
||||
assert.equal(h.get('.memory-model-form input'), model);
|
||||
}
|
||||
}
|
||||
assert.equal(h.previews(), 1);
|
||||
});
|
||||
|
||||
it('does not mutate the document for an unchanged appearance', () => {
|
||||
const { dom } = documentRoot();
|
||||
const document = dom.window.document;
|
||||
applyTheme(document);
|
||||
const observer = new dom.window.MutationObserver(() => {});
|
||||
observer.observe(document.documentElement, { attributes: true });
|
||||
applyTheme(document, 'dark');
|
||||
assert.equal(observer.takeRecords().length, 0);
|
||||
applyTheme(document, 'light');
|
||||
assert.equal(observer.takeRecords().length, 1);
|
||||
observer.disconnect();
|
||||
});
|
||||
});
|
||||
|
||||
describe('shared theme palette', () => {
|
||||
const read = (file: string) =>
|
||||
readFileSync(new URL(`../../renderer/${file}`, import.meta.url), 'utf8');
|
||||
const theme = read('theme.css');
|
||||
const tokens = (body: string) =>
|
||||
Object.fromEntries(
|
||||
Array.from(body.matchAll(/(--live-[\w-]+):\s*([^;]+);/g), (match) => [
|
||||
match[1]!,
|
||||
match[2]!,
|
||||
]),
|
||||
);
|
||||
const dark = tokens(theme.match(/:root\s*\{([^}]+)\}/)![1]!);
|
||||
const light = tokens(
|
||||
theme.match(/:root\[data-theme='light'\]\s*\{([^}]+)\}/)![1]!,
|
||||
);
|
||||
|
||||
it('defines both appearances for every semantic token and leaves literal colors only in the brand orb', () => {
|
||||
assert.deepEqual(Object.keys(dark).sort(), Object.keys(light).sort());
|
||||
for (const name of ['style.css', 'subagents.css']) {
|
||||
const css = read(name);
|
||||
assert.match(css, /@import '\.\/theme\.css'/);
|
||||
for (const match of css.matchAll(/var\((--live-[\w-]+)/g))
|
||||
assert(match[1]! in dark, `Undefined ${match[1]}`);
|
||||
const surfaceRules = css.replace(/[^{}]+\{[^{}]+\}/g, (rule) =>
|
||||
rule.split('{')[0]!.includes('orb-core') ? '' : rule,
|
||||
);
|
||||
assert.doesNotMatch(surfaceRules, /#[\da-f]{3,8}\b|rgba?\(/i);
|
||||
}
|
||||
assert.match(
|
||||
read('style.css'),
|
||||
/linear-gradient\(145deg, #8bd7ed, #8671ce 72%, #b982d4\)/,
|
||||
);
|
||||
assert.match(dark['--live-status-bg']!, /\/ 68%\)/);
|
||||
assert.match(light['--live-status-bg']!, /\/ 96%\)/);
|
||||
});
|
||||
|
||||
it('keeps light text, errors, statuses and primary buttons readable', () => {
|
||||
const rgb = (hex: string) =>
|
||||
hex
|
||||
.replace('#', '')
|
||||
.match(/../g)!
|
||||
.map((v) => parseInt(v, 16));
|
||||
const luminance = (color: number[]) =>
|
||||
color
|
||||
.map((v) => v / 255)
|
||||
.map((v) => (v <= 0.04045 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4))
|
||||
.reduce(
|
||||
(sum, value, i) => sum + value * [0.2126, 0.7152, 0.0722][i]!,
|
||||
0,
|
||||
);
|
||||
const contrast = (fg: number[], bg: number[]) => {
|
||||
const [a, b] = [luminance(fg), luminance(bg)].sort((a, b) => b - a);
|
||||
return (a! + 0.05) / (b! + 0.05);
|
||||
};
|
||||
for (const color of [
|
||||
'text',
|
||||
'muted',
|
||||
'subtle',
|
||||
'error',
|
||||
'success',
|
||||
'warning',
|
||||
]) {
|
||||
assert(
|
||||
contrast(
|
||||
rgb(light[`--live-${color}`]!),
|
||||
rgb(light['--live-surface']!),
|
||||
) >= 4.5,
|
||||
color,
|
||||
);
|
||||
}
|
||||
for (const desktop of [0, 255]) {
|
||||
const background = [249, 250, 255].map((v) => v * 0.96 + desktop * 0.04);
|
||||
assert(contrast(rgb(light['--live-text']!), background) >= 4.5);
|
||||
assert(contrast(rgb(light['--live-error']!), background) >= 4.5);
|
||||
}
|
||||
assert(contrast([255, 255, 255], rgb(light['--live-primary-bg']!)) >= 4.5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, it } from 'node:test';
|
||||
import { readHostTheme, saveHostTheme } from '../theme-store.ts';
|
||||
import { isLiveTheme, LIVE_THEMES } from '../../shared/theme.ts';
|
||||
|
||||
const directories: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
function directory(): string {
|
||||
const path = mkdtempSync(join(tmpdir(), 'live-theme-store-'));
|
||||
directories.push(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
describe('Host-local theme preferences', () => {
|
||||
it('accepts only the three exact preference values', () => {
|
||||
for (const value of LIVE_THEMES) assert.equal(isLiveTheme(value), true);
|
||||
for (const value of [
|
||||
undefined,
|
||||
null,
|
||||
true,
|
||||
1,
|
||||
'',
|
||||
'auto',
|
||||
'Dark',
|
||||
' light ',
|
||||
[],
|
||||
{},
|
||||
{ theme: 'dark' },
|
||||
]) {
|
||||
assert.equal(isLiveTheme(value), false);
|
||||
}
|
||||
});
|
||||
|
||||
it('round-trips each preference privately without touching language settings', () => {
|
||||
const root = directory();
|
||||
const data = join(root, 'user-data');
|
||||
const path = join(data, 'theme.json');
|
||||
assert.equal(readHostTheme(path), 'system');
|
||||
saveHostTheme(path, 'light');
|
||||
const languagePath = join(data, 'language.json');
|
||||
writeFileSync(languagePath, '{"language":"zh-CN"}', { mode: 0o600 });
|
||||
for (const theme of LIVE_THEMES) {
|
||||
saveHostTheme(path, theme);
|
||||
assert.equal(readHostTheme(path), theme);
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, 'utf8')), { theme });
|
||||
assert.equal(readFileSync(languagePath, 'utf8'), '{"language":"zh-CN"}');
|
||||
assert.deepEqual(readdirSync(data).sort(), [
|
||||
'language.json',
|
||||
'theme.json',
|
||||
]);
|
||||
}
|
||||
if (process.platform !== 'win32') {
|
||||
assert.equal(statSync(data).mode & 0o777, 0o700);
|
||||
assert.equal(statSync(path).mode & 0o777, 0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to System for missing, malformed, unknown or non-object preferences', () => {
|
||||
const path = join(directory(), 'theme.json');
|
||||
assert.equal(readHostTheme(path), 'system');
|
||||
for (const contents of [
|
||||
'{',
|
||||
'null',
|
||||
'true',
|
||||
'1',
|
||||
'"dark"',
|
||||
'[]',
|
||||
'{}',
|
||||
'{"language":"zh-CN"}',
|
||||
'{"theme":"auto"}',
|
||||
'{"theme":"Dark"}',
|
||||
'{"theme":null}',
|
||||
'{"theme":false}',
|
||||
'{"theme":1}',
|
||||
'{"theme":[]}',
|
||||
'{"theme":{"theme":"dark"}}',
|
||||
]) {
|
||||
writeFileSync(path, contents);
|
||||
assert.equal(readHostTheme(path), 'system', contents);
|
||||
}
|
||||
assert.equal(readHostTheme(directory()), 'system');
|
||||
});
|
||||
|
||||
it('rejects invalid saves before writing or replacing the previous preference', () => {
|
||||
const root = directory();
|
||||
const path = join(root, 'theme.json');
|
||||
saveHostTheme(path, 'dark');
|
||||
for (const value of [undefined, null, 'auto', 'Dark', 1, true, {}, []]) {
|
||||
assert.throws(() => saveHostTheme(path, value as never), TypeError);
|
||||
assert.equal(readHostTheme(path), 'dark');
|
||||
assert.deepEqual(readdirSync(root), ['theme.json']);
|
||||
}
|
||||
const missing = join(root, 'uncreated', 'theme.json');
|
||||
assert.throws(() => saveHostTheme(missing, 'auto' as never), TypeError);
|
||||
assert.deepEqual(readdirSync(root), ['theme.json']);
|
||||
});
|
||||
|
||||
it('reports filesystem failures and cleans temporary files after failed replacement', () => {
|
||||
const root = directory();
|
||||
const path = join(root, 'theme.json');
|
||||
mkdirSync(path);
|
||||
writeFileSync(join(path, 'keep.txt'), 'Keep this directory intact');
|
||||
assert.throws(() => saveHostTheme(path, 'light'));
|
||||
assert.deepEqual(readdirSync(root), ['theme.json']);
|
||||
assert.equal(
|
||||
readFileSync(join(path, 'keep.txt'), 'utf8'),
|
||||
'Keep this directory intact',
|
||||
);
|
||||
const blockedParent = join(root, 'file-not-directory');
|
||||
writeFileSync(blockedParent, 'unchanged');
|
||||
assert.throws(() =>
|
||||
saveHostTheme(join(blockedParent, 'theme.json'), 'dark'),
|
||||
);
|
||||
assert.equal(readFileSync(blockedParent, 'utf8'), 'unchanged');
|
||||
});
|
||||
});
|
||||
@@ -2,23 +2,35 @@ import { randomUUID } from 'node:crypto';
|
||||
import { lstat, mkdir, readdir, unlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { liveMessage } from '@qwen-code/qwen-live/i18n';
|
||||
import { MAX_CAPTURE_ASSET_BYTES } from '../shared/protocol.ts';
|
||||
import {
|
||||
loadNativeAppshot,
|
||||
type NativeAppshot,
|
||||
type NativeAppshotCapture,
|
||||
type NativeDisplay,
|
||||
type NativeDisplayCapture,
|
||||
} from './native-appshot.ts';
|
||||
|
||||
const MAX_APP_NAME_CHARS = 512;
|
||||
const MAX_WINDOW_TITLE_CHARS = 2_048;
|
||||
const MAX_ACCESSIBILITY_TEXT_CHARS = 32_000;
|
||||
const MAX_SCREENSHOT_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_SCREENSHOT_BYTES = MAX_CAPTURE_ASSET_BYTES;
|
||||
const CAPTURE_FILE_TTL_MS = 60_000;
|
||||
const DISPLAY_UUID =
|
||||
/^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/iu;
|
||||
|
||||
export interface AppshotCapture {
|
||||
function isDisplayUuid(value: unknown): value is string {
|
||||
return (
|
||||
typeof value === 'string' && value.length === 36 && DISPLAY_UUID.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
export interface AppshotFrame {
|
||||
appName: string;
|
||||
windowTitle?: string;
|
||||
accessibilityText: string;
|
||||
screenshotPath: string;
|
||||
screenshot: Uint8Array;
|
||||
}
|
||||
|
||||
function boundedText(value: unknown, maximum: number, field: string): string {
|
||||
@@ -30,7 +42,7 @@ function boundedText(value: unknown, maximum: number, field: string): string {
|
||||
|
||||
export function validateNativeCapture(
|
||||
value: NativeAppshotCapture,
|
||||
): Omit<AppshotCapture, 'screenshotPath'> & { screenshot: Uint8Array } {
|
||||
): AppshotFrame {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
@@ -58,8 +70,45 @@ export function validateNativeCapture(
|
||||
};
|
||||
}
|
||||
|
||||
export function validateNativeDisplayCapture(
|
||||
value: NativeDisplayCapture,
|
||||
requestedDisplay: string,
|
||||
): NativeDisplayCapture {
|
||||
if (
|
||||
!value ||
|
||||
!isDisplayUuid(value.displayId) ||
|
||||
(requestedDisplay !== 'primary' &&
|
||||
value.displayId.toLowerCase() !== requestedDisplay.toLowerCase())
|
||||
)
|
||||
throw new Error(liveMessage('host.error.displayUnavailable'));
|
||||
const screenshot = value.screenshot;
|
||||
const signature = [137, 80, 78, 71, 13, 10, 26, 10];
|
||||
if (
|
||||
!(screenshot instanceof Uint8Array) ||
|
||||
screenshot.byteLength < 33 ||
|
||||
screenshot.byteLength > MAX_SCREENSHOT_BYTES ||
|
||||
signature.some((byte, index) => screenshot[index] !== byte)
|
||||
)
|
||||
throw new Error(liveMessage('host.error.displayCapture'));
|
||||
const header = new DataView(
|
||||
screenshot.buffer,
|
||||
screenshot.byteOffset,
|
||||
screenshot.byteLength,
|
||||
);
|
||||
if (
|
||||
header.getUint32(8) !== 13 ||
|
||||
header.getUint32(12) !== 0x49484452 ||
|
||||
header.getUint32(16) < 1 ||
|
||||
header.getUint32(16) > 1920 ||
|
||||
header.getUint32(20) < 1 ||
|
||||
header.getUint32(20) > 1080
|
||||
)
|
||||
throw new Error(liveMessage('host.error.displayCapture'));
|
||||
return { displayId: value.displayId.toLowerCase(), screenshot };
|
||||
}
|
||||
|
||||
export class AppshotCaptureService {
|
||||
private capturing = false;
|
||||
private captureTail?: Promise<void>;
|
||||
private readonly cleanupTimers = new Map<NodeJS.Timeout, string>();
|
||||
|
||||
constructor(
|
||||
@@ -67,57 +116,119 @@ export class AppshotCaptureService {
|
||||
private readonly native: () => NativeAppshot = loadNativeAppshot,
|
||||
) {}
|
||||
|
||||
async capture(): Promise<AppshotCapture> {
|
||||
if (this.capturing) {
|
||||
throw new Error('An Appshot capture is already in progress.');
|
||||
}
|
||||
this.capturing = true;
|
||||
let screenshotPath: string | undefined;
|
||||
try {
|
||||
await mkdir(this.captureDirectory, { recursive: true, mode: 0o700 });
|
||||
const directoryStat = await lstat(this.captureDirectory);
|
||||
if (
|
||||
!directoryStat.isDirectory() ||
|
||||
directoryStat.isSymbolicLink() ||
|
||||
(directoryStat.mode & 0o077) !== 0
|
||||
) {
|
||||
throw new Error('The Appshot capture directory is not private.');
|
||||
}
|
||||
await this.removeStaleCaptures();
|
||||
captureFrame(): Promise<AppshotFrame> {
|
||||
return this.queueCapture(() => this.captureFrameNow());
|
||||
}
|
||||
|
||||
const capture = validateNativeCapture(
|
||||
await this.native().captureAppshot(),
|
||||
);
|
||||
screenshotPath = join(this.captureDirectory, `${randomUUID()}.png`);
|
||||
await writeFile(screenshotPath, capture.screenshot, {
|
||||
flag: 'wx',
|
||||
mode: 0o600,
|
||||
listDisplays(): NativeDisplay[] {
|
||||
try {
|
||||
const displays = this.native().listDisplays();
|
||||
if (!Array.isArray(displays)) throw new Error('Invalid display list');
|
||||
const seen = new Set<string>();
|
||||
let primary = false;
|
||||
return displays.map((display) => {
|
||||
if (
|
||||
!display ||
|
||||
!isDisplayUuid(display.id) ||
|
||||
seen.has(display.id.toLowerCase()) ||
|
||||
typeof display.name !== 'string' ||
|
||||
!display.name.trim() ||
|
||||
!Number.isSafeInteger(display.width) ||
|
||||
display.width < 1 ||
|
||||
!Number.isSafeInteger(display.height) ||
|
||||
display.height < 1 ||
|
||||
typeof display.primary !== 'boolean' ||
|
||||
(display.primary && primary)
|
||||
)
|
||||
throw new Error('Invalid display');
|
||||
seen.add(display.id.toLowerCase());
|
||||
primary ||= display.primary;
|
||||
return {
|
||||
...display,
|
||||
id: display.id.toLowerCase(),
|
||||
name: display.name.trim().slice(0, 256),
|
||||
};
|
||||
});
|
||||
const stat = await lstat(screenshotPath);
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
stat.size <= 0 ||
|
||||
stat.size > MAX_SCREENSHOT_BYTES ||
|
||||
(stat.mode & 0o077) !== 0
|
||||
) {
|
||||
throw new Error('Native Appshot wrote an invalid screenshot file.');
|
||||
}
|
||||
const result: AppshotCapture = {
|
||||
appName: capture.appName,
|
||||
...(capture.windowTitle ? { windowTitle: capture.windowTitle } : {}),
|
||||
accessibilityText: capture.accessibilityText,
|
||||
screenshotPath,
|
||||
};
|
||||
this.scheduleCleanup(screenshotPath);
|
||||
screenshotPath = undefined;
|
||||
return result;
|
||||
} finally {
|
||||
this.capturing = false;
|
||||
if (screenshotPath) await unlink(screenshotPath).catch(() => undefined);
|
||||
} catch {
|
||||
throw new Error(liveMessage('host.error.displayList'));
|
||||
}
|
||||
}
|
||||
|
||||
captureDisplayFrame(displayId = 'primary'): Promise<NativeDisplayCapture> {
|
||||
return this.queueCapture(async () => {
|
||||
if (displayId !== 'primary' && !isDisplayUuid(displayId))
|
||||
throw new Error(liveMessage('host.error.displayUnavailable'));
|
||||
let capture: NativeDisplayCapture;
|
||||
try {
|
||||
capture = await this.native().captureDisplay(displayId.toLowerCase());
|
||||
} catch (error) {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? error.code
|
||||
: undefined;
|
||||
throw new Error(
|
||||
liveMessage(
|
||||
code === 'DISPLAY_UNAVAILABLE'
|
||||
? 'host.error.displayUnavailable'
|
||||
: code === 'DISPLAY_PERMISSION'
|
||||
? 'runtime.screenPermission'
|
||||
: 'host.error.displayCapture',
|
||||
),
|
||||
);
|
||||
}
|
||||
return validateNativeDisplayCapture(capture, displayId);
|
||||
});
|
||||
}
|
||||
|
||||
private queueCapture<T>(captureNow: () => Promise<T>): Promise<T> {
|
||||
const capture = this.captureTail
|
||||
? this.captureTail.then(captureNow)
|
||||
: captureNow();
|
||||
const tail = capture.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
this.captureTail = tail;
|
||||
void tail.then(() => {
|
||||
if (this.captureTail === tail) this.captureTail = undefined;
|
||||
});
|
||||
return capture;
|
||||
}
|
||||
|
||||
async storeJpeg(image: Uint8Array): Promise<string> {
|
||||
if (
|
||||
image.byteLength < 4 ||
|
||||
image.byteLength > MAX_SCREENSHOT_BYTES ||
|
||||
image[0] !== 0xff ||
|
||||
image[1] !== 0xd8 ||
|
||||
image[image.byteLength - 2] !== 0xff ||
|
||||
image[image.byteLength - 1] !== 0xd9
|
||||
) {
|
||||
throw new Error('Camera returned an invalid JPEG screenshot.');
|
||||
}
|
||||
await this.prepareCaptureDirectory();
|
||||
const path = join(this.captureDirectory, `${randomUUID()}.jpg`);
|
||||
await this.writePrivateCapture(path, image);
|
||||
this.scheduleCleanup(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
async storePng(image: Uint8Array): Promise<string> {
|
||||
const signature = [137, 80, 78, 71, 13, 10, 26, 10];
|
||||
if (
|
||||
image.byteLength <= signature.length ||
|
||||
image.byteLength > MAX_SCREENSHOT_BYTES ||
|
||||
signature.some((byte, index) => image[index] !== byte)
|
||||
) {
|
||||
throw new Error('Appshot returned an invalid PNG screenshot.');
|
||||
}
|
||||
await this.prepareCaptureDirectory();
|
||||
const path = join(this.captureDirectory, `${randomUUID()}.png`);
|
||||
await this.writePrivateCapture(path, image);
|
||||
this.scheduleCleanup(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
for (const [timer, path] of this.cleanupTimers) {
|
||||
clearTimeout(timer);
|
||||
@@ -126,6 +237,10 @@ export class AppshotCaptureService {
|
||||
this.cleanupTimers.clear();
|
||||
}
|
||||
|
||||
private async captureFrameNow(): Promise<AppshotFrame> {
|
||||
return validateNativeCapture(await this.native().captureAppshot());
|
||||
}
|
||||
|
||||
private scheduleCleanup(path: string): void {
|
||||
const timer = setTimeout(() => {
|
||||
this.cleanupTimers.delete(timer);
|
||||
@@ -135,6 +250,41 @@ export class AppshotCaptureService {
|
||||
this.cleanupTimers.set(timer, path);
|
||||
}
|
||||
|
||||
private async prepareCaptureDirectory(): Promise<void> {
|
||||
await mkdir(this.captureDirectory, { recursive: true, mode: 0o700 });
|
||||
const directoryStat = await lstat(this.captureDirectory);
|
||||
if (
|
||||
!directoryStat.isDirectory() ||
|
||||
directoryStat.isSymbolicLink() ||
|
||||
(directoryStat.mode & 0o077) !== 0
|
||||
) {
|
||||
throw new Error('The Appshot capture directory is not private.');
|
||||
}
|
||||
await this.removeStaleCaptures();
|
||||
}
|
||||
|
||||
private async writePrivateCapture(
|
||||
path: string,
|
||||
image: Uint8Array,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await writeFile(path, image, { flag: 'wx', mode: 0o600 });
|
||||
const stat = await lstat(path);
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
stat.size <= 0 ||
|
||||
stat.size > MAX_SCREENSHOT_BYTES ||
|
||||
(stat.mode & 0o077) !== 0
|
||||
) {
|
||||
throw new Error('Appshot wrote an invalid screenshot file.');
|
||||
}
|
||||
} catch (error) {
|
||||
await unlink(path).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async removeStaleCaptures(): Promise<void> {
|
||||
const entries = await readdir(this.captureDirectory, {
|
||||
withFileTypes: true,
|
||||
@@ -142,7 +292,11 @@ export class AppshotCaptureService {
|
||||
const now = Date.now();
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
if (!entry.isFile() || !entry.name.endsWith('.png')) return;
|
||||
if (
|
||||
!entry.isFile() ||
|
||||
(!entry.name.endsWith('.png') && !entry.name.endsWith('.jpg'))
|
||||
)
|
||||
return;
|
||||
const path = join(this.captureDirectory, entry.name);
|
||||
const stat = await lstat(path).catch(() => undefined);
|
||||
if (stat && now - stat.mtimeMs > CAPTURE_FILE_TTL_MS) {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Loaded 100 of 247 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user