mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-11-07 14:38:23 -10:00
Merge origin/main into ci/retain-raw-junit-reports
This commit is contained in:
yiliang114 committed
commit
e7f6f7ae45
35 files changed
+1284
-383
No files matched your search
@@ -4,7 +4,7 @@
|
||||
//
|
||||
// classify_pr carries the routing logic TWICE — the `runs-on` expression
|
||||
// (which selects the classify job's own runner) and the `pick_runner` shell
|
||||
// step (which publishes `ubuntu_runner` for every downstream Linux job). If
|
||||
// step (which publishes `ubuntu_runner` for downstream Linux jobs). If
|
||||
// they drift, classify and the Test job land on different pools. These tests
|
||||
// evaluate BOTH against the same event matrix — including the negative
|
||||
// associations that must stay hosted — and assert they agree.
|
||||
@@ -118,7 +118,14 @@ function evalRunsOn(expression, { ecsDisabled, eventName, sameRepo, assoc }) {
|
||||
|
||||
// Executes the real pick_runner shell with the same inputs and returns the
|
||||
// selected runner exactly as CI would publish it.
|
||||
function runPickRunner({ ecsDisabled, sameRepo, assoc, eventName, dispatch }) {
|
||||
function runPickRunner({
|
||||
ecsDisabled,
|
||||
sameRepo,
|
||||
assoc,
|
||||
eventName,
|
||||
dispatch,
|
||||
fleet,
|
||||
}) {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'pick-runner-'));
|
||||
const outputFile = join(tmp, 'github_output');
|
||||
const result = spawnSync('bash', ['-c', pickRunner.run], {
|
||||
@@ -127,6 +134,7 @@ function runPickRunner({ ecsDisabled, sameRepo, assoc, eventName, dispatch }) {
|
||||
AUTHOR_ASSOCIATION: assoc,
|
||||
ECS_DISABLED: ecsDisabled ? 'true' : '',
|
||||
EVENT_NAME: eventName,
|
||||
LINUX_RUNNER_FLEET: fleet ?? '',
|
||||
DISPATCH_LINUX_RUNNER: dispatch ?? '',
|
||||
GITHUB_OUTPUT: outputFile,
|
||||
},
|
||||
@@ -266,6 +274,73 @@ describe('ci.yml classify_pr runner routing', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('ci.yml web_shell_e2e_smoke runner routing', () => {
|
||||
const expression = String(ciDoc.jobs.web_shell_e2e_smoke['runs-on'])
|
||||
.replace(/^\$\{\{\s*/, '')
|
||||
.replace(/\s*\}\}$/, '');
|
||||
const selectSmokeRunner = new Function(
|
||||
'needs',
|
||||
'fromJSON',
|
||||
'contains',
|
||||
`return (${expression});`,
|
||||
);
|
||||
|
||||
it('uses ARM64 for self-hosted runs while preserving every hosted fallback', () => {
|
||||
for (const ecsDisabled of [false, true]) {
|
||||
for (const fleet of ['', 'x64', 'arm64', 'hosted']) {
|
||||
for (const sameRepo of [false, true]) {
|
||||
for (const assoc of ASSOCIATIONS) {
|
||||
for (const dispatch of [
|
||||
undefined,
|
||||
'self-hosted',
|
||||
'self-hosted-arm64',
|
||||
'hosted',
|
||||
]) {
|
||||
const runner = runPickRunner({
|
||||
ecsDisabled,
|
||||
fleet,
|
||||
sameRepo,
|
||||
assoc,
|
||||
dispatch,
|
||||
eventName: dispatch ? 'workflow_dispatch' : 'pull_request',
|
||||
});
|
||||
assert.deepEqual(
|
||||
selectSmokeRunner(
|
||||
{ classify_pr: { outputs: { ubuntu_runner: runner } } },
|
||||
JSON.parse,
|
||||
(labels, label) => labels.includes(label),
|
||||
),
|
||||
JSON.parse(runner).includes('self-hosted')
|
||||
? ['self-hosted', 'linux', 'arm64']
|
||||
: JSON.parse(HOSTED),
|
||||
JSON.stringify({
|
||||
ecsDisabled,
|
||||
fleet,
|
||||
sameRepo,
|
||||
assoc,
|
||||
dispatch,
|
||||
runner,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('stays hosted when the classifier output is missing', () => {
|
||||
assert.deepEqual(
|
||||
selectSmokeRunner(
|
||||
{ classify_pr: { outputs: {} } },
|
||||
JSON.parse,
|
||||
(labels, label) => labels.includes(label),
|
||||
),
|
||||
JSON.parse(HOSTED),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ci.yml test_windows runner routing', () => {
|
||||
it('keeps every pull request hosted, whoever opens it', () => {
|
||||
// A pull_request run executes the workflow YAML from the PR's own merge
|
||||
|
||||
@@ -1444,14 +1444,16 @@ jobs:
|
||||
needs.classify_pr.outputs.skip_ci != 'true' &&
|
||||
needs.test.outputs.ci_profile == 'full'
|
||||
}}
|
||||
# Routed like test's: trusted runs use the ECS pool, untrusted fork PRs
|
||||
# stay hosted. #11250 moved it hosted after four pool attempts failed
|
||||
# (run 34081409590). Three ran the smoke unpinned at 8 workers, half the
|
||||
# host's 16 cores, against hosted's 2, and timed out broadly on
|
||||
# Keep the classifier's trust and hosted overrides, but use ARM64 for
|
||||
# this lane independently of the ordinary Linux CI fleet. The history
|
||||
# below is x64-specific, not a verdict on ARM64, but it is why this lane
|
||||
# is fragile: #11250 moved it back to hosted after four pool attempts
|
||||
# failed (run 34081409590). Three ran the smoke unpinned at 8 workers,
|
||||
# half the host's 16 cores, against hosted's 2, and timed out broadly on
|
||||
# visibility and clock waits. The fourth's transcript gate took 73 s
|
||||
# against its 60 s budget while npm install alone took 28 min on that
|
||||
# host. The smoke step pins the hosted worker count.
|
||||
runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}'
|
||||
runs-on: '${{ contains(fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]''), ''self-hosted'') && fromJSON(''["self-hosted", "linux", "arm64"]'') || fromJSON(''["ubuntu-latest"]'') }}'
|
||||
# Slow hosted mirrors kept downloading past 23 min; reserve time for
|
||||
# the 208 smoke tests and artifact cleanup after installation.
|
||||
timeout-minutes: 60
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Managed Workspace Web Fetch
|
||||
|
||||
[English](2026-10-23-managed-web-fetch.md) | [简体中文](2026-10-23-managed-web-fetch.zh-CN.md)
|
||||
|
||||
## Problem and scope
|
||||
|
||||
The `web_fetch` part of [#15076](https://github.com/QwenLM/qwen-code/issues/15076) is missing from the main Hosted Workspace profiles. Add it to the existing files, shell, search and team lead profiles. Snapshot read profiles and the separate CSI files contract retain their existing tool vocabulary. `todo_write`, authenticated fetching and binary downloads are outside this change.
|
||||
|
||||
## Execution and permissions
|
||||
|
||||
Declare `url`, `prompt` and optional `format` with the shared tool's existing semantics. Validate arguments before acquiring a Runtime and normalize GitHub blob URLs before approval so the approval preview identifies the actual destination. Keep `default` and `auto-edit` approval; `yolo` retains the deployment's explicit approval bypass. Fetching remains unavailable in plan mode because an outbound request can disclose information.
|
||||
|
||||
Execute through the existing Session-owned Broker Tool v2 path, using the bound Runtime's `WebFetchTool` in an explicit managed mode. The worker returns normalized text and metadata to the conversation model instead of making a side query without model credentials. Managed mode refuses binary content, never spills content to host files, and bounds its text preview to 16 KiB after JSON encoding with a truncation notice. The existing durable size guard remains authoritative for the final encoded envelope.
|
||||
|
||||
## Network policy
|
||||
|
||||
Managed fetching accepts only credential-free HTTPS URLs resolving entirely to public addresses. Reuse the extension network policy's address classification and pinned lookup for every request, redirect hop and retry. Move that shared implementation to Core utilities while preserving the extension import path through a re-export. A dedicated Undici agent and its matching bundled fetch pin the approved DNS result, require TLS certificate verification and bypass the process-wide proxy dispatcher. Refuse non-Node runtimes because their native fetch may silently ignore the dispatcher. Private, loopback, link-local, metadata and reserved addresses are refused, including IPv4-mapped IPv6 and mixed public/private DNS answers. Managed mode never downgrades to HTTP. Cross-host, scheme or port redirects are returned to the model for a new call and approval; they are never followed automatically. Network policy and response limits also apply under `yolo`.
|
||||
|
||||
## Durability and compatibility
|
||||
|
||||
Advertise `web-fetch/1` in the Harness capability envelope. Add the native tool to approval input previews on both sides, approval continuation validation, native batch recovery, and lifecycle recognition of inline results. Keep the shared file-only declaration baseline at Read/Write/Edit for CSI and its saved recovery definitions; compose web fetch only into the main profiles. Reuse ordinary tool result persistence and cancellation; no database, public API or new route is needed. Upgrade Runtime workers with the Harness before offering the capability. Existing local `web_fetch` keeps its side query, private-network access, proxy behavior and download support.
|
||||
|
||||
## Affected areas
|
||||
|
||||
- Core web fetch and bounded HTTP transfer, with focused network and managed-mode tests.
|
||||
- Hosted declarations, argument validation, capabilities, approval continuation and Runtime tool registration.
|
||||
- Java approval previews and inline-result lifecycle validation, with affected tests.
|
||||
- Managed Agent documentation and the local E2E plan.
|
||||
|
||||
## Validation and acceptance
|
||||
|
||||
Use `.qwen/e2e-tests/managed-web-fetch.md` for baseline and post-change checks. Verify public HTML/markdown/text, format negotiation, redirects, HTTP errors, response limits, cancellation, binary refusal, and bounded output without host paths. Test private and mixed DNS rejection, pinned connections across redirects/retries, TLS verification, and approval deny/resume. Verify normal inline result recovery and lifecycle settlement. Run affected Core/CLI tests, root build/typecheck/bundle, Java affected tests and `mvn verify`; report unavailable real-model or deployment checks separately.
|
||||
|
||||
Acceptance requires the main profiles to declare and execute `web_fetch`, approval to show the actual URL, denied calls to make no request, safe public text to reach the model and durable history, and settled calls to permit lifecycle cleanup. Deployment acceptance remains separate from local verification. There are no unresolved design questions.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Managed Workspace 网页抓取
|
||||
|
||||
[English](2026-10-23-managed-web-fetch.md) | [简体中文](2026-10-23-managed-web-fetch.zh-CN.md)
|
||||
|
||||
## 问题与范围
|
||||
|
||||
主 Hosted Workspace profile 尚未实现 [#15076](https://github.com/QwenLM/qwen-code/issues/15076) 中的 `web_fetch` 子需求。本次将其加入现有的 files、shell、search 和 team lead profile。快照 read profile 与独立的 CSI files 契约保留现有工具词表。`todo_write`、认证网页抓取和二进制下载不在本次范围内。
|
||||
|
||||
## 执行与权限
|
||||
|
||||
声明 `url`、`prompt` 与可选的 `format`，沿用共享工具的参数语义。在获取 Runtime 前校验参数，在审批前规范化 GitHub blob URL，使审批预览展示实际访问目标。保留 `default` 与 `auto-edit` 审批；`yolo` 沿用部署显式选择的免审批语义。网页抓取涉及出站信息披露，因此计划模式仍不允许执行。
|
||||
|
||||
沿用 Session 所属的 Broker Tool v2 路径，在绑定的 Runtime 中以显式 managed 模式执行 `WebFetchTool`。Worker 返回规范化文本和元数据，由会话主模型处理，不在缺少模型凭据时调用旁路模型。Managed 模式拒绝二进制内容，不向宿主机文件溢写，将文本预览的 JSON 编码限制为 16 KiB 并附截断提示。最终编码结果仍以现有持久化大小校验为准。
|
||||
|
||||
## 网络策略
|
||||
|
||||
Managed 抓取仅接受不含凭据、全部解析结果均为公网地址的 HTTPS URL。每次请求、重定向跳转与重试均复用扩展网络策略的地址分类和固定解析。将这段共享实现下移到 Core 工具模块，旧的 extension 导入路径通过重新导出保持兼容。独立的 Undici agent 与同版本内置 fetch 固定已校验的 DNS 结果，强制校验证书，并绕过进程全局代理 dispatcher。非 Node 运行时的原生 fetch 可能静默忽略 dispatcher，因此明确拒绝该运行环境。拒绝私网、回环、链路本地、元数据及保留地址，包括 IPv4 映射 IPv6 和混合公网／私网 DNS 结果。Managed 模式不降级到 HTTP。跨主机、协议或端口的重定向返回模型，通过新调用重新审批，不自动跟随。`yolo` 同样受网络策略及响应限制约束。
|
||||
|
||||
## 持久化与兼容性
|
||||
|
||||
在 Harness 能力声明中增加 `web-fetch/1`。将原生工具加入双端审批输入预览、审批 continuation 校验、原生批次恢复与生命周期的内联结果识别。为 CSI 及其已保存的恢复定义保留仅包含 Read/Write/Edit 的共享文件声明基线，仅在主 profile 中组合网页抓取工具。复用普通工具结果持久化与取消机制，不修改数据库、公开 API，也不新增路由。在提供该能力前，应同步升级 Runtime Worker 与 Harness。现有本地 `web_fetch` 保留旁路模型、私网访问、代理行为与下载能力。
|
||||
|
||||
## 影响范围
|
||||
|
||||
- Core 网页抓取与有界 HTTP 传输，以及网络和 managed 模式的定向测试。
|
||||
- Hosted 工具声明、参数校验、能力、审批 continuation 与 Runtime 工具注册。
|
||||
- Java 审批预览及内联结果生命周期校验，以及受影响的测试。
|
||||
- Managed Agent 文档和本地 E2E 计划。
|
||||
|
||||
## 验证与验收
|
||||
|
||||
使用 `.qwen/e2e-tests/managed-web-fetch.md` 记录基线和改后验证。覆盖公网 HTML／Markdown／文本、格式协商、重定向、HTTP 错误、响应限制、取消、二进制拒绝，以及不含宿主路径的有界输出。验证私网和混合 DNS 拒绝、重定向／重试的连接固定、TLS 校验与审批拒绝／恢复。验证普通内联结果恢复和生命周期收敛。运行受影响的 Core／CLI 测试、根目录 build／typecheck／bundle、Java 定向测试和 `mvn verify`；独立报告未能执行的真实模型或部署检查。
|
||||
|
||||
验收要求：主 profile 声明并执行 `web_fetch`，审批展示实际 URL，拒绝后不发请求，安全公网文本进入模型和持久化历史，已完成调用允许生命周期清理。本地验证与部署验收分别记录。无未决设计问题。
|
||||
@@ -61,7 +61,7 @@ const model = await startFakeOpenAIServer(({ body, requestIndex }) => {
|
||||
(body['tools'] as Array<{ function: { name: string } }>)
|
||||
.map((tool) => tool.function.name)
|
||||
.sort(),
|
||||
['edit', 'read_file', 'write_file'],
|
||||
['edit', 'read_file', 'web_fetch', 'write_file'],
|
||||
);
|
||||
const messages = body['messages'] as Array<{
|
||||
role: string;
|
||||
|
||||
@@ -245,27 +245,31 @@ const modelReply: FakeOpenAIHandler = ({ body }) => {
|
||||
// the lead's team tools with them, since H4e-b3a their shutdown
|
||||
// request, and since #13803 (K4) the Workflow tool — its own kind
|
||||
// reader gates the advertisement, so a child Session still hides it.
|
||||
assert.deepEqual(tools.map((tool) => tool.function.name).sort(), [
|
||||
...(shellProfile ? ['agent'] : []),
|
||||
'edit',
|
||||
...(shellProfile ? ['glob'] : []),
|
||||
...(shellProfile ? ['monitor'] : []),
|
||||
'read_file',
|
||||
...(shellProfile
|
||||
? [
|
||||
'request_shutdown',
|
||||
'run_shell_command',
|
||||
'send_message',
|
||||
'task_create',
|
||||
'task_list',
|
||||
'task_update',
|
||||
'team_create',
|
||||
'team_delete',
|
||||
'workflow',
|
||||
]
|
||||
: []),
|
||||
'write_file',
|
||||
]);
|
||||
assert.deepEqual(
|
||||
tools.map((tool) => tool.function.name).sort(),
|
||||
[
|
||||
...(shellProfile ? ['agent'] : []),
|
||||
'edit',
|
||||
...(shellProfile ? ['glob'] : []),
|
||||
...(shellProfile ? ['monitor'] : []),
|
||||
'read_file',
|
||||
...(shellProfile
|
||||
? [
|
||||
'request_shutdown',
|
||||
'run_shell_command',
|
||||
'send_message',
|
||||
'task_create',
|
||||
'task_list',
|
||||
'task_update',
|
||||
'team_create',
|
||||
'team_delete',
|
||||
'workflow',
|
||||
]
|
||||
: []),
|
||||
'web_fetch',
|
||||
'write_file',
|
||||
].sort(),
|
||||
);
|
||||
const messages = body['messages'] as Array<{
|
||||
role: string;
|
||||
content: unknown;
|
||||
|
||||
@@ -7097,45 +7097,51 @@ describe.skipIf(!hasTmux)('capture-tui (real tmux)', () => {
|
||||
// session dies with the server. The detached arm is the limit: it
|
||||
// setsids into its own session, its parent is init before the reap
|
||||
// even runs, and nothing portable reaches it.
|
||||
// The tag has to ride on the LONG-LIVED process itself. Two earlier
|
||||
// shapes did not: a bare `sleep 41` matched any concurrent run of
|
||||
// this suite on a shared host (and its pkill reached into them), and
|
||||
// `sh -c 'sleep N; : <tag>'` tagged only the WRAPPER — sh forks an
|
||||
// untagged sleep and waits, so pgrep found the wrapper, pkill killed
|
||||
// it, and the sleep survived reparented to init: an orphan per run,
|
||||
// from the test that exists to pin orphans. A per-run symlink to the
|
||||
// real sleep puts the tag in argv[0] of the process that actually
|
||||
// lives.
|
||||
// The tag has to ride on the LONG-LIVED process itself, as an ordinary
|
||||
// argument rather than argv[0]. Earlier shapes did not survive contact:
|
||||
// a bare `sleep 41` matched any concurrent run of this suite on a
|
||||
// shared host (and its pkill reached into them); `sh -c 'sleep N; :
|
||||
// <tag>'` tagged only the WRAPPER — sh forks an untagged sleep and
|
||||
// waits, so pgrep found the wrapper, pkill killed it, and the sleep
|
||||
// survived reparented to init: an orphan per run, from the test that
|
||||
// exists to pin orphans; a per-run symlink to `sleep` assumes the
|
||||
// resolved binary ignores argv[0], which uutils coreutils' multicall
|
||||
// build does NOT — it reads argv[0] as the applet name, finds the tag
|
||||
// there instead of `sleep`, and exits before pgrep can ever see it
|
||||
// (measured on the ARM64 runners, where PATH resolves sleep to
|
||||
// /usr/lib/cargo/bin/coreutils/sleep, which is that multicall build).
|
||||
// A Node waiter keeps the tag in a plain argument of the process that
|
||||
// actually lives, and Node is guaranteed present wherever this suite
|
||||
// runs.
|
||||
const tag = (arm: string): string =>
|
||||
`capture-tui-orphan-${process.pid}-${arm}`;
|
||||
// Resolved, never the literal /bin/sleep: NixOS keeps sleep off /bin
|
||||
// entirely, and a dangling link fails the detached arm RED against
|
||||
// healthy code (spawn ENOENT before any sleeper exists) while the
|
||||
// attached arm passes vacuously.
|
||||
const sleepTarget = probes.sleepBin();
|
||||
if (sleepTarget === undefined) {
|
||||
throw new Error(
|
||||
'sleep is not resolvable to an executable on PATH — the per-run ' +
|
||||
'symlink needs a real target',
|
||||
);
|
||||
}
|
||||
const sleeper = (arm: string): string => {
|
||||
const link = join(dir, tag(arm));
|
||||
symlinkSync(sleepTarget, link);
|
||||
return link;
|
||||
};
|
||||
const waiter = join(dir, 'capture-tui-waiter.cjs');
|
||||
writeFileSync(waiter, 'setTimeout(() => {}, 60_000);\n');
|
||||
const detach = join(dir, 'capture-tui-detach.cjs');
|
||||
writeFileSync(
|
||||
detach,
|
||||
[
|
||||
"const { spawn } = require('node:child_process');",
|
||||
'spawn(process.execPath, [process.argv[2], process.argv[3]], {',
|
||||
' detached: true,',
|
||||
" stdio: 'ignore',",
|
||||
'}).unref();',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
const node = JSON.stringify(process.execPath);
|
||||
const arms: Array<[string, string, boolean]> = [
|
||||
// A child in the pane's session: dies with the server. The guarantee.
|
||||
[
|
||||
'attached',
|
||||
`${sleeper('attached')} 41 & printf 'MARK\\n'; sleep 20`,
|
||||
`${node} ${JSON.stringify(waiter)} ${tag('attached')} & printf 'MARK\\n'; sleep 20`,
|
||||
false,
|
||||
],
|
||||
// setsid'd into its own session, parent already init before the reap
|
||||
// runs: nothing portable reaches it. The documented limit.
|
||||
[
|
||||
'detached',
|
||||
`node -e "require('child_process').spawn('${sleeper('detached')}',['42'],{detached:true,stdio:'ignore'}).unref()"; printf 'MARK\\n'; sleep 20`,
|
||||
`${node} ${JSON.stringify(detach)} ${JSON.stringify(waiter)} ${tag('detached')}; printf 'MARK\\n'; sleep 20`,
|
||||
true,
|
||||
],
|
||||
];
|
||||
|
||||
@@ -49,6 +49,16 @@ it.each(
|
||||
),
|
||||
),
|
||||
)
|
||||
.concat(
|
||||
['allow', 'deny', 'cancelled', 'expired'].map((decision) => ({
|
||||
decision,
|
||||
promptId: randomUUID(),
|
||||
round: 1,
|
||||
planMode: false,
|
||||
publicTool: 'web_fetch',
|
||||
profile: 'hosted-workspace-files/1',
|
||||
})),
|
||||
)
|
||||
.concat(
|
||||
['allow', 'deny', 'cancelled', 'expired'].flatMap((decision) =>
|
||||
['hosted-workspace-files/1', 'hosted-workspace-shell/1'].map(
|
||||
@@ -65,14 +75,16 @@ it.each(
|
||||
)
|
||||
.concat(
|
||||
['allow', 'deny', 'cancelled', 'expired'].flatMap((decision) =>
|
||||
['write_file', 'agent', 'run_shell_command'].map((publicTool) => ({
|
||||
decision,
|
||||
promptId: randomUUID(),
|
||||
round: 1,
|
||||
planMode: false,
|
||||
profile: 'hosted-workspace-shell/1',
|
||||
publicTool,
|
||||
})),
|
||||
['write_file', 'web_fetch', 'agent', 'run_shell_command'].map(
|
||||
(publicTool) => ({
|
||||
decision,
|
||||
promptId: randomUUID(),
|
||||
round: 1,
|
||||
planMode: false,
|
||||
profile: 'hosted-workspace-shell/1',
|
||||
publicTool,
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)(
|
||||
@@ -256,7 +268,9 @@ it.each(
|
||||
}
|
||||
: publicTool === 'run_shell_command'
|
||||
? { command: 'printf once' }
|
||||
: { file_path: 'original.txt', content: 'one original batch' },
|
||||
: publicTool === 'web_fetch'
|
||||
? { url: 'https://example.com/docs', prompt: 'Summarize' }
|
||||
: { file_path: 'original.txt', content: 'one original batch' },
|
||||
isClientInitiated: false,
|
||||
prompt_id: promptId,
|
||||
};
|
||||
|
||||
@@ -31,7 +31,13 @@ import {
|
||||
import { hostedRuntimeSessionId } from './hosted-workspace-broker.js';
|
||||
import { readHostedTurnApprovalSelection } from './hosted-approval-selection.js';
|
||||
|
||||
const NATIVE_TOOLS = new Set(['read_file', 'write_file', 'edit', 'glob']);
|
||||
const NATIVE_TOOLS = new Set([
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'glob',
|
||||
'web_fetch',
|
||||
]);
|
||||
|
||||
export interface HostedNativePlannedCall {
|
||||
readonly call: ToolCallRequestInfo;
|
||||
|
||||
@@ -68,6 +68,7 @@ describe('Hosted Harness private contract', () => {
|
||||
'recovery-cleanup/1',
|
||||
'targeted-cancel/1',
|
||||
'event-keepalive/1',
|
||||
'web-fetch/1',
|
||||
],
|
||||
});
|
||||
expect(Object.isFrozen(contract)).toBe(true);
|
||||
|
||||
@@ -90,6 +90,7 @@ export function createHostedHarnessContract(
|
||||
// GET /session/:id/events writes keepalive comments while no event
|
||||
// is committed, so a client may bound a stream by its silence.
|
||||
'event-keepalive/1',
|
||||
'web-fetch/1',
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15926,6 +15926,7 @@ describe('Hosted Harness no-tool session', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'glob',
|
||||
]);
|
||||
await headers(supertest(server).delete(`/session/${SESSION_ID}`)).set(
|
||||
@@ -15975,6 +15976,7 @@ describe('Hosted Harness no-tool session', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'glob',
|
||||
]);
|
||||
expect(state.model).toHaveBeenCalledTimes(2);
|
||||
@@ -16054,6 +16056,7 @@ describe('Hosted Harness no-tool session', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'run_shell_command',
|
||||
'monitor',
|
||||
'glob',
|
||||
@@ -16120,6 +16123,7 @@ describe('Hosted Harness no-tool session', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'run_shell_command',
|
||||
'monitor',
|
||||
'glob',
|
||||
@@ -16590,6 +16594,7 @@ describe('Hosted Harness no-tool session', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
...(profile.includes('shell') ? ['run_shell_command', 'glob'] : []),
|
||||
]);
|
||||
};
|
||||
@@ -28869,6 +28874,7 @@ describe('Hosted Harness Runtime turn takeover', () => {
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
...(toolProfile.endsWith('/2') ? ['glob'] : []),
|
||||
]);
|
||||
// The takeover-built attachment starts with an undefined slot: the
|
||||
|
||||
@@ -859,6 +859,7 @@ describe('hosted-session-command-session funnel (H4e-b3)', () => {
|
||||
expect(record.run.delivery?.state).toBe('rejected');
|
||||
});
|
||||
|
||||
// The 256 real durable commands require over 1,000 filesystem syncs.
|
||||
it('enforces the per-run command budget with budget_exhausted', ({
|
||||
signal,
|
||||
}) => {
|
||||
@@ -926,7 +927,7 @@ describe('hosted-session-command-session funnel (H4e-b3)', () => {
|
||||
})();
|
||||
pendingBudgetTests.push(work);
|
||||
return work;
|
||||
}, 30_000);
|
||||
}, 90_000);
|
||||
|
||||
it("commits the leader's receipt against an attached run and refetches its reply", async () => {
|
||||
const lead = await leadSideSetup();
|
||||
|
||||
@@ -1542,6 +1542,97 @@ it('preserves trimmed and normalized valid file paths', async () => {
|
||||
await turn.finish();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ url: 'http://example.com', prompt: 'Read' },
|
||||
{ url: 'https://user:[email protected]', prompt: 'Read' },
|
||||
{ url: 'https://example.com', prompt: '' },
|
||||
{ url: 'https://example.com', prompt: 'Read', format: 'json' },
|
||||
{ url: 'https://example.com', prompt: 'Read', headers: {} },
|
||||
])(
|
||||
'refuses invalid web_fetch arguments before acquiring a Runtime: %j',
|
||||
async (args) => {
|
||||
const call = { ...calls[0], name: 'web_fetch', args };
|
||||
const result = await turn.execute(
|
||||
[call],
|
||||
[{ functionCall: { id: call.callId, name: call.name, args } }],
|
||||
'model',
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(JSON.stringify(result)).toContain('Hosted web_fetch requires');
|
||||
expect(broker.acquire).not.toHaveBeenCalled();
|
||||
expect(broker.prepare).not.toHaveBeenCalled();
|
||||
expect(broker.execute).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['default', 'auto-edit'] as const)(
|
||||
'requires %s web_fetch approval with the actual destination and persists the result',
|
||||
async (mode) => {
|
||||
turn = createTurn(false, { mode });
|
||||
const call = {
|
||||
...calls[0],
|
||||
name: 'web_fetch',
|
||||
args: {
|
||||
url: 'https://github.com/QwenLM/qwen-code/blob/main/README.md',
|
||||
prompt: 'Summarize',
|
||||
},
|
||||
};
|
||||
const running = turn.execute(
|
||||
[call],
|
||||
[{ functionCall: { id: call.callId, name: call.name, args: call.args } }],
|
||||
'model',
|
||||
new AbortController().signal,
|
||||
);
|
||||
const requestId = await requested();
|
||||
const input = JSON.parse(
|
||||
(
|
||||
await session.resources.read(
|
||||
(await checkpoint()).approval!.invocationRef!,
|
||||
)
|
||||
).toString(),
|
||||
);
|
||||
expect(JSON.parse(input.payloadJson).input.url).toBe(
|
||||
'https://raw.githubusercontent.com/QwenLM/qwen-code/main/README.md',
|
||||
);
|
||||
expect(broker.prepare).not.toHaveBeenCalled();
|
||||
await resolveHostedAction(session, waiters, requestId, answer('allow'));
|
||||
const result = await running;
|
||||
expect(broker.execute).toHaveBeenCalledTimes(1);
|
||||
expect(JSON.parse(broker.execute.mock.calls[0][1]).input).toMatchObject({
|
||||
url: 'https://raw.githubusercontent.com/QwenLM/qwen-code/main/README.md',
|
||||
prompt: 'Summarize',
|
||||
});
|
||||
expect(result[0].functionResponse?.response?.['executionStatus']).toBe(
|
||||
'success',
|
||||
);
|
||||
expect((await session.sink.project()).at(-1)?.message?.parts).toEqual(
|
||||
JSON.parse(JSON.stringify(result)),
|
||||
);
|
||||
await turn.consumeResults();
|
||||
await turn.finish();
|
||||
},
|
||||
);
|
||||
|
||||
it('does not dispatch a denied web_fetch', async () => {
|
||||
turn = createTurn(false, { mode: 'default' });
|
||||
const call = {
|
||||
...calls[0],
|
||||
name: 'web_fetch',
|
||||
args: { url: 'https://example.com/docs', prompt: 'Summarize' },
|
||||
};
|
||||
const running = turn.execute(
|
||||
[call],
|
||||
[{ functionCall: { id: call.callId, name: call.name, args: call.args } }],
|
||||
'model',
|
||||
new AbortController().signal,
|
||||
);
|
||||
const requestId = await requested();
|
||||
await resolveHostedAction(session, waiters, requestId, answer('deny'));
|
||||
expect(JSON.stringify(await running)).toContain('denied');
|
||||
expect(broker.execute).not.toHaveBeenCalled();
|
||||
expect(broker.prepare).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('offers glob under the unified Shell, file-search and read profiles', async () => {
|
||||
const signal = new AbortController().signal;
|
||||
const advertised = async (profile?: string, shell = false) =>
|
||||
@@ -1561,7 +1652,7 @@ it('offers glob under the unified Shell, file-search and read profiles', async (
|
||||
{ profile },
|
||||
).declarations(signal)
|
||||
).map((tool) => tool.name);
|
||||
const file = ['read_file', 'write_file', 'edit'];
|
||||
const file = ['read_file', 'write_file', 'edit', 'web_fetch'];
|
||||
const shellTools = [...file, 'run_shell_command', 'monitor'];
|
||||
await expect(advertised()).resolves.toEqual(file);
|
||||
await expect(advertised('hosted-workspace-files/1')).resolves.toEqual(file);
|
||||
@@ -1631,6 +1722,7 @@ it('dispatches read_file and glob on the read profile', async () => {
|
||||
// #13753 I3: every tool that writes, and every other tool, is refused on
|
||||
// the read profile before the Broker is acquired.
|
||||
it.each([
|
||||
['web_fetch', { url: 'https://example.com/docs', prompt: 'Summarize' }],
|
||||
['write_file', { file_path: 'a.txt', content: 'x' }],
|
||||
['edit', { file_path: 'a.txt', old_string: 'a', new_string: 'b' }],
|
||||
['run_shell_command', { command: 'touch a.txt' }],
|
||||
@@ -2897,6 +2989,7 @@ it.each(['files', 'shell', 'mcp'])(
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'mcp_write',
|
||||
]);
|
||||
expect(declarations.at(-1)).toEqual(remote);
|
||||
@@ -4590,6 +4683,7 @@ it('admits exactly the declared native tools to the version 2 input preview', ()
|
||||
// through its two send_message declarations and the team board through
|
||||
// HOSTED_TEAM_TOOLS, not the shell set.
|
||||
expect(HOSTED_INPUT_PREVIEW_TOOLS).toEqual([
|
||||
'web_fetch',
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
@@ -9944,6 +10038,7 @@ it('keeps the SSH foreground-only declaration and admission free of detached too
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
'web_fetch',
|
||||
'run_shell_command',
|
||||
]);
|
||||
const shell = declarations.find((tool) => tool.name === 'run_shell_command');
|
||||
|
||||
@@ -208,6 +208,7 @@ import {
|
||||
} from './hosted-hook-session.js';
|
||||
import { parseHookExecution } from '@qwen-code/qwen-code-core/managed-runtime/managed-hook-record.js';
|
||||
import { waitForTurn } from './hosted-turn-wait.js';
|
||||
import { rewriteGitHubBlobUrl } from '@qwen-code/qwen-code-core/tools/web-fetch.js';
|
||||
import { boundedShellPreview } from './managed-shell-publisher.js';
|
||||
import {
|
||||
HOSTED_SKILL_APPROVAL_REFUSAL,
|
||||
@@ -602,8 +603,28 @@ export const HOSTED_WORKSPACE_FILE_TOOLS: FunctionDeclaration[] = [
|
||||
},
|
||||
];
|
||||
|
||||
const HOSTED_WEB_FETCH_TOOL: FunctionDeclaration = {
|
||||
name: 'web_fetch',
|
||||
description:
|
||||
'Fetch a public HTTPS web page from the Session Runtime. Returns bounded text and HTTP metadata; HTML is converted to Markdown. Use prompt to describe what to extract and answer using the returned content. Private networks, authenticated URLs and binary downloads are unavailable. Cross-host redirects require a new call. Outbound requests require approval in default and auto-edit modes.',
|
||||
parametersJsonSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: { type: 'string', description: 'The public HTTPS URL to fetch.' },
|
||||
prompt: {
|
||||
type: 'string',
|
||||
description: 'Information to extract from the page.',
|
||||
},
|
||||
format: { type: 'string', enum: ['auto', 'markdown', 'html', 'text'] },
|
||||
},
|
||||
required: ['url', 'prompt'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
};
|
||||
|
||||
export const HOSTED_WORKSPACE_SHELL_TOOLS: FunctionDeclaration[] = [
|
||||
...HOSTED_WORKSPACE_FILE_TOOLS,
|
||||
HOSTED_WEB_FETCH_TOOL,
|
||||
{
|
||||
name: 'run_shell_command',
|
||||
description:
|
||||
@@ -682,6 +703,7 @@ export const HOSTED_WORKSPACE_READ_TOOLS: FunctionDeclaration[] = [
|
||||
// no preview. Both degrade silently into the card's "Tool arguments are
|
||||
// unavailable for this approval." fallback, so a test enumerates the set.
|
||||
export const HOSTED_INPUT_PREVIEW_TOOLS: readonly string[] = [
|
||||
'web_fetch',
|
||||
'read_file',
|
||||
'write_file',
|
||||
'edit',
|
||||
@@ -1971,7 +1993,7 @@ export class HostedWorkspaceToolTurn {
|
||||
...[
|
||||
...(this.publication || this.shell
|
||||
? HOSTED_WORKSPACE_SHELL_TOOLS
|
||||
: HOSTED_WORKSPACE_FILE_TOOLS),
|
||||
: [...HOSTED_WORKSPACE_FILE_TOOLS, HOSTED_WEB_FETCH_TOOL]),
|
||||
...(search ? [HOSTED_GLOB_TOOL] : []),
|
||||
]
|
||||
.filter((tool) => !this.foregroundOnly || tool.name !== 'monitor')
|
||||
@@ -2399,13 +2421,19 @@ export class HostedWorkspaceToolTurn {
|
||||
items.every(
|
||||
(item) =>
|
||||
item.state === 'settled' &&
|
||||
['read_file', 'write_file', 'edit', 'glob'].includes(
|
||||
['read_file', 'write_file', 'edit', 'glob', 'web_fetch'].includes(
|
||||
item.toolName,
|
||||
),
|
||||
) &&
|
||||
(saved ||
|
||||
items.every((item) =>
|
||||
['read_file', 'glob'].includes(item.toolName),
|
||||
[
|
||||
'read_file',
|
||||
'glob',
|
||||
...(isHostedWorkspaceReadProfile(this.profile)
|
||||
? []
|
||||
: ['web_fetch']),
|
||||
].includes(item.toolName),
|
||||
)) &&
|
||||
(await canSettleHostedFileHistory(this.session, {
|
||||
pendingUndo: null,
|
||||
@@ -3356,7 +3384,7 @@ export class HostedWorkspaceToolTurn {
|
||||
singlePlanExit ||
|
||||
(this.publicChildren &&
|
||||
['agent', 'run_shell_command'].includes(request.call.name)) ||
|
||||
['read_file', 'write_file', 'edit', 'glob'].includes(
|
||||
['read_file', 'write_file', 'edit', 'glob', 'web_fetch'].includes(
|
||||
request.call.name,
|
||||
)
|
||||
) ||
|
||||
@@ -3685,6 +3713,37 @@ export class HostedWorkspaceToolTurn {
|
||||
: this.publication
|
||||
? { ...args }
|
||||
: { ...args, is_background: false };
|
||||
} else if (call.name === 'web_fetch') {
|
||||
input = { ...call.args };
|
||||
const args = call.args;
|
||||
try {
|
||||
const url =
|
||||
typeof args['url'] === 'string'
|
||||
? new URL(args['url'])
|
||||
: undefined;
|
||||
if (
|
||||
!url ||
|
||||
url.protocol !== 'https:' ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
typeof args['url'] !== 'string' ||
|
||||
!/^https:\/\//i.test(args['url']) ||
|
||||
typeof args['prompt'] !== 'string' ||
|
||||
!args['prompt'].trim() ||
|
||||
(args['format'] !== undefined &&
|
||||
!['auto', 'markdown', 'html', 'text'].includes(
|
||||
args['format'] as string,
|
||||
)) ||
|
||||
Object.keys(args).some(
|
||||
(key) => !['url', 'prompt', 'format'].includes(key),
|
||||
)
|
||||
)
|
||||
throw new Error('Invalid web_fetch arguments.');
|
||||
input['url'] = rewriteGitHubBlobUrl(url.toString());
|
||||
} catch {
|
||||
validationError =
|
||||
'Hosted web_fetch requires a public HTTPS URL without credentials, a nonempty prompt, and optional format auto, markdown, html or text. Correct the arguments and retry.';
|
||||
}
|
||||
} else if (call.name === 'glob') {
|
||||
input = { ...call.args };
|
||||
const globError =
|
||||
|
||||
@@ -60,6 +60,23 @@ import {
|
||||
type ManagedToolSet,
|
||||
} from './managed-runtime-tool-executor.js';
|
||||
|
||||
vi.mock(
|
||||
'@qwen-code/qwen-code-core/utils/runtimeFetchOptions.js',
|
||||
async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<
|
||||
typeof import('@qwen-code/qwen-code-core/utils/runtimeFetchOptions.js')
|
||||
>();
|
||||
return {
|
||||
...actual,
|
||||
loadUndici: async () => ({
|
||||
...(await actual.loadUndici()),
|
||||
fetch: globalThis.fetch,
|
||||
}),
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// A long-running foreground command the Shell tool admits: a bare
|
||||
// `sleep N` trips its standalone-sleep refusal long before the budget.
|
||||
const LONG_RUN = `"${process.execPath}" -e 'setInterval(()=>{},1000)'`;
|
||||
@@ -2803,6 +2820,64 @@ describe('ManagedToolExecutor acknowledgement', () => {
|
||||
argsDigest: createHash('sha256').update('args').digest('hex'),
|
||||
};
|
||||
|
||||
it('executes web_fetch as text once without a worker model or host files', async () => {
|
||||
const root = workspace();
|
||||
const executor = ManagedToolExecutor.forWorkspace(root, 'runtime-01');
|
||||
const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response('<h1>Public docs</h1>', {
|
||||
headers: { 'Content-Type': 'text/html' },
|
||||
}),
|
||||
);
|
||||
const input = { url: 'https://8.8.8.8/docs', prompt: 'Summarize' };
|
||||
try {
|
||||
expect(executor.hasTool('web_fetch')).toBe(true);
|
||||
const result = await executor.execute(
|
||||
sessionReference,
|
||||
'web_fetch',
|
||||
input,
|
||||
);
|
||||
expect(result.executionStatus).toBe('success');
|
||||
expect(result.responseParts).toEqual([
|
||||
expect.objectContaining({
|
||||
text: expect.stringContaining('Public docs\n==========='),
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(result)).not.toContain('processing failed');
|
||||
expect(
|
||||
await executor.execute(sessionReference, 'web_fetch', input),
|
||||
).toEqual(result);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fs.readdirSync(root)).toEqual(['a.txt']);
|
||||
expect(executor.acknowledge(sessionReference)?.state).toBe(
|
||||
'acknowledged',
|
||||
);
|
||||
expect(() => executor.closeSessionAdmission('session-a')).not.toThrow();
|
||||
} finally {
|
||||
fetch.mockRestore();
|
||||
await executor.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('blocks web_fetch to Runtime-local addresses before making a request', async () => {
|
||||
const executor = ManagedToolExecutor.forWorkspace(
|
||||
workspace(),
|
||||
'runtime-01',
|
||||
);
|
||||
const fetch = vi.spyOn(globalThis, 'fetch');
|
||||
try {
|
||||
const result = await executor.execute(sessionReference, 'web_fetch', {
|
||||
url: 'https://169.254.169.254/latest/meta-data',
|
||||
prompt: 'Read',
|
||||
});
|
||||
expect(result.executionStatus).toBe('error');
|
||||
expect(JSON.stringify(result)).toContain('blocked address');
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
fetch.mockRestore();
|
||||
await executor.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('lets a session close after its settled call is acknowledged', async () => {
|
||||
const executor = ManagedToolExecutor.forWorkspace(
|
||||
workspace(),
|
||||
|
||||
@@ -26,6 +26,7 @@ import { ReadFileTool } from '@qwen-code/qwen-code-core/tools/read-file.js';
|
||||
import { WriteFileTool } from '@qwen-code/qwen-code-core/tools/write-file.js';
|
||||
import { EditTool } from '@qwen-code/qwen-code-core/tools/edit.js';
|
||||
import { GlobTool } from '@qwen-code/qwen-code-core/tools/glob.js';
|
||||
import { WebFetchTool } from '@qwen-code/qwen-code-core/tools/web-fetch.js';
|
||||
import { ShellTool } from '@qwen-code/qwen-code-core/tools/shell.js';
|
||||
import {
|
||||
ManagedAutomationGateTool,
|
||||
@@ -243,6 +244,7 @@ const ADMITTED_TOOL_NAMES: ReadonlySet<string> = new Set([
|
||||
EditTool.Name,
|
||||
ShellTool.Name,
|
||||
GlobTool.Name,
|
||||
WebFetchTool.Name,
|
||||
// The managed automation gate probe: the Harness's own execution, never
|
||||
// a model's (docs/design/2026-10-22-managed-automation-gate.md).
|
||||
ManagedAutomationGateTool.Name,
|
||||
@@ -2719,6 +2721,7 @@ export function createManagedToolSet(
|
||||
new ReadFileTool(config),
|
||||
new WriteFileTool(config),
|
||||
new EditTool(config),
|
||||
new WebFetchTool(config, 'managed'),
|
||||
// The walk never leaves the Session, whatever the pattern spells.
|
||||
new GlobTool(config, {
|
||||
containmentRoot: directory,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
@@ -85,6 +86,51 @@ const renderBlock = (der: Buffer): string => {
|
||||
// so suites that drive the real oracle skip instead of failing.
|
||||
const loaderOracleAvailable = typeof tls.getCACertificates === 'function';
|
||||
|
||||
/**
|
||||
* What the loader takes from `contents`, asked of this suite's own Node.
|
||||
*
|
||||
* A second, independent request of the same oracle on purpose: it lets a
|
||||
* shape whose verdict is not the same on every build compare the module
|
||||
* against the loader instead of against one build's habit. The BOM shapes are
|
||||
* the ones that need it — linux/x64 takes a BOM in the header section while
|
||||
* linux/arm64 refuses it with `PEM routines::bad base64 decode`, at the same
|
||||
* Node patch and OpenSSL version (measured directly on both arches for
|
||||
* v22.23.2 and v22.23.4), so hard-coding either verdict makes the suite
|
||||
* architecture-dependent.
|
||||
*/
|
||||
const loaderBlocks = (contents: string): string[] | undefined => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-ca-expected-'));
|
||||
try {
|
||||
const file = path.join(dir, 'expected.pem');
|
||||
fs.writeFileSync(file, contents, { mode: 0o600 });
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'--no-deprecation',
|
||||
'--input-type=commonjs',
|
||||
'-e',
|
||||
`const tls = require('node:tls');
|
||||
process.stdout.write(JSON.stringify(tls.getCACertificates('extra')));`,
|
||||
],
|
||||
{
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, NODE_EXTRA_CA_CERTS: file },
|
||||
timeout: 10_000,
|
||||
},
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`the loader oracle failed: ${result.stderr}`);
|
||||
}
|
||||
const blocks: unknown = JSON.parse(result.stdout);
|
||||
if (!Array.isArray(blocks) || !blocks.every((b) => typeof b === 'string')) {
|
||||
throw new Error('the loader oracle answered outside its contract');
|
||||
}
|
||||
return blocks.length > 0 ? blocks.map((b) => b.trimEnd()) : undefined;
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
describe.skipIf(!loaderOracleAvailable)('extractCertificateBlocks', () => {
|
||||
it('ignores NODE_OPTIONS that only affect eval-based oracle children', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-ca-preload-'));
|
||||
@@ -324,7 +370,9 @@ describe.skipIf(!loaderOracleAvailable)('extractCertificateBlocks', () => {
|
||||
'-----END CERTIFICATE-----\uFEFF',
|
||||
),
|
||||
]) {
|
||||
expect(extractCertificateBlocks(file)).toEqual([ROOT_PEM.trim()]);
|
||||
// The loader is the oracle, and its BOM verdict is build-specific: take
|
||||
// whatever this Node answers, and hold the module to it.
|
||||
expect(extractCertificateBlocks(file)).toEqual(loaderBlocks(file));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -481,16 +529,13 @@ describe.skipIf(!loaderOracleAvailable)('extractCertificateBlocks', () => {
|
||||
});
|
||||
|
||||
it('takes a block whose header section is empty', () => {
|
||||
// Oracle: authorized=true for both an empty line and a BOM-only line
|
||||
// directly after BEGIN — the loader splits the block there and reads the
|
||||
// rest as data. The BOM-only line is the loader's own separator, not a
|
||||
// body line, which is why it must not be judged as base64.
|
||||
// Both an empty line and a BOM-only line directly after BEGIN split the
|
||||
// block there, and the rest is read as data. The BOM-only line is judged
|
||||
// by the loader, not by us: x64 takes it and arm64 refuses it, so the
|
||||
// expectation is the loader's own answer for this build.
|
||||
for (const separator of ['', '\uFEFF']) {
|
||||
expect(
|
||||
extractCertificateBlocks(
|
||||
`-----BEGIN CERTIFICATE-----\n${separator}\n${bodyLines(ROOT_PEM).join('\n')}\n-----END CERTIFICATE-----\n`,
|
||||
),
|
||||
).toEqual([ROOT_PEM.trim()]);
|
||||
const file = `-----BEGIN CERTIFICATE-----\n${separator}\n${bodyLines(ROOT_PEM).join('\n')}\n-----END CERTIFICATE-----\n`;
|
||||
expect(extractCertificateBlocks(file)).toEqual(loaderBlocks(file));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1690,20 +1690,28 @@ describe('extension tests', () => {
|
||||
it('signals the durable commit before deferred settings finish', async () => {
|
||||
const { manager, prepared } = await readyPrepared('settings-deferred');
|
||||
const settings = deferred();
|
||||
const settingsStarted = deferred();
|
||||
const commitSettings = stubCommitSettings(
|
||||
prepared,
|
||||
vi.fn(async () => await settings.promise),
|
||||
vi.fn(async () => {
|
||||
settingsStarted.resolve();
|
||||
await settings.promise;
|
||||
}),
|
||||
);
|
||||
const onCommitted = vi.fn();
|
||||
|
||||
const committing = manager.commitPreparedExtension(prepared, onCommitted);
|
||||
await vi.waitFor(() => expect(commitSettings).toHaveBeenCalledOnce());
|
||||
|
||||
expect(onCommitted).toHaveBeenCalledOnce();
|
||||
expect(onCommitted.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
commitSettings.mock.invocationCallOrder[0]!,
|
||||
);
|
||||
settings.resolve();
|
||||
try {
|
||||
await Promise.race([settingsStarted.promise, committing]);
|
||||
expect(commitSettings).toHaveBeenCalledOnce();
|
||||
expect(onCommitted).toHaveBeenCalledOnce();
|
||||
expect(onCommitted.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
commitSettings.mock.invocationCallOrder[0]!,
|
||||
);
|
||||
} finally {
|
||||
settings.resolve();
|
||||
await committing;
|
||||
}
|
||||
await expect(committing).resolves.toMatchObject({
|
||||
identity: { name: 'settings-deferred' },
|
||||
});
|
||||
|
||||
@@ -4,168 +4,7 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { promises as dns } from 'node:dns';
|
||||
import { BlockList, isIP, type LookupFunction } from 'node:net';
|
||||
|
||||
import type { ExtensionNetworkPolicy } from '../config/config.js';
|
||||
|
||||
const blockedAddresses = new BlockList();
|
||||
const publicIpv6Addresses = new BlockList();
|
||||
publicIpv6Addresses.addSubnet('2000::', 3, 'ipv6');
|
||||
|
||||
for (const [address, prefix] of [
|
||||
['0.0.0.0', 8],
|
||||
['10.0.0.0', 8],
|
||||
['100.64.0.0', 10],
|
||||
['127.0.0.0', 8],
|
||||
['169.254.0.0', 16],
|
||||
['172.16.0.0', 12],
|
||||
['192.0.0.0', 24],
|
||||
['192.0.2.0', 24],
|
||||
['192.88.99.0', 24],
|
||||
['192.168.0.0', 16],
|
||||
['198.18.0.0', 15],
|
||||
['198.51.100.0', 24],
|
||||
['203.0.113.0', 24],
|
||||
['224.0.0.0', 4],
|
||||
['240.0.0.0', 4],
|
||||
] as const) {
|
||||
blockedAddresses.addSubnet(address, prefix, 'ipv4');
|
||||
}
|
||||
|
||||
for (const [address, prefix] of [
|
||||
['2001::', 23],
|
||||
['2001:db8::', 32],
|
||||
['2002::', 16],
|
||||
['3fff::', 20],
|
||||
] as const) {
|
||||
blockedAddresses.addSubnet(address, prefix, 'ipv6');
|
||||
}
|
||||
|
||||
export interface ResolvedNetworkTarget {
|
||||
url: URL;
|
||||
lookup?: LookupFunction;
|
||||
curlResolve?: string;
|
||||
}
|
||||
|
||||
async function waitForAbortable<T>(
|
||||
promise: Promise<T>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
if (!signal) return await promise;
|
||||
signal.throwIfAborted();
|
||||
return await new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const finish = (callback: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
callback();
|
||||
};
|
||||
const onAbort = () => finish(() => reject(signal.reason));
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
promise.then(
|
||||
(value) => finish(() => resolve(value)),
|
||||
(error: unknown) => finish(() => reject(error)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function stripIpv6Brackets(hostname: string): string {
|
||||
return hostname.startsWith('[') && hostname.endsWith(']')
|
||||
? hostname.slice(1, -1)
|
||||
: hostname;
|
||||
}
|
||||
|
||||
function parseMappedIpv4(address: string): string | undefined {
|
||||
const suffix = address.toLowerCase().slice('::ffff:'.length);
|
||||
if (isIP(suffix) === 4) return suffix;
|
||||
|
||||
const parts = suffix.split(':');
|
||||
if (
|
||||
parts.length !== 2 ||
|
||||
parts.some((part) => !/^[\da-f]{1,4}$/.test(part))
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const upper = Number.parseInt(parts[0], 16);
|
||||
const lower = Number.parseInt(parts[1], 16);
|
||||
return `${upper >> 8}.${upper & 0xff}.${lower >> 8}.${lower & 0xff}`;
|
||||
}
|
||||
|
||||
function isBlockedAddress(address: string, family: number): boolean {
|
||||
if (family === 6 && address.toLowerCase().startsWith('::ffff:')) {
|
||||
const mapped = parseMappedIpv4(address);
|
||||
return mapped === undefined || isBlockedAddress(mapped, 4);
|
||||
}
|
||||
if (family === 6 && !publicIpv6Addresses.check(address, 'ipv6')) return true;
|
||||
return blockedAddresses.check(address, family === 6 ? 'ipv6' : 'ipv4');
|
||||
}
|
||||
|
||||
export async function resolveNetworkTarget(
|
||||
value: string | URL,
|
||||
policy?: ExtensionNetworkPolicy,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ResolvedNetworkTarget> {
|
||||
signal?.throwIfAborted();
|
||||
const url = value instanceof URL ? value : new URL(value);
|
||||
if (policy !== 'public') return { url };
|
||||
if (url.protocol !== 'https:') {
|
||||
throw new Error('Public extension network requests must use HTTPS.');
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new Error(
|
||||
'Public extension network requests must not use credentials.',
|
||||
);
|
||||
}
|
||||
|
||||
const hostname = stripIpv6Brackets(url.hostname);
|
||||
const literalFamily = isIP(hostname);
|
||||
const addresses = literalFamily
|
||||
? [{ address: hostname, family: literalFamily }]
|
||||
: await waitForAbortable(
|
||||
dns.lookup(hostname, { all: true, verbatim: true }),
|
||||
signal,
|
||||
);
|
||||
if (addresses.length === 0) {
|
||||
throw new Error(`Extension network host did not resolve: ${hostname}`);
|
||||
}
|
||||
if (
|
||||
addresses.some(({ address, family }) => isBlockedAddress(address, family))
|
||||
) {
|
||||
throw new Error(
|
||||
`Extension network host resolved to a blocked address: ${hostname}`,
|
||||
);
|
||||
}
|
||||
|
||||
const selected = addresses[0];
|
||||
const lookup: LookupFunction = (requestedHostname, options, callback) => {
|
||||
if (stripIpv6Brackets(requestedHostname) !== hostname) {
|
||||
const error = new Error(
|
||||
'Pinned extension lookup hostname mismatch',
|
||||
) as NodeJS.ErrnoException;
|
||||
error.code = 'ENOTFOUND';
|
||||
callback(error, '', 0);
|
||||
return;
|
||||
}
|
||||
if (options.all) {
|
||||
callback(null, [selected]);
|
||||
} else {
|
||||
callback(null, selected.address, selected.family);
|
||||
}
|
||||
};
|
||||
const port = url.port || '443';
|
||||
const curlHostname = literalFamily === 6 ? `[${hostname}]` : hostname;
|
||||
const curlAddress =
|
||||
selected.family === 6 ? `[${selected.address}]` : selected.address;
|
||||
|
||||
return {
|
||||
url,
|
||||
lookup,
|
||||
curlResolve: `${curlHostname}:${port}:${curlAddress}`,
|
||||
};
|
||||
}
|
||||
export {
|
||||
resolveNetworkTarget,
|
||||
type ResolvedNetworkTarget,
|
||||
} from '../utils/network-policy.js';
|
||||
@@ -128,6 +128,10 @@ async function checkpointOf(session: ManagedSession) {
|
||||
}
|
||||
|
||||
describe('LocalManagedRuntimeOutcomes', () => {
|
||||
// 65 paged admits and a cold reopen: ~9.5 s on a warm laptop, but the ECS
|
||||
// pool runs several suites at once and the flat 60 s budget timed out there —
|
||||
// #15366, #15261 and #15308 each lost their Test gate to this one test.
|
||||
// Only the budget moves; every assertion stays identical.
|
||||
it('continues and cold-opens a paged Turn without losing the earliest receipt', async () => {
|
||||
const root = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'qwen-paged-outcomes-'),
|
||||
@@ -171,7 +175,7 @@ describe('LocalManagedRuntimeOutcomes', () => {
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
}, 60_000);
|
||||
}, 180_000);
|
||||
|
||||
it('admits a call with its intent and an await_runtime checkpoint', async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-outcomes-'));
|
||||
|
||||
@@ -345,17 +345,15 @@ describe('downloadMediaUrl', () => {
|
||||
});
|
||||
|
||||
it('translates resolver errors into download-domain phrasing', async () => {
|
||||
// resolveNetworkTarget phrases its errors for its original caller
|
||||
// ("Extension network host …"); passing that through would misattribute
|
||||
// the refusal (the credentials gate avoids the same problem), so the
|
||||
// reachable resolver errors are translated, echoing only the hostname.
|
||||
// Resolver failures are translated into download-domain phrasing,
|
||||
// echoing only the hostname.
|
||||
const cases: Array<[string, RegExp]> = [
|
||||
[
|
||||
'Extension network host resolved to a blocked address: media.example.com',
|
||||
'Public network host resolved to a blocked address: media.example.com',
|
||||
/URL host resolved to a non-public address/,
|
||||
],
|
||||
[
|
||||
'Extension network host did not resolve: media.example.com',
|
||||
'Public network host did not resolve: media.example.com',
|
||||
/URL host did not resolve/,
|
||||
],
|
||||
['socket hang up', /URL host could not be verified/],
|
||||
@@ -598,8 +596,7 @@ describe('downloadMediaUrl', () => {
|
||||
});
|
||||
|
||||
it('refuses URLs that embed credentials, without echoing them', async () => {
|
||||
// resolveNetworkTarget would also reject these, but naming "extension
|
||||
// network requests". Assert the refusal and that neither user nor password
|
||||
// Assert the download-specific refusal and that neither user nor password
|
||||
// reaches the error text (rendered in the UI, written to the debug log).
|
||||
const fetchFn = vi.fn<typeof fetch>();
|
||||
const err = await dlErr({
|
||||
|
||||
@@ -247,11 +247,8 @@ export async function downloadMediaUrl(params: {
|
||||
);
|
||||
}
|
||||
|
||||
// Credentials in the URL are refused explicitly. `resolveNetworkTarget`
|
||||
// would also reject them, but its message says "extension network requests"
|
||||
// (that helper's original caller), which misattributes the reason here. The
|
||||
// check is deliberately silent about the values themselves: OmniDownloadError
|
||||
// messages reach the UI and the debug log.
|
||||
// Refuse credentials here with download-domain wording; never echo their
|
||||
// values in the UI or debug log.
|
||||
if (parsed.username || parsed.password) {
|
||||
throw new OmniDownloadError(
|
||||
`URL media must not embed credentials (refused for safety): ` +
|
||||
@@ -363,11 +360,8 @@ export async function downloadMediaUrl(params: {
|
||||
// Fail closed: unlike the general web-fetch path, these bytes are
|
||||
// uploaded to a third party, so "could not verify" must mean "do not
|
||||
// fetch" rather than "connect and hope". The resolver's own text is
|
||||
// translated rather than echoed: resolveNetworkTarget phrases its
|
||||
// errors as "Extension network host …" (that helper's original
|
||||
// caller), the same misattribution the credentials check above
|
||||
// avoids — and nothing server-controlled beyond the hostname may
|
||||
// enter the message anyway.
|
||||
// translated into download-domain phrasing rather than echoed, and
|
||||
// nothing server-controlled beyond the hostname may enter the message.
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
const reason = /blocked address/i.test(text)
|
||||
? 'resolved to a non-public address'
|
||||
|
||||
@@ -161,6 +161,136 @@ describe('WebFetchTool', () => {
|
||||
return { result, sent };
|
||||
}
|
||||
|
||||
describe('managed execution', () => {
|
||||
const managedRun = (url = 'https://example.com/docs') =>
|
||||
new WebFetchTool(mockConfig, 'managed')
|
||||
.build({ url, prompt: 'Summarize', format: 'markdown' })
|
||||
.execute(new AbortController().signal);
|
||||
|
||||
it('returns normalized text without a side query, cache or host files', async () => {
|
||||
const fetch = stubFetch({
|
||||
body: Buffer.from('<h1>Docs</h1><p>Details</p>'),
|
||||
});
|
||||
await run('https://example.com/docs');
|
||||
const localFetches = fetch.mock.calls.length;
|
||||
mockGetBaseLlmClient.mockClear();
|
||||
const result = await managedRun();
|
||||
await managedRun();
|
||||
expect(result.llmContent).toContain('Docs\n====');
|
||||
expect(result.llmContent).toContain('Details');
|
||||
expect(result.llmContent).toContain('Status: 200');
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(fetch).toHaveBeenCalledTimes(localFetches + 2);
|
||||
expect(fetch).toHaveBeenLastCalledWith(
|
||||
'https://example.com/docs',
|
||||
expect.objectContaining({
|
||||
networkPolicy: 'public',
|
||||
headers: expect.objectContaining({
|
||||
Accept: 'text/markdown, */*;q=0.1',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mockGetBaseLlmClient).not.toHaveBeenCalled();
|
||||
expect(fs.readdirSync(toolResultsDir)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['http://example.com', 'https://user:[email protected]'])(
|
||||
'refuses %s before fetching',
|
||||
async (url) => {
|
||||
const fetch = stubFetch();
|
||||
expect(() =>
|
||||
new WebFetchTool(mockConfig, 'managed').build({
|
||||
url,
|
||||
prompt: 'Summarize',
|
||||
}),
|
||||
).toThrow();
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('refuses binary downloads without writing a file', async () => {
|
||||
stubFetch(PDF);
|
||||
const result = await managedRun();
|
||||
expect(result.error?.message).toContain('text content only');
|
||||
expect(fs.readdirSync(toolResultsDir)).toEqual([]);
|
||||
expect(mockExtractPDFText).not.toHaveBeenCalled();
|
||||
expect(mockGetBaseLlmClient).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ kind: 'Unicode', content: '文档😀'.repeat(20_000) },
|
||||
{ kind: 'escaped', content: '\u0001"\\'.repeat(20_000) },
|
||||
])(
|
||||
'bounds $kind output without spilling large pages',
|
||||
async ({ content }) => {
|
||||
stubFetch({ contentType: 'text/plain', body: Buffer.from(content) });
|
||||
const result = await managedRun();
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.llmContent).toContain(content.slice(0, 30));
|
||||
expect(result.llmContent).toContain('Content truncated');
|
||||
expect(
|
||||
Buffer.byteLength(JSON.stringify(result.llmContent)),
|
||||
).toBeLessThanOrEqual(16 * 1024);
|
||||
expect(fs.readdirSync(toolResultsDir)).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it('does not downgrade after a TLS failure', async () => {
|
||||
const fetch = vi
|
||||
.spyOn(fetchUtils, 'fetchWithPolicy')
|
||||
.mockRejectedValue(
|
||||
new fetchUtils.FetchError('TLS failed', 'ECONNREFUSED'),
|
||||
);
|
||||
expect((await managedRun()).error).toBeDefined();
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps a redirect destination visible even with a large original URL', async () => {
|
||||
const url = `https://example.com/?q=${'x'.repeat(50_000)}`;
|
||||
vi.spyOn(fetchUtils, 'fetchWithPolicy').mockResolvedValue({
|
||||
kind: 'cross-host-redirect',
|
||||
originalUrl: url,
|
||||
redirectUrl: 'https://other.example/docs',
|
||||
status: 302,
|
||||
});
|
||||
const result = await managedRun(url);
|
||||
expect(result.llmContent).toContain('https://other.example/docs');
|
||||
expect(
|
||||
Buffer.byteLength(JSON.stringify(result.llmContent)),
|
||||
).toBeLessThanOrEqual(16 * 1024);
|
||||
});
|
||||
|
||||
it('bounds failure text and error metadata within the durable result budget', async () => {
|
||||
const url = `https://example.com/?q=${'x'.repeat(33_500)}`;
|
||||
vi.spyOn(fetchUtils, 'fetchWithPolicy').mockRejectedValue(
|
||||
new Error('Connection failed'),
|
||||
);
|
||||
const result = await managedRun(url);
|
||||
expect(result.error).toBeDefined();
|
||||
expect(result.llmContent).toContain('Content truncated');
|
||||
expect(
|
||||
Buffer.byteLength(JSON.stringify(result.llmContent)),
|
||||
).toBeLessThanOrEqual(16 * 1024);
|
||||
expect(
|
||||
Buffer.byteLength(JSON.stringify(result.error!.message)),
|
||||
).toBeLessThanOrEqual(16 * 1024);
|
||||
expect(
|
||||
Buffer.byteLength(
|
||||
JSON.stringify({
|
||||
executionCallId: 'call',
|
||||
functionResponse: {
|
||||
response: {
|
||||
error: result.llmContent,
|
||||
runtimeError: result.error,
|
||||
executionStatus: 'error',
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
).toBeLessThan(64 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe('execute', () => {
|
||||
it('should throw validation error when url parameter is missing', async () => {
|
||||
const tool = new WebFetchTool(mockConfig);
|
||||
|
||||
@@ -65,6 +65,7 @@ export function sideQueryTimeoutMs(): number {
|
||||
// Truncation applies to converted/decoded text, never to raw HTML — cutting
|
||||
// markup before conversion silently destroys content deep in large pages.
|
||||
const MAX_CONTENT_CHARS = 100_000;
|
||||
const MANAGED_MAX_CONTENT_BYTES = 16 * 1024;
|
||||
const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_REDIRECTS = 10;
|
||||
|
||||
@@ -227,6 +228,26 @@ export interface WebFetchToolParams {
|
||||
format?: 'auto' | 'markdown' | 'html' | 'text';
|
||||
}
|
||||
|
||||
function managedContentPreview(content: string): string {
|
||||
if (Buffer.byteLength(JSON.stringify(content)) <= MANAGED_MAX_CONTENT_BYTES)
|
||||
return content;
|
||||
const notice =
|
||||
'\n\n[Content truncated to fit the Managed Session result limit.]';
|
||||
let low = 0;
|
||||
let high = content.length;
|
||||
while (low < high) {
|
||||
const mid = Math.ceil((low + high) / 2);
|
||||
if (
|
||||
Buffer.byteLength(JSON.stringify(content.slice(0, mid) + notice)) <=
|
||||
MANAGED_MAX_CONTENT_BYTES
|
||||
)
|
||||
low = mid;
|
||||
else high = mid - 1;
|
||||
}
|
||||
if (low > 0 && /[\uD800-\uDBFF]/u.test(content[low - 1])) low--;
|
||||
return content.slice(0, low) + notice;
|
||||
}
|
||||
|
||||
/**
|
||||
* Implementation of the WebFetch tool invocation logic
|
||||
*/
|
||||
@@ -239,6 +260,7 @@ class WebFetchToolInvocation extends BaseToolInvocation<
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
params: WebFetchToolParams,
|
||||
private readonly mode: 'local' | 'managed',
|
||||
) {
|
||||
super(params);
|
||||
this.debugLogger = createDebugLogger('WEB_FETCH');
|
||||
@@ -306,7 +328,10 @@ class WebFetchToolInvocation extends BaseToolInvocation<
|
||||
|
||||
private buildRedirectResult(redirect: FetchPolicyRedirect): ToolResult {
|
||||
const redirectHost = hostnameOf(redirect.redirectUrl);
|
||||
const message = `REDIRECT DETECTED: The URL redirects to a different host (or scheme/port), which was not followed automatically.
|
||||
const message =
|
||||
this.mode === 'managed'
|
||||
? `Redirect (${redirect.status}) was not followed. Make a new web_fetch call for this destination, subject to the same network policy and approval:\n${redirect.redirectUrl}`
|
||||
: `REDIRECT DETECTED: The URL redirects to a different host (or scheme/port), which was not followed automatically.
|
||||
|
||||
Original URL: ${redirect.originalUrl}
|
||||
Redirect URL: ${redirect.redirectUrl}
|
||||
@@ -335,7 +360,8 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
): Promise<CacheEntry | FetchPolicyRedirect> {
|
||||
const fetchUrl = this.getFetchUrl();
|
||||
const acceptHeader = this.getAcceptHeader();
|
||||
const urlCache = getUrlCacheFor(this.config.storage);
|
||||
const urlCache =
|
||||
this.mode === 'managed' ? undefined : getUrlCacheFor(this.config.storage);
|
||||
|
||||
// The format param changes the Accept header, which can change what the
|
||||
// server returns — a URL-only key would serve format:'html' content to a
|
||||
@@ -343,7 +369,7 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
// that share one Config/Storage (/clear and /new call startNewSession
|
||||
// with a fresh ID); old-session entries age out of the LRU.
|
||||
const cacheKey = `${this.config.getSessionId?.() ?? ''}|${acceptHeader} ${fetchUrl}`;
|
||||
const cached = urlCache.get(cacheKey);
|
||||
const cached = urlCache?.get(cacheKey);
|
||||
if (cached && Date.now() - cached.fetchedAt < CACHE_TTL_MS) {
|
||||
this.debugLogger.debug(`[WebFetchTool] Cache hit for ${fetchUrl}`);
|
||||
return cached;
|
||||
@@ -353,6 +379,7 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
);
|
||||
|
||||
const policyOptions = {
|
||||
...(this.mode === 'managed' ? { networkPolicy: 'public' as const } : {}),
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxBytes: MAX_RESPONSE_BYTES,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
@@ -375,7 +402,11 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
// caller actually asked for — the user approved the http URL they
|
||||
// supplied, and fallback responses are never cached (see below).
|
||||
const { upgradedFrom } = this.getFetchPlan();
|
||||
if (upgradedFrom && isConnectionLevelError(error)) {
|
||||
if (
|
||||
this.mode !== 'managed' &&
|
||||
upgradedFrom &&
|
||||
isConnectionLevelError(error)
|
||||
) {
|
||||
this.debugLogger.debug(
|
||||
`[WebFetchTool] https upgrade failed (${(error as Error).message}); retrying original ${upgradedFrom}`,
|
||||
);
|
||||
@@ -441,6 +472,10 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
let content: string;
|
||||
|
||||
if (isBinary) {
|
||||
if (this.mode === 'managed')
|
||||
throw new Error(
|
||||
'Managed web_fetch supports text content only; binary downloads are unavailable.',
|
||||
);
|
||||
// Keep the raw bytes on disk so the model (or shell tools) can inspect
|
||||
// them. Binary bytes are never UTF-8 decoded for the side-query —
|
||||
// mojibake wastes tokens and misleads the model. PDFs get real text
|
||||
@@ -514,7 +549,7 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
let persistedTextPath: string | undefined;
|
||||
let rawOutputSize: ToolOutputSize | undefined;
|
||||
let reduced = false;
|
||||
if (content.length > MAX_CONTENT_CHARS) {
|
||||
if (this.mode !== 'managed' && content.length > MAX_CONTENT_CHARS) {
|
||||
rawOutputSize = measureToolOutput(content);
|
||||
const shortened = await truncateToolOutput(
|
||||
this.config,
|
||||
@@ -555,7 +590,7 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
// endpoint — a silent downgrade of exactly the no-downgrade guarantee the
|
||||
// fallback is scoped to preserve.
|
||||
if (!usedHttpFallback && content.length <= MAX_CACHEABLE_CHARS) {
|
||||
urlCache.set(cacheKey, entry);
|
||||
urlCache?.set(cacheKey, entry);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
@@ -586,6 +621,14 @@ Status: ${entry.status} ${entry.statusText || 'OK'} | Content-Type: ${entry.cont
|
||||
|
||||
const displaySummary = `Received ${formatByteSize(entry.byteLength)} (${entry.status} ${entry.statusText || 'OK'}) from ${hostnameOf(this.params.url)}${entry.persistedPath ? ` — binary saved to ${entry.persistedPath}` : ''}`;
|
||||
|
||||
if (this.mode === 'managed') {
|
||||
signal.throwIfAborted();
|
||||
return {
|
||||
llmContent: `${header}\n\n${entry.content}`,
|
||||
returnDisplay: `${displaySummary}${elapsedSuffix()}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Binary with no extractable text: nothing useful to summarize —
|
||||
// return the metadata and file location without a side-query call.
|
||||
if (entry.persistedPath && !entry.content) {
|
||||
@@ -770,7 +813,27 @@ ${entry.content}
|
||||
}
|
||||
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
return this.executeDirectFetch(signal);
|
||||
const result = await this.executeDirectFetch(signal);
|
||||
if (this.mode !== 'managed') return result;
|
||||
return {
|
||||
...result,
|
||||
llmContent:
|
||||
typeof result.llmContent === 'string'
|
||||
? managedContentPreview(result.llmContent)
|
||||
: result.llmContent,
|
||||
returnDisplay:
|
||||
typeof result.returnDisplay === 'string'
|
||||
? managedContentPreview(result.returnDisplay)
|
||||
: result.returnDisplay,
|
||||
...(result.error
|
||||
? {
|
||||
error: {
|
||||
...result.error,
|
||||
message: managedContentPreview(result.error.message),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -783,7 +846,10 @@ export class WebFetchTool extends BaseDeclarativeTool<
|
||||
> {
|
||||
static readonly Name: string = ToolNames.WEB_FETCH;
|
||||
|
||||
constructor(private readonly config: Config) {
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly mode: 'local' | 'managed' = 'local',
|
||||
) {
|
||||
super(
|
||||
WebFetchTool.Name,
|
||||
ToolDisplayNames.WEB_FETCH,
|
||||
@@ -832,6 +898,8 @@ export class WebFetchTool extends BaseDeclarativeTool<
|
||||
if (parsedUrl.username || parsedUrl.password) {
|
||||
return "The 'url' must not include credentials.";
|
||||
}
|
||||
if (this.mode === 'managed' && parsedUrl.protocol !== 'https:')
|
||||
return 'Managed web_fetch requires an HTTPS URL.';
|
||||
} catch {
|
||||
return "The 'url' is malformed and could not be parsed.";
|
||||
}
|
||||
@@ -847,10 +915,14 @@ export class WebFetchTool extends BaseDeclarativeTool<
|
||||
// Normalize before the invocation exists: the scheduler feeds
|
||||
// invocation.params into evaluatePermissionFlow, so this is what makes
|
||||
// domain rules match the host actually contacted.
|
||||
return new WebFetchToolInvocation(this.config, {
|
||||
...params,
|
||||
url: rewriteGitHubBlobUrl(params.url),
|
||||
});
|
||||
return new WebFetchToolInvocation(
|
||||
this.config,
|
||||
{
|
||||
...params,
|
||||
url: rewriteGitHubBlobUrl(params.url),
|
||||
},
|
||||
this.mode,
|
||||
);
|
||||
}
|
||||
|
||||
override toAutoClassifierInput(
|
||||
|
||||
@@ -5,6 +5,26 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { promises as dns } from 'node:dns';
|
||||
import * as undici from 'undici';
|
||||
import * as runtimeFetchOptions from './runtimeFetchOptions.js';
|
||||
|
||||
const agentOptions = vi.hoisted(() => vi.fn());
|
||||
vi.mock('undici', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof undici>();
|
||||
return {
|
||||
...actual,
|
||||
fetch: vi.fn((...args: Parameters<typeof fetch>) =>
|
||||
globalThis.fetch(...args),
|
||||
),
|
||||
Agent: class extends actual.Agent {
|
||||
constructor(options?: undici.Agent.Options) {
|
||||
super(options);
|
||||
agentOptions(options);
|
||||
}
|
||||
},
|
||||
};
|
||||
});
|
||||
import {
|
||||
CONNECTION_LEVEL_ERROR_CODES,
|
||||
FetchError,
|
||||
@@ -53,6 +73,122 @@ afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
describe('public fetch policy', () => {
|
||||
const options: FetchPolicyOptions = {
|
||||
networkPolicy: 'public',
|
||||
timeoutMs: 5_000,
|
||||
maxBytes: 1024,
|
||||
maxRedirects: 3,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
agentOptions.mockClear();
|
||||
vi.mocked(undici.fetch).mockClear();
|
||||
globalThis.fetch = vi
|
||||
.fn()
|
||||
.mockImplementation(async () => new Response('docs'));
|
||||
});
|
||||
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it.each([
|
||||
'http://example.com',
|
||||
'https://user:[email protected]',
|
||||
'https://127.0.0.1',
|
||||
'https://169.254.169.254',
|
||||
'https://100.100.100.200',
|
||||
'https://[::1]',
|
||||
'https://[::ffff:7f00:1]',
|
||||
])('refuses %s before opening a connection', async (url) => {
|
||||
await expect(fetchWithPolicy(url, options)).rejects.toThrow();
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses mixed public/private DNS answers', async () => {
|
||||
vi.spyOn(dns, 'lookup').mockResolvedValue([
|
||||
{ address: '93.184.216.34', family: 4 },
|
||||
{ address: '10.0.0.1', family: 4 },
|
||||
] as never);
|
||||
await expect(
|
||||
fetchWithPolicy('https://example.com', options),
|
||||
).rejects.toThrow('blocked address');
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses runtimes that cannot enforce the pinned dispatcher', async () => {
|
||||
vi.spyOn(runtimeFetchOptions, 'detectRuntime').mockReturnValue('bun');
|
||||
await expect(fetchWithPolicy('https://8.8.8.8', options)).rejects.toThrow(
|
||||
'require Node.js',
|
||||
);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
expect(agentOptions).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['redirect', 'retry'])(
|
||||
'rechecks DNS after a %s and refuses rebinding',
|
||||
async (mode) => {
|
||||
const lookup = vi
|
||||
.spyOn(dns, 'lookup')
|
||||
.mockResolvedValueOnce([
|
||||
{ address: '93.184.216.34', family: 4 },
|
||||
] as never)
|
||||
.mockResolvedValue([{ address: '127.0.0.1', family: 4 }] as never);
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
mode === 'redirect'
|
||||
? redirect(302, '/next')
|
||||
: new Response(null, { status: 429 }),
|
||||
);
|
||||
if (mode === 'redirect')
|
||||
await expect(
|
||||
fetchWithPolicy('https://example.com', options),
|
||||
).rejects.toThrow('blocked address');
|
||||
else
|
||||
expect(
|
||||
(await fetchWithPolicy('https://example.com', options)).kind,
|
||||
).toBe('response');
|
||||
expect(lookup).toHaveBeenCalledTimes(2);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it('pins DNS in a dedicated TLS-verified dispatcher and closes it after the body', async () => {
|
||||
const lookup = vi
|
||||
.spyOn(dns, 'lookup')
|
||||
.mockResolvedValue([{ address: '93.184.216.34', family: 4 }] as never);
|
||||
const response = await fetchResponse('https://example.com/docs', options);
|
||||
expect(response.body.toString()).toBe('docs');
|
||||
expect(undici.fetch).toHaveBeenCalledTimes(1);
|
||||
expect(agentOptions).toHaveBeenCalledWith({
|
||||
connect: { lookup: expect.any(Function), rejectUnauthorized: true },
|
||||
});
|
||||
const pinnedLookup = agentOptions.mock.calls[0][0]!.connect as {
|
||||
lookup: NonNullable<import('node:net').LookupFunction>;
|
||||
};
|
||||
const resolved = vi.fn();
|
||||
pinnedLookup.lookup('example.com', {}, resolved);
|
||||
expect(resolved).toHaveBeenCalledWith(null, '93.184.216.34', 4);
|
||||
expect(lookup).toHaveBeenCalledTimes(1);
|
||||
const init = vi.mocked(fetch).mock.calls[0][1] as RequestInit & {
|
||||
dispatcher: undici.Agent;
|
||||
};
|
||||
expect(init.dispatcher).toBeInstanceOf(undici.Agent);
|
||||
expect(init.dispatcher.closed).toBe(true);
|
||||
});
|
||||
|
||||
it('does not follow a cross-host redirect', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
redirect(302, 'https://127.0.0.1/private'),
|
||||
);
|
||||
expect(
|
||||
await fetchWithPolicy('https://8.8.8.8/docs', options),
|
||||
).toMatchObject({
|
||||
kind: 'cross-host-redirect',
|
||||
redirectUrl: 'https://127.0.0.1/private',
|
||||
});
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatFetchErrorForUser', () => {
|
||||
const saved = {
|
||||
QWEN_TLS_INSECURE: process.env['QWEN_TLS_INSECURE'],
|
||||
|
||||
@@ -6,8 +6,14 @@
|
||||
|
||||
import { getErrorMessage } from './errors.js';
|
||||
import { delay } from './retry.js';
|
||||
import { isTlsVerificationDisabled } from './runtimeFetchOptions.js';
|
||||
import {
|
||||
detectRuntime,
|
||||
isTlsVerificationDisabled,
|
||||
loadUndici,
|
||||
} from './runtimeFetchOptions.js';
|
||||
import { URL } from 'node:url';
|
||||
import type { Agent } from 'undici';
|
||||
import { resolveNetworkTarget } from './network-policy.js';
|
||||
|
||||
const PRIVATE_IP_RANGES = [
|
||||
/^10\./,
|
||||
@@ -189,6 +195,8 @@ export function isPermittedRedirect(
|
||||
}
|
||||
|
||||
export interface FetchPolicyOptions {
|
||||
/** HTTPS with public DNS addresses pinned to each connection. */
|
||||
networkPolicy?: 'public';
|
||||
/** Budget for the entire transfer — headers AND body. */
|
||||
timeoutMs: number;
|
||||
/** Reject responses larger than this, before or during body read. */
|
||||
@@ -237,7 +245,8 @@ const RETRY_DELAY_MS = 500;
|
||||
* Fetch with manual redirect handling, a full-transfer timeout, a byte cap
|
||||
* enforced while streaming, caller-abort wiring, and a single retry on
|
||||
* transient failures (403/429 statuses, connection resets). Uses the global
|
||||
* fetch so the process-wide proxy dispatcher (setGlobalDispatcher) applies.
|
||||
* fetch so the process-wide proxy dispatcher (setGlobalDispatcher) applies
|
||||
* unless a public-only request supplies its own pinned, TLS-verified agent.
|
||||
* The timeout budget spans both attempts.
|
||||
*/
|
||||
export async function fetchWithPolicy(
|
||||
@@ -344,108 +353,128 @@ async function fetchPolicyAttempt(
|
||||
|
||||
let currentUrl = url;
|
||||
for (let hop = 0; hop <= options.maxRedirects; hop++) {
|
||||
let response: Response;
|
||||
let dispatcher: Agent | undefined;
|
||||
try {
|
||||
response = await fetch(currentUrl, {
|
||||
signal,
|
||||
headers: options.headers,
|
||||
redirect: 'manual',
|
||||
});
|
||||
} catch (error) {
|
||||
return wrapAbort(error);
|
||||
}
|
||||
|
||||
if (REDIRECT_STATUSES.has(response.status)) {
|
||||
const location = response.headers.get('location');
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (!location) {
|
||||
throw new FetchError('Redirect response missing Location header');
|
||||
let request = fetch;
|
||||
if (options.networkPolicy === 'public') {
|
||||
if (detectRuntime() !== 'node')
|
||||
throw new FetchError(
|
||||
'Public network requests require Node.js for DNS-pinned connections.',
|
||||
);
|
||||
const target = await resolveNetworkTarget(currentUrl, 'public', signal);
|
||||
const undici = await loadUndici();
|
||||
dispatcher = new undici.Agent({
|
||||
connect: { lookup: target.lookup, rejectUnauthorized: true },
|
||||
});
|
||||
request = undici.fetch as unknown as typeof fetch;
|
||||
}
|
||||
let redirectUrl: string;
|
||||
let response: Response;
|
||||
try {
|
||||
redirectUrl = new URL(location, currentUrl).toString();
|
||||
} catch {
|
||||
throw new FetchError(
|
||||
`Redirect response has a malformed Location header: ${location}`,
|
||||
);
|
||||
const init: RequestInit & { dispatcher?: Agent } = {
|
||||
signal,
|
||||
headers: options.headers,
|
||||
redirect: 'manual',
|
||||
...(dispatcher ? { dispatcher } : {}),
|
||||
};
|
||||
response = await request(currentUrl, init);
|
||||
} catch (error) {
|
||||
return wrapAbort(error);
|
||||
}
|
||||
if (!isPermittedRedirect(currentUrl, redirectUrl)) {
|
||||
|
||||
if (REDIRECT_STATUSES.has(response.status)) {
|
||||
const location = response.headers.get('location');
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (!location) {
|
||||
throw new FetchError('Redirect response missing Location header');
|
||||
}
|
||||
let redirectUrl: string;
|
||||
try {
|
||||
redirectUrl = new URL(location, currentUrl).toString();
|
||||
} catch {
|
||||
throw new FetchError(
|
||||
`Redirect response has a malformed Location header: ${location}`,
|
||||
);
|
||||
}
|
||||
if (!isPermittedRedirect(currentUrl, redirectUrl)) {
|
||||
return {
|
||||
kind: 'cross-host-redirect',
|
||||
originalUrl: currentUrl,
|
||||
redirectUrl,
|
||||
status: response.status,
|
||||
};
|
||||
}
|
||||
currentUrl = redirectUrl;
|
||||
continue;
|
||||
}
|
||||
|
||||
// The sole caller discards the body of every non-2xx response (only the
|
||||
// status/statusText feed its failure message, and the 403/429 retry path
|
||||
// inspects the status alone). Cancel the body without buffering it:
|
||||
// otherwise an oversized error page reports EMSGSIZE instead of its real
|
||||
// status, and a slow-trickling error body can burn the whole timeout for
|
||||
// bytes that are never read.
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
return {
|
||||
kind: 'cross-host-redirect',
|
||||
originalUrl: currentUrl,
|
||||
redirectUrl,
|
||||
kind: 'response',
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
contentType: response.headers.get('content-type') ?? '',
|
||||
contentDisposition: response.headers.get('content-disposition') ?? '',
|
||||
body: Buffer.alloc(0),
|
||||
finalUrl: currentUrl,
|
||||
};
|
||||
}
|
||||
currentUrl = redirectUrl;
|
||||
continue;
|
||||
}
|
||||
|
||||
// The sole caller discards the body of every non-2xx response (only the
|
||||
// status/statusText feed its failure message, and the 403/429 retry path
|
||||
// inspects the status alone). Cancel the body without buffering it:
|
||||
// otherwise an oversized error page reports EMSGSIZE instead of its real
|
||||
// status, and a slow-trickling error body can burn the whole timeout for
|
||||
// bytes that are never read.
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
const contentLength = Number(response.headers.get('content-length'));
|
||||
if (Number.isFinite(contentLength) && contentLength > options.maxBytes) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new FetchError(
|
||||
`Response too large: ${contentLength} bytes exceeds the ${options.maxBytes}-byte limit`,
|
||||
'EMSGSIZE',
|
||||
);
|
||||
}
|
||||
|
||||
// Chunks are retained as-is (undici yields a fresh Uint8Array per read);
|
||||
// Buffer.concat below performs the single necessary copy.
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
if (response.body) {
|
||||
const reader = response.body.getReader();
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > options.maxBytes) {
|
||||
throw new FetchError(
|
||||
`Response too large: exceeded the ${options.maxBytes}-byte limit while streaming`,
|
||||
'EMSGSIZE',
|
||||
);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch (error) {
|
||||
await reader.cancel().catch(() => {});
|
||||
if (error instanceof FetchError) throw error;
|
||||
return wrapAbort(error);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'response',
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
contentType: response.headers.get('content-type') ?? '',
|
||||
contentDisposition: response.headers.get('content-disposition') ?? '',
|
||||
body: Buffer.alloc(0),
|
||||
body: Buffer.concat(chunks, total),
|
||||
finalUrl: currentUrl,
|
||||
};
|
||||
} finally {
|
||||
await dispatcher?.close();
|
||||
}
|
||||
|
||||
const contentLength = Number(response.headers.get('content-length'));
|
||||
if (Number.isFinite(contentLength) && contentLength > options.maxBytes) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new FetchError(
|
||||
`Response too large: ${contentLength} bytes exceeds the ${options.maxBytes}-byte limit`,
|
||||
'EMSGSIZE',
|
||||
);
|
||||
}
|
||||
|
||||
// Chunks are retained as-is (undici yields a fresh Uint8Array per read);
|
||||
// Buffer.concat below performs the single necessary copy.
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
if (response.body) {
|
||||
const reader = response.body.getReader();
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > options.maxBytes) {
|
||||
throw new FetchError(
|
||||
`Response too large: exceeded the ${options.maxBytes}-byte limit while streaming`,
|
||||
'EMSGSIZE',
|
||||
);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch (error) {
|
||||
await reader.cancel().catch(() => {});
|
||||
if (error instanceof FetchError) throw error;
|
||||
return wrapAbort(error);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'response',
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
contentType: response.headers.get('content-type') ?? '',
|
||||
contentDisposition: response.headers.get('content-disposition') ?? '',
|
||||
body: Buffer.concat(chunks, total),
|
||||
finalUrl: currentUrl,
|
||||
};
|
||||
}
|
||||
throw new FetchError(
|
||||
`Too many redirects (exceeded ${options.maxRedirects})`,
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { promises as dns } from 'node:dns';
|
||||
import { BlockList, isIP, type LookupFunction } from 'node:net';
|
||||
|
||||
import type { ExtensionNetworkPolicy } from '../config/config.js';
|
||||
|
||||
const blockedAddresses = new BlockList();
|
||||
const publicIpv6Addresses = new BlockList();
|
||||
publicIpv6Addresses.addSubnet('2000::', 3, 'ipv6');
|
||||
|
||||
for (const [address, prefix] of [
|
||||
['0.0.0.0', 8],
|
||||
['10.0.0.0', 8],
|
||||
['100.64.0.0', 10],
|
||||
['127.0.0.0', 8],
|
||||
['169.254.0.0', 16],
|
||||
['172.16.0.0', 12],
|
||||
['192.0.0.0', 24],
|
||||
['192.0.2.0', 24],
|
||||
['192.88.99.0', 24],
|
||||
['192.168.0.0', 16],
|
||||
['198.18.0.0', 15],
|
||||
['198.51.100.0', 24],
|
||||
['203.0.113.0', 24],
|
||||
['224.0.0.0', 4],
|
||||
['240.0.0.0', 4],
|
||||
] as const) {
|
||||
blockedAddresses.addSubnet(address, prefix, 'ipv4');
|
||||
}
|
||||
|
||||
for (const [address, prefix] of [
|
||||
['2001::', 23],
|
||||
['2001:db8::', 32],
|
||||
['2002::', 16],
|
||||
['3fff::', 20],
|
||||
] as const) {
|
||||
blockedAddresses.addSubnet(address, prefix, 'ipv6');
|
||||
}
|
||||
|
||||
export interface ResolvedNetworkTarget {
|
||||
url: URL;
|
||||
lookup?: LookupFunction;
|
||||
curlResolve?: string;
|
||||
}
|
||||
|
||||
async function waitForAbortable<T>(
|
||||
promise: Promise<T>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
if (!signal) return await promise;
|
||||
signal.throwIfAborted();
|
||||
return await new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const finish = (callback: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
callback();
|
||||
};
|
||||
const onAbort = () => finish(() => reject(signal.reason));
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
promise.then(
|
||||
(value) => finish(() => resolve(value)),
|
||||
(error: unknown) => finish(() => reject(error)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function stripIpv6Brackets(hostname: string): string {
|
||||
return hostname.startsWith('[') && hostname.endsWith(']')
|
||||
? hostname.slice(1, -1)
|
||||
: hostname;
|
||||
}
|
||||
|
||||
function parseMappedIpv4(address: string): string | undefined {
|
||||
const suffix = address.toLowerCase().slice('::ffff:'.length);
|
||||
if (isIP(suffix) === 4) return suffix;
|
||||
|
||||
const parts = suffix.split(':');
|
||||
if (
|
||||
parts.length !== 2 ||
|
||||
parts.some((part) => !/^[\da-f]{1,4}$/.test(part))
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const upper = Number.parseInt(parts[0], 16);
|
||||
const lower = Number.parseInt(parts[1], 16);
|
||||
return `${upper >> 8}.${upper & 0xff}.${lower >> 8}.${lower & 0xff}`;
|
||||
}
|
||||
|
||||
function isBlockedAddress(address: string, family: number): boolean {
|
||||
if (family === 6 && address.toLowerCase().startsWith('::ffff:')) {
|
||||
const mapped = parseMappedIpv4(address);
|
||||
return mapped === undefined || isBlockedAddress(mapped, 4);
|
||||
}
|
||||
if (family === 6 && !publicIpv6Addresses.check(address, 'ipv6')) return true;
|
||||
return blockedAddresses.check(address, family === 6 ? 'ipv6' : 'ipv4');
|
||||
}
|
||||
|
||||
export async function resolveNetworkTarget(
|
||||
value: string | URL,
|
||||
policy?: ExtensionNetworkPolicy,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ResolvedNetworkTarget> {
|
||||
signal?.throwIfAborted();
|
||||
const url = value instanceof URL ? value : new URL(value);
|
||||
if (policy !== 'public') return { url };
|
||||
if (url.protocol !== 'https:') {
|
||||
throw new Error('Public network requests must use HTTPS.');
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new Error('Public network requests must not use credentials.');
|
||||
}
|
||||
|
||||
const hostname = stripIpv6Brackets(url.hostname);
|
||||
const literalFamily = isIP(hostname);
|
||||
const addresses = literalFamily
|
||||
? [{ address: hostname, family: literalFamily }]
|
||||
: await waitForAbortable(
|
||||
dns.lookup(hostname, { all: true, verbatim: true }),
|
||||
signal,
|
||||
);
|
||||
if (addresses.length === 0) {
|
||||
throw new Error(`Public network host did not resolve: ${hostname}`);
|
||||
}
|
||||
if (
|
||||
addresses.some(({ address, family }) => isBlockedAddress(address, family))
|
||||
) {
|
||||
throw new Error(
|
||||
`Public network host resolved to a blocked address: ${hostname}`,
|
||||
);
|
||||
}
|
||||
|
||||
const selected = addresses[0];
|
||||
const lookup: LookupFunction = (requestedHostname, options, callback) => {
|
||||
if (stripIpv6Brackets(requestedHostname) !== hostname) {
|
||||
const error = new Error(
|
||||
'Pinned public network lookup hostname mismatch',
|
||||
) as NodeJS.ErrnoException;
|
||||
error.code = 'ENOTFOUND';
|
||||
callback(error, '', 0);
|
||||
return;
|
||||
}
|
||||
if (options.all) {
|
||||
callback(null, [selected]);
|
||||
} else {
|
||||
callback(null, selected.address, selected.family);
|
||||
}
|
||||
};
|
||||
const port = url.port || '443';
|
||||
const curlHostname = literalFamily === 6 ? `[${hostname}]` : hostname;
|
||||
const curlAddress =
|
||||
selected.family === 6 ? `[${selected.address}]` : selected.address;
|
||||
|
||||
return {
|
||||
url,
|
||||
lookup,
|
||||
curlResolve: `${curlHostname}:${port}:${curlAddress}`,
|
||||
};
|
||||
}
|
||||
@@ -567,6 +567,31 @@ the tenant and writer headers are scope and fencing inputs, not a substitute
|
||||
for transport identity. Responses under the private prefix use
|
||||
`Cache-Control: no-store`.
|
||||
|
||||
### Web fetch in Workspace Sessions
|
||||
|
||||
The main Hosted files, shell, search and team lead profiles provide `web_fetch`
|
||||
with `url`, `prompt` and optional `format` (`auto`, `markdown`, `html` or `text`).
|
||||
The bound Runtime fetches public HTTPS content and returns normalized text and
|
||||
HTTP metadata to the conversation model. `default` and `auto-edit` require
|
||||
approval with the actual destination URL; `yolo` retains its approval bypass.
|
||||
Plan mode, snapshot read profiles and the dedicated CSI files profile do not
|
||||
admit this tool.
|
||||
|
||||
The Runtime worker must use Node.js; other JavaScript runtimes are refused
|
||||
because they may ignore the pinned connection dispatcher.
|
||||
Every request, retry and same-host redirect checks and pins public DNS
|
||||
addresses, verifies TLS and uses a dedicated connection agent. Private and
|
||||
metadata addresses, URL credentials, plain HTTP and binary downloads are
|
||||
refused. The request does not inherit the process-wide proxy dispatcher.
|
||||
Cross-host, scheme or port redirects are returned for a new call and approval.
|
||||
Text is bounded to 16 KiB after JSON encoding with a truncation notice; no
|
||||
download or overflow file is written on the worker host. The worker does not
|
||||
make a separate model call or retain a web cache.
|
||||
|
||||
Upgrade Runtime workers together with the Harness before relying on its
|
||||
`web-fetch/1` capability. See the [English design](../../docs/design/2026-10-23-managed-web-fetch.md)
|
||||
and [Chinese design](../../docs/design/2026-10-23-managed-web-fetch.zh-CN.md).
|
||||
|
||||
### Wake recovery after Harness loss
|
||||
|
||||
With the Session Store enabled, an independent control-plane scan discovers
|
||||
|
||||
+1
-1
@@ -292,7 +292,7 @@ public class WorkspaceProfileLifecycleVerifier {
|
||||
byte[] payload = input.path("payloadJson").textValue().getBytes(StandardCharsets.UTF_8);
|
||||
JsonNode call = ToolPublicationContract.readJson(payload);
|
||||
if (call.size() == 2 && call.path("input").isObject()
|
||||
&& List.of("read_file", "write_file", "edit", "glob", "managed_mcp_call").contains(call.path("toolName").asText())) {
|
||||
&& List.of("read_file", "write_file", "edit", "glob", "web_fetch", "managed_mcp_call").contains(call.path("toolName").asText())) {
|
||||
inputs.add(input.path("runtimeSessionId").textValue() + ":sha256:" + ToolPublicationContract.sha256(payload));
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ public class ManagedActionService {
|
||||
LoggerFactory.getLogger(ManagedActionService.class);
|
||||
private static final int INPUT_PREVIEW_BYTES = 8192;
|
||||
private static final Set<String> PREVIEW_TOOLS =
|
||||
Set.of("read_file", "write_file", "edit", "run_shell_command",
|
||||
Set.of("web_fetch", "read_file", "write_file", "edit", "run_shell_command",
|
||||
"agent", "send_message", "team_create", "task_create",
|
||||
"task_update", "workflow", "request_shutdown", "exit_plan_mode",
|
||||
"team_plan_approval");
|
||||
|
||||
+2
-1
@@ -175,13 +175,14 @@ class ManagedActionsTest {
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"read_file", "write_file", "edit", "run_shell_command",
|
||||
@ValueSource(strings = {"web_fetch", "read_file", "write_file", "edit", "run_shell_command",
|
||||
"team_create", "task_create", "task_update", "workflow",
|
||||
"request_shutdown", "exit_plan_mode", "team_plan_approval"})
|
||||
void previewsExactInputOnPublicAndWebShellListsAndDetails(String tool) throws Exception {
|
||||
String tenant = tenant();
|
||||
String session = session(tenant);
|
||||
String arguments = switch (tool) {
|
||||
case "web_fetch" -> "{ \"url\" : \"https://example.com/docs\", \"prompt\" : \"Summarize\" }";
|
||||
case "read_file" -> "{ \"file_path\" : \"notes.md\", \"offset\" : 1e2 }";
|
||||
case "write_file" -> "{ \"file_path\" : \"notes.md\", \"content\" : \"\\u4e2d\\n\" }";
|
||||
case "edit" -> "{ \"file_path\" : \"notes.md\", \"old_string\" : \"a\", \"new_string\" : \"b\" }";
|
||||
|
||||
+6
-4
@@ -280,14 +280,16 @@ class WorkspaceProfileLifecycleTest {
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"read_file", "write_file", "edit", "glob", "wrong_digest", "wrong_runtime", "wrong_harness",
|
||||
@ValueSource(strings = {"read_file", "write_file", "edit", "glob", "web_fetch", "wrong_digest", "wrong_runtime", "wrong_harness",
|
||||
"shell", "v3", "publication", "unreferenced", "corrupt", "unsettled"})
|
||||
void shellProfileNativeFileInputsRequireOriginalIdentityAndKnownSettlement(String mode) throws Exception {
|
||||
var f = fixture();
|
||||
var binding = f.bindings.findOrCreate(new RuntimeProvisionRequest(f.scope(), f.session));
|
||||
String tool = java.util.List.of("read_file", "write_file", "edit", "glob").contains(mode) ? mode
|
||||
String tool = java.util.List.of("read_file", "write_file", "edit", "glob", "web_fetch").contains(mode) ? mode
|
||||
: "shell".equals(mode) ? "run_shell_command" : "read_file";
|
||||
String payload = f.json.writeValueAsString(Map.of("toolName", tool, "input", Map.of("file_path", "proof.txt")));
|
||||
Map<String, String> input = "web_fetch".equals(tool)
|
||||
? Map.of("url", "https://example.com/docs", "prompt", "Summarize") : Map.of("file_path", "proof.txt");
|
||||
String payload = f.json.writeValueAsString(Map.of("toolName", tool, "input", input));
|
||||
String requestDigest = "sha256:" + ("wrong_digest".equals(mode) ? "a".repeat(64)
|
||||
: ToolPublicationContract.sha256(payload.getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
var reference = new java.util.HashMap<String, Object>(Map.of("sessionId", "original-owner", "promptId", "turn-1",
|
||||
@@ -320,7 +322,7 @@ class WorkspaceProfileLifecycleTest {
|
||||
beans.getBeanProvider(ManagedToolResultStore.class));
|
||||
Runnable verify = () -> f.transactions.executeWithoutResult(ignored -> verifier.verifyResources(operation,
|
||||
java.util.List.of(binding), WorkspaceToolProfiles.SHELL));
|
||||
if (java.util.List.of("read_file", "write_file", "edit", "glob").contains(mode)) {
|
||||
if (java.util.List.of("read_file", "write_file", "edit", "glob", "web_fetch").contains(mode)) {
|
||||
verify.run();
|
||||
} else {
|
||||
assertThatThrownBy(verify::run).isInstanceOf(ManagedAgentException.class);
|
||||
|
||||
@@ -138,6 +138,26 @@ describe('raw JUnit report retention', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// The smoke lane keeps the classifier's trust decision and hosted overrides
|
||||
// but picks its own architecture, so its `runs-on` cannot be compared with
|
||||
// another lane's. Evaluate the real expression the way
|
||||
// `.github/scripts/ci-runner-routing.test.mjs` does.
|
||||
const runnerOn = (job, ubuntuRunner) => {
|
||||
const expression = String(ci.jobs[job]['runs-on'])
|
||||
.replace(/^\$\{\{\s*/, '')
|
||||
.replace(/\s*\}\}$/, '');
|
||||
return new Function(
|
||||
'needs',
|
||||
'fromJSON',
|
||||
'contains',
|
||||
`return (${expression});`,
|
||||
)(
|
||||
{ classify_pr: { outputs: { ubuntu_runner: ubuntuRunner } } },
|
||||
JSON.parse,
|
||||
(labels, label) => labels.includes(label),
|
||||
);
|
||||
};
|
||||
|
||||
it('gives the shared Linux test lane one ceiling on both pools', () => {
|
||||
// The full suite outgrew 60 minutes on hosted runners too: on 2026-10-17,
|
||||
// 17 of the 30 hosted runs that ended on their own were cancelled at 60.
|
||||
@@ -168,8 +188,19 @@ it('keeps lint_and_static sized for cold-cache pool runs', () => {
|
||||
expect(timeoutMinutesOn('lint_and_static', '')).toBe(45);
|
||||
});
|
||||
|
||||
it('routes browser gates like the shared Linux test lane, at the hosted worker count', () => {
|
||||
expect(ci.jobs.web_shell_e2e_smoke['runs-on']).toBe(ci.jobs.test['runs-on']);
|
||||
it('routes the browser smoke to ARM64 independently of the shared Linux test lane', () => {
|
||||
// The lane keeps the classifier's trust decision and hosted overrides, but
|
||||
// it no longer inherits the ordinary Linux fleet's architecture.
|
||||
expect(runnerOn('web_shell_e2e_smoke', ECS_RUNNER)).toEqual([
|
||||
'self-hosted',
|
||||
'linux',
|
||||
'arm64',
|
||||
]);
|
||||
expect(runnerOn('web_shell_e2e_smoke', HOSTED_RUNNER)).toEqual([
|
||||
'ubuntu-latest',
|
||||
]);
|
||||
// A missing classifier output (classify_pr skipped) is the hosted routing.
|
||||
expect(runnerOn('web_shell_e2e_smoke', '')).toEqual(['ubuntu-latest']);
|
||||
const smoke = ci.jobs.web_shell_e2e_smoke.steps.find(
|
||||
(step) => step.name === 'Run web-shell browser smoke',
|
||||
);
|
||||
|
||||
@@ -1042,8 +1042,11 @@ describe('no-AK integration CI wiring', () => {
|
||||
);
|
||||
const webShellJob = getWorkflowJob(workflow, 'web_shell_e2e_smoke');
|
||||
|
||||
expect(webShellJob).toContain(
|
||||
"runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''[\"ubuntu-latest\"]'') }}'",
|
||||
// The lane still takes its runner from the classifier, so an untrusted
|
||||
// fork PR cannot reach a self-hosted host; the architecture override it
|
||||
// applies on top is pinned in scripts/tests/ci-platform-lanes.test.js.
|
||||
expect(webShellJob).toMatch(
|
||||
/runs-on: .*needs\.classify_pr\.outputs\.ubuntu_runner/,
|
||||
);
|
||||
const hostedInstall = getWorkflowStep(
|
||||
webShellJob,
|
||||
|
||||
Reference in new issue
Block a user