From 3059d8a40f6dcf1e0709937ed1be2166d704bbaf Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:29:00 +0800 Subject: [PATCH] test(clients): verify native DeepSeek Harness integration (#1524) * test(clients): verify native DeepSeek Harness integration * fix(ci): avoid load-sensitive escape regression timing --- docs/installation.md | 38 ++++ docs/testing.md | 15 ++ package.json | 1 + scripts/verify-deepseek-client.mjs | 346 +++++++++++++++++++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 scripts/verify-deepseek-client.mjs diff --git a/docs/installation.md b/docs/installation.md index 86fffb28f..da726182b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -217,6 +217,44 @@ REA follows `HERMES_HOME` whitespace trimming, environment-variable expansion, and home expansion before resolving that profile. Setup previews the resolved configuration and skill destinations; doctor and uninstall use those same paths. +### DeepSeek Harness + +DeepSeek Harness uses a Cordis YAML patch sequence rather than a `mcpServers` +object. REA's `setup --client` does not edit this format. For Harness +`0.2.0-rc.2`, append the following entry to `$DSH_HOME/cordis.patch.yml` +(`~/.dsh/cordis.patch.yml` when `DSH_HOME` is unset). Preserve existing patches +and replace the executable paths with absolute paths from your installation: + +```yaml +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: /absolute/path/to/node + args: ["/absolute/path/to/rea/scripts/rea.mjs", mcp] + cwd: !!js process.cwd() +``` + +For a global npm installation, the launcher is under +`$(npm root -g)/rea-agents/scripts/rea.mjs`. Check the Node executable with +`command -v node`. Register `serverName: rea` only once in each active profile, +then restart Harness. It exposes tools as `mcp__rea__`. +Run `rea setup --skill --dry-run`, review the destination, then +`rea setup --skill --yes` to install the workflow in `~/.agents/skills`. +Harness discovers that directory, or `$DSH_AGENTS_HOME/skills` when configured; +its native `skill` tool can load `reverse-engineer-anything`. + +REA's optional [native client verification](testing.md#end-to-end-integration-and-golden-evidence) +checks headless chat, skill loading, all tool input schemas and actual analysis +using the installed Harness and a local model fixture. This establishes the +Harness integration, with live DeepSeek API compatibility remaining unverified. +See the [Harness MCP guide](https://github.com/deepseek-ai/deepseek-harness/blob/639ed015397290b3745d163aafe02ffee4aa3f84/packages/mcp/mcp-client/README.md) +for its native configuration contract. + +### Other client settings + For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2 both load. If the configuration already uses OpenCode V2's native `mcp.servers` table, setup registers REA there instead and replaces any earlier diff --git a/docs/testing.md b/docs/testing.md index 5b5943e18..031baa824 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -105,6 +105,21 @@ home; this does not prove their default home-directory discovery. Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of a named sticky profile with `HERMES_HOME` still pointing to its root. +`verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`; +verified with `@deepseek-ai/dsh@0.2.0-rc.2`) and Git. Set +`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane +installs the skill through `rea setup --skill`, writes the native Cordis MCP +patch into an isolated `DSH_HOME`, and uses `DSH_AGENTS_HOME` for native personal +skill discovery. A separate Git root prevents inherited project skills from +masking a missing installation. The default `call` mode checks skill loading, +complete catalog discovery, the actual forwarded input schemas and regexes, +and full JavaScript Evidence for a Unicode path. Use `-- chat` for ordinary +chat with REA enabled, or `REA_VERIFY_RUNTIME_ROOT` for an installed package. +The fixture uses Harness's custom OpenAI adapter and the one-million-token +context capacity declared by its default DeepSeek Flash model; it does not +establish live DeepSeek API or native Windows coverage. Receipts include +request artifacts, result digests and owned-process lineage. + Full E2E tests invoke the production command dispatcher and real providers, without fake launchers, runners or responses. `verify:keyed-archive` writes an actual Foundation binary archive, runs the CLI and a separate stdio MCP diff --git a/package.json b/package.json index 0c47e5901..663055d64 100644 --- a/package.json +++ b/package.json @@ -182,6 +182,7 @@ "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs", "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi", "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes", + "verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs", "probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs", "verify:published": "node scripts/verify-published-package.mjs", "config:print": "node scripts/print-mcp-config.mjs", diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs new file mode 100644 index 000000000..d53131a88 --- /dev/null +++ b/scripts/verify-deepseek-client.mjs @@ -0,0 +1,346 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../dist/contracts/toolContracts.js"; +import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; + +const mode = process.argv[2] ?? "call"; +assert(["chat", "call"].includes(mode), "Select chat or call."); +assert(process.platform !== "win32", "Native Windows coverage is unverified."); +const repo = fileURLToPath(new URL("..", import.meta.url)); +const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; +const command = process.env.REA_VERIFY_DEEPSEEK_COMMAND ?? "dsh"; +const lab = await mkdtemp(join(tmpdir(), "rea-deepseek-client-")); +const account = join(lab, "account"); +const profile = join(lab, "selected profile"); +const workspace = join(lab, "workspace"); +const target = join(workspace, "fixture α测试"); +const skillPath = join( + account, + ".agents", + "skills", + "reverse-engineer-anything", + "SKILL.md", +); +const cli = join(runtimeRoot, "scripts", "rea.mjs"); +const verifier = createVerifierRun(); +await Promise.all([ + mkdir(account), + mkdir(profile), + mkdir(target, { recursive: true }), +]); +const environment = { + PATH: process.env.PATH ?? "", + USERPROFILE: account, + DSH_HOME: profile, + DSH_AGENTS_HOME: join(account, ".agents"), + OPENAI_API_KEY: "local-fixture-key", + REA_PROCESS_RUN_ID: verifier.run_id, +}; +// A temporary directory under the checkout must not inherit its project skills or AGENTS.md. +await exec( + "git", + ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], + { + env: environment, + timeout: 30_000, + }, +); +await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', +); +const clientVersion = ( + await exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + }) +).stdout.trim(); +for (const args of [ + ["setup", "--skill", "--dry-run"], + ["setup", "--skill", "--yes"], +]) { + const result = await exec(process.execPath, [cli, ...args, "--json"], { + env: environment, + cwd: workspace, + timeout: 60_000, + }); + const parsed = JSON.parse(result.stdout); + assert.equal(parsed.status, args.includes("--dry-run") ? "planned" : "ready"); + if (args.includes("--dry-run")) { + assert.equal(parsed.plannedActions.length, 1); + assert.equal(parsed.plannedActions[0].kind, "install_skill"); + assert.equal(parsed.plannedActions[0].target, dirname(skillPath)); + } + await writeFile( + join(lab, args.includes("--dry-run") ? "setup-plan.json" : "setup.json"), + result.stdout, + ); +} +assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), +); + +let catalog; +let patternCount = 0; +let skillRequested = false; +let skillLoaded = false; +let evidenceSeen = false; +let finalSeen = false; +let artifactDigest; +let mainRequests = 0; +const ajv = new Ajv2020({ strict: false, validateFormats: false }); +const patterns = new Set(); +const validatePatterns = (value) => { + if (!value || typeof value !== "object") return; + for (const [key, child] of Object.entries(value)) { + if (key === "pattern" && typeof child === "string") { + // Check the representation the native client actually forwards to its model adapter. + for (const flags of ["", "u", "v"]) new RegExp(child, flags); + patterns.add(child); + patternCount++; + } else validatePatterns(child); + } +}; +const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes + ) + return value; + for (const child of Object.values(value)) { + const found = extractEvidence(child); + if (found) return found; + } + } +}; +const fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `deepseek-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + // Harness generates a session title in a separate, tool-free model request. + if (tools.length === 0) + return { role: "assistant", content: "REA validation" }; + assert( + ++mainRequests <= 4, + "Harness did not complete the bounded fixture workflow.", + ); + const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); + const observedCatalog = rea + .map((tool) => tool.name.slice("mcp__rea__".length)) + .sort(); + assert.deepEqual( + observedCatalog, + TOOL_CONTRACTS.map((tool) => tool.name).sort(), + ); + if (catalog === undefined) { + catalog = observedCatalog; + for (const tool of rea) { + assert( + ajv.validateSchema(tool.parameters), + `${tool.name}: ${ajv.errorsText(ajv.errors)}`, + ); + validatePatterns(tool.parameters); + } + assert( + patternCount > 0, + "The advertised regex constraints must reach the model.", + ); + } + const toolMessages = (body.messages ?? []).filter( + (message) => message.role === "tool", + ); + const serialized = JSON.stringify(toolMessages); + skillLoaded ||= + serialized.includes( + '', + ) && serialized.includes(dirname(skillPath)); + for (const message of toolMessages) { + const found = extractEvidence(message.content); + if (found === undefined) continue; + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(found); + assert.equal(evidence.subject.local_path, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .find( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap( + (entry) => entry.fields, + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + const text = JSON.stringify(evidence); + await writeFile(join(lab, "evidence.json"), text); + artifactDigest = createHash("sha256").update(text).digest("hex"); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + assert(tools.some((tool) => tool.name === "skill")); + skillRequested = true; + return call("skill", { name: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Harness must load the installed skill before analysis.", + ); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + }, +}); +let failure; +try { + // Use the client's native Cordis patch DSL; generic `rea setup --client` does not edit it. + await writeFile( + join(profile, "cordis.patch.yml"), + ` +- id: llm-pi-ai + config: + providers: + rea-fixture: + apiKeyEnv: OPENAI_API_KEY + api: openai-completions + baseURL: ${fixture.baseUrl} + defaultMaxTokens: 4096 + retryPolicy: + mode: normal + maxRetries: 0 + models: + - id: rea-client-fixture + contextWindow: 1000000 +- id: agent-default-model + config: + provider: rea-fixture + model: rea-client-fixture +- insert: + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: + serverName: rea + transport: stdio + command: ${JSON.stringify(process.execPath)} + args: ${JSON.stringify([cli, "mcp"])} + cwd: !!js process.cwd() + env: + REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)} +`, + ); + const execution = exec( + command, + [ + "--profile", + "headless", + "--json", + mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + }, + ); + execution.child.stdin.end(); + const result = await execution; + await writeFile(join(lab, "client.stdout"), result.stdout); + await writeFile(join(lab, "client.stderr"), result.stderr); + assert.equal(fixture.failure, undefined); + assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); + if (mode === "call") assert(evidenceSeen && skillLoaded); +} catch (cause) { + failure = cause; + if (typeof cause?.stdout === "string") + await writeFile(join(lab, "client.stdout"), cause.stdout); + if (typeof cause?.stderr === "string") + await writeFile(join(lab, "client.stderr"), cause.stderr); +} finally { + await fixture.close(); + const receipt = { + client: "deepseek-harness", + clientVersion, + mode, + passed: failure === undefined, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", + modelContextWindow: 1000000, + skillDiscovery: + "native DSH_AGENTS_HOME personal directory; isolated project root", + catalogSize: catalog?.length, + patternCount, + uniquePatterns: patterns.size, + skillLoaded, + evidenceSeen, + artifactDigest, + finalSeen, + requests: fixture.requests, + fixtureFailure: fixture.failure, + verifier: await completeVerifierRun(verifier), + }; + await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); + console.log(JSON.stringify(receipt)); +} +if (failure !== undefined) throw failure;