mirror of
https://github.com/anomalyco/opencode.git
synced 2026-07-25 12:45:51 -04:00
test(llm): record chronological system updates
This commit is contained in:
Kit Langton
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"name": "anthropic-messages/anthropic-haiku-4-5-chronological-system-update",
|
||||
"recordedAt": "2026-06-04T19:14:47.473Z",
|
||||
"provider": "anthropic",
|
||||
"route": "anthropic-messages",
|
||||
"transport": "http",
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"tags": [
|
||||
"prefix:anthropic-messages",
|
||||
"provider:anthropic",
|
||||
"system",
|
||||
"chronological-system-update",
|
||||
"golden"
|
||||
]
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Follow the latest instruction exactly. Reply only with the requested word.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the latest instruction for your reply.\"},{\"type\":\"text\",\"text\":\"<system-update>\\nFor this reply, respond exactly with: UPDATED\\n</system-update>\"}]}],\"stream\":true,\"max_tokens\":80,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01QxsUPda7mfZ9zGSgw4JD54\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":49,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":5,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"UPDATED\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":49,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":5} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"name": "gemini/gemini-2-5-flash-chronological-system-update",
|
||||
"recordedAt": "2026-06-04T19:14:48.065Z",
|
||||
"provider": "google",
|
||||
"route": "gemini",
|
||||
"transport": "http",
|
||||
"model": "gemini-2.5-flash",
|
||||
"tags": [
|
||||
"prefix:gemini",
|
||||
"provider:google",
|
||||
"system",
|
||||
"chronological-system-update",
|
||||
"golden"
|
||||
]
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use the latest instruction for your reply.\"},{\"text\":\"<system-update>\\nFor this reply, respond exactly with: UPDATED\\n</system-update>\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Follow the latest instruction exactly. Reply only with the requested word.\"}]},\"generationConfig\":{\"maxOutputTokens\":80,\"temperature\":0,\"thinkingConfig\":{\"thinkingBudget\":0}}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"UPDATED\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 44,\"candidatesTokenCount\": 2,\"totalTokenCount\": 46,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 44}],\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-2.5-flash\",\"responseId\": \"p84hasmqJL2I-sAPtZnUkA0\"}\r\n\r\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"name": "openai-chat/openai-chat-gpt-4o-mini-chronological-system-update",
|
||||
"recordedAt": "2026-06-04T19:14:43.750Z",
|
||||
"provider": "openai",
|
||||
"route": "openai-chat",
|
||||
"transport": "http",
|
||||
"model": "gpt-4o-mini",
|
||||
"tags": [
|
||||
"prefix:openai-chat",
|
||||
"provider:openai",
|
||||
"system",
|
||||
"chronological-system-update",
|
||||
"golden"
|
||||
]
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.openai.com/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Follow the latest instruction exactly. Reply only with the requested word.\"},{\"role\":\"user\",\"content\":\"Use the latest instruction for your reply.\\n<system-update>\\nFor this reply, respond exactly with: UPDATED\\n</system-update>\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_tokens\":80,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "data: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"uIKJFKJXo\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"UPDATED\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6HB3\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"UlWDE\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[],\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":1,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"w348ZVmuZE6\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+39
File diff suppressed because one or more lines are too long
@@ -73,7 +73,7 @@ describeRecordedGoldenScenarios([
|
||||
prefix: "openai-chat",
|
||||
model: openAIChat,
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
scenarios: ["text", "tool-call", "tool-loop"],
|
||||
scenarios: ["text", "chronological-system-update", "tool-call", "tool-loop"],
|
||||
},
|
||||
{
|
||||
name: "OpenAI Responses gpt-5.5",
|
||||
@@ -83,6 +83,7 @@ describeRecordedGoldenScenarios([
|
||||
tags: ["flagship"],
|
||||
scenarios: [
|
||||
{ id: "text", temperature: false },
|
||||
{ id: "chronological-system-update", temperature: false },
|
||||
{ id: "reasoning", temperature: false },
|
||||
{ id: "reasoning-continuation", temperature: false },
|
||||
{ id: "tool-call", temperature: false },
|
||||
@@ -104,7 +105,7 @@ describeRecordedGoldenScenarios([
|
||||
model: anthropicHaiku,
|
||||
requires: ["ANTHROPIC_API_KEY"],
|
||||
options: { redactor: Redactor.defaults({ requestHeaders: { allow: ["content-type", "anthropic-version"] } }) },
|
||||
scenarios: ["text", "tool-call"],
|
||||
scenarios: ["text", "chronological-system-update", "tool-call"],
|
||||
},
|
||||
{
|
||||
name: "Anthropic Opus 4.7",
|
||||
@@ -123,7 +124,12 @@ describeRecordedGoldenScenarios([
|
||||
prefix: "gemini",
|
||||
model: gemini,
|
||||
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
|
||||
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }],
|
||||
scenarios: [
|
||||
{ id: "text", maxTokens: 80 },
|
||||
"chronological-system-update",
|
||||
"tool-call",
|
||||
{ id: "image", maxTokens: 160 },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "xAI Grok 3 Mini",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { Effect, Schema } from "effect"
|
||||
import {
|
||||
LLM,
|
||||
LLMEvent,
|
||||
@@ -332,6 +332,28 @@ const runTextScenario = (context: GoldenScenarioContext) =>
|
||||
}),
|
||||
])
|
||||
|
||||
export const runChronologicalSystemUpdateScenario = (context: GoldenScenarioContext) =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* generate(
|
||||
LLM.request({
|
||||
id: `${context.id}_chronological_system_update`,
|
||||
model: context.model,
|
||||
system: "Follow the latest instruction exactly. Reply only with the requested word.",
|
||||
cache: "none",
|
||||
messages: [
|
||||
Message.user("Use the latest instruction for your reply."),
|
||||
Message.system("For this reply, respond exactly with: UPDATED"),
|
||||
],
|
||||
providerOptions:
|
||||
context.model.route.id === "gemini" ? { gemini: { thinkingConfig: { thinkingBudget: 0 } } } : undefined,
|
||||
generation: generation(context, context.maxTokens ?? 80),
|
||||
}),
|
||||
)
|
||||
|
||||
expectFinish(response.events, "stop")
|
||||
expect(response.text.trim()).toMatch(/^UPDATED[.!]?$/i)
|
||||
})
|
||||
|
||||
const runToolCallScenario = (context: GoldenScenarioContext) =>
|
||||
runGeneratedConversation(context, [
|
||||
user("Call get_weather with city exactly Paris."),
|
||||
@@ -445,6 +467,11 @@ const runToolLoopScenario = (context: GoldenScenarioContext) =>
|
||||
|
||||
const goldenScenarios = {
|
||||
text: { title: "streams text", tags: ["text", "golden"], run: runTextScenario },
|
||||
"chronological-system-update": {
|
||||
title: "uses chronological system update",
|
||||
tags: ["system", "chronological-system-update", "golden"],
|
||||
run: runChronologicalSystemUpdateScenario,
|
||||
},
|
||||
"tool-call": { title: "streams tool call", tags: ["tool", "tool-call", "golden"], run: runToolCallScenario },
|
||||
"tool-loop": { title: "drives a tool loop", tags: ["tool", "tool-loop", "golden"], run: runToolLoopScenario },
|
||||
image: { title: "reads image text", tags: ["media", "image", "vision", "golden"], run: runImageScenario },
|
||||
|
||||
Reference in New Issue
Block a user