diff --git a/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-chronological-system-update.json b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-chronological-system-update.json new file mode 100644 index 00000000000..2708b2ec6f8 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-chronological-system-update.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "metadata": { + "name": "anthropic-messages/anthropic-haiku-4-5-chronological-system-update", + "recordedAt": "2026-06-04T19:14:47.473Z", + "provider": "anthropic", + "route": "anthropic-messages", + "transport": "http", + "model": "claude-haiku-4-5-20251001", + "tags": [ + "prefix:anthropic-messages", + "provider:anthropic", + "system", + "chronological-system-update", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Follow the latest instruction exactly. Reply only with the requested word.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the latest instruction for your reply.\"},{\"type\":\"text\",\"text\":\"\\nFor this reply, respond exactly with: UPDATED\\n\"}]}],\"stream\":true,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01QxsUPda7mfZ9zGSgw4JD54\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":49,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":5,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"UPDATED\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":49,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":5} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-chronological-system-update.json b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-chronological-system-update.json new file mode 100644 index 00000000000..e7bb189294b --- /dev/null +++ b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-chronological-system-update.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "gemini/gemini-2-5-flash-chronological-system-update", + "recordedAt": "2026-06-04T19:14:48.065Z", + "provider": "google", + "route": "gemini", + "transport": "http", + "model": "gemini-2.5-flash", + "tags": [ + "prefix:gemini", + "provider:google", + "system", + "chronological-system-update", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use the latest instruction for your reply.\"},{\"text\":\"\\nFor this reply, respond exactly with: UPDATED\\n\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Follow the latest instruction exactly. Reply only with the requested word.\"}]},\"generationConfig\":{\"maxOutputTokens\":80,\"temperature\":0,\"thinkingConfig\":{\"thinkingBudget\":0}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"UPDATED\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 44,\"candidatesTokenCount\": 2,\"totalTokenCount\": 46,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 44}],\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-2.5-flash\",\"responseId\": \"p84hasmqJL2I-sAPtZnUkA0\"}\r\n\r\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-chronological-system-update.json b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-chronological-system-update.json new file mode 100644 index 00000000000..6d002e4b94e --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-chronological-system-update.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "openai-chat/openai-chat-gpt-4o-mini-chronological-system-update", + "recordedAt": "2026-06-04T19:14:43.750Z", + "provider": "openai", + "route": "openai-chat", + "transport": "http", + "model": "gpt-4o-mini", + "tags": [ + "prefix:openai-chat", + "provider:openai", + "system", + "chronological-system-update", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Follow the latest instruction exactly. Reply only with the requested word.\"},{\"role\":\"user\",\"content\":\"Use the latest instruction for your reply.\\n\\nFor this reply, respond exactly with: UPDATED\\n\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"uIKJFKJXo\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"UPDATED\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6HB3\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"UlWDE\"}\n\ndata: {\"id\":\"chatcmpl-Dn7oZjOtsXXzajgbNQ17HQXOPABbd\",\"object\":\"chat.completion.chunk\",\"created\":1780600483,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e0de90b008\",\"choices\":[],\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":1,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"w348ZVmuZE6\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-chronological-system-update.json b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-chronological-system-update.json new file mode 100644 index 00000000000..f434c08f21c --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-chronological-system-update.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses/openai-responses-gpt-5-5-chronological-system-update", + "recordedAt": "2026-06-04T19:14:46.794Z", + "provider": "openai", + "route": "openai-responses", + "transport": "http", + "model": "gpt-5.5", + "tags": [ + "prefix:openai-responses", + "provider:openai", + "flagship", + "system", + "chronological-system-update", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.5\",\"input\":[{\"role\":\"system\",\"content\":\"Follow the latest instruction exactly. Reply only with the requested word.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Use the latest instruction for your reply.\"},{\"type\":\"input_text\",\"text\":\"\\nFor this reply, respond exactly with: UPDATED\\n\"}]}],\"store\":false,\"include\":[\"reasoning.encrypted_content\"],\"reasoning\":{\"effort\":\"medium\",\"summary\":\"auto\"},\"text\":{\"verbosity\":\"low\"},\"max_output_tokens\":80,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0b058cc7f4ec9168016a21cea3d244819da4789560e119a7ac\",\"object\":\"response\",\"created_at\":1780600483,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"low\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0b058cc7f4ec9168016a21cea3d244819da4789560e119a7ac\",\"object\":\"response\",\"created_at\":1780600483,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"low\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_0b058cc7f4ec9168016a21cea4a180819dab9efa31556dff80\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqIc6kqld4jlRSpxnmJIcQxeZVqwQ9w073E8j7To6pXvbKYQTT2P_KD86yGR20IrGeredD67SvjdCe32pPRqNcDgit4XCJuSKWzvQt8EZOrs5ZbY7EjnRWOWv7WGLRFyzIJG2oQVbEwrJZzV-msWBXOj45GobymmHAJT1yj76ks33WcKtOSyGXWfmzIx1tfeVFULriBFnnWJ45iYFJS2VdwvkHyLci6uitNfwHWOALkA00aavmw9oCac8LSDrKz7aIOYMHekFI4IosCjJ0GgTrPvGm_D0EIzeAcNn_ofL31u5i1r_v--RNC9dog0Xc1DxoXeXDoLUj1YZ3JxWjFhhyAlraZ-t5S4kHsAqIo_21usyZf2u59DSdIvA1JnzDYQO24-LhsnMsHjqVvh0tYh9EZ75U7-7arr8KOupXzrFDY4SrNFmdTdyT4AdDF8CamDfHszti2P5a0q0E_0lBCehiMPBHlg8yKvVN8oSkf3svUIyJfaQ80g_K0tOgwkOW54rMbBalKNRKOCQnP7_eN16XldJbhcPY1K00XUBjfWOxLR-mXv1-20CINZKsy6a_lp-yrR7fcGaYykY7v75-svaj-6nEpVTelFiepcnHySP3vBzEzlP9n7eQf6CEcb8IUNh8sYQSnnvpEba9kWrTz7laA-EseH1tIN_GHxd1aTPyg_hW1mjN3IajA38Lvsd7RT1rcE644zSbz2-mqh5NTkQ9SGkvW3yhTLJWageRvT_qSqPVnEv18qtO0KgSOLgmsij43kfPvrcXFmnSnwtuf6kebY_T88il919MIRSuLEzI4vocXiL3sJ34pp4HPb1nfdu6N8IP_m28ti47LOQi5FZd9oFm8aYJgUaijlu5O7loDcI5Mz1uNsSqAd0V6Vs-mPFPLj00WqF4wFaYsp4qjmr9uNVaug==\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_0b058cc7f4ec9168016a21cea4a180819dab9efa31556dff80\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqIc6lpWyK-ZtXIPgHUsad6JFwBf2WpDu7lyu-ISjz0dAs-NPdkHl21o6eo84i9i-kuwPidonWRG35JNnedZ-4k17N5iJC81DwW8Gpb83rQ6VN1YqNQeWEY0Y56euNz4A13EzP84g-_X-6NDxS9WvQWTBwNrdkDKsRGn62CGxaz6EiO8Gkeb9ZGUxLc0TdLtM4fBK3Kpy1mvFOyp1kv4CSw1VE3g3iN2p8CskF1BaduprfT9WDPRfO4CHJs8_YlEFDN_THnIy-2vXsCJlqoMZ7xg0i1syW0osiISNXWimhRRg5wFME50ZQV0F6WjETCFcMsFrkEHAdjxGhhrWroPT_79niuE1iA74iiwUpeF1Bp5dTYBbwijJrDnhFG5CpOYtgUb5-5lPdwmUK0nbYDBL6dQC-PntXU6vi0u3pVRcil0dOqkzhzy56BtBM4tJagYnhuAaAkW86nbq31qq1eH8XmUSKItZ3Um2eOlGHwRdV_0iZVYYvqquoByy4Wptznw0y30XvMSeSDp2deLqzU4dCLYgxcYczlREEd1pVCah-HjpSipHNcUUMshP8h0wKBcy3C5ciBzT-cmgTIgCEEWkAatSx3ADDM9MOnk6fjogmkj-FETMIebgI02zN4Qh1GXJZahseuhouM8O5ei-17XqHWwlRvEZfolUfHSiBnHGH1Hk2mdnqLm3WfAlULQ1d0SEYrZlztdMuQW-4Fw619fixbatUvp_p0VvlOPGEKXmgKMy_ZOkJSVIS0O0KphVRCebnVWRsg5oGGezVormxKaZj5yKYtgSaODXM_q0GTgo5OxaaIz7RHX_c3e8URJ5auUwZGXh3mzNgsB7VDo9wkawCeHG0BVJ1akwF7J1uNcwfMD1tFNrO5zQwF1NfiwFo-uVots8AHK5pZqrFpY64LULoiNVmqD1EZpB_sUCwpQdVSvgox5ZvW4xzcdNEDsuYklCB25xruiIo9qufGrrSjSK8DNxB2w==\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":4}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"UPDATED\",\"item_id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"logprobs\":[],\"obfuscation\":\"mgR1eK6bQ\",\"output_index\":1,\"sequence_number\":6}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":7,\"text\":\"UPDATED\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"UPDATED\"},\"sequence_number\":8}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"UPDATED\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":9}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0b058cc7f4ec9168016a21cea3d244819da4789560e119a7ac\",\"object\":\"response\",\"created_at\":1780600483,\"status\":\"completed\",\"background\":false,\"completed_at\":1780600486,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[{\"id\":\"rs_0b058cc7f4ec9168016a21cea4a180819dab9efa31556dff80\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqIc6m-feiwQGp0dCrntYnYC-tRtrFG_e34B0XI6-cuPjU9c7PhWAjN7pyE8sIXfe8t1Nxi8aqXwfKbMgFX-lycUFo6Fyv209ZOpWwUf4gL5KIvCnHvISI13qmgiIzti2HHBO3HYSHNdFvrj8B3AxU1KEMjn7Pnv1-c23i0H9mYWp6TTO9K1vDCWkGZYpGe9OP0aFBZmxkdB5fWWwi2X6naLOpvHn2ssTesecwQ2lRyrdY_EV1rtL8YL0zG78azgQdJPNRL8gv_BQ1HLHS1xUDgvYnVgQb8U8Tu-42scVWP6qBXJk3-01d6sjUj8RYVgrsjLIsHZq1QlnBTSNnynGLoALtt7cC4g3LhbTCjSTP5kG-KxDQ8RiBd34CP8voN8oaM_2Ycs0t7mCtBZdmWjw92iViVlW7wVV5PNU8fPz0BqGQqKWnb6xawvEqPxhxiiNuaby26mGdMOpilDNJVyQV06NVAMtNW7givFKxT-cRuvWxecfFgGdSfbK3V03k1NlN5Pard8ZqZgqZSpOEk_MpV59outeqvKozOl1jX6E8SXPtPW4nNfHz_vUb57hKGWx6meJj2kZlQ1xpb3Tb4MEMLltKj_OPbfyv-GfXyhm_enzRCu0iepQAhU1fFHfns8z0tT4xndX_5mEMgiDkXDBg0bfXpBA8uOuiJIlV5s1wXi85zJQ6hpnqG6fKMNI2iGgQihIm4OiiG73eyePSVto8nAiq_Q4WnDMX_k1w6V1D5IIeTIc8MLZjDL5kw3pm6GnnoxlNoYoBnmdupW_ckMKMzNDg0dGLQ17TsaEiGr1dfAxStBiP1hIx_fFYebksXtFr-OVxCN_Z09hkEQ6BOvrDtE02VPI_7vjz9EroixIKYdJDWwmvXwiXYGLSzL4uyN1Z4aLS5MrfWeG3AQNSMjOBBQ_SsPUFFqskhMiy1keO5-htbgV1gbOl9BlnNO1VaACCAKdbweOdosCUiRojQ3KKhhlVhg==\",\"summary\":[]},{\"id\":\"msg_0b058cc7f4ec9168016a21cea52760819d927c7a1d6538a62a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"UPDATED\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"low\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":48,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":17,\"output_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":65},\"user\":null,\"metadata\":{}},\"sequence_number\":10}\n\n" + } + } + ] +} diff --git a/packages/llm/test/provider/golden.recorded.test.ts b/packages/llm/test/provider/golden.recorded.test.ts index 79d8589cf8c..f82df3b5eb9 100644 --- a/packages/llm/test/provider/golden.recorded.test.ts +++ b/packages/llm/test/provider/golden.recorded.test.ts @@ -73,7 +73,7 @@ describeRecordedGoldenScenarios([ prefix: "openai-chat", model: openAIChat, requires: ["OPENAI_API_KEY"], - scenarios: ["text", "tool-call", "tool-loop"], + scenarios: ["text", "chronological-system-update", "tool-call", "tool-loop"], }, { name: "OpenAI Responses gpt-5.5", @@ -83,6 +83,7 @@ describeRecordedGoldenScenarios([ tags: ["flagship"], scenarios: [ { id: "text", temperature: false }, + { id: "chronological-system-update", temperature: false }, { id: "reasoning", temperature: false }, { id: "reasoning-continuation", temperature: false }, { id: "tool-call", temperature: false }, @@ -104,7 +105,7 @@ describeRecordedGoldenScenarios([ model: anthropicHaiku, requires: ["ANTHROPIC_API_KEY"], options: { redactor: Redactor.defaults({ requestHeaders: { allow: ["content-type", "anthropic-version"] } }) }, - scenarios: ["text", "tool-call"], + scenarios: ["text", "chronological-system-update", "tool-call"], }, { name: "Anthropic Opus 4.7", @@ -123,7 +124,12 @@ describeRecordedGoldenScenarios([ prefix: "gemini", model: gemini, requires: ["GOOGLE_GENERATIVE_AI_API_KEY"], - scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }], + scenarios: [ + { id: "text", maxTokens: 80 }, + "chronological-system-update", + "tool-call", + { id: "image", maxTokens: 160 }, + ], }, { name: "xAI Grok 3 Mini", diff --git a/packages/llm/test/recorded-scenarios.ts b/packages/llm/test/recorded-scenarios.ts index 1fc01ae76e1..1aa9998bfc5 100644 --- a/packages/llm/test/recorded-scenarios.ts +++ b/packages/llm/test/recorded-scenarios.ts @@ -1,5 +1,5 @@ import { expect } from "bun:test" -import { Effect, Schema, Stream } from "effect" +import { Effect, Schema } from "effect" import { LLM, LLMEvent, @@ -332,6 +332,28 @@ const runTextScenario = (context: GoldenScenarioContext) => }), ]) +export const runChronologicalSystemUpdateScenario = (context: GoldenScenarioContext) => + Effect.gen(function* () { + const response = yield* generate( + LLM.request({ + id: `${context.id}_chronological_system_update`, + model: context.model, + system: "Follow the latest instruction exactly. Reply only with the requested word.", + cache: "none", + messages: [ + Message.user("Use the latest instruction for your reply."), + Message.system("For this reply, respond exactly with: UPDATED"), + ], + providerOptions: + context.model.route.id === "gemini" ? { gemini: { thinkingConfig: { thinkingBudget: 0 } } } : undefined, + generation: generation(context, context.maxTokens ?? 80), + }), + ) + + expectFinish(response.events, "stop") + expect(response.text.trim()).toMatch(/^UPDATED[.!]?$/i) + }) + const runToolCallScenario = (context: GoldenScenarioContext) => runGeneratedConversation(context, [ user("Call get_weather with city exactly Paris."), @@ -445,6 +467,11 @@ const runToolLoopScenario = (context: GoldenScenarioContext) => const goldenScenarios = { text: { title: "streams text", tags: ["text", "golden"], run: runTextScenario }, + "chronological-system-update": { + title: "uses chronological system update", + tags: ["system", "chronological-system-update", "golden"], + run: runChronologicalSystemUpdateScenario, + }, "tool-call": { title: "streams tool call", tags: ["tool", "tool-call", "golden"], run: runToolCallScenario }, "tool-loop": { title: "drives a tool loop", tags: ["tool", "tool-loop", "golden"], run: runToolLoopScenario }, image: { title: "reads image text", tags: ["media", "image", "vision", "golden"], run: runImageScenario },