feat(llm-observability): Show tools in conversation view (#29503)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

frontend/src/scenes/session-recordings/player/inspector/components/AIEventItems.tsx

@@ -19,6 +19,7 @@ export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.
             {event.event === '$ai_generation' ? (
                 <ConversationMessagesDisplay
                     input={event.properties.$ai_input}
+                    tools={event.properties.$ai_tools}
                     output={
                         event.properties.$ai_is_error
                             ? event.properties.$ai_error

products/llm_observability/frontend/ConversationDisplay/ConversationDisplay.stories.tsx

@@ -96,6 +96,26 @@ def meaning_of_life():
 export const Tools = Template.bind({})
 Tools.args = {
     eventProperties: {
+        $ai_tools: [
+            {
+                function: {
+                    name: 'foo',
+                    parameters: {
+                        additionalProperties: false,
+                        properties: {
+                            thing: {
+                                description: 'The thing to thingify.',
+                                type: 'string',
+                            },
+                        },
+                        required: ['thing'],
+                        type: 'object',
+                    },
+                    strict: true,
+                },
+                type: 'function',
+            },
+        ],
         $ai_input: [
             { role: 'system', content: 'You are a good bot.' },
             { role: 'user', content: 'Please foo "Bar!"' },

products/llm_observability/frontend/ConversationDisplay/ConversationDisplay.tsx

@@ -18,6 +18,7 @@ export function ConversationDisplay({ eventProperties }: { eventProperties: Even
             <ConversationMessagesDisplay
                 input={eventProperties.$ai_input}
                 output={eventProperties.$ai_output_choices ?? eventProperties.$ai_output ?? eventProperties.$ai_error}
+                tools={eventProperties.$ai_tools}
                 httpStatus={eventProperties.$ai_http_status}
                 raisedError={eventProperties.$ai_is_error}
                 bordered

products/llm_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx

@@ -15,17 +15,19 @@ import { normalizeMessages } from '../utils'
 export function ConversationMessagesDisplay({
     input,
     output,
+    tools,
     httpStatus,
     raisedError,
     bordered = false,
 }: {
     input: any
     output: any
+    tools?: any
     httpStatus?: number
     raisedError?: boolean
     bordered?: boolean
 }): JSX.Element {
-    const inputNormalized = normalizeMessages(input, 'user')
+    const inputNormalized = normalizeMessages(input, 'user', tools)
     const outputNormalized = normalizeMessages(output, 'assistant')
 
     const outputDisplay = raisedError ? (
@@ -50,23 +52,25 @@ export function ConversationMessagesDisplay({
                 </span>
             )}
         </div>
+    ) : outputNormalized.length > 0 ? (
+        outputNormalized.map((message, i) => <LLMMessageDisplay key={i} message={message} isOutput />)
     ) : (
-        outputNormalized?.map((message, i) => <LLMMessageDisplay key={i} message={message} isOutput />) || (
-            <div className="rounded border text-default p-2 italic bg-[var(--bg-fill-error-tertiary)]">No output</div>
-        )
+        <div className="rounded border text-default p-2 italic bg-[var(--bg-fill-error-tertiary)]">No output</div>
     )
 
     return (
         <LLMInputOutput
             inputDisplay={
-                inputNormalized?.map((message, i) => (
-                    <>
-                        <LLMMessageDisplay key={i} message={message} />
-                        {i < inputNormalized.length - 1 && (
-                            <div className="border-l ml-2 h-2" /> /* Spacer connecting messages visually */
-                        )}
-                    </>
-                )) || (
+                inputNormalized.length > 0 ? (
+                    inputNormalized.map((message, i) => (
+                        <React.Fragment key={i}>
+                            <LLMMessageDisplay message={message} />
+                            {i < inputNormalized.length - 1 && (
+                                <div className="border-l ml-2 h-2" /> /* Spacer connecting messages visually */
+                            )}
+                        </React.Fragment>
+                    ))
+                ) : (
                     <div className="rounded border text-default p-2 italic bg-[var(--bg-fill-error-tertiary)]">
                         No input
                     </div>
@@ -76,7 +80,7 @@ export function ConversationMessagesDisplay({
             outputHeading={
                 raisedError
                     ? `Error (${httpStatus})`
-                    : `Output${outputNormalized && outputNormalized.length > 1 ? ' (multiple choices)' : ''}`
+                    : `Output${outputNormalized.length > 1 ? ' (multiple choices)' : ''}`
             }
             bordered={bordered}
         />
@@ -94,9 +98,13 @@ export const LLMMessageDisplay = React.memo(
         const isMarkdownCandidate = content ? /(\n\s*```|^>\s|#{1,6}\s)/.test(content) : false
 
         // Render any additional keyword arguments as JSON.
-        const additionalKwargsEntries = Object.fromEntries(
-            Object.entries(additionalKwargs).filter(([, value]) => value !== undefined)
-        )
+        const additionalKwargsEntries = Array.isArray(additionalKwargs.tools)
+            ? // Tools are a special case of input - and we want name and description to show first for them!
+              additionalKwargs.tools.map(({ function: { name, description, ...func }, ...tool }) => ({
+                  function: { name, description, ...func },
+                  ...tool,
+              }))
+            : Object.fromEntries(Object.entries(additionalKwargs).filter(([, value]) => value !== undefined))
 
         const renderMessageContent = (content: string): JSX.Element | null => {
             if (!content) {

products/llm_observability/frontend/LLMObservabilityTraceScene.tsx

@@ -401,6 +401,7 @@ const EventContent = React.memo(({ event }: { event: LLMTrace | LLMTraceEvent |
                     {isLLMTraceEvent(event) ? (
                         event.event === '$ai_generation' ? (
                             <ConversationMessagesDisplay
+                                tools={event.properties.$ai_tools}
                                 input={event.properties.$ai_input}
                                 output={
                                     event.properties.$ai_is_error

products/llm_observability/frontend/utils.ts

@@ -226,20 +226,26 @@ export function normalizeMessage(output: unknown, defaultRole?: string): CompatM
     ]
 }
 
-export function normalizeMessages(output: unknown, defaultRole?: string): CompatMessage[] | null {
-    if (!output) {
-        return null
+export function normalizeMessages(messages: unknown, defaultRole?: string, tools?: unknown): CompatMessage[] {
+    const normalizedMessages: CompatMessage[] = []
+
+    if (tools) {
+        normalizedMessages.push({
+            role: 'tools',
+            content: '',
+            tools,
+        })
     }
 
-    if (Array.isArray(output)) {
-        return output.map((message) => normalizeMessage(message, defaultRole)).flat()
+    if (Array.isArray(messages)) {
+        normalizedMessages.push(...messages.map((message) => normalizeMessage(message, defaultRole)).flat())
     }
 
-    if (typeof output === 'object' && 'choices' in output && Array.isArray(output.choices)) {
-        return output.choices.map((message) => normalizeMessage(message, defaultRole)).flat()
+    if (typeof messages === 'object' && messages && 'choices' in messages && Array.isArray(messages.choices)) {
+        normalizedMessages.push(...messages.choices.map((message) => normalizeMessage(message, defaultRole)).flat())
     }
 
-    return null
+    return normalizedMessages
 }
 
 export function removeMilliseconds(timestamp: string): string {

requirements.in

@@ -70,7 +70,7 @@ pandas==2.2.0
 paramiko==3.4.0
 Pillow==10.2.0
 pdpyras==5.2.0
-posthoganalytics==3.16.0
+posthoganalytics==3.18.1
 psutil==6.0.0
 psycopg2-binary==2.9.7
 pymssql==2.3.1

requirements.txt

@@ -568,7 +568,7 @@ pluggy==1.5.0
     # via dlt
 ply==3.11
     # via jsonpath-ng
-posthoganalytics==3.16.0
+posthoganalytics==3.18.1
     # via -r requirements.in
 prometheus-client==0.14.1
     # via django-prometheus