Improve image support for tool calls (#15765)

- Introduce ToolCallResult type for better of known MCP result types
- Add visual playwright MCP server to AppTester (can use screenshots)
- Improve rendering for tool call results

Author: martin-fleck-at

packages/ai-anthropic/src/node/anthropic-language-model.ts

@@ -26,11 +26,12 @@ import {
     TokenUsageParams,
     UserRequest,
     ImageContent,
+    ToolCallResult,
     ImageMimeType
 } from '@theia/ai-core';
 import { CancellationToken, isArray } from '@theia/core';
 import { Anthropic } from '@anthropic-ai/sdk';
-import { Message, MessageParam, Base64ImageSource } from '@anthropic-ai/sdk/resources';
+import type { Base64ImageSource, ImageBlockParam, Message, MessageParam, TextBlockParam, ToolResultBlockParam } from '@anthropic-ai/sdk/resources';
 
 export const DEFAULT_MAX_TOKENS = 4096;
 
@@ -193,10 +194,17 @@ export class AnthropicModel implements LanguageModel {
         }
     }
 
-    protected formatToolCallResult(result: unknown): string | Array<{ type: 'text', text: string }> {
-        if (typeof result === 'object' && result && 'content' in result && Array.isArray(result.content) &&
-            result.content.every(item => typeof item === 'object' && item && 'type' in item && 'text' in item)) {
-            return result.content;
+    protected formatToolCallResult(result: ToolCallResult): ToolResultBlockParam['content'] {
+        if (typeof result === 'object' && result && 'content' in result && Array.isArray(result.content)) {
+            return result.content.map<TextBlockParam | ImageBlockParam>(content => {
+                if (content.type === 'text') {
+                    return { type: 'text', text: content.text };
+                } else if (content.type === 'image') {
+                    return { type: 'image', source: { type: 'base64', data: content.base64data, media_type: mimeTypeToMediaType(content.mimeType) } };
+                } else {
+                    return { type: 'text', text: content.data };
+                }
+            });
         }
 
         if (isArray(result)) {
@@ -321,10 +329,7 @@ export class AnthropicModel implements LanguageModel {
 
                     }));
 
-                    const calls = toolResult.map(tr => {
-                        const resultAsString = typeof tr.result === 'string' ? tr.result : JSON.stringify(tr.result);
-                        return { finished: true, id: tr.id, result: resultAsString, function: { name: tr.name, arguments: tr.arguments } };
-                    });
+                    const calls = toolResult.map(tr => ({ finished: true, id: tr.id, result: tr.result, function: { name: tr.name, arguments: tr.arguments } }));
                     yield { tool_calls: calls };
 
                     const toolResponseMessage: Anthropic.Messages.MessageParam = {

packages/ai-chat-ui/src/browser/chat-response-renderer/toolcall-part-renderer.tsx

@@ -19,18 +19,23 @@ import { inject, injectable } from '@theia/core/shared/inversify';
 import { ChatResponseContent, ToolCallChatResponseContent } from '@theia/ai-chat/lib/common';
 import { ReactNode } from '@theia/core/shared/react';
 import { nls } from '@theia/core/lib/common/nls';
-import { codicon } from '@theia/core/lib/browser';
+import { codicon, OpenerService } from '@theia/core/lib/browser';
 import * as React from '@theia/core/shared/react';
 import { ToolConfirmation, ToolConfirmationState } from './tool-confirmation';
 import { ToolConfirmationManager, ToolConfirmationMode } from '@theia/ai-chat/lib/browser/chat-tool-preferences';
 import { ResponseNode } from '../chat-tree-view';
+import { useMarkdownRendering } from './markdown-part-renderer';
+import { ToolCallResult } from '@theia/ai-core';
 
 @injectable()
 export class ToolCallPartRenderer implements ChatResponsePartRenderer<ToolCallChatResponseContent> {
 
     @inject(ToolConfirmationManager)
     protected toolConfirmationManager: ToolConfirmationManager;
 
+    @inject(OpenerService)
+    protected openerService: OpenerService;
+
     canHandle(response: ChatResponseContent): number {
         if (ToolCallChatResponseContent.is(response)) {
             return 10;
@@ -47,7 +52,52 @@ export class ToolCallPartRenderer implements ChatResponsePartRenderer<ToolCallCh
             toolConfirmationManager={this.toolConfirmationManager}
             chatId={chatId}
             renderCollapsibleArguments={this.renderCollapsibleArguments.bind(this)}
-            tryPrettyPrintJson={this.tryPrettyPrintJson.bind(this)} />;
+            responseRenderer={this.renderResult.bind(this)} />;
+    }
+
+    protected renderResult(response: ToolCallChatResponseContent): ReactNode {
+        const result = this.tryParse(response.result);
+        if (!result) {
+            return undefined;
+        }
+        if (typeof result === 'string') {
+            return <pre>{JSON.stringify(result, undefined, 2)}</pre>;
+        }
+        if ('content' in result) {
+            return <div className='theia-toolCall-response-content'>
+                {result.content.map((content, idx) => {
+                    switch (content.type) {
+                        case 'image': {
+                            return <div key={`content-${idx}-${content.type}`} className='theia-toolCall-image-result'>
+                                <img src={`data:${content.mimeType};base64,${content.base64data}`} />
+                            </div>;
+                        }
+                        case 'text': {
+                            return <div key={`content-${idx}-${content.type}`} className='theia-toolCall-text-result'>
+                                <MarkdownRender text={content.text} openerService={this.openerService} />
+                            </div>;
+                        }
+                        case 'audio':
+                        case 'error':
+                        default: {
+                            return <div key={`content-${idx}-${content.type}`} className='theia-toolCall-default-result'><pre>{JSON.stringify(response, undefined, 2)}</pre></div>;
+                        }
+                    }
+                })}
+            </div>;
+        }
+        return <pre>{JSON.stringify(result, undefined, 2)}</pre>;
+    }
+
+    private tryParse(result: ToolCallResult): ToolCallResult {
+        if (!result) {
+            return undefined;
+        }
+        try {
+            return typeof result === 'string' ? JSON.parse(result) : result;
+        } catch (error) {
+            return result;
+        }
     }
 
     protected getToolConfirmationSettings(responseId: string, chatId: string): ToolConfirmationMode {
@@ -75,29 +125,6 @@ export class ToolCallPartRenderer implements ChatResponsePartRenderer<ToolCallCh
             return args;
         }
     }
-
-    private tryPrettyPrintJson(response: ToolCallChatResponseContent): string | undefined {
-        let responseContent = response.result;
-        try {
-            if (responseContent) {
-                if (typeof responseContent === 'string') {
-                    responseContent = JSON.parse(responseContent);
-                }
-                responseContent = JSON.stringify(responseContent, undefined, 2);
-            }
-        } catch (e) {
-            if (typeof responseContent !== 'string') {
-                responseContent = nls.localize(
-                    'theia/ai/chat-ui/toolcall-part-renderer/prettyPrintError',
-                    "The content could not be converted to string: '{0}'. This is the original content: '{1}'.",
-                    e.message,
-                    responseContent
-                );
-            }
-            // fall through
-        }
-        return responseContent;
-    }
 }
 
 const Spinner = () => (
@@ -110,13 +137,13 @@ interface ToolCallContentProps {
     toolConfirmationManager: ToolConfirmationManager;
     chatId: string;
     renderCollapsibleArguments: (args: string | undefined) => ReactNode;
-    tryPrettyPrintJson: (response: ToolCallChatResponseContent) => string | undefined;
+    responseRenderer: (response: ToolCallChatResponseContent) => ReactNode | undefined;
 }
 
 /**
  * A function component to handle tool call rendering and confirmation
  */
-const ToolCallContent: React.FC<ToolCallContentProps> = ({ response, confirmationMode, toolConfirmationManager, chatId, tryPrettyPrintJson, renderCollapsibleArguments }) => {
+const ToolCallContent: React.FC<ToolCallContentProps> = ({ response, confirmationMode, toolConfirmationManager, chatId, responseRenderer, renderCollapsibleArguments }) => {
     const [confirmationState, setConfirmationState] = React.useState<ToolConfirmationState>('waiting');
 
     React.useEffect(() => {
@@ -163,35 +190,43 @@ const ToolCallContent: React.FC<ToolCallContentProps> = ({ response, confirmatio
 
     return (
         <div className='theia-toolCall'>
-            <h4>
-                {confirmationState === 'denied' ? (
-                    <span className="theia-tool-denied">
-                        <span className={codicon('error')}></span> {nls.localize('theia/ai/chat-ui/toolcall-part-renderer/denied', 'Execution denied')}: {response.name}
+            {confirmationState === 'denied' ? (
+                <span className='theia-toolCall-denied'>
+                    <span className={codicon('error')}></span> {nls.localize('theia/ai/chat-ui/toolcall-part-renderer/denied', 'Execution denied')}: {response.name}
+                </span>
+            ) : response.finished ? (
+                <details className='theia-toolCall-finished'>
+                    <summary>
+                        {nls.localize('theia/ai/chat-ui/toolcall-part-renderer/finished', 'Ran')} {response.name}
+                        ({renderCollapsibleArguments(response.arguments)})
+                    </summary>
+                    <div className='theia-toolCall-response-result'>
+                        {responseRenderer(response)}
+                    </div>
+                </details>
+            ) : (
+                confirmationState === 'allowed' && (
+                    <span className='theia-toolCall-allowed'>
+                        <Spinner /> {nls.localizeByDefault('Running')} {response.name}
                     </span>
-                ) : response.finished ? (
-                    <details>
-                        <summary>{nls.localize('theia/ai/chat-ui/toolcall-part-renderer/finished', 'Ran')} {response.name}
-                            ({renderCollapsibleArguments(response.arguments)})
-                        </summary>
-                        <pre>{tryPrettyPrintJson(response)}</pre>
-                    </details>
-                ) : (
-                    confirmationState === 'allowed' && (
-                        <span>
-                            <Spinner /> {nls.localizeByDefault('Running')} {response.name}
-                        </span>
-                    )
-                )}
-            </h4>
+                )
+            )}
 
             {/* Show confirmation UI when waiting for allow */}
             {confirmationState === 'waiting' && (
-                <ToolConfirmation
-                    response={response}
-                    onAllow={handleAllow}
-                    onDeny={handleDeny}
-                />
+                <span className='theia-toolCall-waiting'>
+                    <ToolConfirmation
+                        response={response}
+                        onAllow={handleAllow}
+                        onDeny={handleDeny}
+                    />
+                </span>
             )}
         </div>
     );
 };
+
+const MarkdownRender = ({ text, openerService }: { text: string; openerService: OpenerService }) => {
+    const ref = useMarkdownRendering(text, openerService);
+    return <div ref={ref}></div>;
+};

packages/ai-chat-ui/src/browser/style/index.css

@@ -7,8 +7,8 @@
   flex: 1;
 }
 
-.chat-input-widget > .ps__rail-x,
-.chat-input-widget > .ps__rail-y {
+.chat-input-widget>.ps__rail-x,
+.chat-input-widget>.ps__rail-y {
   display: none !important;
 }
 
@@ -23,7 +23,7 @@
   overflow-wrap: break-word;
 }
 
-div:last-child > .theia-ChatNode {
+div:last-child>.theia-ChatNode {
   border: none;
 }
 
@@ -59,6 +59,7 @@ div:last-child > .theia-ChatNode {
 }
 
 @keyframes dots {
+
   0%,
   20% {
     content: "";
@@ -121,7 +122,7 @@ div:last-child > .theia-ChatNode {
   padding-inline-start: 1rem;
 }
 
-.theia-ChatNode li > p {
+.theia-ChatNode li>p {
   margin-top: 0;
   margin-bottom: 0;
 }
@@ -135,7 +136,7 @@ div:last-child > .theia-ChatNode {
   font-size: var(--theia-code-font-size);
 }
 
-.theia-RequestNode > p div {
+.theia-RequestNode>p div {
   display: inline;
 }
 
@@ -357,8 +358,7 @@ div:last-child > .theia-ChatNode {
   text-align: center;
 }
 
-.theia-ChatInput-ChangeSet-List
-  .theia-ChatInput-ChangeSet-Icon.codicon::before {
+.theia-ChatInput-ChangeSet-List .theia-ChatInput-ChangeSet-Icon.codicon::before {
   font-size: var(--theia-ui-font-size1);
 }
 
@@ -375,8 +375,7 @@ div:last-child > .theia-ChatNode {
   color: var(--theia-disabledForeground);
 }
 
-.theia-ChatInput-ChangeSet-List
-  .theia-ChatInput-ChangeSet-AdditionalInfo-SuffixIcon {
+.theia-ChatInput-ChangeSet-List .theia-ChatInput-ChangeSet-AdditionalInfo-SuffixIcon {
   font-size: var(--theia-ui-font-size0) px;
   margin-left: 4px;
 }
@@ -619,8 +618,7 @@ div:last-child > .theia-ChatNode {
   display: flex;
   flex-direction: column;
   gap: 8px;
-  border: var(--theia-border-width) solid
-    var(--theia-sideBarSectionHeader-border);
+  border: var(--theia-border-width) solid var(--theia-sideBarSectionHeader-border);
   padding: 8px 12px 12px;
   border-radius: 5px;
   margin: 0 0 8px 0;
@@ -821,14 +819,29 @@ div:last-child > .theia-ChatNode {
   color: var(--theia-errorForeground);
 }
 
-.theia-tool-pending {
+.theia-toolCall-denied,
+.theia-toolCall-finished summary,
+.theia-toolCall-allowed,
+.theia-toolCall-waiting {
+  font-weight: bold;
+}
+
+.theia-toolCall-allowed .codicon-loading {
+  font-size: 1em;
+}
+
+.theia-toolCall-pending {
   color: var(--theia-descriptionForeground);
 }
 
-.theia-tool-denied {
+.theia-toolCall-denied {
   color: var(--theia-errorForeground);
 }
 
+.theia-toolCall-response-result {
+  font-weight: normal;
+}
+
 .theia-toolCall .fa,
 .theia-toolCall details summary::marker,
 .theia-thinking .fa,

packages/ai-chat/src/common/chat-model.ts

@@ -25,6 +25,7 @@ import {
     ResolvedAIContextVariable,
     TextMessage,
     ThinkingMessage,
+    ToolCallResult,
     ToolResultMessage,
     ToolUseMessage
 } from '@theia/ai-core';
@@ -389,7 +390,7 @@ export interface ToolCallChatResponseContent extends Required<ChatResponseConten
     name?: string;
     arguments?: string;
     finished: boolean;
-    result?: string;
+    result?: ToolCallResult;
     confirmed: Promise<boolean>;
     confirm(): void;
     deny(): void;
@@ -1497,12 +1498,12 @@ export class ToolCallChatResponseContentImpl implements ToolCallChatResponseCont
     protected _name?: string;
     protected _arguments?: string;
     protected _finished?: boolean;
-    protected _result?: string;
+    protected _result?: ToolCallResult;
     protected _confirmed: Promise<boolean>;
     protected _confirmationResolver?: (value: boolean) => void;
     protected _confirmationRejecter?: (reason?: unknown) => void;
 
-    constructor(id?: string, name?: string, arg_string?: string, finished?: boolean, result?: string) {
+    constructor(id?: string, name?: string, arg_string?: string, finished?: boolean, result?: ToolCallResult) {
         this._id = id;
         this._name = name;
         this._arguments = arg_string;
@@ -1527,7 +1528,7 @@ export class ToolCallChatResponseContentImpl implements ToolCallChatResponseCont
     get finished(): boolean {
         return this._finished === undefined ? false : this._finished;
     }
-    get result(): string | undefined {
+    get result(): ToolCallResult | undefined {
         return this._result;
     }