feat(compose): add enhanced tool interfaces with multimodal support (#760)

Introduce EnhancedInvokableTool and EnhancedStreamableTool interfaces that support
returning structured multimodal results (ToolResult) instead of plain strings.

Key changes:

1. New Tool Interfaces (components/tool/interface.go):
   - EnhancedInvokableTool: accepts ToolArgument, returns *ToolResult
   - EnhancedStreamableTool: accepts ToolArgument, returns StreamReader[*ToolResult]

2. Tool Utils (components/tool/utils/):
   - Added InferEnhancedTool and InferEnhancedStreamTool for type inference
   - Added NewEnhancedTool and NewEnhancedStreamTool factory functions
   - Support both regular and optionable variants

3. ToolsNode Enhancement (compose/tool_node.go):
   - Added EnhancedInvokableToolMiddleware and EnhancedStreamableToolMiddleware
   - Auto-conversion between enhanced invokable and streamable endpoints
   - Priority given to enhanced tools when both interfaces are implemented
   - Updated interrupt/rerun state to track enhanced tool results separately

4. Schema Extensions (schema/message.go):
   - Added ToolPartType enum (text, image, audio, video, file)
   - Added ToolArgument and ToolResult structs for structured I/O
   - Added ToolOutputPart with multimodal content support
   - Added ConcatToolResults for stream chunk merging
   - Added ToMessageInputParts() for ToolResult to Message conversion
   - Enhanced Message.String() to display multimodal content

5. React Agent Integration (flow/agent/react/):
   - Added enhanced tool result senders for MessageFuture
   - Added middleware support for enhanced tool result collection

This enables tools to return rich content like images, audio, video, and files
alongside text, supporting advanced multimodal AI agent scenarios.

Author: zhuangjie1125

components/tool/callback_extra.go

@@ -19,6 +19,7 @@ package tool
 
 import (
 	"github.com/cloudwego/eino/callbacks"
+	"github.com/cloudwego/eino/schema"
 )
 
 // CallbackInput is the input for the tool callback.
@@ -33,6 +34,8 @@ type CallbackInput struct {
 type CallbackOutput struct {
 	// Response is the response for the tool.
 	Response string
+	// ToolOutput is the multimodal output for the tool. Used when the tool returns structured data.
+	ToolOutput *schema.ToolResult
 	// Extra is the extra information for the tool.
 	Extra map[string]any
 }
@@ -44,6 +47,8 @@ func ConvCallbackInput(src callbacks.CallbackInput) *CallbackInput {
 		return t
 	case string:
 		return &CallbackInput{ArgumentsInJSON: t}
+	case *schema.ToolArgument:
+		return &CallbackInput{ArgumentsInJSON: t.TextArgument}
 	default:
 		return nil
 	}
@@ -56,6 +61,8 @@ func ConvCallbackOutput(src callbacks.CallbackOutput) *CallbackOutput {
 		return t
 	case string:
 		return &CallbackOutput{Response: t}
+	case *schema.ToolResult:
+		return &CallbackOutput{ToolOutput: t}
 	default:
 		return nil
 	}

components/tool/interface.go

@@ -41,3 +41,18 @@ type StreamableTool interface {
 
 	StreamableRun(ctx context.Context, argumentsInJSON string, opts ...Option) (*schema.StreamReader[string], error)
 }
+
+// EnhancedInvokableTool is a tool interface that supports returning structured multimodal results.
+// Unlike InvokableTool which returns a string, this interface returns *schema.ToolResult
+// which can contain text, images, audio, video, and files.
+type EnhancedInvokableTool interface {
+	BaseTool
+	InvokableRun(ctx context.Context, toolArgument *schema.ToolArgument, opts ...Option) (*schema.ToolResult, error)
+}
+
+// EnhancedStreamableTool is a streaming tool interface that supports returning structured multimodal results.
+// It provides a stream reader for accessing multimodal content progressively.
+type EnhancedStreamableTool interface {
+	BaseTool
+	StreamableRun(ctx context.Context, toolArgument *schema.ToolArgument, opts ...Option) (*schema.StreamReader[*schema.ToolResult], error)
+}

components/tool/utils/invokable_func.go

@@ -56,6 +56,33 @@ func InferOptionableTool[T, D any](toolName, toolDesc string, i OptionableInvoke
 	return newOptionableTool(ti, i, opts...), nil
 }
 
+// EnhancedInvokeFunc is the function type for the enhanced tool.
+type EnhancedInvokeFunc[T any] func(ctx context.Context, input T) (output *schema.ToolResult, err error)
+
+// OptionableEnhancedInvokeFunc is the function type for the enhanced tool with tool option.
+type OptionableEnhancedInvokeFunc[T any] func(ctx context.Context, input T, opts ...tool.Option) (output *schema.ToolResult, err error)
+
+// InferEnhancedTool creates an EnhancedInvokableTool from a given function by inferring the ToolInfo from the function's request parameters.
+// End-user can pass a SchemaCustomizerFn in opts to customize the go struct tag parsing process, overriding default behavior.
+func InferEnhancedTool[T any](toolName, toolDesc string, i EnhancedInvokeFunc[T], opts ...Option) (tool.EnhancedInvokableTool, error) {
+	ti, err := goStruct2ToolInfo[T](toolName, toolDesc, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewEnhancedTool(ti, i, opts...), nil
+}
+
+// InferOptionableEnhancedTool creates an EnhancedInvokableTool from a given function by inferring the ToolInfo from the function's request parameters, with tool option.
+func InferOptionableEnhancedTool[T any](toolName, toolDesc string, i OptionableEnhancedInvokeFunc[T], opts ...Option) (tool.EnhancedInvokableTool, error) {
+	ti, err := goStruct2ToolInfo[T](toolName, toolDesc, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return newOptionableEnhancedTool(ti, i, opts...), nil
+}
+
 // GoStruct2ParamsOneOf converts a go struct to a ParamsOneOf.
 // if you attempt to use ResponseFormat of some ChatModel to get StructuredOutput, you can infer the JSONSchema from the go struct.
 func GoStruct2ParamsOneOf[T any](opts ...Option) (*schema.ParamsOneOf, error) {
@@ -200,3 +227,76 @@ func snakeToCamel(s string) string {
 
 	return strings.Join(parts, "")
 }
+
+// NewEnhancedTool Create an enhanced tool, where the input is in JSON format and output is *schema.ToolResult.
+func NewEnhancedTool[T any](desc *schema.ToolInfo, i EnhancedInvokeFunc[T], opts ...Option) tool.EnhancedInvokableTool {
+	return newOptionableEnhancedTool(desc, func(ctx context.Context, input T, _ ...tool.Option) (*schema.ToolResult, error) {
+		return i(ctx, input)
+	}, opts...)
+}
+
+func newOptionableEnhancedTool[T any](desc *schema.ToolInfo, i OptionableEnhancedInvokeFunc[T], opts ...Option) tool.EnhancedInvokableTool {
+	to := getToolOptions(opts...)
+
+	return &enhancedInvokableTool[T]{
+		info: desc,
+		um:   to.um,
+		Fn:   i,
+	}
+}
+
+type enhancedInvokableTool[T any] struct {
+	info *schema.ToolInfo
+
+	um UnmarshalArguments
+
+	Fn OptionableEnhancedInvokeFunc[T]
+}
+
+func (e *enhancedInvokableTool[T]) Info(ctx context.Context) (*schema.ToolInfo, error) {
+	return e.info, nil
+}
+
+func (e *enhancedInvokableTool[T]) InvokableRun(ctx context.Context, toolArgument *schema.ToolArgument, opts ...tool.Option) (*schema.ToolResult, error) {
+	var inst T
+	var err error
+
+	if e.um != nil {
+		var val any
+		val, err = e.um(ctx, toolArgument.TextArgument)
+		if err != nil {
+			return nil, fmt.Errorf("[EnhancedLocalFunc] failed to unmarshal arguments, toolName=%s, err=%w", e.getToolName(), err)
+		}
+		gt, ok := val.(T)
+		if !ok {
+			return nil, fmt.Errorf("[EnhancedLocalFunc] invalid type, toolName=%s, expected=%T, given=%T", e.getToolName(), inst, val)
+		}
+		inst = gt
+	} else {
+		inst = generic.NewInstance[T]()
+
+		err = sonic.UnmarshalString(toolArgument.TextArgument, &inst)
+		if err != nil {
+			return nil, fmt.Errorf("[EnhancedLocalFunc] failed to unmarshal arguments in json, toolName=%s, err=%w", e.getToolName(), err)
+		}
+	}
+
+	resp, err := e.Fn(ctx, inst, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("[EnhancedLocalFunc] failed to invoke tool, toolName=%s, err=%w", e.getToolName(), err)
+	}
+
+	return resp, nil
+}
+
+func (e *enhancedInvokableTool[T]) GetType() string {
+	return snakeToCamel(e.getToolName())
+}
+
+func (e *enhancedInvokableTool[T]) getToolName() string {
+	if e.info == nil {
+		return ""
+	}
+
+	return e.info.Name
+}

components/tool/utils/streamable_func.go

@@ -155,3 +155,101 @@ func (s *streamableTool[T, D]) getToolName() string {
 
 	return s.info.Name
 }
+
+// EnhancedStreamFunc is the function type for the enhanced streamable tool.
+type EnhancedStreamFunc[T any] func(ctx context.Context, input T) (output *schema.StreamReader[*schema.ToolResult], err error)
+
+// OptionableEnhancedStreamFunc is the function type for the enhanced streamable tool with tool option.
+type OptionableEnhancedStreamFunc[T any] func(ctx context.Context, input T, opts ...tool.Option) (output *schema.StreamReader[*schema.ToolResult], err error)
+
+// InferEnhancedStreamTool creates an EnhancedStreamableTool from a given function by inferring the ToolInfo from the function's request parameters.
+// End-user can pass a SchemaCustomizerFn in opts to customize the go struct tag parsing process, overriding default behavior.
+func InferEnhancedStreamTool[T any](toolName, toolDesc string, s EnhancedStreamFunc[T], opts ...Option) (tool.EnhancedStreamableTool, error) {
+	ti, err := goStruct2ToolInfo[T](toolName, toolDesc, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewEnhancedStreamTool(ti, s, opts...), nil
+}
+
+// InferOptionableEnhancedStreamTool creates an EnhancedStreamableTool from a given function by inferring the ToolInfo from the function's request parameters, with tool option.
+func InferOptionableEnhancedStreamTool[T any](toolName, toolDesc string, s OptionableEnhancedStreamFunc[T], opts ...Option) (tool.EnhancedStreamableTool, error) {
+	ti, err := goStruct2ToolInfo[T](toolName, toolDesc, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return newOptionableEnhancedStreamTool(ti, s, opts...), nil
+}
+
+// NewEnhancedStreamTool Create an enhanced streaming tool, where the input is in JSON format and output is *schema.StreamReader[*schema.ToolResult].
+func NewEnhancedStreamTool[T any](desc *schema.ToolInfo, s EnhancedStreamFunc[T], opts ...Option) tool.EnhancedStreamableTool {
+	return newOptionableEnhancedStreamTool(desc,
+		func(ctx context.Context, input T, _ ...tool.Option) (output *schema.StreamReader[*schema.ToolResult], err error) {
+			return s(ctx, input)
+		},
+		opts...)
+}
+
+func newOptionableEnhancedStreamTool[T any](desc *schema.ToolInfo, s OptionableEnhancedStreamFunc[T], opts ...Option) tool.EnhancedStreamableTool {
+	to := getToolOptions(opts...)
+
+	return &enhancedStreamableTool[T]{
+		info: desc,
+		um:   to.um,
+		Fn:   s,
+	}
+}
+
+type enhancedStreamableTool[T any] struct {
+	info *schema.ToolInfo
+
+	um UnmarshalArguments
+
+	Fn OptionableEnhancedStreamFunc[T]
+}
+
+func (s *enhancedStreamableTool[T]) Info(ctx context.Context) (*schema.ToolInfo, error) {
+	return s.info, nil
+}
+
+func (s *enhancedStreamableTool[T]) StreamableRun(ctx context.Context, toolArgument *schema.ToolArgument, opts ...tool.Option) (
+	outStream *schema.StreamReader[*schema.ToolResult], err error) {
+
+	var inst T
+	if s.um != nil {
+		var val any
+		val, err = s.um(ctx, toolArgument.TextArgument)
+		if err != nil {
+			return nil, fmt.Errorf("[EnhancedLocalStreamFunc] failed to unmarshal arguments, toolName=%s, err=%w", s.getToolName(), err)
+		}
+
+		gt, ok := val.(T)
+		if !ok {
+			return nil, fmt.Errorf("[EnhancedLocalStreamFunc] type err, toolName=%s, expected=%T, given=%T", s.getToolName(), inst, val)
+		}
+		inst = gt
+	} else {
+		inst = generic.NewInstance[T]()
+
+		err = sonic.UnmarshalString(toolArgument.TextArgument, &inst)
+		if err != nil {
+			return nil, fmt.Errorf("[EnhancedLocalStreamFunc] failed to unmarshal arguments in json, toolName=%s, err=%w", s.getToolName(), err)
+		}
+	}
+
+	return s.Fn(ctx, inst, opts...)
+}
+
+func (s *enhancedStreamableTool[T]) GetType() string {
+	return snakeToCamel(s.getToolName())
+}
+
+func (s *enhancedStreamableTool[T]) getToolName() string {
+	if s.info == nil {
+		return ""
+	}
+
+	return s.info.Name
+}