yao/agent/assistant/build_content.go

package assistant

import (
	"fmt"

	"github.com/yaoapp/yao/agent/content"
	"github.com/yaoapp/yao/agent/content/text"
	contentTypes "github.com/yaoapp/yao/agent/content/types"
	"github.com/yaoapp/yao/agent/context"
)

// BuildContent processes messages through Vision function to convert extended content types
// (file, data) to standard LLM-compatible types (text, image_url, input_audio)
//
// This should be called after BuildRequest and before executing LLM call
func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, opts *context.Options) ([]context.Message, error) {
	// Skip complex content parsing if requested (for internal calls like needsearch)
	// Still convert file attachments to raw text
	if opts != nil && opts.Skip != nil && opts.Skip.ContentParsing {
		return convertFilesToText(ctx, messages), nil
	}

	// Set AssistantID in context for file info tracking in Space
	// This ensures hooks can access file information using the correct namespace
	if ctx.AssistantID == "" {
		ctx.AssistantID = ast.ID
	}

	// Get connector and capabilities
	connector, capabilities, err := ast.GetConnector(ctx, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to get connector: %w", err)
	}

	// Build parse options
	parseOptions := &contentTypes.Options{
		Capabilities:      capabilities,
		CompletionOptions: options,
		Connector:         connector,
		StreamOptions:     options.StreamOptions,
	}

	contentMessages, referenceContext, err := content.ParseUserInput(ctx, messages, parseOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to parse content: %w", err)
	}

	// Inject reference context into messages
	if referenceContext != nil {
		contentMessages = ast.injectSearchContext(contentMessages, referenceContext)
	}

	return contentMessages, nil
}

// convertFilesToText converts file attachments in messages to raw text
// Used when SkipContentParsing is enabled - simple text extraction without vision/PDF processing
func convertFilesToText(ctx *context.Context, messages []context.Message) []context.Message {
	result := make([]context.Message, 0, len(messages))
	textHandler := text.New(nil)

	for _, msg := range messages {
		// Only process user messages
		if msg.Role != context.RoleUser {
			result = append(result, msg)
			continue
		}

		// Handle content parts
		parts, ok := msg.Content.([]context.ContentPart)
		if !ok {
			// Try []interface{} (from history/JSON)
			if iparts, ok := msg.Content.([]interface{}); ok {
				parts = convertInterfaceToParts(iparts)
			}
		}

		if len(parts) == 0 {
			result = append(result, msg)
			continue
		}

		// Convert file parts to text
		newParts := make([]context.ContentPart, 0, len(parts))
		for _, part := range parts {
			switch part.Type {
			case context.ContentFile:
				// Convert file to raw text
				if part.File != nil && part.File.URL != "" {
					textPart, _, err := textHandler.ParseRaw(ctx, part)
					if err == nil {
						newParts = append(newParts, textPart)
						continue
					}
				}
				newParts = append(newParts, part)

			case context.ContentImageURL:
				// Skip images - cannot convert to text without vision
				continue

			default:
				newParts = append(newParts, part)
			}
		}

		newMsg := msg
		newMsg.Content = newParts
		result = append(result, newMsg)
	}

	return result
}

// convertInterfaceToParts converts []interface{} to []ContentPart for file extraction
func convertInterfaceToParts(items []interface{}) []context.ContentPart {
	parts := make([]context.ContentPart, 0, len(items))
	for _, item := range items {
		m, ok := item.(map[string]interface{})
		if !ok {
			continue
		}

		typeStr, _ := m["type"].(string)
		part := context.ContentPart{
			Type: context.ContentPartType(typeStr),
		}

		switch typeStr {
		case "text":
			if t, ok := m["text"].(string); ok {
				part.Text = t
			}
		case "file":
			if fileData, ok := m["file"].(map[string]interface{}); ok {
				part.File = &context.FileAttachment{}
				if url, ok := fileData["url"].(string); ok {
					part.File.URL = url
				}
				if filename, ok := fileData["filename"].(string); ok {
					part.File.Filename = filename
				}
			}
		case "image_url":
			part.Type = context.ContentImageURL
		default:
			continue
		}

		parts = append(parts, part)
	}
	return parts
}