feat(providers): add streaming reasoning_content and multimodal media support

- openai_compat: extract reasoning_content from streaming SSE deltas, benefiting providers like DeepSeek, Mimo, and Kimi that return chain-of-thought content in streamed chunks Media handling refactor (addresses review feedback): - agent: centralize video/audio inline decisions in resolveMediaRefs with provider capability gating via VideoCapable/AudioCapable interfaces - agent: add mediaInlinePolicy + mediaInlinePolicyFromProvider() to probe provider capabilities before encoding media as data URLs - agent: add size guard via encodeMediaToDataURL (respects MaxMediaSize) - providers: add VideoCapable and AudioCapable interfaces (same pattern as ThinkingCapable/NativeSearchCapable) - openai_compat: implement SupportsVideo/SupportsAudio for Mimo, Qwen, OpenAI (host and providerName detection) - httpapi: delegate SupportsVideo/SupportsAudio through HTTPProvider - httpapi: GeminiProvider declares video + audio support - common: add video_url serialization in SerializeMessages (now reachable via the centralized pipeline); fix missing continue after audio branch - Tests for capability detection, inline policy, oversized media skip, and provider-unsupported media fallback to path tags
2026-05-03 22:44:03 +08:00 · 2026-05-03 22:44:03 +08:00 · aeba8f7142
commit aeba8f7142
parent be67aed4dc
18 changed files with 625 additions and 55 deletions
--- a/pkg/agent/agent_init.go
+++ b/pkg/agent/agent_init.go
@ -248,7 +248,7 @@ func registerSharedTools(
 			// This keeps subagent vision support working even when the optimized
 			// sub-turn spawner path is unavailable.
 			subagentManager.SetMediaResolver(func(msgs []providers.Message) []providers.Message {
-				return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize())
+				return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize(), mediaInlinePolicy{})
 			})

 			// Set the spawner that links into AgentLoop's turnState
--- a/pkg/agent/agent_media.go
+++ b/pkg/agent/agent_media.go
@ -31,16 +31,44 @@ var (
 	filePlaceholderRegex  = regexp.MustCompile(`\[file(:\s+[^\]]*)?\]`)
 )

+// mediaInlinePolicy controls which media types are encoded as inline data URLs.
+type mediaInlinePolicy struct {
+	video bool
+	audio bool
+}
+
+// mediaInlinePolicyFromProvider probes the provider's capability interfaces
+// to build an inline policy. Providers that do not implement VideoCapable or
+// AudioCapable default to false (path tags only).
+func mediaInlinePolicyFromProvider(p providers.LLMProvider) mediaInlinePolicy {
+	var policy mediaInlinePolicy
+	if vc, ok := p.(providers.VideoCapable); ok {
+		policy.video = vc.SupportsVideo()
+	}
+	if ac, ok := p.(providers.AudioCapable); ok {
+		policy.audio = ac.SupportsAudio()
+	}
+	return policy
+}
+
 // resolveMediaRefs resolves media:// refs in messages.
 // For user messages: images get path tags only ([image:/path]) so the LLM
 // can decide whether to view them via load_image or operate on the file.
+// Video and audio refs are encoded as inline data URLs (with size guard)
+// only when the active provider declares support via the policy parameter.
 // For tool messages: images are base64-encoded and appended as a synthetic
 // user message only after the contiguous tool-message block ends, so we don't
 // break the tool-results-must-immediately-follow-assistant constraint that
-// LLM APIs enforce.
-// Non-image files always get path tags regardless of role.
+// LLM APIs enforce. Video and audio in tool messages follow the same pattern
+// when the provider supports them.
+// Non-image/video/audio files always get path tags regardless of role.
 // Returns a new slice; original messages are not mutated.
-func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxSize int) []providers.Message {
+func resolveMediaRefs(
+	messages []providers.Message,
+	store media.MediaStore,
+	maxSize int,
+	policy mediaInlinePolicy,
+) []providers.Message {
 	if store == nil {
 		return messages
 	}
@ -104,11 +132,24 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
 			mime := detectMIME(localPath, meta)
 			pathTags = append(pathTags, buildPathTag(mime, localPath))

-			if m.Role == "tool" && strings.HasPrefix(mime, "image/") {
-				dataURL := encodeImageToDataURL(localPath, mime, info, maxSize)
+			isImage := strings.HasPrefix(mime, "image/")
+			isVideo := strings.HasPrefix(mime, "video/")
+			isAudio := strings.HasPrefix(mime, "audio/")
+			shouldInline := (isVideo && policy.video) || (isAudio && policy.audio)
+
+			if m.Role == "tool" && (isImage || shouldInline) {
+				// Tool-role media: encode and defer as synthetic user message
+				dataURL := encodeMediaToDataURL(localPath, mime, info, maxSize)
 				if dataURL != "" {
 					pendingToolImages = append(pendingToolImages, dataURL)
 				}
+			} else if shouldInline {
+				// User/assistant-role video & audio: encode inline as data URL
+				// only when the active provider declares support
+				dataURL := encodeMediaToDataURL(localPath, mime, info, maxSize)
+				if dataURL != "" {
+					resolved = append(resolved, dataURL)
+				}
 			}
 		}

@ -132,9 +173,9 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
 	return result
 }

-// encodeImageToDataURL base64-encodes an image file into a data URL.
+// encodeMediaToDataURL base64-encodes a media file (image, video, audio) into a data URL.
 // Returns empty string if the file exceeds maxSize or encoding fails.
-func encodeImageToDataURL(localPath, mime string, info os.FileInfo, maxSize int) string {
+func encodeMediaToDataURL(localPath, mime string, info os.FileInfo, maxSize int) string {
 	if info.Size() > int64(maxSize) {
 		logger.WarnCF("agent", "Media file too large, skipping", map[string]any{
 			"path":     localPath,
--- a/pkg/agent/agent_test.go
+++ b/pkg/agent/agent_test.go
@ -4688,7 +4688,7 @@ func TestResolveMediaRefs_ImageInjectsPathTag(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "describe this", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media (images use path tags), got %d", len(result[0].Media))
@ -4721,7 +4721,7 @@ func TestResolveMediaRefs_ToolRoleImageAppendedAsUserMessage(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "tool", Content: "Image loaded", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	// Tool message should have path tag but no base64
 	if len(result[0].Media) != 0 {
@ -4768,7 +4768,7 @@ func TestResolveMediaRefs_MultiToolCallPreservesOrdering(t *testing.T) {
 		{Role: "tool", Content: "Image loaded [image: photo]", Media: []string{imgRef}},
 		{Role: "tool", Content: "file contents here"},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	// assistant, tool#1, tool#2 must remain contiguous — no user in between
 	if result[0].Role != "assistant" {
@ -4810,7 +4810,7 @@ func TestResolveMediaRefs_OversizedImageSkipsBase64KeepsPathTag(t *testing.T) {
 		{Role: "user", Content: "hi", Media: []string{ref}},
 	}
 	// Use a tiny limit (1KB) so the file is oversized
-	result := resolveMediaRefs(messages, store, 1024)
+	result := resolveMediaRefs(messages, store, 1024, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media))
@ -4835,7 +4835,7 @@ func TestResolveMediaRefs_UnknownTypeInjectsPath(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "hi", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media entries, got %d", len(result[0].Media))
@ -4850,7 +4850,7 @@ func TestResolveMediaRefs_PassesThroughNonMediaRefs(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "hi", Media: []string{"https://example.com/img.png"}},
 	}
-	result := resolveMediaRefs(messages, nil, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, nil, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 1 || result[0].Media[0] != "https://example.com/img.png" {
 		t.Fatalf("expected passthrough of non-media:// URL, got %v", result[0].Media)
@ -4875,7 +4875,7 @@ func TestResolveMediaRefs_DoesNotMutateOriginal(t *testing.T) {
 	}
 	originalRef := original[0].Media[0]

-	resolveMediaRefs(original, store, config.DefaultMaxMediaSize)
+	resolveMediaRefs(original, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if original[0].Media[0] != originalRef {
 		t.Fatal("resolveMediaRefs mutated original message slice")
@ -4895,7 +4895,7 @@ func TestResolveMediaRefs_UsesMetaContentType(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "hi", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media (images use path tags), got %d", len(result[0].Media))
@ -4919,7 +4919,7 @@ func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "report.pdf [file]", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media (non-image), got %d", len(result[0].Media))
@ -4930,29 +4930,56 @@ func TestResolveMediaRefs_PDFInjectsFilePath(t *testing.T) {
 	}
 }

-func TestResolveMediaRefs_AudioInjectsAudioPath(t *testing.T) {
-	store := media.NewFileMediaStore()
-	dir := t.TempDir()
-
-	oggPath := filepath.Join(dir, "voice.ogg")
-	os.WriteFile(oggPath, []byte("fake audio"), 0o644)
-	ref, _ := store.Store(oggPath, media.MediaMeta{ContentType: "audio/ogg"}, "test")
-
-	messages := []providers.Message{
-		{Role: "user", Content: "voice.ogg [audio]", Media: []string{ref}},
+func TestResolveMediaRefs_AudioVideoInlinesWithCapability(t *testing.T) {
+	cases := []struct {
+		name        string
+		filename    string
+		fakeData    string
+		contentType string
+		tag         string // "audio" or "video"
+		dataPrefix  string
+		policy      mediaInlinePolicy
+	}{
+		{
+			name: "audio/ogg", filename: "voice.ogg", fakeData: "fake audio",
+			contentType: "audio/ogg", tag: "audio", dataPrefix: "data:audio/ogg;base64,",
+			policy: mediaInlinePolicy{audio: true},
+		},
+		{
+			name: "video/mp4", filename: "clip.mp4", fakeData: "fake video",
+			contentType: "video/mp4", tag: "video", dataPrefix: "data:video/mp4;base64,",
+			policy: mediaInlinePolicy{video: true},
+		},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)

-	if len(result[0].Media) != 0 {
-		t.Fatalf("expected 0 media, got %d", len(result[0].Media))
-	}
-	expected := "voice.ogg [audio:" + oggPath + "]"
-	if result[0].Content != expected {
-		t.Fatalf("expected content %q, got %q", expected, result[0].Content)
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			store := media.NewFileMediaStore()
+			dir := t.TempDir()
+			filePath := filepath.Join(dir, tc.filename)
+			os.WriteFile(filePath, []byte(tc.fakeData), 0o644)
+			ref, _ := store.Store(filePath, media.MediaMeta{ContentType: tc.contentType}, "test")
+
+			messages := []providers.Message{
+				{Role: "user", Content: tc.filename + " [" + tc.tag + "]", Media: []string{ref}},
+			}
+			result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, tc.policy)
+
+			if len(result[0].Media) != 1 {
+				t.Fatalf("expected 1 media (inline data URL), got %d", len(result[0].Media))
+			}
+			if !strings.HasPrefix(result[0].Media[0], tc.dataPrefix) {
+				t.Fatalf("expected %s data URL, got %q", tc.contentType, result[0].Media[0])
+			}
+			expected := tc.filename + " [" + tc.tag + ":" + filePath + "]"
+			if result[0].Content != expected {
+				t.Fatalf("expected content %q, got %q", expected, result[0].Content)
+			}
+		})
 	}
 }

-func TestResolveMediaRefs_VideoInjectsVideoPath(t *testing.T) {
+func TestResolveMediaRefs_VideoNotInlinedWithoutCapability(t *testing.T) {
 	store := media.NewFileMediaStore()
 	dir := t.TempDir()

@ -4963,10 +4990,11 @@ func TestResolveMediaRefs_VideoInjectsVideoPath(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "clip.mp4 [video]", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	// Empty policy: provider does not support video
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
-		t.Fatalf("expected 0 media, got %d", len(result[0].Media))
+		t.Fatalf("expected 0 media (provider does not support video), got %d", len(result[0].Media))
 	}
 	expected := "clip.mp4 [video:" + mp4Path + "]"
 	if result[0].Content != expected {
@ -4974,6 +5002,29 @@ func TestResolveMediaRefs_VideoInjectsVideoPath(t *testing.T) {
 	}
 }

+func TestResolveMediaRefs_OversizedVideoSkipsInline(t *testing.T) {
+	store := media.NewFileMediaStore()
+	dir := t.TempDir()
+
+	mp4Path := filepath.Join(dir, "big.mp4")
+	os.WriteFile(mp4Path, []byte("fake video content"), 0o644)
+	ref, _ := store.Store(mp4Path, media.MediaMeta{ContentType: "video/mp4"}, "test")
+
+	messages := []providers.Message{
+		{Role: "user", Content: "big.mp4 [video]", Media: []string{ref}},
+	}
+	// Use a tiny limit (1 byte) so the file is oversized
+	result := resolveMediaRefs(messages, store, 1, mediaInlinePolicy{video: true})
+
+	if len(result[0].Media) != 0 {
+		t.Fatalf("expected 0 media (oversized), got %d", len(result[0].Media))
+	}
+	expected := "big.mp4 [video:" + mp4Path + "]"
+	if result[0].Content != expected {
+		t.Fatalf("expected content %q, got %q", expected, result[0].Content)
+	}
+}
+
 func TestResolveMediaRefs_NoGenericTagAppendsPath(t *testing.T) {
 	store := media.NewFileMediaStore()
 	dir := t.TempDir()
@ -4985,7 +5036,7 @@ func TestResolveMediaRefs_NoGenericTagAppendsPath(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "here is my data", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	expected := "here is my data [file:" + csvPath + "]"
 	if result[0].Content != expected {
@ -5077,7 +5128,7 @@ func TestResolveMediaRefs_JSONContentPrependsPathTag(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: jsonContent, Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	want := "[image:" + pngPath + "]\n" + jsonContent
 	if result[0].Content != want {
@ -5097,7 +5148,7 @@ func TestResolveMediaRefs_EmptyContentGetsPathTag(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "", Media: []string{ref}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	expected := "[file:" + docPath + "]"
 	if result[0].Content != expected {
@ -5126,7 +5177,7 @@ func TestResolveMediaRefs_MixedImageAndFile(t *testing.T) {
 	messages := []providers.Message{
 		{Role: "user", Content: "check these [file]", Media: []string{imgRef, fileRef}},
 	}
-	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize)
+	result := resolveMediaRefs(messages, store, config.DefaultMaxMediaSize, mediaInlinePolicy{})

 	if len(result[0].Media) != 0 {
 		t.Fatalf("expected 0 media (all types use path tags), got %d", len(result[0].Media))
--- a/pkg/agent/pipeline_llm.go
+++ b/pkg/agent/pipeline_llm.go
@ -31,7 +31,8 @@ func (p *Pipeline) CallLLM(

 	// PreLLM: resolve media refs (except on iteration 1 where user media is already resolved)
 	if iteration > 1 {
-		exec.messages = resolveMediaRefs(exec.messages, p.MediaStore, maxMediaSize)
+		policy := mediaInlinePolicyFromProvider(ts.agent.Provider)
+		exec.messages = resolveMediaRefs(exec.messages, p.MediaStore, maxMediaSize, policy)
 	}

 	// PreLLM: graceful terminal handling
@ -292,7 +293,7 @@ func (p *Pipeline) CallLLM(
 		if isNetworkError && retry < maxRetries {
 			backoff := time.Duration(retry+1) * time.Duration(backoffSecs) * time.Second
 			al.emitEvent(
-				EventKindLLMRetry,
+				runtimeevents.KindAgentLLMRetry,
 				ts.eventMeta("runTurn", "turn.llm.retry"),
 				LLMRetryPayload{
 					Attempt:    retry + 1,
--- a/pkg/agent/pipeline_setup.go
+++ b/pkg/agent/pipeline_setup.go
@ -16,6 +16,7 @@ import (
 func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution, error) {
 	cfg := p.Cfg
 	maxMediaSize := cfg.Agents.Defaults.GetMaxMediaSize()
+	policy := mediaInlinePolicyFromProvider(ts.agent.Provider)

 	var history []providers.Message
 	var summary string
@ -35,7 +36,7 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution
 		promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media),
 	)

-	messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize)
+	messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize, policy)

 	if !ts.opts.NoHistory {
 		toolDefs := ts.agent.Tools.ToProviderDefs()
@ -64,7 +65,7 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution
 			messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt(
 				promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media),
 			)
-			messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize)
+			messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize, policy)
 		}
 	}

--- a/pkg/agent/turn_coord.go
+++ b/pkg/agent/turn_coord.go
@ -122,7 +122,8 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel

 		// Inject pending steering messages
 		if len(pendingMessages) > 0 {
-			resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize)
+			policy := mediaInlinePolicyFromProvider(ts.agent.Provider)
+			resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize, policy)
 			totalContentLen := 0
 			for i, pm := range pendingMessages {
 				messages = append(messages, resolvedPending[i])
@ -371,7 +372,8 @@ func (al *AgentLoop) askSideQuestion(
 	)

 	maxMediaSize := al.GetConfig().Agents.Defaults.GetMaxMediaSize()
-	messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize)
+	policy := mediaInlinePolicyFromProvider(agent.Provider)
+	messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize, policy)

 	activeCandidates, activeModel, usedLight := al.selectCandidates(agent, question, messages)
 	selectedModelName := sideQuestionModelName(agent, usedLight)
--- a/pkg/config/migration.go
+++ b/pkg/config/migration.go
@ -84,6 +84,11 @@ func migrateLegacyAgentDefaultsModel(m map[string]any) {
 func loadConfig(data []byte) (*Config, error) {
 	cfg := DefaultConfig()

+	// Sanitize deprecated fields before strict unknown-field validation.
+	// This handles configs written by older versions or frontends that still
+	// use removed fields (e.g. session.dm_scope → session.dimensions).
+	data, _ = sanitizeDeprecatedFields(data)
+
 	// Pre-scan the JSON to check how many model_list entries the user provided.
 	// Go's JSON decoder reuses existing slice backing-array elements rather than
 	// zero-initializing them, so fields absent from the user's JSON (e.g. api_base)
@ -498,3 +503,77 @@ func mergeModelListsWithMap(mainML []any, secML map[string]any) error {

 	return nil
 }
+
+// sanitizeDeprecatedFields removes known deprecated fields from raw config
+// JSON so that the strict unknown-field validator does not reject them.
+// When possible it migrates deprecated values into their replacements.
+//
+// Known deprecated fields:
+//   - session.dm_scope → session.dimensions (removed in ca9652e1)
+//   - channels → channel_list (renamed in V2→V3 migration)
+//   - bindings (removed in V2→V3 migration)
+//   - providers (removed in V0→V1 migration, replaced by model_list)
+func sanitizeDeprecatedFields(data []byte) ([]byte, error) {
+	var m map[string]any
+	if err := json.Unmarshal(data, &m); err != nil {
+		return data, err
+	}
+
+	changed := false
+
+	// session.dm_scope → session.dimensions
+	if session, ok := m["session"].(map[string]any); ok {
+		if dmScope, hasDM := session["dm_scope"]; hasDM {
+			if _, hasDims := session["dimensions"]; !hasDims {
+				if scope, ok := dmScope.(string); ok {
+					session["dimensions"] = dmScopeToDimensions(scope)
+				}
+			}
+			delete(session, "dm_scope")
+			changed = true
+		}
+	}
+
+	// channels → channel_list (V2 legacy)
+	if channels, hasChannels := m["channels"]; hasChannels {
+		if _, hasChannelList := m["channel_list"]; !hasChannelList {
+			m["channel_list"] = channels
+		}
+		delete(m, "channels")
+		changed = true
+	}
+
+	// bindings (removed in V2→V3, handled by applyLegacyBindingsMigration)
+	if _, hasBindings := m["bindings"]; hasBindings {
+		delete(m, "bindings")
+		changed = true
+	}
+
+	// providers (V0 legacy, replaced by model_list)
+	if _, hasProviders := m["providers"]; hasProviders {
+		delete(m, "providers")
+		changed = true
+	}
+
+	if !changed {
+		return data, nil
+	}
+	return json.Marshal(m)
+}
+
+// dmScopeToDimensions converts a legacy dm_scope value to the new
+// session dimensions slice.
+func dmScopeToDimensions(scope string) []string {
+	switch scope {
+	case "per-channel-peer":
+		return []string{"chat", "sender"}
+	case "per-channel":
+		return []string{"chat"}
+	case "per-peer":
+		return []string{"sender"}
+	case "global":
+		return []string{}
+	default:
+		return []string{"chat"}
+	}
+}
--- a/pkg/config/migration_test.go
+++ b/pkg/config/migration_test.go
@ -6,6 +6,7 @@
 package config

 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"testing"
@ -395,3 +396,142 @@ func TestMigrateV1ToV3_AlreadyNestedFormat(t *testing.T) {
 	// Should NOT have nested settings inside settings
 	require.NotContains(t, settings, "settings")
 }
+
+func TestSanitizeDeprecatedFields_MigratesDMScope(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		wantDims []string
+	}{
+		{"per-channel-peer", `{"session":{"dm_scope":"per-channel-peer"}}`, []string{"chat", "sender"}},
+		{"per-channel", `{"session":{"dm_scope":"per-channel"}}`, []string{"chat"}},
+		{"per-peer", `{"session":{"dm_scope":"per-peer"}}`, []string{"sender"}},
+		{"global", `{"session":{"dm_scope":"global"}}`, []string{}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := sanitizeDeprecatedFields([]byte(tt.input))
+			require.NoError(t, err)
+
+			var m map[string]any
+			require.NoError(t, json.Unmarshal(out, &m))
+
+			session := m["session"].(map[string]any)
+			require.NotContains(t, session, "dm_scope", "dm_scope should be removed")
+
+			dims, ok := session["dimensions"].([]any)
+			require.True(t, ok, "dimensions should be a slice")
+			got := make([]string, len(dims))
+			for i, d := range dims {
+				got[i] = d.(string)
+			}
+			require.Equal(t, tt.wantDims, got)
+		})
+	}
+}
+
+func TestSanitizeDeprecatedFields_PreservesExistingDimensions(t *testing.T) {
+	input := `{"session":{"dm_scope":"global","dimensions":["chat","sender"]}}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(out, &m))
+
+	session := m["session"].(map[string]any)
+	require.NotContains(t, session, "dm_scope")
+	dims := session["dimensions"].([]any)
+	require.Equal(t, 2, len(dims), "existing dimensions should be preserved")
+}
+
+func TestSanitizeDeprecatedFields_NoSessionUnchanged(t *testing.T) {
+	input := `{"version":3,"gateway":{"host":"localhost"}}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+	require.JSONEq(t, input, string(out))
+}
+
+func TestSanitizeDeprecatedFields_MigratesChannelsToChannelList(t *testing.T) {
+	input := `{"version":3,"channels":{"telegram":{"type":"telegram","enabled":true}}}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(out, &m))
+	require.NotContains(t, m, "channels", "channels should be removed")
+	require.Contains(t, m, "channel_list", "channel_list should be present")
+}
+
+func TestSanitizeDeprecatedFields_ChannelsDoesNotOverwriteExistingChannelList(t *testing.T) {
+	input := `{"version":3,"channels":{"old":{"type":"old"}},"channel_list":{"telegram":{"type":"telegram"}}}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(out, &m))
+	require.NotContains(t, m, "channels")
+	cl := m["channel_list"].(map[string]any)
+	require.Contains(t, cl, "telegram", "existing channel_list should be preserved")
+	require.NotContains(t, cl, "old", "old channels should not overwrite channel_list")
+}
+
+func TestSanitizeDeprecatedFields_RemovesBindings(t *testing.T) {
+	input := `{"version":3,"bindings":[{"agent":"main"}]}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(out, &m))
+	require.NotContains(t, m, "bindings")
+}
+
+func TestSanitizeDeprecatedFields_RemovesProviders(t *testing.T) {
+	input := `{"version":3,"providers":{"openai":{"api_key":"sk-test"}}}`
+	out, err := sanitizeDeprecatedFields([]byte(input))
+	require.NoError(t, err)
+
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(out, &m))
+	require.NotContains(t, m, "providers")
+}
+
+func TestLoadConfig_WithLegacyDMScope(t *testing.T) {
+	tmpDir := t.TempDir()
+	configPath := filepath.Join(tmpDir, "config.json")
+
+	raw := `{
+		"version": 3,
+		"session": {
+			"dm_scope": "per-channel-peer"
+		},
+		"model_list": []
+	}`
+	require.NoError(t, os.WriteFile(configPath, []byte(raw), 0o600))
+
+	cfg, err := LoadConfig(configPath)
+	require.NoError(t, err, "LoadConfig should not fail with legacy dm_scope")
+	require.Equal(t, []string{"chat", "sender"}, cfg.Session.Dimensions)
+}
+
+func TestLoadConfig_WithLegacyChannelsField(t *testing.T) {
+	tmpDir := t.TempDir()
+	configPath := filepath.Join(tmpDir, "config.json")
+
+	raw := `{
+		"version": 3,
+		"channels": {
+			"telegram": {
+				"type": "telegram",
+				"enabled": true,
+				"settings": {"token": "test-token"}
+			}
+		},
+		"model_list": []
+	}`
+	require.NoError(t, os.WriteFile(configPath, []byte(raw), 0o600))
+
+	cfg, err := LoadConfig(configPath)
+	require.NoError(t, err, "LoadConfig should not fail with legacy channels field")
+	require.NotNil(t, cfg)
+}
--- a/pkg/providers/common/common.go
+++ b/pkg/providers/common/common.go
@ -135,6 +135,17 @@ func SerializeMessages(messages []Message) []any {
 						"format": format,
 					},
 				})
+				continue
+			}
+
+			if strings.HasPrefix(mediaURL, "data:video/") {
+				parts = append(parts, map[string]any{
+					"type": "video_url",
+					"video_url": map[string]any{
+						"url": mediaURL,
+					},
+				})
+				continue
 			}
 		}

--- a/pkg/providers/common/common_test.go
+++ b/pkg/providers/common/common_test.go
@ -129,6 +129,41 @@ func TestSerializeMessages_WithAudioMedia(t *testing.T) {
 	}
 }

+func TestSerializeMessages_WithVideoMedia(t *testing.T) {
+	messages := []Message{
+		{Role: "user", Content: "describe this video", Media: []string{"data:video/mp4;base64,AAAAAA"}},
+	}
+	result := SerializeMessages(messages)
+
+	data, _ := json.Marshal(result)
+	var msgs []map[string]any
+	json.Unmarshal(data, &msgs)
+
+	content, ok := msgs[0]["content"].([]any)
+	if !ok {
+		t.Fatalf("expected array content for media message, got %T", msgs[0]["content"])
+	}
+	if len(content) != 2 {
+		t.Fatalf("expected 2 content parts, got %d", len(content))
+	}
+
+	videoPart, ok := content[1].(map[string]any)
+	if !ok {
+		t.Fatalf("expected video content part to be an object, got %T", content[1])
+	}
+	if videoPart["type"] != "video_url" {
+		t.Fatalf("video part type = %v, want video_url", videoPart["type"])
+	}
+
+	videoURL, ok := videoPart["video_url"].(map[string]any)
+	if !ok {
+		t.Fatalf("expected video_url object, got %T", videoPart["video_url"])
+	}
+	if videoURL["url"] != "data:video/mp4;base64,AAAAAA" {
+		t.Fatalf("video url = %v, want data:video/mp4;base64,AAAAAA", videoURL["url"])
+	}
+}
+
 func TestSerializeMessages_MediaWithToolCallID(t *testing.T) {
 	messages := []Message{
 		{Role: "tool", Content: "result", Media: []string{"data:image/png;base64,xyz"}, ToolCallID: "call_1"},
--- a/pkg/providers/httpapi/gemini_provider.go
+++ b/pkg/providers/httpapi/gemini_provider.go
@ -63,6 +63,14 @@ func (p *GeminiProvider) SupportsThinking() bool {
 	return true
 }

+func (p *GeminiProvider) SupportsVideo() bool {
+	return true
+}
+
+func (p *GeminiProvider) SupportsAudio() bool {
+	return true
+}
+
 func (p *GeminiProvider) Chat(
 	ctx context.Context,
 	messages []Message,
--- a/pkg/providers/httpapi/http_provider.go
+++ b/pkg/providers/httpapi/http_provider.go
@ -78,6 +78,14 @@ func (p *HTTPProvider) SupportsNativeSearch() bool {
 	return p.delegate.SupportsNativeSearch()
 }

+func (p *HTTPProvider) SupportsVideo() bool {
+	return p.delegate.SupportsVideo()
+}
+
+func (p *HTTPProvider) SupportsAudio() bool {
+	return p.delegate.SupportsAudio()
+}
+
 func (p *HTTPProvider) SetProviderName(providerName string) {
 	if p == nil || p.delegate == nil {
 		return
--- a/pkg/providers/openai_compat/provider.go
+++ b/pkg/providers/openai_compat/provider.go
@ -419,6 +419,7 @@ func parseStreamResponse(
 	onChunk func(accumulated string),
 ) (*LLMResponse, error) {
 	var textContent strings.Builder
+	var reasoningContent strings.Builder
 	var finishReason string
 	var usage *UsageInfo

@ -451,8 +452,9 @@ func parseStreamResponse(
 		var chunk struct {
 			Choices []struct {
 				Delta struct {
-					Content   string `json:"content"`
-					ToolCalls []struct {
+					Content          string `json:"content"`
+					ReasoningContent string `json:"reasoning_content"`
+					ToolCalls        []struct {
 						Index    int    `json:"index"`
 						ID       string `json:"id"`
 						Function *struct {
@ -480,6 +482,11 @@ func parseStreamResponse(

 		choice := chunk.Choices[0]

+		// Accumulate reasoning content (DeepSeek, Mimo, Kimi, etc.)
+		if choice.Delta.ReasoningContent != "" {
+			reasoningContent.WriteString(choice.Delta.ReasoningContent)
+		}
+
 		// Accumulate text content
 		if choice.Delta.Content != "" {
 			textContent.WriteString(choice.Delta.Content)
@ -544,10 +551,11 @@ func parseStreamResponse(
 	}

 	return &LLMResponse{
-		Content:      textContent.String(),
-		ToolCalls:    toolCalls,
-		FinishReason: finishReason,
-		Usage:        usage,
+		Content:          textContent.String(),
+		ReasoningContent: reasoningContent.String(),
+		ToolCalls:        toolCalls,
+		FinishReason:     finishReason,
+		Usage:            usage,
 	}, nil
 }

@ -587,6 +595,34 @@ func (p *Provider) SupportsNativeSearch() bool {
 	return isNativeSearchHost(p.apiBase)
 }

+// SupportsVideo implements providers.VideoCapable.
+func (p *Provider) SupportsVideo() bool {
+	switch p.providerName {
+	case "mimo", "qwen", "qwen-portal", "qwen-intl", "qwen-international",
+		"dashscope-intl", "qwen-us", "dashscope-us":
+		return true
+	}
+	return isMimoHost(p.apiBase)
+}
+
+// SupportsAudio implements providers.AudioCapable.
+func (p *Provider) SupportsAudio() bool {
+	switch p.providerName {
+	case "mimo", "openai", "qwen", "qwen-portal", "qwen-intl", "qwen-international",
+		"dashscope-intl", "qwen-us", "dashscope-us":
+		return true
+	}
+	return isNativeOpenAIOrAzureEndpoint(p.apiBase) || isMimoHost(p.apiBase)
+}
+
+func isMimoHost(apiBase string) bool {
+	u, err := url.Parse(apiBase)
+	if err != nil {
+		return false
+	}
+	return u.Hostname() == "api.xiaomimimo.com"
+}
+
 // isNativeOpenAIOrAzureEndpoint reports whether the given API base points to
 // OpenAI's own API or an Azure OpenAI deployment.
 func isNativeOpenAIOrAzureEndpoint(apiBase string) bool {
--- a/pkg/providers/openai_compat/provider_test.go
+++ b/pkg/providers/openai_compat/provider_test.go
@ -1494,6 +1494,45 @@ func TestIsNativeSearchHost(t *testing.T) {
 	}
 }

+func TestSupportsVideo_Mimo(t *testing.T) {
+	p := NewProvider("key", "https://api.xiaomimimo.com/v1", "")
+	if !p.SupportsVideo() {
+		t.Fatal("Mimo provider should support video")
+	}
+}
+
+func TestSupportsVideo_DeepSeek(t *testing.T) {
+	p := NewProvider("key", "https://api.deepseek.com/v1", "")
+	if p.SupportsVideo() {
+		t.Fatal("DeepSeek provider should not support video")
+	}
+}
+
+func TestSupportsAudio_OpenAI(t *testing.T) {
+	p := NewProvider("key", "https://api.openai.com/v1", "")
+	if !p.SupportsAudio() {
+		t.Fatal("OpenAI provider should support audio")
+	}
+}
+
+func TestSupportsAudio_DeepSeek(t *testing.T) {
+	p := NewProvider("key", "https://api.deepseek.com/v1", "")
+	if p.SupportsAudio() {
+		t.Fatal("DeepSeek provider should not support audio")
+	}
+}
+
+func TestSupportsVideo_QwenByProviderName(t *testing.T) {
+	p := NewProvider("key", "https://dashscope.aliyuncs.com/compatible-mode/v1", "",
+		WithProviderName("qwen"))
+	if !p.SupportsVideo() {
+		t.Fatal("Qwen provider should support video")
+	}
+	if !p.SupportsAudio() {
+		t.Fatal("Qwen provider should support audio")
+	}
+}
+
 func TestSupportsNativeSearch_OpenAI(t *testing.T) {
 	p := NewProvider("key", "https://api.openai.com/v1", "")
 	if !p.SupportsNativeSearch() {
@ -1656,6 +1695,39 @@ func TestProviderChat_NativeSearchIgnoredOnNonOpenAI(t *testing.T) {
 	}
 }

+func TestProviderChatStream_ParsesReasoningContent(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		fmt.Fprintln(w, `data: {"choices":[{"delta":{"reasoning_content":"Let me think"},"finish_reason":null}]}`)
+		fmt.Fprintln(w)
+		fmt.Fprintln(w, `data: {"choices":[{"delta":{"reasoning_content":"... 1+1=2"},"finish_reason":null}]}`)
+		fmt.Fprintln(w)
+		fmt.Fprintln(w, `data: {"choices":[{"delta":{"content":"The answer is 2"},"finish_reason":"stop"}]}`)
+		fmt.Fprintln(w)
+		fmt.Fprintln(w, "data: [DONE]")
+	}))
+	defer server.Close()
+
+	p := NewProvider("key", server.URL, "")
+	out, err := p.ChatStream(
+		t.Context(),
+		[]Message{{Role: "user", Content: "1+1=?"}},
+		nil,
+		"mimo-v2.5",
+		nil,
+		nil,
+	)
+	if err != nil {
+		t.Fatalf("ChatStream() error = %v", err)
+	}
+	if out.ReasoningContent != "Let me think... 1+1=2" {
+		t.Fatalf("ReasoningContent = %q, want %q", out.ReasoningContent, "Let me think... 1+1=2")
+	}
+	if out.Content != "The answer is 2" {
+		t.Fatalf("Content = %q, want %q", out.Content, "The answer is 2")
+	}
+}
+
 func TestSerializeMessages_StripsSystemParts(t *testing.T) {
 	messages := []protocoltypes.Message{
 		{
--- a/pkg/providers/types.go
+++ b/pkg/providers/types.go
@ -68,6 +68,21 @@ type NativeSearchCapable interface {
 	SupportsNativeSearch() bool
 }

+// VideoCapable is an optional interface for providers that support inline
+// video content (e.g. Mimo, Qwen). When a provider does not implement this
+// interface, video media refs are resolved to path tags only.
+type VideoCapable interface {
+	SupportsVideo() bool
+}
+
+// AudioCapable is an optional interface for providers that support inline
+// audio content (e.g. OpenAI GPT-4o-audio, Mimo, Qwen). When a provider
+// does not implement this interface, audio media refs are resolved to path
+// tags only.
+type AudioCapable interface {
+	SupportsAudio() bool
+}
+
 // FailoverReason classifies why an LLM request failed for fallback decisions.
 type FailoverReason string

--- a/web/backend/api/config.go
+++ b/web/backend/api/config.go
@ -167,6 +167,7 @@ func (h *Handler) handlePatchConfig(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, fmt.Sprintf("Invalid channel array field: %v", err), http.StatusBadRequest)
 		return
 	}
+	migrateDeprecatedSessionFields(base)

 	// Convert merged map back to Config struct
 	merged, err := json.Marshal(base)
@ -386,6 +387,40 @@ func mergeMap(dst, src map[string]any) {
 	}
 }

+// migrateDeprecatedSessionFields converts deprecated session fields in a raw
+// config map. Currently handles session.dm_scope → session.dimensions.
+func migrateDeprecatedSessionFields(m map[string]any) {
+	session, ok := m["session"].(map[string]any)
+	if !ok {
+		return
+	}
+	dmScope, hasDM := session["dm_scope"]
+	if !hasDM {
+		return
+	}
+	if _, hasDims := session["dimensions"]; !hasDims {
+		if scope, ok := dmScope.(string); ok {
+			session["dimensions"] = dmScopeToDimensions(scope)
+		}
+	}
+	delete(session, "dm_scope")
+}
+
+func dmScopeToDimensions(scope string) []string {
+	switch scope {
+	case "per-channel-peer":
+		return []string{"chat", "sender"}
+	case "per-channel":
+		return []string{"chat"}
+	case "per-peer":
+		return []string{"sender"}
+	case "global":
+		return []string{}
+	default:
+		return []string{"chat"}
+	}
+}
+
 func asMapField(value map[string]any, key string) (map[string]any, bool) {
 	raw, exists := value[key]
 	if !exists {
--- a/web/frontend/src/components/config/config-page.tsx
+++ b/web/frontend/src/components/config/config-page.tsx
@ -30,6 +30,7 @@ import {
  EMPTY_LAUNCHER_FORM,
  type LauncherForm,
  buildFormFromConfig,
+  dmScopeToDimensions,
  parseCIDRText,
  parseIntField,
  parseMultilineList,
@ -256,7 +257,7 @@ export function ConfigPage() {
            },
          },
          session: {
-            dm_scope: dmScope,
+            dimensions: dmScopeToDimensions(dmScope),
          },
          tools: {
            cron: {
--- a/web/frontend/src/components/config/form-model.ts
+++ b/web/frontend/src/components/config/form-model.ts
@ -66,6 +66,38 @@ export const DM_SCOPE_OPTIONS = [
  },
 ] as const

+/**
+ * Convert a legacy dm_scope value to the new dimensions array.
+ */
+export function dmScopeToDimensions(scope: string): string[] {
+  switch (scope) {
+    case "per-channel-peer":
+      return ["chat", "sender"]
+    case "per-channel":
+      return ["chat"]
+    case "per-peer":
+      return ["sender"]
+    case "global":
+      return []
+    default:
+      return ["chat"]
+  }
+}
+
+/**
+ * Convert a dimensions array back to a legacy dm_scope value for display.
+ */
+export function dimensionsToDmScope(dimensions: unknown): string {
+  if (!Array.isArray(dimensions)) return "per-channel-peer"
+  const dims = dimensions.filter((d): d is string => typeof d === "string")
+  const hasChat = dims.includes("chat")
+  const hasSender = dims.includes("sender")
+  if (hasChat && hasSender) return "per-channel-peer"
+  if (hasChat && !hasSender) return "per-channel"
+  if (!hasChat && hasSender) return "per-peer"
+  return "global"
+}
+
 export const EMPTY_FORM: CoreConfigForm = {
  workspace: "",
  restrictToWorkspace: true,
@ -211,7 +243,9 @@ export function buildFormFromConfig(config: unknown): CoreConfigForm {
      defaults.summarize_token_percent,
      EMPTY_FORM.summarizeTokenPercent,
    ),
-    dmScope: asString(session.dm_scope) || EMPTY_FORM.dmScope,
+    dmScope: session.dimensions !== undefined
+      ? dimensionsToDmScope(session.dimensions)
+      : (asString(session.dm_scope) || EMPTY_FORM.dmScope),
    heartbeatEnabled:
      heartbeat.enabled === undefined
        ? EMPTY_FORM.heartbeatEnabled