c1e92b726d
- Added functionality to build and manage MCP configuration for sandbox environments, allowing for dynamic tool execution. - Enhanced the Assistant's Stream method to skip MCP tool calls in sandbox mode, with internal handling by Claude CLI. - Introduced unit tests for MCP configuration building and skills directory resolution, ensuring robust integration. - Updated sandbox manager to create IPC sessions and manage tool exposure dynamically, improving interaction with external agents. - Enhanced documentation to reflect new features and integration points for MCP and skills within the sandbox.
481 lines
15 KiB
Go
481 lines
15 KiB
Go
package assistant_test
|
|
|
|
import (
|
|
stdContext "context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/yaoapp/yao/agent/assistant"
|
|
"github.com/yaoapp/yao/agent/context"
|
|
"github.com/yaoapp/yao/agent/testutils"
|
|
"github.com/yaoapp/yao/openapi/oauth/types"
|
|
)
|
|
|
|
// newSandboxE2EContext creates a Context for sandbox E2E testing
|
|
// Uses unique chatID to avoid container name conflicts
|
|
func newSandboxE2EContext(chatIDPrefix, assistantID string) *context.Context {
|
|
// Generate unique chatID using timestamp to avoid container conflicts
|
|
chatID := fmt.Sprintf("%s-%d", chatIDPrefix, time.Now().UnixNano())
|
|
|
|
authorized := &types.AuthorizedInfo{
|
|
Subject: "sandbox-e2e-test-user",
|
|
ClientID: "sandbox-e2e-test-client",
|
|
Scope: "openid profile",
|
|
SessionID: "sandbox-e2e-test-session",
|
|
UserID: "sandbox-user-123",
|
|
TeamID: "sandbox-team-456",
|
|
TenantID: "sandbox-tenant-789",
|
|
}
|
|
|
|
ctx := context.New(stdContext.Background(), authorized, chatID)
|
|
ctx.AssistantID = assistantID
|
|
ctx.Locale = "en-us"
|
|
ctx.Theme = "light"
|
|
ctx.Client = context.Client{
|
|
Type: "web",
|
|
UserAgent: "SandboxE2ETest/1.0",
|
|
IP: "127.0.0.1",
|
|
}
|
|
ctx.Referer = context.RefererAPI
|
|
ctx.Accept = context.AcceptWebCUI
|
|
ctx.Route = ""
|
|
ctx.Metadata = make(map[string]interface{})
|
|
return ctx
|
|
}
|
|
|
|
// TestSandboxBasicE2E tests the basic sandbox assistant end-to-end
|
|
// This test verifies that:
|
|
// 1. Sandbox is correctly initialized
|
|
// 2. Claude CLI command is built correctly
|
|
// 3. Docker container is created and managed
|
|
func TestSandboxBasicE2E(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox E2E test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the basic sandbox assistant
|
|
ast, err := assistant.Get("tests.sandbox.basic")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: sandbox assistant not available: %v", err)
|
|
}
|
|
|
|
// Verify sandbox is configured
|
|
require.NotNil(t, ast.Sandbox, "Sandbox should be configured")
|
|
assert.Equal(t, "claude", ast.Sandbox.Command)
|
|
t.Logf("✓ Sandbox configured with command: %s", ast.Sandbox.Command)
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-basic-e2e", "tests.sandbox.basic")
|
|
|
|
// Test messages
|
|
messages := []context.Message{
|
|
{Role: context.RoleUser, Content: "echo hello sandbox"},
|
|
}
|
|
|
|
// Execute stream
|
|
// Note: This will fail if Docker/Claude image is not available, which is expected in CI
|
|
response, err := ast.Stream(ctx, messages)
|
|
if err != nil {
|
|
// Check if it's a Docker/sandbox availability issue
|
|
errStr := err.Error()
|
|
if strings.Contains(errStr, "Docker") ||
|
|
strings.Contains(errStr, "sandbox") ||
|
|
strings.Contains(errStr, "container") ||
|
|
strings.Contains(errStr, "image") {
|
|
t.Skipf("Skipping test: Docker/sandbox not available: %v", err)
|
|
}
|
|
t.Fatalf("Stream failed: %v", err)
|
|
}
|
|
|
|
// Verify response
|
|
require.NotNil(t, response, "Response should not be nil")
|
|
|
|
// Verify response completion (Claude CLI should return some response)
|
|
if response.Completion != nil && response.Completion.Content != nil {
|
|
if contentStr, ok := response.Completion.Content.(string); ok && contentStr != "" {
|
|
t.Logf("✓ Response content: %s", truncateString(contentStr, 200))
|
|
} else {
|
|
t.Logf("⚠ Response content type: %T", response.Completion.Content)
|
|
}
|
|
} else {
|
|
t.Log("⚠ Response content is empty (might be expected for some commands)")
|
|
}
|
|
|
|
t.Log("✓ Basic sandbox E2E test passed")
|
|
}
|
|
|
|
// truncateString truncates a string to maxLen and adds "..." if truncated
|
|
func truncateString(s string, maxLen int) string {
|
|
if len(s) <= maxLen {
|
|
return s
|
|
}
|
|
return s[:maxLen] + "..."
|
|
}
|
|
|
|
// TestSandboxHooksE2E tests the sandbox assistant with hooks
|
|
func TestSandboxHooksE2E(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox E2E test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the hooks sandbox assistant
|
|
ast, err := assistant.Get("tests.sandbox.hooks")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: sandbox hooks assistant not available: %v", err)
|
|
}
|
|
|
|
// Verify sandbox and hooks are configured
|
|
require.NotNil(t, ast.Sandbox, "Sandbox should be configured")
|
|
require.NotNil(t, ast.HookScript, "HookScript should be loaded")
|
|
t.Logf("✓ Sandbox and hooks configured")
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-hooks-e2e", "tests.sandbox.hooks")
|
|
|
|
// Test messages
|
|
messages := []context.Message{
|
|
{Role: context.RoleUser, Content: "test hooks integration"},
|
|
}
|
|
|
|
// Execute stream
|
|
response, err := ast.Stream(ctx, messages)
|
|
if err != nil {
|
|
errStr := err.Error()
|
|
if strings.Contains(errStr, "Docker") ||
|
|
strings.Contains(errStr, "sandbox") ||
|
|
strings.Contains(errStr, "container") ||
|
|
strings.Contains(errStr, "image") {
|
|
t.Skipf("Skipping test: Docker/sandbox not available: %v", err)
|
|
}
|
|
t.Fatalf("Stream failed: %v", err)
|
|
}
|
|
|
|
require.NotNil(t, response, "Response should not be nil")
|
|
t.Log("✓ Sandbox hooks E2E test passed")
|
|
}
|
|
|
|
// TestSandboxFullE2E tests the full sandbox assistant with MCPs and Skills
|
|
func TestSandboxFullE2E(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox E2E test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the full sandbox assistant
|
|
ast, err := assistant.Get("tests.sandbox.full")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: full sandbox assistant not available: %v", err)
|
|
}
|
|
|
|
// Verify all components are configured
|
|
require.NotNil(t, ast.Sandbox, "Sandbox should be configured")
|
|
require.NotNil(t, ast.MCP, "MCP should be configured")
|
|
require.NotNil(t, ast.HookScript, "HookScript should be loaded")
|
|
t.Logf("✓ Full sandbox configured: command=%s, MCP servers=%d",
|
|
ast.Sandbox.Command, len(ast.MCP.Servers))
|
|
|
|
// Verify MCP configuration
|
|
assert.Len(t, ast.MCP.Servers, 1)
|
|
assert.Equal(t, "echo", ast.MCP.Servers[0].ServerID)
|
|
t.Logf("✓ MCP server: %s with tools %v", ast.MCP.Servers[0].ServerID, ast.MCP.Servers[0].Tools)
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-full-e2e", "tests.sandbox.full")
|
|
|
|
// Test messages
|
|
messages := []context.Message{
|
|
{Role: context.RoleUser, Content: "test full sandbox with MCP and skills"},
|
|
}
|
|
|
|
// Execute stream
|
|
response, err := ast.Stream(ctx, messages)
|
|
if err != nil {
|
|
errStr := err.Error()
|
|
if strings.Contains(errStr, "Docker") ||
|
|
strings.Contains(errStr, "sandbox") ||
|
|
strings.Contains(errStr, "container") ||
|
|
strings.Contains(errStr, "image") {
|
|
t.Skipf("Skipping test: Docker/sandbox not available: %v", err)
|
|
}
|
|
t.Fatalf("Stream failed: %v", err)
|
|
}
|
|
|
|
require.NotNil(t, response, "Response should not be nil")
|
|
t.Log("✓ Full sandbox E2E test passed")
|
|
}
|
|
|
|
// TestSandboxContextAccess tests that sandbox is accessible in hooks via ctx.sandbox
|
|
func TestSandboxContextAccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox context access test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the hooks sandbox assistant
|
|
ast, err := assistant.Get("tests.sandbox.hooks")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: sandbox hooks assistant not available: %v", err)
|
|
}
|
|
|
|
require.NotNil(t, ast.HookScript, "HookScript should be loaded")
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-ctx-access", "tests.sandbox.hooks")
|
|
|
|
// Test Create Hook - it should have access to ctx.sandbox
|
|
messages := []context.Message{
|
|
{Role: context.RoleUser, Content: "test sandbox context access"},
|
|
}
|
|
|
|
// Execute Create hook directly
|
|
// This tests that the hook runs without error (sandbox operations tested within)
|
|
opts := &context.Options{}
|
|
response, _, err := ast.HookScript.Create(ctx, messages, opts)
|
|
|
|
// The hook might fail if sandbox isn't initialized yet (that's done in Stream)
|
|
// But we can at least verify the hook exists and can be called
|
|
if err != nil {
|
|
// If the error is about sandbox not being available, that's expected
|
|
// because we haven't initialized the sandbox yet
|
|
if strings.Contains(err.Error(), "sandbox") {
|
|
t.Logf("Expected error: sandbox not available in direct hook call: %v", err)
|
|
} else {
|
|
t.Fatalf("Unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
// Response might be nil, that's okay
|
|
t.Logf("Create hook response: %v", response)
|
|
t.Log("✓ Sandbox context access test passed")
|
|
}
|
|
|
|
// TestSandboxMCPToolCall tests that Claude actually calls MCP tools via IPC
|
|
// This test specifically asks Claude to use the echo tool and verifies the result
|
|
func TestSandboxMCPToolCall(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox MCP tool call test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the full sandbox assistant (has MCP echo tool)
|
|
ast, err := assistant.Get("tests.sandbox.full")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: full sandbox assistant not available: %v", err)
|
|
}
|
|
|
|
// Verify MCP is configured with echo tools
|
|
require.NotNil(t, ast.MCP, "MCP should be configured")
|
|
require.NotEmpty(t, ast.MCP.Servers, "MCP servers should be configured")
|
|
t.Logf("✓ MCP configured with server: %s, tools: %v",
|
|
ast.MCP.Servers[0].ServerID, ast.MCP.Servers[0].Tools)
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-mcp-tool", "tests.sandbox.full")
|
|
|
|
// Explicit prompt to use echo tool
|
|
// This tells Claude to use the MCP tool specifically
|
|
messages := []context.Message{
|
|
{
|
|
Role: context.RoleUser,
|
|
Content: `Please use the 'ping' MCP tool to send a ping with message "MCP_TEST_SUCCESS".
|
|
Just call the tool and show me the result. Do not explain, just use the tool.`,
|
|
},
|
|
}
|
|
|
|
// Collect all response content
|
|
var responseContent strings.Builder
|
|
|
|
// Execute stream
|
|
response, err := ast.Stream(ctx, messages)
|
|
if err != nil {
|
|
errStr := err.Error()
|
|
if strings.Contains(errStr, "Docker") ||
|
|
strings.Contains(errStr, "sandbox") ||
|
|
strings.Contains(errStr, "container") ||
|
|
strings.Contains(errStr, "image") {
|
|
t.Skipf("Skipping test: Docker/sandbox not available: %v", err)
|
|
}
|
|
t.Fatalf("Stream failed: %v", err)
|
|
}
|
|
|
|
require.NotNil(t, response, "Response should not be nil")
|
|
|
|
// Get the response content
|
|
fullResponse := ""
|
|
if response.Completion != nil && response.Completion.Content != nil {
|
|
if contentStr, ok := response.Completion.Content.(string); ok {
|
|
fullResponse = contentStr
|
|
responseContent.WriteString(contentStr)
|
|
}
|
|
}
|
|
|
|
t.Logf("Claude response: %s", fullResponse)
|
|
|
|
// Check if Claude acknowledged using the tool or returned tool results
|
|
// The response should contain either:
|
|
// 1. Evidence of tool call (tool_use block in response)
|
|
// 2. The ping result "pong" or "MCP_TEST_SUCCESS"
|
|
// 3. Some indication that it attempted to use the MCP tool
|
|
|
|
hasToolEvidence := strings.Contains(fullResponse, "pong") ||
|
|
strings.Contains(fullResponse, "MCP_TEST_SUCCESS") ||
|
|
strings.Contains(fullResponse, "ping") ||
|
|
strings.Contains(fullResponse, "tool")
|
|
|
|
if hasToolEvidence {
|
|
t.Log("✓ Claude appears to have used the MCP tool")
|
|
} else {
|
|
t.Logf("⚠ Claude response does not clearly show MCP tool usage")
|
|
t.Logf("Response: %s", fullResponse)
|
|
}
|
|
|
|
// At minimum, verify we got a response
|
|
if fullResponse == "" {
|
|
t.Log("⚠ Response content is empty")
|
|
}
|
|
t.Log("✓ Sandbox MCP tool call test completed")
|
|
}
|
|
|
|
// TestSandboxMCPEchoTool tests the echo MCP tool specifically
|
|
// This test uses a more explicit prompt to force tool usage
|
|
func TestSandboxMCPEchoTool(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping sandbox MCP echo test in short mode")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
// Load the full sandbox assistant
|
|
ast, err := assistant.Get("tests.sandbox.full")
|
|
if err != nil {
|
|
t.Skipf("Skipping test: full sandbox assistant not available: %v", err)
|
|
}
|
|
|
|
// Create context
|
|
ctx := newSandboxE2EContext("sandbox-mcp-echo", "tests.sandbox.full")
|
|
|
|
// Very explicit prompt for echo tool
|
|
messages := []context.Message{
|
|
{
|
|
Role: context.RoleUser,
|
|
Content: `Call the 'echo' MCP tool with message "ECHO_VERIFICATION_12345" and uppercase=true.
|
|
Show me the exact response from the tool.`,
|
|
},
|
|
}
|
|
|
|
response, err := ast.Stream(ctx, messages)
|
|
if err != nil {
|
|
errStr := err.Error()
|
|
if strings.Contains(errStr, "Docker") ||
|
|
strings.Contains(errStr, "sandbox") ||
|
|
strings.Contains(errStr, "container") ||
|
|
strings.Contains(errStr, "image") {
|
|
t.Skipf("Skipping test: Docker/sandbox not available: %v", err)
|
|
}
|
|
t.Fatalf("Stream failed: %v", err)
|
|
}
|
|
|
|
require.NotNil(t, response)
|
|
|
|
fullResponse := ""
|
|
if response.Completion != nil && response.Completion.Content != nil {
|
|
if contentStr, ok := response.Completion.Content.(string); ok {
|
|
fullResponse = contentStr
|
|
}
|
|
}
|
|
t.Logf("Claude response for echo tool: %s", fullResponse)
|
|
|
|
// The echo tool with uppercase=true should return "ECHO_VERIFICATION_12345"
|
|
// Check if this appears in the response
|
|
if strings.Contains(fullResponse, "ECHO_VERIFICATION_12345") {
|
|
t.Log("✓ MCP echo tool executed successfully - found verification string in response")
|
|
} else if strings.Contains(fullResponse, "echo") || strings.Contains(fullResponse, "ECHO") {
|
|
t.Log("✓ MCP echo tool appears to have been used (found 'echo' in response)")
|
|
} else {
|
|
t.Logf("⚠ Could not verify echo tool execution. Response: %s", fullResponse)
|
|
}
|
|
|
|
t.Log("✓ Sandbox MCP echo tool test completed")
|
|
}
|
|
|
|
// TestSandboxLoadConfiguration verifies that sandbox assistants load correctly
|
|
func TestSandboxLoadConfiguration(t *testing.T) {
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
testCases := []struct {
|
|
name string
|
|
assistantID string
|
|
expectSandbox bool
|
|
expectMCP bool
|
|
expectHooks bool
|
|
}{
|
|
{
|
|
name: "BasicSandbox",
|
|
assistantID: "tests.sandbox.basic",
|
|
expectSandbox: true,
|
|
expectMCP: false,
|
|
expectHooks: false,
|
|
},
|
|
{
|
|
name: "HooksSandbox",
|
|
assistantID: "tests.sandbox.hooks",
|
|
expectSandbox: true,
|
|
expectMCP: false,
|
|
expectHooks: true,
|
|
},
|
|
{
|
|
name: "FullSandbox",
|
|
assistantID: "tests.sandbox.full",
|
|
expectSandbox: true,
|
|
expectMCP: true,
|
|
expectHooks: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ast, err := assistant.Get(tc.assistantID)
|
|
if err != nil {
|
|
t.Skipf("Skipping: assistant %s not available: %v", tc.assistantID, err)
|
|
}
|
|
|
|
// Check sandbox
|
|
if tc.expectSandbox {
|
|
require.NotNil(t, ast.Sandbox, "Expected sandbox to be configured")
|
|
assert.Equal(t, "claude", ast.Sandbox.Command)
|
|
t.Logf("✓ %s: Sandbox configured with command=%s", tc.name, ast.Sandbox.Command)
|
|
}
|
|
|
|
// Check MCP
|
|
if tc.expectMCP {
|
|
require.NotNil(t, ast.MCP, "Expected MCP to be configured")
|
|
assert.True(t, len(ast.MCP.Servers) > 0, "Expected at least one MCP server")
|
|
t.Logf("✓ %s: MCP configured with %d servers", tc.name, len(ast.MCP.Servers))
|
|
}
|
|
|
|
// Check hooks
|
|
if tc.expectHooks {
|
|
require.NotNil(t, ast.HookScript, "Expected hooks to be loaded")
|
|
t.Logf("✓ %s: Hooks loaded", tc.name)
|
|
}
|
|
})
|
|
}
|
|
}
|