From 69058787f9df51f10045a77d0e2e75fd4acd82ca Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 5 Feb 2026 14:45:54 +0800 Subject: [PATCH] Add VNC remote desktop support for sandbox containers - Add VNC-enabled Docker images (playwright, desktop) with Xvfb, x11vnc, noVNC - Implement VNC proxy service for WebSocket-based VNC access - Add API endpoints: /sandbox/{id}/vnc, /vnc/client, /vnc/ws - Support dynamic VNC port mapping for Docker Desktop (macOS/Windows) - Add YAO_SANDBOX_VNC_PORT_MAPPING config option for local development - Update build.sh to support building VNC images - Include design document and updated README Co-authored-by: Cursor --- .gitignore | 1 + openapi/openapi.go | 4 + openapi/sandbox/sandbox.go | 98 ++ sandbox/DESIGN-PLAYWRIGHT-VNC.md | 1498 ++++++++++++++++++++++++++ sandbox/README.md | 97 +- sandbox/config.go | 8 + sandbox/docker/build.sh | 36 +- sandbox/docker/desktop/Dockerfile | 91 ++ sandbox/docker/playwright/Dockerfile | 89 ++ sandbox/docker/vnc/entrypoint-vnc.sh | 40 + sandbox/docker/vnc/start-vnc.sh | 94 ++ sandbox/manager.go | 36 + sandbox/vncproxy/config.go | 81 ++ sandbox/vncproxy/proxy.go | 565 ++++++++++ sandbox/vncproxy/proxy_test.go | 90 ++ 15 files changed, 2811 insertions(+), 17 deletions(-) create mode 100644 openapi/sandbox/sandbox.go create mode 100644 sandbox/DESIGN-PLAYWRIGHT-VNC.md create mode 100644 sandbox/docker/desktop/Dockerfile create mode 100644 sandbox/docker/playwright/Dockerfile create mode 100644 sandbox/docker/vnc/entrypoint-vnc.sh create mode 100644 sandbox/docker/vnc/start-vnc.sh create mode 100644 sandbox/vncproxy/config.go create mode 100644 sandbox/vncproxy/proxy.go create mode 100644 sandbox/vncproxy/proxy_test.go diff --git a/.gitignore b/.gitignore index 8ddfece5..eaab7002 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ agent/test/MULTI_TURN_DESIGN.md agent/test/UPGRADE_PLAN.md introduction/* !sandbox/docker/build.sh +!sandbox/docker/vnc/*.sh sandbox/docker/yao-bridge-* sandbox/docker/claude-proxy-* sandbox/docker/claude/claude-proxy-* diff --git a/openapi/openapi.go b/openapi/openapi.go index 6a6113b7..3d3da032 100644 --- a/openapi/openapi.go +++ b/openapi/openapi.go @@ -22,6 +22,7 @@ import ( "github.com/yaoapp/yao/openapi/oauth/acl" "github.com/yaoapp/yao/openapi/oauth/types" "github.com/yaoapp/yao/openapi/response" + "github.com/yaoapp/yao/openapi/sandbox" "github.com/yaoapp/yao/openapi/team" openapiTrace "github.com/yaoapp/yao/openapi/trace" "github.com/yaoapp/yao/openapi/user" @@ -159,6 +160,9 @@ func (openapi *OpenAPI) Attach(router *gin.Engine) { // App handlers (menu, etc.) app.Attach(group.Group("/app"), openapi.OAuth) + // Sandbox handlers (VNC proxy for visual browser automation) + sandbox.Attach(group.Group("/sandbox"), openapi.OAuth) + // Custom handlers (Defined by developer) } diff --git a/openapi/sandbox/sandbox.go b/openapi/sandbox/sandbox.go new file mode 100644 index 00000000..eecaf437 --- /dev/null +++ b/openapi/sandbox/sandbox.go @@ -0,0 +1,98 @@ +package sandbox + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/yaoapp/yao/openapi/oauth/types" + "github.com/yaoapp/yao/sandbox/vncproxy" +) + +var vncProxy *vncproxy.Proxy + +// Attach attaches sandbox handlers to the router group +// Routes: +// - GET /sandbox/:id/vnc - Get VNC status +// - GET /sandbox/:id/vnc/client - Get noVNC client page +// - GET /sandbox/:id/vnc/ws - WebSocket proxy to container VNC +func Attach(group *gin.RouterGroup, oauth types.OAuth) { + // Initialize VNC proxy lazily on first request + // This avoids startup errors if Docker is not available + + // VNC status endpoint (requires auth) + group.GET("/:id/vnc", oauth.Guard, handleVNCStatus) + + // VNC client page (requires auth) + group.GET("/:id/vnc/client", oauth.Guard, handleVNCClient) + + // VNC WebSocket proxy (requires auth) + // Note: WebSocket upgrade happens after auth middleware + group.GET("/:id/vnc/ws", oauth.Guard, handleVNCWebSocket) +} + +// ensureProxy ensures the VNC proxy is initialized +func ensureProxy() error { + if vncProxy != nil { + return nil + } + + var err error + vncProxy, err = vncproxy.NewProxy(nil) + return err +} + +// handleVNCStatus returns VNC status for a sandbox container +func handleVNCStatus(c *gin.Context) { + if err := ensureProxy(); err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "VNC service not available", + }) + return + } + + // Rewrite path to match vncproxy expected format + sandboxID := c.Param("id") + c.Request.URL.Path = "/v1/sandbox/" + sandboxID + "/vnc" + + vncProxy.HandleVNCStatus(c.Writer, c.Request) +} + +// handleVNCClient serves the noVNC client page +func handleVNCClient(c *gin.Context) { + if err := ensureProxy(); err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "VNC service not available", + }) + return + } + + // Rewrite path to match vncproxy expected format + sandboxID := c.Param("id") + c.Request.URL.Path = "/v1/sandbox/" + sandboxID + "/vnc/client" + + vncProxy.HandleVNCClient(c.Writer, c.Request) +} + +// handleVNCWebSocket proxies WebSocket to container VNC +func handleVNCWebSocket(c *gin.Context) { + if err := ensureProxy(); err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "VNC service not available", + }) + return + } + + // Rewrite path to match vncproxy expected format + sandboxID := c.Param("id") + c.Request.URL.Path = "/v1/sandbox/" + sandboxID + "/vnc/ws" + + vncProxy.HandleVNCWebSocket(c.Writer, c.Request) +} + +// Close closes the VNC proxy and releases resources +func Close() error { + if vncProxy != nil { + return vncProxy.Close() + } + return nil +} diff --git a/sandbox/DESIGN-PLAYWRIGHT-VNC.md b/sandbox/DESIGN-PLAYWRIGHT-VNC.md new file mode 100644 index 00000000..51a9e834 --- /dev/null +++ b/sandbox/DESIGN-PLAYWRIGHT-VNC.md @@ -0,0 +1,1498 @@ +# Sandbox VNC Integration Design Document + +## Overview + +This document describes the design for integrating VNC remote desktop access into the Yao Sandbox system. This enables users to **observe Claude's operations in real-time** through a web-based VNC client, providing full transparency and building trust. + +The design provides **multiple sandbox image variants** with VNC support. Users can choose the appropriate image type when configuring their assistants based on their needs. + +## Goals + +1. **Transparency**: Let users see exactly what Claude is doing in the sandbox in real-time +2. **Multiple Image Options**: Provide different sandbox images for different use cases +3. **User Choice**: Allow users to select sandbox image type when building assistants +4. **Web-Based Access**: Use noVNC for browser-based VNC access (no client installation required) +5. **Unified Entry Point**: Single proxy endpoint to access any container's VNC session +6. **Security**: Proper authentication and isolation between users +7. **Minimal Core Changes**: Leverage existing sandbox infrastructure with minimal modifications + +## Non-Goals + +1. Persistent VNC sessions across container restarts +2. Multi-user access to the same VNC session +3. Audio support + +## Architecture + +### High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ User Browser │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Yao Web UI │ │ +│ │ │ │ +│ │ ┌─────────────────────┐ ┌─────────────────────────────────┐ │ │ +│ │ │ 💬 Chat Window │ │ 📺 VNC Preview (iframe) │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ User: Help me... │ │ Real-time view of Claude's │ │ │ +│ │ │ │ │ operations in sandbox │ │ │ +│ │ │ Claude: Working... │ │ │ │ │ +│ │ └─────────────────────┘ └─────────────────────────────────┘ │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + │ WebSocket (VNC) + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Yao Server (Host) │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ VNC Proxy Service │ │ +│ │ (sandbox/vncproxy) │ │ +│ │ │ │ +│ │ Endpoints: │ │ +│ │ ├── GET /v1/sandbox/{id}/vnc → VNC status │ │ +│ │ ├── GET /v1/sandbox/{id}/vnc/client → noVNC client │ │ +│ │ └── GET /v1/sandbox/{id}/vnc/ws → WebSocket │ │ +│ │ │ │ +│ │ Internal Flow: │ │ +│ │ 1. Authenticate request (JWT/session) │ │ +│ │ 2. Resolve container name: yao-sandbox-{id} │ │ +│ │ 3. Get container IP from Docker API │ │ +│ │ 4. Proxy WebSocket to container_ip:6080 │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ Docker Bridge Network │ +│ │ │ +│ ┌──────────────────────────────────────┼──────────────────────────────┐ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────────┐ ┌──────────────────────────┐ ┌──────────────────┐ │ +│ │ sandbox-claude │ │ sandbox-claude-playwright│ │ sandbox-claude- │ │ +│ │ (No VNC) │ │ (Browser + VNC) │ │ desktop (Full) │ │ +│ │ │ │ │ │ │ │ +│ │ • Claude CLI │ │ • Claude CLI │ │ • Claude CLI │ │ +│ │ • Node.js │ │ • Node.js │ │ • Node.js │ │ +│ │ • Python │ │ • Python │ │ • Python │ │ +│ │ │ │ • Playwright + Browsers │ │ • XFCE Desktop │ │ +│ │ │ │ • Xvfb + VNC │ │ • File Manager │ │ +│ │ │ │ • Fluxbox (minimal WM) │ │ • Terminal │ │ +│ │ │ │ │ │ • Xvfb + VNC │ │ +│ └──────────────────┘ └──────────────────────────┘ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Image Variants + +| Image | VNC | Use Case | Size | Memory | +|-------|-----|----------|------|--------| +| `sandbox-claude` | ❌ | Code execution, scripts, CLI tasks | ~700MB | 2GB | +| `sandbox-claude-playwright` | ✅ | Browser automation, web scraping | ~1.8GB | 4GB | +| `sandbox-claude-desktop` | ✅ | Full visibility, any GUI app | ~2.5GB | 4GB | + +### User Selection Flow + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Assistant Configuration UI │ +│ │ +│ Assistant Name: [My Web Scraper ] │ +│ │ +│ Sandbox Environment: │ +│ ┌─────────────────────────────────────────────────────────────────┐│ +│ │ ○ Standard (sandbox-claude) ││ +│ │ Code execution, no GUI. Lightweight and fast. ││ +│ │ ││ +│ │ ○ Browser (sandbox-claude-playwright) ⭐ ││ +│ │ Playwright browser automation with VNC preview. ││ +│ │ See browser operations in real-time. ││ +│ │ ││ +│ │ ● Desktop (sandbox-claude-desktop) ││ +│ │ Full Ubuntu desktop with VNC preview. ││ +│ │ See ALL operations: terminal, files, browser, etc. ││ +│ └─────────────────────────────────────────────────────────────────┘│ +│ │ +│ [ Save Assistant ] │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Components + +### 1. Docker Images + +**Location**: `sandbox/docker/` + +Three image variants sharing the same VNC infrastructure: + +``` +ubuntu:24.04 + └── sandbox-base:latest (~200MB) + └── sandbox-claude:latest (~700MB) # No VNC + ├── sandbox-claude-playwright:latest (~1.8GB) # VNC + Browser + └── sandbox-claude-desktop:latest (~2.5GB) # VNC + Full Desktop +``` + +#### 1.1 sandbox-claude-playwright (Browser + VNC) + +For browser automation tasks with real-time visibility. + +**Includes**: +- Everything from `sandbox-claude` +- Xvfb (virtual display) +- x11vnc + noVNC +- Fluxbox (minimal window manager) +- Playwright + Chromium/Firefox + +#### 1.2 sandbox-claude-desktop (Full Desktop + VNC) + +For maximum transparency - users can see everything Claude does. + +**Includes**: +- Everything from `sandbox-claude` +- Xvfb (virtual display) +- x11vnc + noVNC +- XFCE desktop environment +- Thunar file manager +- xfce4-terminal +- Playwright + browsers (optional) + +### 2. VNC Proxy Service + +**Location**: `sandbox/vncproxy/` + +A unified Go service that provides VNC access to all VNC-enabled containers. + +**Key Features**: +- Single entry point for all containers +- WebSocket proxy to container VNC +- Container IP resolution via Docker API +- Authentication and authorization +- Works with any VNC-enabled image + +**Key Interfaces**: + +```go +// VNCProxy handles VNC connections to sandbox containers +type VNCProxy struct { + docker *client.Client + manager *sandbox.Manager + config *Config +} + +// Config for VNC proxy +type Config struct { + // Container VNC port (fixed, internal) + ContainerVNCPort int // default: 5900 + + // Container noVNC/websockify port (fixed, internal) + ContainerNoVNCPort int // default: 6080 + + // Connection timeout + Timeout time.Duration +} + +// ServeHTTP handles HTTP requests +func (p *VNCProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) + +// GetVNCURL returns the VNC URL for a container +func (p *VNCProxy) GetVNCURL(sandboxID string) (string, error) + +// GetContainerIP returns the internal IP of a container +func (p *VNCProxy) GetContainerIP(containerName string) (string, error) +``` + +### 3. Manager Extensions (Optional) + +**Location**: `sandbox/manager.go` + +**Note**: These extensions are optional. The core sandbox functionality works without changes because: +- Image is specified in assistant config, passed to existing `GetOrCreate()` +- VNC status is determined by checking container env vars at runtime + +Optional helper types for convenience: + +```go +// ImageType represents the sandbox image variant (optional, for reference) +type ImageType string + +const ( + ImageTypeClaude ImageType = "claude" // No VNC + ImageTypePlaywright ImageType = "playwright" // Browser + VNC + ImageTypeDesktop ImageType = "desktop" // Full desktop + VNC +) + +// ImageConfig holds configuration for each image type (optional, for reference) +var ImageConfigs = map[ImageType]struct { + Image string + VNCEnabled bool + Memory string + CPU float64 +}{ + ImageTypeClaude: {"yaoapp/sandbox-claude:latest", false, "2g", 1.0}, + ImageTypePlaywright: {"yaoapp/sandbox-claude-playwright:latest", true, "4g", 2.0}, + ImageTypeDesktop: {"yaoapp/sandbox-claude-desktop:latest", true, "4g", 2.0}, +} +``` + +VNC access is determined at runtime by VNC Proxy checking container env vars - no Manager changes needed. + +### 4. API Endpoints + +All endpoints under `/v1/sandbox/`. Each sandbox has its own unique ID (generated by the caller/business layer). + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/v1/sandbox/{id}` | POST | Create container (with image) | +| `/v1/sandbox/{id}` | GET | Get container status | +| `/v1/sandbox/{id}` | DELETE | Stop/remove container | +| `/v1/sandbox/{id}/vnc` | GET | Get VNC access info | +| `/v1/sandbox/{id}/vnc/client` | GET | Serve noVNC HTML client (supports `?viewonly=true`) | +| `/v1/sandbox/{id}/vnc/ws` | GET | WebSocket proxy to container VNC | + +**Sandbox ID**: +- Generated by the caller (business layer) +- Format: any unique string (e.g., UUID, `{userID}-{chatID}`, `{assistantID}-{sessionID}`) +- Container name: `yao-sandbox-{id}` + +**Create Container Request**: + +```json +// POST /v1/sandbox/abc123-def456 +{ + "image": "yaoapp/sandbox-claude-desktop:latest" // Optional, defaults based on config +} +``` + +**VNC Access Response**: + +```json +// GET /v1/sandbox/abc123-def456/vnc +// VNC ready: +{ + "available": true, + "status": "ready", + "sandbox_id": "abc123-def456", + "container": "yao-sandbox-abc123-def456", + "client_url": "/v1/sandbox/abc123-def456/vnc/client", + "websocket_url": "/v1/sandbox/abc123-def456/vnc/ws" +} + +// VNC starting (container running but VNC services not ready yet): +{ + "available": false, + "status": "starting", + "sandbox_id": "abc123-def456", + "container": "yao-sandbox-abc123-def456", + "message": "VNC services are starting..." +} + +// VNC not supported (sandbox-claude image): +{ + "available": false, + "status": "not_supported", + "sandbox_id": "abc123-def456", + "container": "yao-sandbox-abc123-def456", + "message": "VNC not available for this container type" +} + +// Container not found/running: +{ + "available": false, + "status": "unavailable", + "sandbox_id": "abc123-def456", + "message": "Container not available" +} +``` + +**Full API Structure**: + +``` +/v1/sandbox/ +├── {id} +│ ├── POST # Create container +│ ├── GET # Get container status +│ ├── DELETE # Stop/remove container +│ ├── /exec # Execute command +│ ├── /files # File operations +│ └── /vnc # VNC access (if available) +│ ├── GET # VNC status & URLs +│ ├── /client # noVNC HTML client +│ └── /ws # WebSocket proxy +``` + +**Business Layer Integration Example**: + +```go +// Agent executor generates sandbox ID +sandboxID := fmt.Sprintf("%s-%s", userID, chatID) + +// Or use UUID for more isolation +sandboxID := uuid.New().String() + +// Or per-assistant session +sandboxID := fmt.Sprintf("%s-%s", assistantID, sessionID) +``` + +### 5. Assistant Configuration (Developer Side) + +Developers configure sandbox image type in the assistant's `package.yao` file: + +```yaml +# assistants/my-assistant/package.yao +name: My Web Assistant +description: Web scraping assistant with browser preview + +sandbox: + command: claude + image: "yaoapp/sandbox-claude-desktop:latest" # Choose image variant + max_memory: "4g" + max_cpu: 2.0 +``` + +**Available Images**: +- `yaoapp/sandbox-claude:latest` - No VNC, lightweight +- `yaoapp/sandbox-claude-playwright:latest` - Browser + VNC +- `yaoapp/sandbox-claude-desktop:latest` - Full desktop + VNC + +**Note**: No changes required to `agent/sandbox/` code. The existing `Image` field in `SandboxConfig` already supports custom images. + +### 6. CUI Integration (User Side) + +Users interact with VNC preview through CUI's action system. The VNC preview opens as a **sidebar iframe** via the `navigate` action. + +#### 6.1 Roles and Responsibilities + +| Role | Action | Interface | +|------|--------|-----------| +| **Developer** | Configure `sandbox.image` in `package.yao` | YAML config file | +| **User** | View VNC preview during chat | CUI chat interface | + +#### 6.2 No CUI Page Needed + +The CUI `navigate` action already supports loading any URL via iframe in the sidebar. The `/v1/sandbox/{id}/vnc/client` API returns a complete HTML page with noVNC, so we can use it directly. + +**Navigate action route types** (from `cui/packages/cui/chatbox/messages/Action/actions/navigate.ts`): +- `$dashboard/xxx` → CUI Dashboard pages +- `/xxx` → Loaded via iframe in sidebar +- `http(s)://xxx` → External URLs via iframe + +Since `/v1/sandbox/{id}/vnc/client` starts with `/`, it will be loaded in an iframe automatically. + +#### 6.3 Opening VNC Preview via Action + +When the sandbox starts and VNC is available, Claude can return a `navigate` action to open the preview: + +```json +{ + "type": "action", + "actions": [{ + "name": "navigate", + "payload": { + "route": "/v1/sandbox/abc123-def456/vnc/client", + "title": "实时预览", + "icon": "material-desktop_windows" + } + }] +} +``` + +Or as a clickable button in the chat: + +```json +{ + "type": "action", + "actions": [{ + "name": "button", + "payload": { + "text": "📺 查看实时预览", + "action": { + "name": "navigate", + "payload": { + "route": "/v1/sandbox/abc123-def456/vnc/client", + "title": "实时预览", + "icon": "material-desktop_windows" + } + } + } + }] +} +``` + +#### 6.4 User Experience Flow + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Step 1: User starts chat with sandbox-enabled assistant │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 💬 Chat │ │ +│ │ │ │ +│ │ User: 帮我爬取这个网站的数据 │ │ +│ │ │ │ +│ │ Claude: 好的，我正在启动浏览器环境... │ │ +│ │ [📺 查看实时预览] ← Action button │ │ +│ │ │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + │ User clicks button + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Step 2: VNC preview opens in sidebar (iframe loads /vnc/client API) │ +│ │ +│ ┌──────────────────────────┐ ┌────────────────────────────────────┐ │ +│ │ 💬 Chat │ │ 📺 实时预览 [×] │ │ +│ │ │ │ ┌────────────────────────────────┐│ │ +│ │ User: 帮我爬取... │ │ │ ││ │ +│ │ │ │ │ noVNC (from API response) ││ │ +│ │ Claude: 正在打开 │ │ │ ││ │ +│ │ 浏览器，访问目标网站... │ │ │ User can see Claude ││ │ +│ │ │ │ │ operating the browser ││ │ +│ │ [📺 查看实时预览] │ │ │ ││ │ +│ │ │ │ └────────────────────────────────┘│ │ +│ └──────────────────────────┘ └────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +#### 6.5 How Claude Knows to Show VNC Button + +The VNC preview button is triggered by the **Agent executor**, not by Claude itself. When the sandbox starts with a VNC-enabled image, the executor can inject a system message or action. + +**Option A: Agent Executor Injects Action** (Recommended) + +In `agent/sandbox/claude/executor.go`, when sandbox starts with VNC: + +```go +func (e *Executor) Stream(...) { + // After sandbox container is ready + if e.isVNCEnabled() { + // Send VNC preview action to frontend + handler(message.StreamEvent{ + Type: "action", + Data: map[string]interface{}{ + "actions": []map[string]interface{}{{ + "name": "button", + "payload": map[string]interface{}{ + "text": "📺 查看实时预览", + "action": map[string]interface{}{ + "name": "navigate", + "payload": map[string]interface{}{ + "route": fmt.Sprintf("/v1/sandbox/%s/vnc/client", sandboxID), + "title": "实时预览", + }, + }, + }, + }}, + }, + }) + } + // ... continue with Claude execution +} +``` + +**Option B: System Prompt Hint** + +Add to system prompt when VNC is enabled: +``` +当你在沙盒中执行可视化任务时（如浏览器操作），可以告知用户点击"查看实时预览"按钮观看操作过程。 +``` + +#### 6.6 VNC Interaction Modes + +The VNC preview supports two modes controlled by the `viewonly` query parameter: + +| Mode | URL | Description | +|------|-----|-------------| +| **Interactive** (default) | `/vnc/client` | User can use keyboard and mouse | +| **View-only** | `/vnc/client?viewonly=true` | User can only watch | + +**Use Cases**: + +| Scenario | Mode | Example | +|----------|------|---------| +| Watch Claude browse web | View-only | `?viewonly=true` | +| User needs to login | Interactive | (default) | +| User needs to solve CAPTCHA | Interactive | (default) | +| Sensitive operation | View-only | `?viewonly=true` | + +**Action Examples**: + +```json +// View-only mode (just watching) +{ + "name": "navigate", + "payload": { + "route": "/v1/sandbox/abc123/vnc/client?viewonly=true", + "title": "实时预览" + } +} + +// Interactive mode (user needs to login) +{ + "name": "navigate", + "payload": { + "route": "/v1/sandbox/abc123/vnc/client", + "title": "请在此登录" + } +} +``` + +**How Claude Waits for User Input**: + +When user interaction is needed (e.g., login), Claude can: + +1. **Wait for user confirmation** (simple): + ``` + Claude: 请在 VNC 窗口中登录，完成后告诉我 + User: 登录好了 + Claude: 好的，继续执行... + ``` + +2. **Auto-detect via script** (advanced): + ```python + # Wait for login success indicator + page.wait_for_selector("#user-avatar", timeout=300000) # 5 min timeout + print("Login detected, continuing...") + ``` + +#### 6.7 CUI Changes + +**No CUI changes required.** The existing `navigate` action + `app/openSidebar` event already handles loading the VNC client API response in an iframe. + +## Implementation Details + +### Dockerfile.playwright + +```dockerfile +ARG REGISTRY=yaoapp +FROM ${REGISTRY}/sandbox-claude:latest + +USER root + +# Install X11, VNC, and minimal window manager +RUN apt-get update && apt-get install -y --no-install-recommends \ + xvfb \ + x11vnc \ + fluxbox \ + novnc \ + python3-websockify \ + fonts-liberation \ + fonts-noto-cjk \ + x11-utils \ + xdotool \ + && rm -rf /var/lib/apt/lists/* + +# Install Playwright system dependencies (requires root) +RUN npx playwright install-deps chromium firefox + +# Install Playwright and browsers as sandbox user +USER sandbox +RUN npm install -g playwright && \ + npx playwright install chromium firefox + +USER root + +# VNC startup script +COPY start-vnc.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start-vnc.sh + +# Update entrypoint to start VNC (includes original claude entrypoint logic) +COPY entrypoint-vnc.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Environment +ENV DISPLAY=:99 +ENV VNC_PORT=5900 +ENV NOVNC_PORT=6080 +ENV RESOLUTION=1920x1080x24 +ENV SANDBOX_VNC_ENABLED=true + +EXPOSE 5900 6080 + +USER sandbox +WORKDIR /workspace + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["sleep", "infinity"] +``` + +### Dockerfile.desktop + +```dockerfile +ARG REGISTRY=yaoapp +FROM ${REGISTRY}/sandbox-claude:latest + +USER root + +# Install X11, VNC, and XFCE desktop +RUN apt-get update && apt-get install -y --no-install-recommends \ + xvfb \ + x11vnc \ + novnc \ + python3-websockify \ + # XFCE Desktop + xfce4 \ + xfce4-terminal \ + thunar \ + # Fonts + fonts-liberation \ + fonts-noto-cjk \ + # Utilities + x11-utils \ + xdotool \ + && apt-get remove -y xfce4-screensaver xscreensaver || true \ + && rm -rf /var/lib/apt/lists/* + +# Optional: Install Playwright system dependencies (requires root) +RUN npx playwright install-deps chromium || true + +# Optional: Install Playwright for browser automation +USER sandbox +RUN npm install -g playwright && \ + npx playwright install chromium || true + +USER root + +# VNC startup script +COPY start-vnc.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start-vnc.sh + +# Update entrypoint (includes original claude entrypoint logic) +COPY entrypoint-vnc.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Environment +ENV DISPLAY=:99 +ENV VNC_PORT=5900 +ENV NOVNC_PORT=6080 +ENV RESOLUTION=1920x1080x24 +ENV SANDBOX_VNC_ENABLED=true +ENV SANDBOX_DESKTOP=xfce + +EXPOSE 5900 6080 + +USER sandbox +WORKDIR /workspace + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["sleep", "infinity"] +``` + +### start-vnc.sh (Shared) + +```bash +#!/bin/bash +set -e + +DISPLAY_NUM="${DISPLAY_NUM:-99}" +RESOLUTION="${RESOLUTION:-1920x1080x24}" +VNC_PORT="${VNC_PORT:-5900}" +NOVNC_PORT="${NOVNC_PORT:-6080}" +VNC_PASSWORD="${VNC_PASSWORD:-}" +DESKTOP="${SANDBOX_DESKTOP:-fluxbox}" + +export DISPLAY=:${DISPLAY_NUM} + +# Start Xvfb (virtual framebuffer) +echo "Starting Xvfb on display :${DISPLAY_NUM}..." +Xvfb :${DISPLAY_NUM} -screen 0 ${RESOLUTION} & +XVFB_PID=$! +sleep 1 + +if ! kill -0 $XVFB_PID 2>/dev/null; then + echo "ERROR: Xvfb failed to start" + exit 1 +fi + +# Start window manager / desktop +echo "Starting ${DESKTOP}..." +case "$DESKTOP" in + xfce|xfce4) + startxfce4 & + ;; + *) + fluxbox & + ;; +esac + +# Start VNC server +echo "Starting x11vnc on port ${VNC_PORT}..." +VNC_ARGS="-display :${DISPLAY_NUM} -forever -shared -rfbport ${VNC_PORT} -noxdamage" +if [ -n "$VNC_PASSWORD" ]; then + mkdir -p ~/.vnc + x11vnc -storepasswd "$VNC_PASSWORD" ~/.vnc/passwd + VNC_ARGS="$VNC_ARGS -rfbauth ~/.vnc/passwd" +else + VNC_ARGS="$VNC_ARGS -nopw" +fi +x11vnc $VNC_ARGS & + +# Start noVNC (websockify) +echo "Starting noVNC on port ${NOVNC_PORT}..." +websockify --web=/usr/share/novnc/ ${NOVNC_PORT} localhost:${VNC_PORT} & + +echo "VNC services started successfully" +echo " - Desktop: ${DESKTOP}" +echo " - VNC port: ${VNC_PORT}" +echo " - noVNC port: ${NOVNC_PORT}" + +# Note: Don't wait here - let the entrypoint continue +# Background processes will keep running +``` + +### entrypoint-vnc.sh + +```bash +#!/bin/bash +# Container entrypoint for VNC-enabled images +# This extends the original sandbox-claude entrypoint with VNC support + +# ============================================ +# VNC Services Startup +# ============================================ +if [ "$SANDBOX_VNC_ENABLED" = "true" ]; then + echo "Starting VNC services..." + /usr/local/bin/start-vnc.sh & + sleep 2 +fi + +# ============================================ +# Original sandbox-claude entrypoint logic +# (copied from sandbox-claude Dockerfile) +# ============================================ +WORKSPACE="${WORKSPACE:-/workspace}" +PORT="${CLAUDE_PROXY_PORT:-3456}" +ENV_FILE="/tmp/claude-proxy-env" + +# If proxy env vars are set AND proxy is not running, start it +# This supports docker run -e CLAUDE_PROXY_BACKEND=... usage +if [ -n "$CLAUDE_PROXY_BACKEND" ] && [ -n "$CLAUDE_PROXY_API_KEY" ] && [ -n "$CLAUDE_PROXY_MODEL" ]; then + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + /usr/local/bin/start-claude-proxy + fi + + # Write env vars to a file that can be sourced + if curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + echo "export ANTHROPIC_BASE_URL=http://127.0.0.1:${PORT}" > "$ENV_FILE" + echo "export ANTHROPIC_API_KEY=dummy" >> "$ENV_FILE" + chmod 644 "$ENV_FILE" + fi +fi + +# Execute the command passed to docker run +exec "$@" +``` + +### VNC Proxy Implementation + +```go +// sandbox/vncproxy/proxy.go +package vncproxy + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/docker/docker/client" + "github.com/gorilla/websocket" +) + +type Proxy struct { + docker *client.Client + config *Config + + // IP cache with TTL support + ipCache map[string]ipCacheEntry + ipCacheMu sync.RWMutex +} + +type Config struct { + ContainerVNCPort int // default: 5900 + ContainerNoVNCPort int // default: 6080 + Timeout time.Duration // default: 30s +} + +func New(docker *client.Client, config *Config) *Proxy { + if config.ContainerVNCPort == 0 { + config.ContainerVNCPort = 5900 + } + if config.ContainerNoVNCPort == 0 { + config.ContainerNoVNCPort = 6080 + } + if config.Timeout == 0 { + config.Timeout = 30 * time.Second + } + + return &Proxy{ + docker: docker, + config: config, + ipCache: make(map[string]ipCacheEntry), + } +} + +// HandleVNCStatus returns VNC status for a container +// GET /v1/sandbox/{id}/vnc +func (p *Proxy) HandleVNCStatus(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := fmt.Sprintf("yao-sandbox-%s", sandboxID) + + response := map[string]interface{}{ + "sandbox_id": sandboxID, + "container": containerName, + } + + // Check if container exists and is running + ip, err := p.getContainerIP(r.Context(), containerName) + if err != nil { + response["available"] = false + response["status"] = "unavailable" + response["message"] = "Container not available" + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // Check if VNC is enabled for this container + if !p.checkVNCEnabled(r.Context(), containerName) { + response["available"] = false + response["status"] = "not_supported" + response["message"] = "VNC not available for this container type" + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // Check if VNC services are ready (try to connect to websockify port) + if !p.checkVNCReady(r.Context(), ip) { + response["available"] = false + response["status"] = "starting" + response["message"] = "VNC services are starting..." + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // VNC is ready + response["available"] = true + response["status"] = "ready" + response["client_url"] = fmt.Sprintf("/v1/sandbox/%s/vnc/client", sandboxID) + response["websocket_url"] = fmt.Sprintf("/v1/sandbox/%s/vnc/ws", sandboxID) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +// checkVNCReady tests if VNC services are ready by attempting TCP connection +func (p *Proxy) checkVNCReady(ctx context.Context, containerIP string) bool { + addr := fmt.Sprintf("%s:%d", containerIP, p.config.ContainerNoVNCPort) + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + return false + } + conn.Close() + return true +} + +// HandleVNCClient serves the noVNC client page +// GET /v1/sandbox/{id}/vnc/client?viewonly=true|false +func (p *Proxy) HandleVNCClient(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := fmt.Sprintf("yao-sandbox-%s", sandboxID) + + // Verify container exists, is running, and has VNC + _, err := p.getContainerIP(r.Context(), containerName) + if err != nil { + http.Error(w, "Container not available", http.StatusNotFound) + return + } + + if !p.checkVNCEnabled(r.Context(), containerName) { + http.Error(w, "VNC not available for this container", http.StatusBadRequest) + return + } + + // Get viewonly parameter (default: false = interactive) + viewOnly := r.URL.Query().Get("viewonly") == "true" + + // Serve inline noVNC HTML page with status checking + // This embeds the noVNC client directly, with retry logic for VNC startup delay + wsURL := fmt.Sprintf("/v1/sandbox/%s/vnc/ws", sandboxID) + p.serveNoVNCPage(w, sandboxID, wsURL, viewOnly) +} + +// serveNoVNCPage serves an inline HTML page that loads noVNC +// Includes status checking and retry logic for VNC startup delay +// viewOnly: if true, user can only watch; if false, user can interact with keyboard/mouse +func (p *Proxy) serveNoVNCPage(w http.ResponseWriter, sandboxID string, wsPath string, viewOnly bool) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + viewOnlyJS := "false" + if viewOnly { + viewOnlyJS = "true" + } + + html := fmt.Sprintf(` + + + Sandbox Preview + + + +
+ +
正在连接 VNC 服务...
+
+
+ + + + +`, sandboxID, wsPath, viewOnlyJS) + w.Write([]byte(html)) +} + +// HandleVNCWebSocket proxies WebSocket to container VNC +// GET /v1/sandbox/{id}/vnc/ws +func (p *Proxy) HandleVNCWebSocket(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := fmt.Sprintf("yao-sandbox-%s", sandboxID) + + ip, err := p.getContainerIP(r.Context(), containerName) + if err != nil { + http.Error(w, "Container not available", http.StatusNotFound) + return + } + + if !p.checkVNCEnabled(r.Context(), containerName) { + http.Error(w, "VNC not available for this container", http.StatusBadRequest) + return + } + + // Proxy WebSocket to container's websockify port + targetURL := fmt.Sprintf("ws://%s:%d", ip, p.config.ContainerNoVNCPort) + p.proxyWebSocket(w, r, targetURL) +} + +func (p *Proxy) checkVNCEnabled(ctx context.Context, containerName string) bool { + inspect, err := p.docker.ContainerInspect(ctx, containerName) + if err != nil { + return false + } + + // Check environment variable SANDBOX_VNC_ENABLED + for _, env := range inspect.Config.Env { + if env == "SANDBOX_VNC_ENABLED=true" { + return true + } + } + return false +} + +// ipCacheEntry holds cached IP with expiration +type ipCacheEntry struct { + IP string + ExpiresAt time.Time +} + +func (p *Proxy) getContainerIP(ctx context.Context, containerName string) (string, error) { + // Check cache first (with TTL) + p.ipCacheMu.RLock() + if entry, ok := p.ipCache[containerName]; ok { + if time.Now().Before(entry.ExpiresAt) { + p.ipCacheMu.RUnlock() + return entry.IP, nil + } + } + p.ipCacheMu.RUnlock() + + // Cache miss or expired, fetch from Docker + inspect, err := p.docker.ContainerInspect(ctx, containerName) + if err != nil { + // Remove stale cache entry + p.ipCacheMu.Lock() + delete(p.ipCache, containerName) + p.ipCacheMu.Unlock() + return "", fmt.Errorf("container not found: %w", err) + } + + if !inspect.State.Running { + // Remove stale cache entry + p.ipCacheMu.Lock() + delete(p.ipCache, containerName) + p.ipCacheMu.Unlock() + return "", fmt.Errorf("container not running") + } + + ip := inspect.NetworkSettings.IPAddress + if ip == "" { + if networks := inspect.NetworkSettings.Networks; networks != nil { + if bridge, ok := networks["bridge"]; ok { + ip = bridge.IPAddress + } + } + } + + if ip == "" { + return "", fmt.Errorf("container has no IP address") + } + + // Cache with 30 second TTL + p.ipCacheMu.Lock() + p.ipCache[containerName] = ipCacheEntry{ + IP: ip, + ExpiresAt: time.Now().Add(30 * time.Second), + } + p.ipCacheMu.Unlock() + + return ip, nil +} + +// InvalidateCache removes a container from the IP cache +// Call this when container state changes (stop/restart) +func (p *Proxy) InvalidateCache(containerName string) { + p.ipCacheMu.Lock() + delete(p.ipCache, containerName) + p.ipCacheMu.Unlock() +} + +func (p *Proxy) proxyWebSocket(w http.ResponseWriter, r *http.Request, targetURL string) { + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, + Subprotocols: []string{"binary"}, // Required for noVNC + } + + clientConn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer clientConn.Close() + + dialer := websocket.Dialer{ + HandshakeTimeout: p.config.Timeout, + } + + targetConn, _, err := dialer.Dial(targetURL, nil) + if err != nil { + return + } + defer targetConn.Close() + + errChan := make(chan error, 2) + + // Client -> Target + go func() { + for { + msgType, data, err := clientConn.ReadMessage() + if err != nil { + errChan <- err + return + } + if err := targetConn.WriteMessage(msgType, data); err != nil { + errChan <- err + return + } + } + }() + + // Target -> Client + go func() { + for { + msgType, data, err := targetConn.ReadMessage() + if err != nil { + errChan <- err + return + } + if err := clientConn.WriteMessage(msgType, data); err != nil { + errChan <- err + return + } + } + }() + + <-errChan +} + +func extractSandboxID(r *http.Request) string { + // Extract from path: /v1/sandbox/{id}/vnc/... + path := r.URL.Path + path = strings.TrimPrefix(path, "/v1/sandbox/") + parts := strings.Split(path, "/") + if len(parts) >= 1 { + return parts[0] + } + return "" +} +``` + +## Security Considerations + +### 1. Authentication + +All VNC endpoints verify user authentication: + +```go +func (p *Proxy) authMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + + // Verify the requesting user owns this sandbox + // Implementation depends on how sandboxID maps to users: + // - If sandboxID = "{userID}-{chatID}", extract userID and compare with session + // - If sandboxID = UUID, lookup in database + // - Delegate to business layer authorization service + + // Example: extract userID from sandboxID pattern "{userID}-{chatID}" + // parts := strings.SplitN(sandboxID, "-", 2) + // if len(parts) >= 1 { + // ownerID := parts[0] + // sessionUserID := getSessionUserID(r) + // if sessionUserID != ownerID { + // http.Error(w, "Unauthorized", http.StatusUnauthorized) + // return + // } + // } + + // TODO: Implement authorization logic based on your sandboxID scheme + + next.ServeHTTP(w, r) + }) +} +``` + +### 2. Network Isolation + +- Containers use Docker bridge network (internal only) +- No VNC ports exposed to host +- All access through authenticated proxy +- Each user can only access their own containers + +### 3. Resource Limits by Image Type + +| Image Type | Memory | CPU | Disk | +|------------|--------|-----|------| +| claude | 2GB | 1.0 | - | +| playwright | 4GB | 2.0 | - | +| desktop | 4GB | 2.0 | - | + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `YAO_SANDBOX_IMAGE` | `yaoapp/sandbox-claude:latest` | Default sandbox image | +| `YAO_SANDBOX_VNC_PORT_MAPPING` | `false` | Enable VNC port mapping to host (for Docker Desktop) | +| `YAO_VNC_PROXY_ENABLED` | `true` | Enable VNC proxy | +| `YAO_VNC_RESOLUTION` | `1920x1080x24` | VNC screen resolution | + +### Docker Desktop Support (macOS/Windows) + +Docker Desktop runs containers inside a LinuxKit VM, so container IPs (`172.17.0.x`) are not directly accessible from the host. To enable VNC access on Docker Desktop: + +```bash +# Enable VNC port mapping for local development +export YAO_SANDBOX_VNC_PORT_MAPPING=true +export YAO_SANDBOX_IMAGE="yaoapp/sandbox-claude-playwright:latest" +``` + +When `YAO_SANDBOX_VNC_PORT_MAPPING=true`: +- Container ports `6080/tcp` (noVNC) and `5900/tcp` (VNC) are mapped to random available host ports +- Ports are bound to `127.0.0.1` for security +- VNC Proxy automatically detects and uses the mapped host ports + +On Linux (native Docker), this option is not needed as container IPs are directly accessible. + +## Implementation Checklist + +### Yao Backend ✅ 完成 + +- [x] `sandbox/docker/playwright/Dockerfile` - Playwright + VNC image +- [x] `sandbox/docker/desktop/Dockerfile` - Full desktop + VNC image +- [x] `sandbox/docker/vnc/start-vnc.sh` - Shared VNC startup script +- [x] `sandbox/docker/vnc/entrypoint-vnc.sh` - VNC entrypoint +- [x] `sandbox/vncproxy/proxy.go` - VNC WebSocket proxy +- [x] `sandbox/vncproxy/config.go` - Proxy configuration +- [x] API router integration - VNC endpoints (`openapi/sandbox/sandbox.go`) +- [x] `sandbox/docker/build.sh` - Update build script +- [x] `sandbox/config.go` - VNC port mapping configuration +- [x] `sandbox/manager.go` - Dynamic VNC port mapping for Docker Desktop + +### No Changes Needed + +- `agent/sandbox/` - existing `Image` field already supports custom images +- `cui/` - existing `navigate` action handles iframe loading via `app/openSidebar` + +## File Structure + +### Yao (Backend) + +``` +yao/sandbox/ +├── vncproxy/ # VNC Proxy Service +│ ├── proxy.go # Main proxy implementation (with port mapping detection) +│ ├── proxy_test.go # Unit tests +│ └── config.go # Configuration +├── docker/ +│ ├── base/ +│ │ └── Dockerfile.base +│ ├── claude/ +│ │ ├── Dockerfile +│ │ └── Dockerfile.full +│ ├── playwright/ # Playwright + VNC image +│ │ └── Dockerfile +│ ├── desktop/ # XFCE Desktop + VNC image +│ │ └── Dockerfile +│ ├── vnc/ # Shared VNC scripts +│ │ ├── start-vnc.sh +│ │ └── entrypoint-vnc.sh +│ └── build.sh # Build script for all images +├── manager.go # Container management (with VNC port mapping) +├── config.go # Configuration (VNCPortMapping option) +├── DESIGN-PLAYWRIGHT-VNC.md # This document +├── TODO-VNC.md # Implementation checklist +└── README.md # Quick start guide +``` + +### CUI (Frontend) + +``` +cui/packages/cui/ +└── ... # No changes needed +``` + +The CUI `navigate` action already supports loading URLs via iframe in sidebar. The `/v1/sandbox/{id}/vnc/client` API returns a complete HTML page that will be loaded directly. + +### Agent (No Changes) + +``` +yao/agent/ +├── sandbox/ +│ ├── types.go # Already supports custom Image +│ └── ... # No changes needed +└── ... +``` + +## Command Execution + +### Overview + +Commands execute identically across all sandbox images. The `Manager.Exec()` and `Manager.Stream()` methods remain unchanged. + +### No Manager Changes Required + +```go +// Manager.Exec() and Manager.Stream() remain unchanged +// Commands run the same way on all images +// DISPLAY=:99 is set in container env, GUI apps (browsers) use it automatically +``` + +### Behavior by Image Type + +| Image | DISPLAY | VNC Visible | Agent Gets Output | +|-------|---------|-------------|-------------------| +| sandbox-claude | ❌ | N/A | ✅ | +| sandbox-claude-playwright | ✅ :99 | Browser window | ✅ | +| sandbox-claude-desktop | ✅ :99 | Browser + Desktop apps | ✅ | + +### What Users See in VNC + +| Operation | sandbox-claude-playwright | sandbox-claude-desktop | +|-----------|--------------------------|------------------------| +| Browser automation | ✅ Visible | ✅ Visible | +| File operations | ❌ | ✅ (open Thunar) | +| Terminal commands | ❌ | ❌ (output to Agent) | + +**Note**: Terminal command output goes to Agent, not to VNC terminal window. This is by design - `docker exec` runs commands directly in the container, not through a terminal emulator. Users can manually open a terminal in VNC if they want to run commands interactively. + +### Why This Design + +1. **100% backward compatible**: No changes to Manager.go +2. **Agent output intact**: stdout/stderr captured normally +3. **Browser visible**: Main use case (Playwright) works perfectly +4. **Low risk**: No code changes = no bugs +5. **Future improvement**: Terminal visibility can be added later if needed + +--- + +## Appendix + +### A. Image Comparison + +| Feature | sandbox-claude | sandbox-claude-playwright | sandbox-claude-desktop | +|---------|---------------|--------------------------|------------------------| +| Claude CLI | ✅ | ✅ | ✅ | +| Node.js | ✅ | ✅ | ✅ | +| Python | ✅ | ✅ | ✅ | +| VNC Access | ❌ | ✅ | ✅ | +| Playwright | ❌ | ✅ | ✅ (optional) | +| File Manager | ❌ | ❌ | ✅ | +| Terminal GUI | ❌ | ❌ | ✅ | +| Desktop | ❌ | Minimal (Fluxbox) | Full (XFCE) | +| Image Size | ~700MB | ~1.8GB | ~2.5GB | +| Memory | 2GB | 4GB | 4GB | +| **Best For** | Scripts, CLI | Browser automation | Full transparency | + +### B. User Visibility & Interaction + +What users can see and do in VNC: + +| Operation | sandbox-claude-playwright | sandbox-claude-desktop | +|-----------|--------------------------|------------------------| +| Browser navigation | ✅ See | ✅ See | +| Browser clicks/typing | ✅ See | ✅ See | +| File creation | ❌ (log only) | ✅ (file manager) | +| Command execution | ❌ (output to Agent) | ❌ (output to Agent) | +| Code editing | ❌ | ✅ (if editor installed) | +| **Trust Level** | Medium | High | + +**User Interaction Modes**: + +| Mode | URL Parameter | User Can | +|------|---------------|----------| +| View-only | `?viewonly=true` | Watch only | +| Interactive | (default) | Keyboard, mouse, typing | + +**Typical Interactive Scenarios**: +- User login (accounts, passwords) +- CAPTCHA solving +- Two-factor authentication +- Manual form filling + +**Note**: Command output goes to Agent (via docker exec), not to a visible terminal in VNC. Users can manually open a terminal in `sandbox-claude-desktop` if needed. + +### C. Implementation Summary + +| Component | Location | Changes | +|-----------|----------|---------| +| Docker Images | `sandbox/docker/playwright/`, `sandbox/docker/desktop/` | NEW | +| VNC Proxy | `sandbox/vncproxy/` | NEW | +| VNC API | Yao router | NEW endpoints | +| CUI | `cui/` | **No changes** (navigate action + iframe) | +| Sandbox Manager | `sandbox/manager.go` | **No changes** | +| Agent Sandbox | `agent/sandbox/` | **No changes** | + +### D. References + +- [Playwright Docker Documentation](https://playwright.dev/docs/docker) +- [noVNC GitHub](https://github.com/novnc/noVNC) +- [XFCE Documentation](https://docs.xfce.org/) +- [CUI Action System](../../cui/packages/cui/chatbox/messages/Action/actions/navigate.ts) +- [Yao Sandbox README](./README.md) +- [Yao Sandbox DESIGN](./DESIGN.md) diff --git a/sandbox/README.md b/sandbox/README.md index b2a7ca05..cd51ed35 100644 --- a/sandbox/README.md +++ b/sandbox/README.md @@ -10,6 +10,7 @@ The sandbox module enables Yao to safely run external AI coding agents (like Cla - IPC communication via Unix sockets - Resource limits (CPU, memory) - Security isolation +- **VNC remote desktop** for visual transparency (optional) ## Architecture @@ -26,11 +27,20 @@ The sandbox module enables Yao to safely run external AI coding agents (like Cla │ │ │ │ │ └────────────────────────┬────────────────────────────────┘ │ │ │ │ +│ ┌────────────────────────┴────────────────────────────────┐ │ +│ │ VNC Proxy Service │ │ +│ │ │ │ +│ │ - GET /v1/sandbox/{id}/vnc → VNC status │ │ +│ │ - GET /v1/sandbox/{id}/vnc/client → noVNC page │ │ +│ │ - GET /v1/sandbox/{id}/vnc/ws → WebSocket proxy │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ │ │ ┌───────────────┼───────────────┐ │ │ ▼ ▼ ▼ │ │ ┌────────────┐ ┌────────────┐ ┌────────────┐ │ -│ │ Container │ │ Container │ │ Container │ │ -│ │ (user1) │ │ (user2) │ │ (user3) │ │ +│ │ sandbox- │ │ sandbox- │ │ sandbox- │ │ +│ │ claude │ │ playwright │ │ desktop │ │ +│ │ (No VNC) │ │ (VNC) │ │ (VNC) │ │ │ └─────┬──────┘ └─────┬──────┘ └─────┬──────┘ │ │ │ │ │ │ │ ──────┴───────────────┴───────────────┴──── │ @@ -45,7 +55,16 @@ The sandbox module enables Yao to safely run external AI coding agents (like Cla ```bash cd sandbox/docker + +# Build base image ./build.sh claude + +# Build VNC-enabled images +./build.sh playwright # Playwright + Fluxbox + VNC +./build.sh desktop # XFCE Desktop + VNC + +# Build all images +./build.sh all ``` ### Usage @@ -84,23 +103,37 @@ data, err := manager.ReadFile(ctx, container.Name, "/workspace/test.txt") ### Environment Variables -| Variable | Default | Description | -| -------------------------- | ----------------------------------- | ------------------------- | -| `YAO_SANDBOX_IMAGE` | `yao/sandbox-claude:latest` | Docker image | -| `YAO_SANDBOX_WORKSPACE` | `{YAO_DATA_ROOT}/sandbox/workspace` | Workspace directory | -| `YAO_SANDBOX_IPC` | `{YAO_DATA_ROOT}/sandbox/ipc` | IPC socket directory | -| `YAO_SANDBOX_MAX` | `100` | Max concurrent containers | -| `YAO_SANDBOX_IDLE_TIMEOUT` | `30m` | Idle timeout | -| `YAO_SANDBOX_MEMORY` | `2g` | Memory limit | -| `YAO_SANDBOX_CPU` | `1.0` | CPU limit | +| Variable | Default | Description | +| ------------------------------ | ----------------------------------- | ---------------------------------------------- | +| `YAO_SANDBOX_IMAGE` | `yao/sandbox-claude:latest` | Docker image | +| `YAO_SANDBOX_WORKSPACE` | `{YAO_DATA_ROOT}/sandbox/workspace` | Workspace directory | +| `YAO_SANDBOX_IPC` | `{YAO_DATA_ROOT}/sandbox/ipc` | IPC socket directory | +| `YAO_SANDBOX_MAX` | `100` | Max concurrent containers | +| `YAO_SANDBOX_IDLE_TIMEOUT` | `30m` | Idle timeout | +| `YAO_SANDBOX_MEMORY` | `2g` | Memory limit | +| `YAO_SANDBOX_CPU` | `1.0` | CPU limit | +| `YAO_SANDBOX_VNC_PORT_MAPPING` | `false` | Enable VNC port mapping (for Docker Desktop) | + +### Docker Desktop (macOS/Windows) + +Docker Desktop runs containers in a LinuxKit VM, so container IPs are not directly accessible from the host. Enable VNC port mapping for local development: + +```bash +export YAO_SANDBOX_VNC_PORT_MAPPING=true +export YAO_SANDBOX_IMAGE="yaoapp/sandbox-claude-playwright:latest" +``` + +When enabled, VNC ports (6080, 5900) are automatically mapped to random available host ports on `127.0.0.1`. ## Docker Images -| Image | Description | -| --------------------------- | ------------------------------------- | -| `yao/sandbox-base:latest` | Base image with git, curl, yao-bridge | -| `yao/sandbox-claude:latest` | + Claude CLI, Node.js 20, Python 3.11 | -| `yao/sandbox-claude:full` | + Go 1.23 | +| Image | VNC | Description | +| ------------------------------------------ | --- | ------------------------------------- | +| `yaoapp/sandbox-base:latest` | ❌ | Base image with git, curl, yao-bridge | +| `yaoapp/sandbox-claude:latest` | ❌ | + Claude CLI, Node.js 20, Python 3.11 | +| `yaoapp/sandbox-claude:full` | ❌ | + Go 1.23 | +| `yaoapp/sandbox-claude-playwright:latest` | ✅ | + Playwright, Fluxbox, VNC (~3.4GB) | +| `yaoapp/sandbox-claude-desktop:latest` | ✅ | + XFCE Desktop, VNC (~3.1GB) | ## IPC Communication @@ -112,6 +145,25 @@ Supported methods: - `tools/list` - List available tools - `tools/call` - Execute a tool +## VNC Remote Desktop + +VNC-enabled images (playwright, desktop) provide real-time visibility into Claude's operations. + +### API Endpoints + +| Endpoint | Description | +| ------------------------------- | ---------------------------------- | +| `GET /v1/sandbox/{id}/vnc` | VNC status (ready/starting/unavailable) | +| `GET /v1/sandbox/{id}/vnc/client` | noVNC HTML client page | +| `GET /v1/sandbox/{id}/vnc/ws` | WebSocket proxy to container VNC | + +### View Modes + +- **Interactive** (default): User can use keyboard and mouse +- **View-only** (`?viewonly=true`): User can only watch + +For detailed design, see [DESIGN-PLAYWRIGHT-VNC.md](./DESIGN-PLAYWRIGHT-VNC.md). + ## Directory Structure ``` @@ -120,11 +172,18 @@ sandbox/ ├── docker/ # Dockerfiles and build script │ ├── base/ │ ├── claude/ +│ ├── playwright/ # Playwright + VNC image +│ ├── desktop/ # XFCE Desktop + VNC image +│ ├── vnc/ # Shared VNC scripts │ └── build.sh ├── ipc/ # IPC system │ ├── manager.go │ ├── session.go │ └── types.go +├── vncproxy/ # VNC proxy service +│ ├── proxy.go +│ ├── config.go +│ └── proxy_test.go ├── config.go # Configuration ├── errors.go # Error types ├── helpers.go # Helper functions @@ -135,11 +194,17 @@ sandbox/ ## Testing ```bash +# Load environment variables first +source env.local.sh + # Unit tests (no Docker required) go test -v ./sandbox/... -run "^Test.*Validation|^Test.*Generation|^Test.*Parsing" # All tests (requires Docker) go test -v ./sandbox/... + +# VNC proxy tests only +go test -v ./sandbox/vncproxy/... ``` ## Security diff --git a/sandbox/config.go b/sandbox/config.go index f6335c5e..ccb653e9 100644 --- a/sandbox/config.go +++ b/sandbox/config.go @@ -21,6 +21,9 @@ type Config struct { ContainerWorkDir string `json:"container_workdir,omitempty"` // Container working directory, default: /workspace ContainerIPCSocket string `json:"container_ipc_socket,omitempty"` // Container IPC socket path, default: /tmp/yao.sock ContainerUser string `json:"container_user,omitempty"` // Container user, default: "" (use image default). Set to "0" for root. + + // VNC port mapping (for Docker Desktop on macOS/Windows where container IPs are not directly accessible) + VNCPortMapping bool `json:"vnc_port_mapping,omitempty"` // Enable VNC port mapping to host, default: false } // DefaultConfig returns a Config with default values @@ -116,4 +119,9 @@ func (c *Config) Init(dataRoot string) { if env := os.Getenv("YAO_SANDBOX_CONTAINER_USER"); env != "" { c.ContainerUser = env } + + // VNC port mapping (for Docker Desktop on macOS/Windows) + if env := os.Getenv("YAO_SANDBOX_VNC_PORT_MAPPING"); env != "" { + c.VNCPortMapping = env == "true" || env == "1" || env == "yes" + } } diff --git a/sandbox/docker/build.sh b/sandbox/docker/build.sh index af99e3bd..c1a126de 100755 --- a/sandbox/docker/build.sh +++ b/sandbox/docker/build.sh @@ -105,6 +105,22 @@ case $TOOL in build_multiarch "sandbox-claude" "claude/Dockerfile" "$PUSH" build_multiarch "sandbox-claude-full" "claude/Dockerfile.full" "$PUSH" ;; + claude-vnc) + echo "" + echo "=== Building Claude VNC images (Playwright + Desktop) ===" + build_multiarch "sandbox-claude-playwright" "playwright/Dockerfile" "$PUSH" + build_multiarch "sandbox-claude-desktop" "desktop/Dockerfile" "$PUSH" + ;; + playwright) + echo "" + echo "=== Building Claude Playwright image ===" + build_multiarch "sandbox-claude-playwright" "playwright/Dockerfile" "$PUSH" + ;; + desktop) + echo "" + echo "=== Building Claude Desktop image ===" + build_multiarch "sandbox-claude-desktop" "desktop/Dockerfile" "$PUSH" + ;; cursor) echo "" echo "=== Building Cursor images ===" @@ -116,14 +132,20 @@ case $TOOL in # Claude build_multiarch "sandbox-claude" "claude/Dockerfile" "$PUSH" build_multiarch "sandbox-claude-full" "claude/Dockerfile.full" "$PUSH" + # Claude VNC variants + build_multiarch "sandbox-claude-playwright" "playwright/Dockerfile" "$PUSH" + build_multiarch "sandbox-claude-desktop" "desktop/Dockerfile" "$PUSH" # Cursor (uncomment when ready) # build_multiarch "sandbox-cursor" "cursor/Dockerfile" "$PUSH" ;; *) echo "Unknown tool: $TOOL" - echo "Usage: $0 [claude|cursor|all] [true|false]" + echo "Usage: $0 [claude|claude-vnc|playwright|desktop|cursor|all] [true|false]" echo " $0 claude # Build Claude images locally" echo " $0 claude true # Build and push Claude images" + echo " $0 claude-vnc # Build Claude VNC images (Playwright + Desktop)" + echo " $0 playwright # Build Claude Playwright image only" + echo " $0 desktop # Build Claude Desktop image only" echo " $0 all true # Build and push all images" exit 1 ;; @@ -142,9 +164,21 @@ if [ "$PUSH" = "true" ]; then echo " - ${REGISTRY}/sandbox-claude:latest" echo " - ${REGISTRY}/sandbox-claude-full:latest" ;; + claude-vnc) + echo " - ${REGISTRY}/sandbox-claude-playwright:latest" + echo " - ${REGISTRY}/sandbox-claude-desktop:latest" + ;; + playwright) + echo " - ${REGISTRY}/sandbox-claude-playwright:latest" + ;; + desktop) + echo " - ${REGISTRY}/sandbox-claude-desktop:latest" + ;; all) echo " - ${REGISTRY}/sandbox-claude:latest" echo " - ${REGISTRY}/sandbox-claude-full:latest" + echo " - ${REGISTRY}/sandbox-claude-playwright:latest" + echo " - ${REGISTRY}/sandbox-claude-desktop:latest" ;; esac fi diff --git a/sandbox/docker/desktop/Dockerfile b/sandbox/docker/desktop/Dockerfile new file mode 100644 index 00000000..6fcd389c --- /dev/null +++ b/sandbox/docker/desktop/Dockerfile @@ -0,0 +1,91 @@ +# Claude sandbox with full XFCE desktop + VNC preview +# Image: sandbox-claude-desktop +# Base: sandbox-claude (Ubuntu 24.04 + Node.js + Python + Claude CLI) +# Adds: Xvfb + x11vnc + noVNC + XFCE desktop + File Manager + Terminal +# +# Supports both amd64 and arm64 architectures + +ARG REGISTRY=yaoapp +FROM ${REGISTRY}/sandbox-claude:latest + +USER root + +# Use MIT mirror (USA) for ARM64 +RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.mit.edu/ubuntu-ports|g' /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || \ + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.mit.edu/ubuntu-ports|g' /etc/apt/sources.list 2>/dev/null || true + +# Install X11, VNC, and XFCE desktop environment +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Virtual display + xvfb \ + # VNC server + x11vnc \ + # noVNC (HTML5 VNC client) and websockify + novnc \ + python3-websockify \ + # XFCE Desktop (full-featured but lightweight) + xfce4 \ + xfce4-terminal \ + thunar \ + # Fonts (required for proper rendering) + fonts-liberation \ + fonts-noto-cjk \ + fonts-noto-color-emoji \ + # X11 utilities + x11-utils \ + xdotool \ + # Audio + pulseaudio \ + # Remove screensaver (causes issues in container) + && apt-get remove -y xfce4-screensaver xscreensaver || true \ + && rm -rf /var/lib/apt/lists/* + +# Optional: Install Playwright system dependencies (requires root) +# Users can run browser automation in desktop mode too +RUN npx playwright install-deps chromium || true + +# Optional: Install Playwright for browser automation +USER sandbox +RUN npm install -g playwright && \ + pip install --user --break-system-packages playwright && \ + npx playwright install chromium || true + +USER root + +# Copy VNC startup scripts +# Note: Build context should be sandbox/docker/, so paths are relative to that +COPY vnc/start-vnc.sh /usr/local/bin/start-vnc.sh +COPY vnc/entrypoint-vnc.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/start-vnc.sh /usr/local/bin/entrypoint.sh + +# Environment variables for VNC +ENV DISPLAY=:99 +ENV VNC_PORT=5900 +ENV NOVNC_PORT=6080 +ENV RESOLUTION=1920x1080x24 +ENV SANDBOX_VNC_ENABLED=true +ENV SANDBOX_DESKTOP=xfce + +# Node.js environment - ensure global modules are accessible +ENV NODE_PATH=/home/sandbox/.npm-global/lib/node_modules + +# Expose VNC ports (internal use only, accessed via proxy) +EXPOSE 5900 6080 + +USER sandbox +WORKDIR /workspace + +# Verify installations +RUN echo "=== Verifying installations ===" && \ + node --version && \ + npm --version && \ + python3 --version && \ + which startxfce4 && \ + which thunar && \ + which xfce4-terminal && \ + which x11vnc && \ + which Xvfb && \ + echo "=== All installations verified ===" + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["sleep", "infinity"] diff --git a/sandbox/docker/playwright/Dockerfile b/sandbox/docker/playwright/Dockerfile new file mode 100644 index 00000000..be6faa46 --- /dev/null +++ b/sandbox/docker/playwright/Dockerfile @@ -0,0 +1,89 @@ +# Claude sandbox with Playwright browser automation + VNC preview +# Image: sandbox-claude-playwright +# Base: sandbox-claude (Ubuntu 24.04 + Node.js + Python + Claude CLI) +# Adds: Xvfb + x11vnc + noVNC + Fluxbox + Playwright browsers +# +# Supports both amd64 and arm64 architectures + +ARG REGISTRY=yaoapp +FROM ${REGISTRY}/sandbox-claude:latest + +USER root + +# Use MIT mirror (USA) for ARM64 +RUN sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.mit.edu/ubuntu-ports|g' /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || \ + sed -i 's|http://ports.ubuntu.com/ubuntu-ports|http://mirrors.mit.edu/ubuntu-ports|g' /etc/apt/sources.list 2>/dev/null || true + +# Install X11, VNC, and minimal window manager +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Virtual display + xvfb \ + # VNC server + x11vnc \ + # noVNC (HTML5 VNC client) and websockify + novnc \ + python3-websockify \ + # Minimal window manager (lightweight, perfect for Playwright) + fluxbox \ + # Fonts (required for proper browser rendering) + fonts-liberation \ + fonts-noto-cjk \ + fonts-noto-color-emoji \ + # X11 utilities + x11-utils \ + xdotool \ + # Audio (for video playback in browsers, can be disabled) + pulseaudio \ + && rm -rf /var/lib/apt/lists/* + +# Install Playwright system dependencies (requires root) +# This installs system libraries needed by Chromium/Firefox +RUN npx playwright install-deps chromium firefox || true + +# Install Playwright and browsers as sandbox user +USER sandbox + +# Install Playwright for Node.js (global) and Python +RUN npm install -g playwright && \ + pip install --user --break-system-packages playwright && \ + npx playwright install chromium firefox + +USER root + +# Copy VNC startup scripts +# Note: Build context should be sandbox/docker/, so paths are relative to that +COPY vnc/start-vnc.sh /usr/local/bin/start-vnc.sh +COPY vnc/entrypoint-vnc.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/start-vnc.sh /usr/local/bin/entrypoint.sh + +# Environment variables for VNC +ENV DISPLAY=:99 +ENV VNC_PORT=5900 +ENV NOVNC_PORT=6080 +ENV RESOLUTION=1920x1080x24 +ENV SANDBOX_VNC_ENABLED=true +ENV SANDBOX_DESKTOP=fluxbox + +# Node.js environment - ensure global modules are accessible +ENV NODE_PATH=/home/sandbox/.npm-global/lib/node_modules + +# Expose VNC ports (internal use only, accessed via proxy) +EXPOSE 5900 6080 + +USER sandbox +WORKDIR /workspace + +# Verify installations +RUN echo "=== Verifying installations ===" && \ + node --version && \ + npm --version && \ + python3 --version && \ + npx playwright --version && \ + python3 -c "from playwright.sync_api import sync_playwright; print('Python Playwright: OK')" && \ + which fluxbox && \ + which x11vnc && \ + which Xvfb && \ + echo "=== All installations verified ===" + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["sleep", "infinity"] diff --git a/sandbox/docker/vnc/entrypoint-vnc.sh b/sandbox/docker/vnc/entrypoint-vnc.sh new file mode 100644 index 00000000..62e48f50 --- /dev/null +++ b/sandbox/docker/vnc/entrypoint-vnc.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Container entrypoint for VNC-enabled sandbox images +# This extends the original sandbox-claude entrypoint with VNC support + +# ============================================ +# VNC Services Startup +# ============================================ +if [ "$SANDBOX_VNC_ENABLED" = "true" ]; then + echo "[Entrypoint] Starting VNC services..." + /usr/local/bin/start-vnc.sh & + # Wait for VNC to initialize + sleep 3 + echo "[Entrypoint] VNC services started in background" +fi + +# ============================================ +# Original sandbox-claude entrypoint logic +# (from sandbox-claude Dockerfile) +# ============================================ +WORKSPACE="${WORKSPACE:-/workspace}" +PORT="${CLAUDE_PROXY_PORT:-3456}" +ENV_FILE="/tmp/claude-proxy-env" + +# If proxy env vars are set AND proxy is not running, start it +# This supports docker run -e CLAUDE_PROXY_BACKEND=... usage +if [ -n "$CLAUDE_PROXY_BACKEND" ] && [ -n "$CLAUDE_PROXY_API_KEY" ] && [ -n "$CLAUDE_PROXY_MODEL" ]; then + if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + /usr/local/bin/start-claude-proxy + fi + + # Write env vars to a file that can be sourced + if curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then + echo "export ANTHROPIC_BASE_URL=http://127.0.0.1:${PORT}" > "$ENV_FILE" + echo "export ANTHROPIC_API_KEY=dummy" >> "$ENV_FILE" + chmod 644 "$ENV_FILE" + fi +fi + +# Execute the command passed to docker run +exec "$@" diff --git a/sandbox/docker/vnc/start-vnc.sh b/sandbox/docker/vnc/start-vnc.sh new file mode 100644 index 00000000..d7d0677b --- /dev/null +++ b/sandbox/docker/vnc/start-vnc.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# VNC services startup script +# Shared by sandbox-claude-playwright and sandbox-claude-desktop +# Starts: Xvfb (virtual display) + Window Manager + x11vnc + websockify (noVNC) + +set -e + +DISPLAY_NUM="${DISPLAY_NUM:-99}" +RESOLUTION="${RESOLUTION:-1920x1080x24}" +VNC_PORT="${VNC_PORT:-5900}" +NOVNC_PORT="${NOVNC_PORT:-6080}" +VNC_PASSWORD="${VNC_PASSWORD:-}" +DESKTOP="${SANDBOX_DESKTOP:-fluxbox}" + +export DISPLAY=:${DISPLAY_NUM} + +echo "[VNC] Starting VNC services..." +echo "[VNC] Display: :${DISPLAY_NUM}" +echo "[VNC] Resolution: ${RESOLUTION}" +echo "[VNC] Desktop: ${DESKTOP}" + +# Start Xvfb (virtual framebuffer) +echo "[VNC] Starting Xvfb..." +Xvfb :${DISPLAY_NUM} -screen 0 ${RESOLUTION} & +XVFB_PID=$! +sleep 1 + +if ! kill -0 $XVFB_PID 2>/dev/null; then + echo "[VNC] ERROR: Xvfb failed to start" + exit 1 +fi +echo "[VNC] Xvfb started (PID: $XVFB_PID)" + +# Start window manager / desktop environment +echo "[VNC] Starting ${DESKTOP}..." +case "$DESKTOP" in + xfce|xfce4) + # XFCE desktop environment + startxfce4 & + ;; + fluxbox) + # Minimal window manager for Playwright + fluxbox & + ;; + *) + # Default to fluxbox + fluxbox & + ;; +esac +sleep 2 + +# Start x11vnc server +echo "[VNC] Starting x11vnc on port ${VNC_PORT}..." +VNC_ARGS="-display :${DISPLAY_NUM} -forever -shared -rfbport ${VNC_PORT} -noxdamage" + +if [ -n "$VNC_PASSWORD" ]; then + mkdir -p ~/.vnc + x11vnc -storepasswd "$VNC_PASSWORD" ~/.vnc/passwd + VNC_ARGS="$VNC_ARGS -rfbauth ~/.vnc/passwd" +else + VNC_ARGS="$VNC_ARGS -nopw" +fi + +x11vnc $VNC_ARGS & +X11VNC_PID=$! +sleep 1 + +if ! kill -0 $X11VNC_PID 2>/dev/null; then + echo "[VNC] ERROR: x11vnc failed to start" + exit 1 +fi +echo "[VNC] x11vnc started (PID: $X11VNC_PID)" + +# Start websockify (noVNC WebSocket proxy) +echo "[VNC] Starting websockify on port ${NOVNC_PORT}..." +websockify --web=/usr/share/novnc/ ${NOVNC_PORT} localhost:${VNC_PORT} & +WEBSOCKIFY_PID=$! +sleep 1 + +if ! kill -0 $WEBSOCKIFY_PID 2>/dev/null; then + echo "[VNC] ERROR: websockify failed to start" + exit 1 +fi +echo "[VNC] websockify started (PID: $WEBSOCKIFY_PID)" + +echo "[VNC] ==================================" +echo "[VNC] VNC services started successfully" +echo "[VNC] Desktop: ${DESKTOP}" +echo "[VNC] VNC port: ${VNC_PORT}" +echo "[VNC] noVNC port: ${NOVNC_PORT}" +echo "[VNC] ==================================" + +# Note: Don't wait here - let the entrypoint continue +# Background processes will keep running diff --git a/sandbox/manager.go b/sandbox/manager.go index 056b99bc..b0fb56d3 100644 --- a/sandbox/manager.go +++ b/sandbox/manager.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "io" + "net" "os" "path/filepath" "strings" @@ -17,6 +18,7 @@ import ( "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-connections/nat" "github.com/yaoapp/yao/sandbox/ipc" ) @@ -306,6 +308,24 @@ func (m *Manager) createContainer(ctx context.Context, userID, chatID string) (* CapDrop: []string{"ALL"}, } + // VNC port mapping for Docker Desktop (macOS/Windows) + // Only enable for VNC-capable images (playwright/desktop) when config is enabled + if m.config.VNCPortMapping && isVNCImage(m.config.Image) { + // Expose VNC ports in container config + containerConfig.ExposedPorts = nat.PortSet{ + "6080/tcp": struct{}{}, // noVNC websockify + "5900/tcp": struct{}{}, // VNC + } + // Enable SANDBOX_VNC_ENABLED environment variable + containerConfig.Env = append(containerConfig.Env, "SANDBOX_VNC_ENABLED=true") + + // Map to random available ports on 127.0.0.1 + hostConfig.PortBindings = nat.PortMap{ + "6080/tcp": []nat.PortBinding{{HostIP: "127.0.0.1", HostPort: ""}}, // empty = random port + "5900/tcp": []nat.PortBinding{{HostIP: "127.0.0.1", HostPort: ""}}, + } + } + // Create container resp, err := m.dockerClient.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, name) if err != nil { @@ -905,3 +925,19 @@ func (m *Manager) fixIPCSocketPermissions(ctx context.Context, containerID strin // Wait briefly for the chmod to complete time.Sleep(50 * time.Millisecond) } + +// isVNCImage checks if the image is VNC-capable (playwright or desktop variants) +func isVNCImage(imageName string) bool { + return strings.Contains(imageName, "playwright") || strings.Contains(imageName, "desktop") +} + +// findAvailablePort finds an available port on the host +// This is used as a fallback; Docker can auto-assign ports when HostPort is empty +func findAvailablePort() (int, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer listener.Close() + return listener.Addr().(*net.TCPAddr).Port, nil +} diff --git a/sandbox/vncproxy/config.go b/sandbox/vncproxy/config.go new file mode 100644 index 00000000..4f63492b --- /dev/null +++ b/sandbox/vncproxy/config.go @@ -0,0 +1,81 @@ +package vncproxy + +import ( + "os" + "strconv" + "time" +) + +// Config holds VNC proxy configuration +type Config struct { + // Network settings + DockerNetwork string `json:"docker_network,omitempty"` // Docker network name (default: bridge) + ContainerNoVNCPort int `json:"container_novnc_port,omitempty"` // noVNC port inside container (default: 6080) + ContainerVNCPort int `json:"container_vnc_port,omitempty"` // VNC port inside container (default: 5900) + ContainerNamePrefix string `json:"container_name_prefix,omitempty"` // Container name prefix (default: yao-sandbox-) + + // Cache settings + IPCacheTTL time.Duration `json:"ip_cache_ttl,omitempty"` // IP cache TTL (default: 30s) + + // VNC status check + VNCCheckTimeout time.Duration `json:"vnc_check_timeout,omitempty"` // Timeout for VNC ready check (default: 2s) +} + +// DefaultConfig returns default configuration +func DefaultConfig() *Config { + return &Config{ + DockerNetwork: "bridge", + ContainerNoVNCPort: 6080, + ContainerVNCPort: 5900, + ContainerNamePrefix: "yao-sandbox-", + IPCacheTTL: 30 * time.Second, + VNCCheckTimeout: 2 * time.Second, + } +} + +// Init initializes config from environment variables +func (c *Config) Init() { + if env := os.Getenv("YAO_VNC_DOCKER_NETWORK"); env != "" { + c.DockerNetwork = env + } else if c.DockerNetwork == "" { + c.DockerNetwork = "bridge" + } + + if env := os.Getenv("YAO_VNC_CONTAINER_NOVNC_PORT"); env != "" { + if v, err := strconv.Atoi(env); err == nil && v > 0 { + c.ContainerNoVNCPort = v + } + } else if c.ContainerNoVNCPort == 0 { + c.ContainerNoVNCPort = 6080 + } + + if env := os.Getenv("YAO_VNC_CONTAINER_VNC_PORT"); env != "" { + if v, err := strconv.Atoi(env); err == nil && v > 0 { + c.ContainerVNCPort = v + } + } else if c.ContainerVNCPort == 0 { + c.ContainerVNCPort = 5900 + } + + if env := os.Getenv("YAO_VNC_CONTAINER_NAME_PREFIX"); env != "" { + c.ContainerNamePrefix = env + } else if c.ContainerNamePrefix == "" { + c.ContainerNamePrefix = "yao-sandbox-" + } + + if env := os.Getenv("YAO_VNC_IP_CACHE_TTL"); env != "" { + if v, err := time.ParseDuration(env); err == nil && v > 0 { + c.IPCacheTTL = v + } + } else if c.IPCacheTTL == 0 { + c.IPCacheTTL = 30 * time.Second + } + + if env := os.Getenv("YAO_VNC_CHECK_TIMEOUT"); env != "" { + if v, err := time.ParseDuration(env); err == nil && v > 0 { + c.VNCCheckTimeout = v + } + } else if c.VNCCheckTimeout == 0 { + c.VNCCheckTimeout = 2 * time.Second + } +} diff --git a/sandbox/vncproxy/proxy.go b/sandbox/vncproxy/proxy.go new file mode 100644 index 00000000..6ec1e4c2 --- /dev/null +++ b/sandbox/vncproxy/proxy.go @@ -0,0 +1,565 @@ +package vncproxy + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" + "github.com/gorilla/websocket" +) + +// ipCacheEntry holds cached container IP with expiration +type ipCacheEntry struct { + IP string + ExpiresAt time.Time +} + +// Proxy handles VNC proxy requests +type Proxy struct { + config *Config + dockerClient *client.Client + ipCache sync.Map // containerName -> *ipCacheEntry + upgrader websocket.Upgrader +} + +// NewProxy creates a new VNC proxy +func NewProxy(config *Config) (*Proxy, error) { + if config == nil { + config = DefaultConfig() + } + config.Init() + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("failed to create Docker client: %w", err) + } + + // Verify Docker connection + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := cli.Ping(ctx); err != nil { + cli.Close() + return nil, fmt.Errorf("Docker not available: %w", err) + } + + return &Proxy{ + config: config, + dockerClient: cli, + upgrader: websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true // Allow all origins for VNC + }, + Subprotocols: []string{"binary"}, // noVNC uses binary subprotocol + }, + }, nil +} + +// Close closes the proxy and releases resources +func (p *Proxy) Close() error { + return p.dockerClient.Close() +} + +// extractSandboxID extracts sandbox ID from request path +// Expected format: /v1/sandbox/{id}/vnc/... +func extractSandboxID(r *http.Request) string { + path := r.URL.Path + // Remove prefix /v1/sandbox/ + path = strings.TrimPrefix(path, "/v1/sandbox/") + // Get ID (first segment before next /) + if idx := strings.Index(path, "/"); idx > 0 { + return path[:idx] + } + return path +} + +// HandleVNCStatus returns VNC status for a container +// GET /v1/sandbox/{id}/vnc +func (p *Proxy) HandleVNCStatus(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := p.config.ContainerNamePrefix + sandboxID + + response := map[string]interface{}{ + "sandbox_id": sandboxID, + "container": containerName, + } + + // Check if container exists and is running + _, err := p.getContainerIP(r.Context(), containerName) + if err != nil { + response["available"] = false + response["status"] = "unavailable" + response["message"] = "Container not available" + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // Check if VNC is enabled for this container + if !p.checkVNCEnabled(r.Context(), containerName) { + response["available"] = false + response["status"] = "not_supported" + response["message"] = "VNC not available for this container type" + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // Check if VNC services are ready (try to connect to websockify port) + if !p.checkVNCReady(r.Context(), containerName) { + response["available"] = false + response["status"] = "starting" + response["message"] = "VNC services are starting..." + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + + // VNC is ready + response["available"] = true + response["status"] = "ready" + response["client_url"] = fmt.Sprintf("/v1/sandbox/%s/vnc/client", sandboxID) + response["websocket_url"] = fmt.Sprintf("/v1/sandbox/%s/vnc/ws", sandboxID) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +// HandleVNCClient serves the noVNC client page +// GET /v1/sandbox/{id}/vnc/client?viewonly=true|false +func (p *Proxy) HandleVNCClient(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := p.config.ContainerNamePrefix + sandboxID + + // Verify container exists and is running + _, err := p.getContainerIP(r.Context(), containerName) + if err != nil { + http.Error(w, "Container not available", http.StatusNotFound) + return + } + + if !p.checkVNCEnabled(r.Context(), containerName) { + http.Error(w, "VNC not available for this container", http.StatusBadRequest) + return + } + + // Get viewonly parameter (default: false = interactive) + viewOnly := r.URL.Query().Get("viewonly") == "true" + + // Serve inline noVNC HTML page with status checking + wsPath := fmt.Sprintf("/v1/sandbox/%s/vnc/ws", sandboxID) + p.serveNoVNCPage(w, sandboxID, wsPath, viewOnly) +} + +// HandleVNCWebSocket proxies WebSocket connection to container VNC +// GET /v1/sandbox/{id}/vnc/ws +func (p *Proxy) HandleVNCWebSocket(w http.ResponseWriter, r *http.Request) { + sandboxID := extractSandboxID(r) + containerName := p.config.ContainerNamePrefix + sandboxID + + // Get VNC endpoint (uses port mapping if available, otherwise container IP) + targetAddr, err := p.getVNCEndpoint(r.Context(), containerName) + if err != nil { + http.Error(w, "Container not available", http.StatusNotFound) + return + } + + // Upgrade HTTP to WebSocket + clientConn, err := p.upgrader.Upgrade(w, r, nil) + if err != nil { + return // Upgrader already sent error response + } + defer clientConn.Close() + + // Connect to container's websockify + targetConn, err := net.DialTimeout("tcp", targetAddr, 5*time.Second) + if err != nil { + clientConn.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "VNC connection failed")) + return + } + defer targetConn.Close() + + // Bidirectional proxy + done := make(chan struct{}) + + // Client -> Container + go func() { + defer func() { done <- struct{}{} }() + for { + messageType, data, err := clientConn.ReadMessage() + if err != nil { + return + } + if messageType == websocket.BinaryMessage { + if _, err := targetConn.Write(data); err != nil { + return + } + } + } + }() + + // Container -> Client + go func() { + defer func() { done <- struct{}{} }() + buf := make([]byte, 32*1024) + for { + n, err := targetConn.Read(buf) + if err != nil { + return + } + if err := clientConn.WriteMessage(websocket.BinaryMessage, buf[:n]); err != nil { + return + } + } + }() + + // Wait for either direction to close + <-done +} + +// getContainerIP gets the IP address of a container, using cache with TTL +func (p *Proxy) getContainerIP(ctx context.Context, containerName string) (string, error) { + // Check cache + if cached, ok := p.ipCache.Load(containerName); ok { + entry := cached.(*ipCacheEntry) + if time.Now().Before(entry.ExpiresAt) { + return entry.IP, nil + } + // Cache expired, delete it + p.ipCache.Delete(containerName) + } + + // Get from Docker + info, err := p.dockerClient.ContainerInspect(ctx, containerName) + if err != nil { + return "", fmt.Errorf("container not found: %w", err) + } + + if !info.State.Running { + return "", fmt.Errorf("container not running") + } + + // Get IP from the specified network or default bridge + var ip string + if info.NetworkSettings != nil && info.NetworkSettings.Networks != nil { + if net, ok := info.NetworkSettings.Networks[p.config.DockerNetwork]; ok { + ip = net.IPAddress + } else { + // Try to get IP from any network + for _, net := range info.NetworkSettings.Networks { + if net.IPAddress != "" { + ip = net.IPAddress + break + } + } + } + } + + if ip == "" { + return "", fmt.Errorf("container has no IP address") + } + + // Cache the result + p.ipCache.Store(containerName, &ipCacheEntry{ + IP: ip, + ExpiresAt: time.Now().Add(p.config.IPCacheTTL), + }) + + return ip, nil +} + +// getVNCEndpoint returns the host:port to connect to for VNC +// It first checks for port mapping (for Docker Desktop), then falls back to container IP +func (p *Proxy) getVNCEndpoint(ctx context.Context, containerName string) (string, error) { + info, err := p.dockerClient.ContainerInspect(ctx, containerName) + if err != nil { + return "", fmt.Errorf("container not found: %w", err) + } + + if !info.State.Running { + return "", fmt.Errorf("container not running") + } + + // Check for port mapping first (for Docker Desktop on macOS/Windows) + if info.NetworkSettings != nil && info.NetworkSettings.Ports != nil { + portKey := nat.Port(fmt.Sprintf("%d/tcp", p.config.ContainerNoVNCPort)) + if bindings, ok := info.NetworkSettings.Ports[portKey]; ok && len(bindings) > 0 { + binding := bindings[0] + if binding.HostPort != "" { + // Use mapped port on localhost + host := binding.HostIP + if host == "" || host == "0.0.0.0" { + host = "127.0.0.1" + } + return net.JoinHostPort(host, binding.HostPort), nil + } + } + } + + // Fall back to container IP (works on Linux with native Docker) + ip, err := p.getContainerIP(ctx, containerName) + if err != nil { + return "", err + } + return net.JoinHostPort(ip, fmt.Sprintf("%d", p.config.ContainerNoVNCPort)), nil +} + +// checkVNCEnabled checks if container has VNC enabled by checking env vars +func (p *Proxy) checkVNCEnabled(ctx context.Context, containerName string) bool { + info, err := p.dockerClient.ContainerInspect(ctx, containerName) + if err != nil { + return false + } + + // Check environment variables for VNC_ENABLED or SANDBOX_VNC_ENABLED + for _, env := range info.Config.Env { + if strings.HasPrefix(env, "SANDBOX_VNC_ENABLED=true") || + strings.HasPrefix(env, "VNC_ENABLED=true") { + return true + } + } + + // Also check if container image is a VNC-enabled variant + imageName := info.Config.Image + if strings.Contains(imageName, "playwright") || + strings.Contains(imageName, "desktop") { + return true + } + + return false +} + +// checkVNCReady tests if VNC services are ready +// Uses docker exec to test port connectivity (works across platforms including macOS Docker Desktop) +func (p *Proxy) checkVNCReady(ctx context.Context, containerName string) bool { + // Use docker exec to test port connectivity from inside the container + // This approach works regardless of host network configuration + execConfig := container.ExecOptions{ + Cmd: []string{"sh", "-c", fmt.Sprintf("nc -z localhost %d 2>/dev/null || (echo | timeout 1 cat < /dev/tcp/localhost/%d > /dev/null 2>&1)", p.config.ContainerNoVNCPort, p.config.ContainerNoVNCPort)}, + AttachStdout: false, + AttachStderr: false, + } + + execResp, err := p.dockerClient.ContainerExecCreate(ctx, containerName, execConfig) + if err != nil { + return false + } + + err = p.dockerClient.ContainerExecStart(ctx, execResp.ID, container.ExecStartOptions{}) + if err != nil { + return false + } + + // Wait for exec to complete and check exit code + for i := 0; i < 10; i++ { + inspect, err := p.dockerClient.ContainerExecInspect(ctx, execResp.ID) + if err != nil { + return false + } + if !inspect.Running { + return inspect.ExitCode == 0 + } + time.Sleep(100 * time.Millisecond) + } + + return false +} + +// serveNoVNCPage serves an inline HTML page with noVNC client +func (p *Proxy) serveNoVNCPage(w http.ResponseWriter, sandboxID, wsPath string, viewOnly bool) { + viewOnlyStr := "false" + modeIndicator := "可交互" + modeColor := "#4CAF50" + if viewOnly { + viewOnlyStr = "true" + modeIndicator = "只读模式" + modeColor = "#FF9800" + } + + html := fmt.Sprintf(` + + + + + VNC - %s + + + +
+ +
正在连接 VNC...
+
+
+
+
%s
+
+ + + +`, sandboxID, modeColor, modeIndicator, sandboxID, wsPath, viewOnlyStr) + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, html) +} + +// RegisterRoutes registers VNC proxy routes to an HTTP mux +func (p *Proxy) RegisterRoutes(mux *http.ServeMux) { + mux.HandleFunc("/v1/sandbox/", func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + + // Match /v1/sandbox/{id}/vnc + if strings.HasSuffix(path, "/vnc") { + p.HandleVNCStatus(w, r) + return + } + + // Match /v1/sandbox/{id}/vnc/client + if strings.HasSuffix(path, "/vnc/client") { + p.HandleVNCClient(w, r) + return + } + + // Match /v1/sandbox/{id}/vnc/ws + if strings.HasSuffix(path, "/vnc/ws") { + p.HandleVNCWebSocket(w, r) + return + } + + http.NotFound(w, r) + }) +} + +// Helper function to check if request requires VNC container +func (p *Proxy) isVNCRequest(r *http.Request) bool { + path := r.URL.Path + return strings.Contains(path, "/vnc") +} diff --git a/sandbox/vncproxy/proxy_test.go b/sandbox/vncproxy/proxy_test.go new file mode 100644 index 00000000..cd13dad4 --- /dev/null +++ b/sandbox/vncproxy/proxy_test.go @@ -0,0 +1,90 @@ +package vncproxy + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestExtractSandboxID(t *testing.T) { + tests := []struct { + name string + path string + expected string + }{ + { + name: "VNC status path", + path: "/v1/sandbox/abc123/vnc", + expected: "abc123", + }, + { + name: "VNC client path", + path: "/v1/sandbox/user-chat-123/vnc/client", + expected: "user-chat-123", + }, + { + name: "VNC websocket path", + path: "/v1/sandbox/test-sandbox-id/vnc/ws", + expected: "test-sandbox-id", + }, + { + name: "Complex ID", + path: "/v1/sandbox/user_123-chat_456/vnc", + expected: "user_123-chat_456", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tt.path, nil) + got := extractSandboxID(req) + if got != tt.expected { + t.Errorf("extractSandboxID() = %q, want %q", got, tt.expected) + } + }) + } +} + +func TestConfigDefaults(t *testing.T) { + config := DefaultConfig() + + if config.DockerNetwork != "bridge" { + t.Errorf("DockerNetwork = %q, want %q", config.DockerNetwork, "bridge") + } + if config.ContainerNoVNCPort != 6080 { + t.Errorf("ContainerNoVNCPort = %d, want %d", config.ContainerNoVNCPort, 6080) + } + if config.ContainerVNCPort != 5900 { + t.Errorf("ContainerVNCPort = %d, want %d", config.ContainerVNCPort, 5900) + } + if config.ContainerNamePrefix != "yao-sandbox-" { + t.Errorf("ContainerNamePrefix = %q, want %q", config.ContainerNamePrefix, "yao-sandbox-") + } +} + +func TestConfigInit(t *testing.T) { + config := &Config{} + config.Init() + + // Should have defaults after Init + if config.DockerNetwork != "bridge" { + t.Errorf("DockerNetwork = %q, want %q", config.DockerNetwork, "bridge") + } + if config.ContainerNoVNCPort != 6080 { + t.Errorf("ContainerNoVNCPort = %d, want %d", config.ContainerNoVNCPort, 6080) + } +} + +// Integration tests require Docker - skip if not available +func TestProxyCreation(t *testing.T) { + // This will fail if Docker is not available, which is expected in CI + proxy, err := NewProxy(nil) + if err != nil { + t.Skipf("Skipping test: Docker not available: %v", err) + } + defer proxy.Close() + + if proxy.config == nil { + t.Error("Proxy config should not be nil") + } +}