yao/sandbox/v2/watcher.go

package sandbox

import (
	"context"
	"fmt"
	"time"

	"github.com/yaoapp/yao/monitor"
)

func init() {
	monitor.Register(&sandboxWatcher{})
}

type sandboxWatcher struct{}

func (w *sandboxWatcher) Name() string            { return "sandbox" }
func (w *sandboxWatcher) Interval() time.Duration { return 30 * time.Second }

func (w *sandboxWatcher) Check(ctx context.Context) []monitor.Alert {
	if mgr == nil {
		return nil
	}

	var alerts []monitor.Alert
	mgr.boxes.Range(func(_, v any) bool {
		b := v.(*Box)

		status := b.inspectStatus(ctx)
		old, _ := b.status.Swap(status).(string)
		if old != "" && old != status {
			alerts = append(alerts, monitor.Alert{
				Level:   monitor.Info,
				Target:  "box:" + b.id,
				Message: fmt.Sprintf("status %s → %s", old, status),
			})
			if status == "running" && old != "running" {
				b.touch()
				alerts = append(alerts, monitor.Alert{
					Level:  monitor.Trace,
					Target: "box:" + b.id,
					Message: fmt.Sprintf("touch on resume, lastCall reset to %s",
						time.UnixMilli(b.lastCall.Load()).Format(time.RFC3339)),
				})
			}
		}

		if status != "running" {
			return true
		}

		// maxLifetime: independent of idle — prevents indefinitely running containers
		if b.policy == LongRunning {
			if lifetime := b.maxLifetime(); lifetime > 0 {
				age := time.Since(b.createdAt)
				if age > lifetime {
					alerts = append(alerts, monitor.Alert{
						Level:   monitor.Warn,
						Target:  "box:" + b.id,
						Message: fmt.Sprintf("lifetime expired (%s), removing", lifetime),
						Action:  func(ctx context.Context) { mgr.Remove(ctx, b.id) },
					})
				} else {
					alerts = append(alerts, monitor.Alert{
						Level:  monitor.Trace,
						Target: "box:" + b.id,
						Message: fmt.Sprintf("lifetime remaining %s (max=%s)",
							(lifetime - age).Round(time.Second), lifetime),
					})
				}
			}
		}

		// OneShot safety net: these containers should have been removed by
		// LifecycleAction right after execution. If they still exist after
		// DefaultOneShotMaxAge it means cleanup failed (e.g. process crash,
		// cfg.ID race before the fix). Remove them based on createdAt so we
		// never kill a container that is still actively executing.
		if b.policy == OneShot {
			age := time.Since(b.createdAt)
			if age > DefaultOneShotMaxAge {
				alerts = append(alerts, monitor.Alert{
					Level:   monitor.Warn,
					Target:  "box:" + b.id,
					Message: fmt.Sprintf("oneshot exceeded max age (%s > %s), removing", age.Round(time.Second), DefaultOneShotMaxAge),
					Action:  func(ctx context.Context) { mgr.Remove(ctx, b.id) },
				})
			} else {
				alerts = append(alerts, monitor.Alert{
					Level:  monitor.Trace,
					Target: "box:" + b.id,
					Message: fmt.Sprintf("oneshot age %s (max=%s)",
						age.Round(time.Second), DefaultOneShotMaxAge),
				})
			}
			return true
		}

		idle := time.Since(b.idleSince())
		timeout := b.idleTimeout()

		alerts = append(alerts, monitor.Alert{
			Level:  monitor.Trace,
			Target: "box:" + b.id,
			Message: fmt.Sprintf("heartbeat status=%s policy=%s idle=%s timeout=%s",
				status, b.policy, idle.Round(time.Second), timeout),
		})

		if timeout <= 0 || idle <= timeout {
			return true
		}

		switch b.policy {
		case Session:
			alerts = append(alerts, monitor.Alert{
				Level:   monitor.Warn,
				Target:  "box:" + b.id,
				Message: fmt.Sprintf("session idle expired (idle=%s, timeout=%s), removing", idle.Round(time.Second), timeout),
				Action: func(ctx context.Context) {
					mgr.Remove(ctx, b.id)
				},
			})

		case LongRunning:
			alerts = append(alerts, monitor.Alert{
				Level:   monitor.Warn,
				Target:  "box:" + b.id,
				Message: fmt.Sprintf("longrunning idle expired (idle=%s, timeout=%s), stopping", idle.Round(time.Second), timeout),
				Action: func(ctx context.Context) {
					b.Stop(ctx)
				},
			})
		}

		return true
	})
	return alerts
}