yao/tools/image/providers.go
Max af0d4edd74 feat(image): add image generation and enhanced reading capabilities
- Introduced `image_generate` tool for generating images from text prompts, with options for specifying output file paths and image dimensions.
- Updated `image_read` functionality to allow optional provider specification for enhanced image analysis.
- Implemented new `GenerateImage` method in the LLM API for seamless integration of image generation capabilities.
- Enhanced documentation to include detailed usage examples for both image reading and generation tools.
- Updated tests to validate new image generation features and ensure robust functionality across image tools.
2026-05-06 10:59:35 +08:00

142 lines
3.5 KiB
Go

package image
import (
_ "embed"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/yao/llmprovider"
"github.com/yaoapp/yao/openapi/oauth/authorized"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
//go:embed providers_schema.json
var ProvidersSchemaJSON []byte
type providerResult struct {
Key string `json:"key"`
Name string `json:"name"`
Models []modelResult `json:"models"`
}
type modelResult struct {
ID string `json:"id"`
Name string `json:"name"`
ConnectorID string `json:"connector_id"`
}
// ProvidersHandler is the tools.image_providers process handler.
func ProvidersHandler(proc *process.Process) interface{} {
capability := proc.ArgsString(0, "image_generation")
authInfo := authorized.ProcessAuthInfo(proc)
if authInfo == nil {
return map[string]interface{}{"error": "unauthorized: no auth info in request"}
}
if llmprovider.Global == nil {
return map[string]interface{}{"error": "llmprovider registry not initialized"}
}
providers, err := listProvidersByCapability(capability, authInfo)
if err != nil {
return map[string]interface{}{"error": err.Error()}
}
return map[string]interface{}{
"capability": capability,
"providers": providers,
}
}
// listProvidersByCapability returns providers matching the given capability,
// filtered by owner scope (builtin always included, dynamic filtered by auth).
func listProvidersByCapability(capability string, authInfo *oauthTypes.AuthorizedInfo) ([]providerResult, error) {
enabled := true
filter := &llmprovider.ProviderFilter{
Capabilities: []string{capability},
Enabled: &enabled,
Source: llmprovider.ProviderSourceAll,
}
allProviders, err := llmprovider.Global.List(filter)
if err != nil {
return nil, err
}
var results []providerResult
for _, p := range allProviders {
if p.Source == llmprovider.ProviderSourceDynamic && authInfo != nil {
if !dynamicOwnerMatch(&p.Owner, authInfo) {
continue
}
}
var models []modelResult
for _, m := range p.Models {
if !m.Enabled {
continue
}
if !modelHasCapability(m.Capabilities, capability) {
continue
}
cid := p.ConnectorID
if p.Source == llmprovider.ProviderSourceDynamic && m.ID != "" {
cid = p.ConnectorID + ":" + m.ID
}
name := m.Name
if name == "" {
name = m.ID
}
models = append(models, modelResult{
ID: m.ID,
Name: name,
ConnectorID: cid,
})
}
if len(models) == 0 {
continue
}
results = append(results, providerResult{
Key: p.Key,
Name: p.Name,
Models: models,
})
}
return results, nil
}
// findFirstImageGenConnector returns the connector ID of the first available
// image generation provider, or empty string if none found.
func findFirstImageGenConnector(authInfo *oauthTypes.AuthorizedInfo) string {
if llmprovider.Global == nil {
return ""
}
providers, err := listProvidersByCapability("image_generation", authInfo)
if err != nil || len(providers) == 0 {
return ""
}
if len(providers[0].Models) == 0 {
return ""
}
return providers[0].Models[0].ConnectorID
}
func dynamicOwnerMatch(owner *llmprovider.ProviderOwner, authInfo *oauthTypes.AuthorizedInfo) bool {
if authInfo.GetTeamID() != "" {
return owner.Type == "team" && owner.TeamID == authInfo.GetTeamID()
}
return owner.Type == "user" && owner.UserID == authInfo.GetUserID()
}
func modelHasCapability(caps []string, target string) bool {
for _, c := range caps {
if c == target {
return true
}
}
return false
}