opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enhance CI workflows and Knowledge Base integration

Browse source 
- Updated GitHub Actions workflows to use the latest Ubuntu version and added services for Neo4j, Qdrant, and FastEmbed.
- Integrated FFmpeg and additional utilities into the CI environment, ensuring they are installed and tested during the workflow.
- Implemented environment variable resolution in the Knowledge Base configuration, allowing dynamic configuration based on environment settings.
- Added tests for environment variable resolution in the Knowledge Base configuration, ensuring correct parsing and handling of variables.
This commit is contained in:
Max 2025-07-23 17:42:10 +08:00
parent b76afe23bf
commit d25ce68ab9
9 changed files with 345 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
.github/workflows/pr-test.yml vendored
View file

@ -64,9 +64,46 @@ env:
YAO_RUNTIME_HEAP_AVAILABLE: 550000000
YAO_RUNTIME_PRECOMPILE: true
# Neo4j
NEO4J_TEST_URL: "neo4j://localhost:7686"
NEO4J_TEST_USER: "neo4j"
NEO4J_TEST_PASS: "Yao2026Neo4j"
# Qdrant
QDRANT_TEST_HOST: "127.0.0.1"
QDRANT_TEST_PORT: "6334"
jobs:
UnitTest:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:latest
ports:
- 6333:6333 # HTTP API
- 6334:6334 # gRPC
fastembed:
image: yaoapp/fastembed:latest-amd64
env:
FASTEMBED_PASSWORD: Yao@2026
ports:
- 6001:8000
neo4j:
image: neo4j:latest
ports:
- "7687:7687"
env:
NEO4J_AUTH: neo4j/Yao2026Neo4j
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
strategy:
matrix:
go: [1.24]
@ -202,6 +239,28 @@ jobs:
with:
go-version: ${{ matrix.go }}
- name: Install FFmpeg 7.x
run: |
wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
- name: Test FFmpeg
run: ffmpeg -version
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.8.0
with:

60
.github/workflows/unit-test.yml vendored
View file

@ -68,9 +68,45 @@ env:
YAO_RUNTIME_HEAP_AVAILABLE: 550000000
YAO_RUNTIME_PRECOMPILE: true
# Neo4j
NEO4J_TEST_URL: "neo4j://localhost:7686"
NEO4J_TEST_USER: "neo4j"
NEO4J_TEST_PASS: "Yao2026Neo4j"
# Qdrant
QDRANT_TEST_HOST: "127.0.0.1"
QDRANT_TEST_PORT: "6334"
jobs:
unit-test:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:latest
ports:
- 6333:6333 # HTTP API
- 6334:6334 # gRPC
fastembed:
image: yaoapp/fastembed:latest-amd64
env:
FASTEMBED_PASSWORD: Yao@2026
ports:
- 6001:8000
neo4j:
image: neo4j:latest
ports:
- "7687:7687"
env:
NEO4J_AUTH: neo4j/Yao2026Neo4j
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
strategy:
matrix:
go: [1.24]
@ -152,6 +188,28 @@ jobs:
with:
go-version: ${{ matrix.go }}
- name: Install FFmpeg 7.x
run: |
wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/local/bin/
sudo cp ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/local/bin/
sudo chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe
- name: Test FFmpeg
run: ffmpeg -version
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Start Redis
uses: supercharge/redis-github-action@1.4.0
with:

8
engine/load.go
View file

@ -19,6 +19,7 @@ import (
"github.com/yaoapp/yao/flow"
"github.com/yaoapp/yao/fs"
"github.com/yaoapp/yao/i18n"
"github.com/yaoapp/yao/kb"
"github.com/yaoapp/yao/moapi"
"github.com/yaoapp/yao/model"
"github.com/yaoapp/yao/neo"
@ -280,6 +281,13 @@ func Load(cfg config.Config, options LoadOption) (warnings []Warning, err error)
}
}
// Load Knowledge Base
_, err = kb.Load(cfg)
if err != nil {
// printErr(cfg.Mode, "Knowledge Base", err)
warnings = append(warnings, Warning{Widget: "Knowledge Base", Error: err})
}
// Load OpenAPI
_, err = openapi.Load(cfg)
if err != nil {

64
kb/config.go
View file

@ -3,6 +3,8 @@ package kb
import (
"encoding/json"
"os"
"regexp"
"strings"
"github.com/yaoapp/gou/graphrag"
"github.com/yaoapp/gou/graphrag/graph/neo4j"
@ -114,8 +116,14 @@ func (c *Config) createGraphStore() (types.GraphStore, error) {
// toVectorStoreConfig converts the vector config to VectorStoreConfig
func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) {
// Convert map[string]interface{} to types.VectorStoreConfig via JSON
jsonData, err := json.Marshal(c.Vector.Config)
// Parse environment variables in config
resolvedConfig, err := c.resolveEnvVars(c.Vector.Config)
if err != nil {
return types.VectorStoreConfig{}, err
}
// Convert resolved config to types.VectorStoreConfig via JSON
jsonData, err := json.Marshal(resolvedConfig)
if err != nil {
return types.VectorStoreConfig{}, err
}
@ -130,8 +138,14 @@ func (c *Config) toVectorStoreConfig() (types.VectorStoreConfig, error) {
// toGraphStoreConfig converts the graph config to GraphStoreConfig
func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) {
// Convert map[string]interface{} to types.GraphStoreConfig via JSON
jsonData, err := json.Marshal(c.Graph.Config)
// Parse environment variables in config
resolvedConfig, err := c.resolveEnvVars(c.Graph.Config)
if err != nil {
return types.GraphStoreConfig{}, err
}
// Convert resolved config to types.GraphStoreConfig via JSON
jsonData, err := json.Marshal(resolvedConfig)
if err != nil {
return types.GraphStoreConfig{}, err
}
@ -144,6 +158,48 @@ func (c *Config) toGraphStoreConfig() (types.GraphStoreConfig, error) {
return config, nil
}
// resolveEnvVars resolves environment variables in configuration values
func (c *Config) resolveEnvVars(config map[string]interface{}) (map[string]interface{}, error) {
resolved := make(map[string]interface{})
for key, value := range config {
switch v := value.(type) {
case string:
resolved[key] = c.parseEnvVar(v)
case map[string]interface{}:
// Recursively resolve nested maps
nestedResolved, err := c.resolveEnvVars(v)
if err != nil {
return nil, err
}
resolved[key] = nestedResolved
default:
resolved[key] = value
}
}
return resolved, nil
}
// parseEnvVar parses environment variable pattern $ENV.VAR_NAME
func (c *Config) parseEnvVar(value string) string {
// Simple pattern to match $ENV.VAR_NAME
envPattern := regexp.MustCompile(`\$ENV\.([A-Za-z_][A-Za-z0-9_]*)`)
return envPattern.ReplaceAllStringFunc(value, func(match string) string {
// Extract variable name (remove $ENV. prefix)
varName := strings.TrimPrefix(match, "$ENV.")
// Get environment variable value
if envValue := os.Getenv(varName); envValue != "" {
return envValue
}
// Return original if environment variable is not set
return match
})
}
// UnmarshalJSON implements json.Unmarshaler interface
func (c *Config) UnmarshalJSON(data []byte) error {
// Use alias type to avoid infinite recursion

133
kb/config_test.go
View file

@ -373,3 +373,136 @@ func TestRoundTrip(t *testing.T) {
t.Errorf("Features mismatch: %+v != %+v", originalConfig.Features, roundTripConfig.Features)
}
}
func TestConfig_ResolveEnvVars(t *testing.T) {
// Set test environment variables
os.Setenv("TEST_HOST", "localhost")
os.Setenv("TEST_PORT", "6333")
defer func() {
os.Unsetenv("TEST_HOST")
os.Unsetenv("TEST_PORT")
}()
config := &Config{}
tests := []struct {
name string
input map[string]interface{}
expected map[string]interface{}
}{
{
name: "simple environment variable",
input: map[string]interface{}{
"host": "$ENV.TEST_HOST",
"port": "$ENV.TEST_PORT",
},
expected: map[string]interface{}{
"host": "localhost",
"port": "6333",
},
},
{
name: "mixed values",
input: map[string]interface{}{
"host": "$ENV.TEST_HOST",
"port": 6333,
"prefix": "test-$ENV.TEST_HOST-suffix",
},
expected: map[string]interface{}{
"host": "localhost",
"port": 6333,
"prefix": "test-localhost-suffix",
},
},
{
name: "nested configuration",
input: map[string]interface{}{
"database": map[string]interface{}{
"host": "$ENV.TEST_HOST",
"port": "$ENV.TEST_PORT",
},
"name": "test",
},
expected: map[string]interface{}{
"database": map[string]interface{}{
"host": "localhost",
"port": "6333",
},
"name": "test",
},
},
{
name: "undefined environment variable",
input: map[string]interface{}{
"host": "$ENV.UNDEFINED_VAR",
"port": "$ENV.TEST_PORT",
},
expected: map[string]interface{}{
"host": "$ENV.UNDEFINED_VAR", // Should remain unchanged
"port": "6333",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := config.resolveEnvVars(tt.input)
if err != nil {
t.Errorf("resolveEnvVars() error = %v", err)
return
}
if !reflect.DeepEqual(result, tt.expected) {
t.Errorf("resolveEnvVars() = %+v, want %+v", result, tt.expected)
}
})
}
}
func TestConfig_ParseEnvVar(t *testing.T) {
// Set test environment variables
os.Setenv("TEST_HOST", "test_value")
defer os.Unsetenv("TEST_HOST")
config := &Config{}
tests := []struct {
name string
input string
expected string
}{
{
name: "simple env var",
input: "$ENV.TEST_HOST",
expected: "test_value",
},
{
name: "env var in string",
input: "prefix-$ENV.TEST_HOST-suffix",
expected: "prefix-test_value-suffix",
},
{
name: "multiple env vars",
input: "$ENV.TEST_HOST-$ENV.TEST_HOST",
expected: "test_value-test_value",
},
{
name: "undefined env var",
input: "$ENV.UNDEFINED_VAR",
expected: "$ENV.UNDEFINED_VAR", // Should remain unchanged
},
{
name: "no env var",
input: "plain_string",
expected: "plain_string",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := config.parseEnvVar(tt.input)
if result != tt.expected {
t.Errorf("parseEnvVar() = %v, want %v", result, tt.expected)
}
})
}
}

2
kb/kb.go
View file

@ -39,7 +39,7 @@ func Load(appConfig config.Config) (*KnowledgeBase, error) {
return nil, err
}
err = application.Parse("kb", raw, &config)
err = application.Parse("kb.yao", raw, &config)
if err != nil {
return nil, err
}

19
kb/kb_test.go Normal file
View file

@ -0,0 +1,19 @@
package kb
import (
"testing"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestLoad(t *testing.T) {
// Setup
test.Prepare(&testing.T{}, config.Conf)
defer test.Clean()
_, err := Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load knowledge base: %v", err)
}
}

2
store/store.go
View file

@ -21,6 +21,8 @@ var systemStores = map[string]string{
"__yao.oauth.client": "yao/stores/oauth/client.badger.yao", // for OAuth client store
"__yao.agent.memory": "yao/stores/agent/memory.badger.yao", // for agent memory store (for agent memory)
"__yao.agent.cache": "yao/stores/agent/cache.lru.yao", // for agent cache store (for agent cache)
"__yao.kb.store": "yao/stores/kb/store.badger.yao", // for knowledge base store
"__yao.kb.cache": "yao/stores/kb/cache.lru.yao", // for knowledge base cache store
}
// replaceVars replaces template variables in the JSON string

3
test/utils.go
View file

@ -54,6 +54,8 @@ var testSystemStores = map[string]string{
"__yao.oauth.client": "yao/stores/oauth/client.badger.yao",
"__yao.oauth.cache": "yao/stores/oauth/cache.lru.yao",
"__yao.agent.memory": "yao/stores/agent/memory.badger.yao",
"__yao.kb.store": "yao/stores/kb/store.badger.yao",
"__yao.kb.cache": "yao/stores/kb/cache.lru.yao",
}
func loadSystemStores(t *testing.T, cfg config.Config) error {
@ -255,6 +257,7 @@ func Prepare(t *testing.T, cfg config.Config, rootEnv ...string) {
dbconnect(t, cfg)
load(t, cfg)
startRuntime(t, cfg)
}
// Clean the test environment