Refactor DSL list retrieval to support conditional data source selection

- Enhanced the List method in the DSL struct to conditionally retrieve data from either the database or file system based on the StoreType specified in ListOptions.
- Updated the Info struct to include a new Source field for optional source content retrieval.
- Modified the List methods in both DB and FS to include source information when requested, improving data accessibility and flexibility.

dsl/dsl.go

@@ -127,15 +127,24 @@ func (dsl *DSL) Source(ctx context.Context, id string) (string, error) {
 // List DSLs
 func (dsl *DSL) List(ctx context.Context, opts *types.ListOptions) ([]*types.Info, error) {
 	// Get the list from the db
-	dbList, err := dsl.db.List(opts)
-	if err != nil {
-		return nil, err
+	var dbList []*types.Info
+	var fileList []*types.Info
+	var err error
+
+	// If StoreType is not specified or is DB, get from db
+	if opts.Store == "" || opts.Store == types.StoreTypeDB {
+		dbList, err = dsl.db.List(opts)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	// Get the list from the file
-	fileList, err := dsl.fs.List(opts)
-	if err != nil {
-		return nil, err
+	// If StoreType is not specified or is File, get from file
+	if opts.Store == "" || opts.Store == types.StoreTypeFile {
+		fileList, err = dsl.fs.List(opts)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// Merge the list and unique

dsl/io/db.go

@@ -118,10 +118,16 @@ func (db *DB) List(options *types.ListOptions) ([]*types.Info, error) {
 		wheres = append(wheres, model.QueryWhere{Wheres: orwheres})
 	}
 
+	// Select fields
+	fields := []interface{}{"dsl_id", "label", "path", "sort", "tags", "description", "status", "store", "mtime", "ctime"}
+	if options.Source {
+		fields = append(fields, "source")
+	}
+
 	// Get the list
 	rows, err := m.Get(model.QueryParam{
-		Wheres: []model.QueryWhere{{Column: "type", Value: db.Type}},
-		Select: []interface{}{"dsl_id", "label", "path", "sort", "tags", "description", "status", "store", "mtime", "ctime"},
+		Wheres: wheres,
+		Select: fields,
 		Orders: orders,
 	})
 	if err != nil {

dsl/io/fs.go

@@ -101,6 +101,14 @@ func (fs *FS) List(options *types.ListOptions) ([]*types.Info, error) {
 			for _, tag := range options.Tags {
 				for _, t := range info.Tags {
 					if t == tag {
+						if options.Source {
+							source, _, err := fs.Source(id)
+							if err != nil {
+								errs = append(errs, err)
+								return nil
+							}
+							info.Source = source
+						}
 						infos = append(infos, info)
 						return nil
 					}
@@ -109,6 +117,14 @@ func (fs *FS) List(options *types.ListOptions) ([]*types.Info, error) {
 		}
 
 		// Add to the list
+		if options.Source {
+			source, _, err := fs.Source(id)
+			if err != nil {
+				errs = append(errs, err)
+				return nil
+			}
+			info.Source = source
+		}
 		infos = append(infos, info)
 		return err
 	}, patterns...)

dsl/types/types.go

@@ -78,30 +78,34 @@ const (
 
 // Info for DSL
 type Info struct {
-	ID string
+	ID string `json:"id" yaml:"id"` // Unique identifier for the DSL instance
 
-	Type        Type
-	Label       string
-	Description string
-	Tags        []string
+	Type        Type     `json:"type" yaml:"type"`                                   // DSL type (model, api, table, form, list, chart, dashboard, etc.)
+	Label       string   `json:"label,omitempty" yaml:"label,omitempty"`             // Display name for the DSL
+	Description string   `json:"description,omitempty" yaml:"description,omitempty"` // Detailed description of the DSL
+	Tags        []string `json:"tags,omitempty" yaml:"tags,omitempty"`               // Tags for categorization and filtering
 
-	Sort  int
-	Path  string
-	Store StoreType
+	Sort  int       `json:"sort,omitempty" yaml:"sort,omitempty"` // Sort order for display, default is 0
+	Path  string    `json:"path" yaml:"path"`                     // File system path or identifier
+	Store StoreType `json:"store" yaml:"store"`                   // Storage type (file or database)
 
-	Readable bool
-	Builtin  bool
+	Readable bool `json:"readable,omitempty" yaml:"readable,omitempty"` // Whether the DSL is readable
+	Builtin  bool `json:"built_in,omitempty" yaml:"built_in,omitempty"` // Whether this is a built-in DSL
 
-	Status Status
-	Mtime  time.Time
-	Ctime  time.Time
+	Status Status    `json:"status,omitempty" yaml:"status,omitempty"` // Current status (loading, loaded, error)
+	Mtime  time.Time `json:"mtime" yaml:"mtime"`                       // Last modification timestamp
+	Ctime  time.Time `json:"ctime" yaml:"ctime"`                       // Creation timestamp
+
+	Source string `json:"source,omitempty" yaml:"source,omitempty"` // Source content, only available when explicitly requested
 }
 
 // ListOptions for DSL list
 type ListOptions struct {
-	Sort  string
-	Order string
-	Tags  []string
+	Sort   string
+	Order  string
+	Store  StoreType
+	Source bool
+	Tags   []string
 }
 
 // CreateOptions for DSL upsert

model/model.go

@@ -1,9 +1,11 @@
 package model
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"strings"
+	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/yaoapp/gou/application"
@@ -11,6 +13,8 @@ import (
 	"github.com/yaoapp/kun/log"
 	"github.com/yaoapp/yao/config"
 	"github.com/yaoapp/yao/data"
+	"github.com/yaoapp/yao/dsl"
+	"github.com/yaoapp/yao/dsl/types"
 	"github.com/yaoapp/yao/share"
 )
 
@@ -54,18 +58,23 @@ func Load(cfg config.Config) error {
 	}, exts...)
 
 	if len(messages) > 0 {
+		for _, message := range messages {
+			log.Error("Load filesystem models error: %s", message)
+		}
 		return fmt.Errorf(strings.Join(messages, ";\n"))
 	}
 
 	// Load database models ( ignore error)
-	err = loadDatabaseModels()
-	if err != nil {
-		log.Error("load database models error: %s", err.Error())
+	errs := loadDatabaseModels()
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error("Load database models error: %s", err.Error())
+		}
 	}
-
 	return err
 }
 
+// LoadSystemModels load system models
 func loadSystemModels() error {
 	for id, path := range systemModels {
 		content, err := data.Read(path)
@@ -100,7 +109,7 @@ func loadSystemModels() error {
 		}
 
 		// Auto migrate
-		err = mod.Migrate(true, model.WithDonotInsertValues(true))
+		err = mod.Migrate(false, model.WithDonotInsertValues(true))
 		if err != nil {
 			log.Error("migrate system model %s error: %s", id, err.Error())
 			return err
@@ -110,6 +119,33 @@ func loadSystemModels() error {
 	return nil
 }
 
-func loadDatabaseModels() error {
-	return nil
+// LoadDatabaseModels load database models
+func loadDatabaseModels() []error {
+
+	var errs []error = []error{}
+	manager, err := dsl.New(types.TypeModel)
+	if err != nil {
+		errs = append(errs, err)
+		return errs
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	models, err := manager.List(ctx, &types.ListOptions{Store: types.StoreTypeDB, Source: true})
+	if err != nil {
+		errs = append(errs, err)
+		return errs
+	}
+
+	// Load models
+	for _, info := range models {
+		_, err := model.LoadSource([]byte(info.Source), info.ID, info.Path)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+	}
+
+	return errs
 }