feat(database): add PostgreSQL support and enhance JSON handling
- Updated database configuration to include PostgreSQL 14.0 in CI workflows. - Enhanced JSON null detection in the Xun store to support PostgreSQL dialect. - Refactored time conversion utilities to handle multiple database formats, including PostgreSQL. - Improved cleanup logic in the DataCleaner to ensure proper order of operations for referential integrity. - Added utility functions for converting between Unix timestamps and database timestamps.
This commit is contained in:
Max
parent
93641dea58
commit
8786aa9a6a
14 changed files with 503 additions and 185 deletions
8
.github/workflows/pr-test.yml
vendored
8
.github/workflows/pr-test.yml
vendored
|
|
@ -1227,7 +1227,7 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
db: [MySQL8.0, SQLite3]
|
||||
db: [MySQL8.0, SQLite3, Postgres14.0]
|
||||
if: >
|
||||
${{ github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.conclusion == 'success' }}
|
||||
|
|
@ -1358,6 +1358,8 @@ jobs:
|
|||
echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
|
||||
if [ "$DB_DRIVER" = "mysql" ]; then
|
||||
echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
|
||||
elif [ "$DB_DRIVER" = "postgres" ]; then
|
||||
echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
|
||||
else
|
||||
echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
|
||||
mkdir -p ${{ github.WORKSPACE }}/../app/db
|
||||
|
|
@ -1404,8 +1406,8 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
db: [MySQL8.0, SQLite3]
|
||||
redis: [4, 5, 6]
|
||||
db: [MySQL8.0, SQLite3, Postgres14.0]
|
||||
redis: [6]
|
||||
mongo: ["6.0"]
|
||||
if: >
|
||||
${{ github.event.workflow_run.event == 'pull_request' &&
|
||||
|
|
|
|||
8
.github/workflows/unit-test.yml
vendored
8
.github/workflows/unit-test.yml
vendored
|
|
@ -924,7 +924,7 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
db: [MySQL8.0, SQLite3]
|
||||
db: [MySQL8.0, SQLite3, Postgres14.0]
|
||||
steps:
|
||||
- name: Checkout Kun
|
||||
uses: actions/checkout@v4
|
||||
|
|
@ -1021,6 +1021,8 @@ jobs:
|
|||
echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV
|
||||
if [ "$DB_DRIVER" = "mysql" ]; then
|
||||
echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
|
||||
elif [ "$DB_DRIVER" = "postgres" ]; then
|
||||
echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV
|
||||
else
|
||||
echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
|
||||
mkdir -p ${{ github.WORKSPACE }}/../app/db
|
||||
|
|
@ -1066,8 +1068,8 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
db: [MySQL8.0, SQLite3]
|
||||
redis: [4, 5, 6]
|
||||
db: [MySQL8.0, SQLite3, Postgres14.0]
|
||||
redis: [6]
|
||||
mongo: ["6.0"]
|
||||
steps:
|
||||
- name: Checkout Kun
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -82,3 +82,4 @@ agent/robot/ROBOT-IM-INTEGRATION-IMPROVEMENT.md
|
|||
agent/robot/ROBOT-CACHE-IMPROVEMENT.md
|
||||
sandbox/v2/PID-KILL-UPGRADE.md
|
||||
sandbox/v2/*.md
|
||||
POSTGRESQL_COMPAT.md
|
||||
|
|
|
|||
|
|
@ -382,16 +382,15 @@ func (store *Xun) GetAssistants(filter types.AssistantFilter, locale ...string)
|
|||
}
|
||||
|
||||
// Apply sandbox filter (true = has sandbox config, false = no sandbox config)
|
||||
// MySQL JSON columns distinguish between SQL NULL and JSON literal null.
|
||||
// CAST(sandbox AS CHAR) returns 'null' for JSON null and NULL for SQL NULL.
|
||||
// DB JSON columns distinguish between SQL NULL and JSON literal null.
|
||||
// Dialect-specific: MySQL uses CAST(... AS CHAR), PG uses ::text, SQLite uses CAST(... AS TEXT).
|
||||
if filter.Sandbox != nil {
|
||||
notNull, isNull := store.sandboxRawSQL()
|
||||
if *filter.Sandbox {
|
||||
qb.WhereNotNull("sandbox").
|
||||
WhereRaw("CAST(`sandbox` AS CHAR) <> 'null'")
|
||||
qb.WhereNotNull("sandbox").WhereRaw(notNull)
|
||||
} else {
|
||||
qb.Where(func(qb query.Query) {
|
||||
qb.WhereNull("sandbox").
|
||||
OrWhereRaw("CAST(`sandbox` AS CHAR) = 'null'")
|
||||
qb.WhereNull("sandbox").OrWhereRaw(isNull)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -906,3 +905,16 @@ func (store *Xun) translate(model *types.AssistantModel, assistantID string, loc
|
|||
// in their original (English) form so that filter.tags round-trips
|
||||
// correctly through the LIKE query on the DB column.
|
||||
}
|
||||
|
||||
// sandboxRawSQL returns dialect-specific raw SQL fragments for sandbox JSON null detection.
|
||||
// Returns (notNullExpr, isNullExpr) for filtering sandbox field.
|
||||
func (store *Xun) sandboxRawSQL() (string, string) {
|
||||
switch store.getDriver() {
|
||||
case "postgres":
|
||||
return `"sandbox"::text <> 'null'`, `"sandbox"::text = 'null'`
|
||||
case "sqlite3":
|
||||
return `CAST(sandbox AS TEXT) <> 'null'`, `CAST(sandbox AS TEXT) = 'null'`
|
||||
default:
|
||||
return "CAST(`sandbox` AS CHAR) <> 'null'", "CAST(`sandbox` AS CHAR) = 'null'"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/yaoapp/yao/openapi/utils"
|
||||
)
|
||||
|
||||
// isNil checks whether a value is truly nil, handling the Go typed-nil-in-interface pitfall.
|
||||
|
|
@ -49,61 +50,30 @@ func getString(data map[string]interface{}, key string) string {
|
|||
}
|
||||
|
||||
func getBool(data map[string]interface{}, key string) bool {
|
||||
switch v := data[key].(type) {
|
||||
case bool:
|
||||
return v
|
||||
case int64:
|
||||
return v != 0
|
||||
case int:
|
||||
return v != 0
|
||||
case float64:
|
||||
return v != 0
|
||||
}
|
||||
return false
|
||||
return utils.ToBool(data[key])
|
||||
}
|
||||
|
||||
func getInt(data map[string]interface{}, key string) int {
|
||||
switch v := data[key].(type) {
|
||||
case int:
|
||||
return v
|
||||
case int64:
|
||||
return int(v)
|
||||
case float64:
|
||||
return int(v)
|
||||
}
|
||||
return 0
|
||||
return utils.ToInt(data[key])
|
||||
}
|
||||
|
||||
func getInt64(data map[string]interface{}, key string) int64 {
|
||||
switch v := data[key].(type) {
|
||||
case int64:
|
||||
return v
|
||||
case int:
|
||||
return int64(v)
|
||||
case float64:
|
||||
return int64(v)
|
||||
case string:
|
||||
// Handle string representation of numbers (common with MySQL BIGINT)
|
||||
var result int64
|
||||
if _, err := fmt.Sscanf(v, "%d", &result); err == nil {
|
||||
return result
|
||||
v := data[key]
|
||||
if t, ok := v.(time.Time); ok {
|
||||
return t.UnixNano()
|
||||
}
|
||||
case time.Time:
|
||||
// Handle time.Time from database
|
||||
return v.UnixNano()
|
||||
}
|
||||
return 0
|
||||
return utils.ToInt64(v)
|
||||
}
|
||||
|
||||
// toMySQLTime converts UnixNano timestamp to MySQL BIGINT format
|
||||
func toMySQLTime(unixNano int64) int64 {
|
||||
// toDBTime converts UnixNano timestamp to database BIGINT format
|
||||
func toDBTime(unixNano int64) int64 {
|
||||
if unixNano == 0 {
|
||||
return 0
|
||||
}
|
||||
return unixNano
|
||||
}
|
||||
|
||||
// fromMySQLTime converts MySQL BIGINT timestamp to UnixNano
|
||||
func fromMySQLTime(mysqlTime int64) int64 {
|
||||
return mysqlTime
|
||||
// fromDBTime converts database BIGINT timestamp to UnixNano
|
||||
func fromDBTime(dbTime int64) int64 {
|
||||
return dbTime
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,17 @@ func (store *Xun) parseJSONFields(data map[string]interface{}, fields []string)
|
|||
}
|
||||
}
|
||||
|
||||
// getDriver returns the database driver name for dialect-aware SQL.
|
||||
// Defaults to "mysql" if the driver cannot be determined.
|
||||
func (store *Xun) getDriver() string {
|
||||
if store.query != nil {
|
||||
if driver, err := store.query.Driver(); err == nil {
|
||||
return driver
|
||||
}
|
||||
}
|
||||
return "mysql"
|
||||
}
|
||||
|
||||
// GenerateAssistantID generates a random-looking 6-digit ID
|
||||
func (store *Xun) GenerateAssistantID() (string, error) {
|
||||
maxAttempts := 10 // Maximum number of attempts to generate a unique ID
|
||||
|
|
|
|||
98
agent/store/xun/xun_internal_test.go
Normal file
98
agent/store/xun/xun_internal_test.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package xun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/xun/capsule"
|
||||
"github.com/yaoapp/yao/agent/store/types"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/test"
|
||||
)
|
||||
|
||||
func TestGetDriver(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
store, err := NewXun(typeSetting("default"))
|
||||
assert.NoError(t, err)
|
||||
xunStore := store.(*Xun)
|
||||
|
||||
driver := xunStore.getDriver()
|
||||
assert.Contains(t, []string{"mysql", "postgres", "sqlite3"}, driver)
|
||||
|
||||
cfg := config.Conf
|
||||
if cfg.DB.Driver != "" {
|
||||
assert.Equal(t, cfg.DB.Driver, driver)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxRawSQL(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
store, err := NewXun(typeSetting("default"))
|
||||
assert.NoError(t, err)
|
||||
xunStore := store.(*Xun)
|
||||
|
||||
notNull, isNull := xunStore.sandboxRawSQL()
|
||||
assert.NotEmpty(t, notNull)
|
||||
assert.NotEmpty(t, isNull)
|
||||
assert.Contains(t, notNull, "null")
|
||||
assert.Contains(t, isNull, "null")
|
||||
|
||||
driver := xunStore.getDriver()
|
||||
switch driver {
|
||||
case "postgres":
|
||||
assert.Contains(t, notNull, `"sandbox"::text`)
|
||||
assert.Contains(t, isNull, `"sandbox"::text`)
|
||||
case "sqlite3":
|
||||
assert.Contains(t, notNull, "CAST(sandbox AS TEXT)")
|
||||
assert.Contains(t, isNull, "CAST(sandbox AS TEXT)")
|
||||
default:
|
||||
assert.Contains(t, notNull, "CAST(`sandbox` AS CHAR)")
|
||||
assert.Contains(t, isNull, "CAST(`sandbox` AS CHAR)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDriverFallback(t *testing.T) {
|
||||
store := &Xun{}
|
||||
driver := store.getDriver()
|
||||
assert.Equal(t, "mysql", driver)
|
||||
}
|
||||
|
||||
func typeSetting(connector string) types.Setting {
|
||||
return types.Setting{Connector: connector}
|
||||
}
|
||||
|
||||
// TestSandboxRawSQLAllDialects verifies SQL fragments for all three dialects
|
||||
// without requiring a live database connection.
|
||||
func TestSandboxRawSQLAllDialects(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
s, err := NewXun(typeSetting("default"))
|
||||
assert.NoError(t, err)
|
||||
xunStore := s.(*Xun)
|
||||
|
||||
// Verify the current driver produces valid SQL fragments
|
||||
notNull, isNull := xunStore.sandboxRawSQL()
|
||||
assert.Contains(t, notNull, "<>")
|
||||
assert.Contains(t, isNull, "=")
|
||||
}
|
||||
|
||||
func TestToDBTime(t *testing.T) {
|
||||
assert.Equal(t, int64(0), toDBTime(0))
|
||||
assert.Equal(t, int64(1234567890), toDBTime(1234567890))
|
||||
assert.Equal(t, int64(-1), toDBTime(-1))
|
||||
}
|
||||
|
||||
func TestFromDBTime(t *testing.T) {
|
||||
assert.Equal(t, int64(0), fromDBTime(0))
|
||||
assert.Equal(t, int64(1234567890), fromDBTime(1234567890))
|
||||
assert.Equal(t, int64(-1), fromDBTime(-1))
|
||||
}
|
||||
|
||||
func init() {
|
||||
_ = capsule.Global
|
||||
}
|
||||
11
job/data.go
11
job/data.go
|
|
@ -1204,7 +1204,7 @@ func mapToStruct(m maps.MapStr, v interface{}) error {
|
|||
case float64:
|
||||
cleanMap[key] = val != 0
|
||||
case string:
|
||||
cleanMap[key] = val == "true" || val == "1"
|
||||
cleanMap[key] = val == "true" || val == "1" || val == "t"
|
||||
default:
|
||||
cleanMap[key] = value
|
||||
}
|
||||
|
|
@ -1213,9 +1213,12 @@ func mapToStruct(m maps.MapStr, v interface{}) error {
|
|||
if str, ok := value.(string); ok && str != "" {
|
||||
// Try multiple time formats
|
||||
formats := []string{
|
||||
"2006-01-02 15:04:05", // MySQL format
|
||||
"2006-01-02T15:04:05Z07:00", // RFC3339
|
||||
"2006-01-02T15:04:05.999999999Z07:00", // RFC3339 with nanoseconds
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02 15:04:05.999999",
|
||||
"2006-01-02 15:04:05.999999-07",
|
||||
"2006-01-02 15:04:05.999999+00",
|
||||
"2006-01-02T15:04:05Z07:00",
|
||||
"2006-01-02T15:04:05.999999999Z07:00",
|
||||
time.RFC3339,
|
||||
time.RFC3339Nano,
|
||||
}
|
||||
|
|
|
|||
97
job/data_internal_test.go
Normal file
97
job/data_internal_test.go
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
package job
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/kun/maps"
|
||||
)
|
||||
|
||||
func TestMapToStructBooleanFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
value interface{}
|
||||
expected bool
|
||||
}{
|
||||
{"bool true", "enabled", true, true},
|
||||
{"bool false", "enabled", false, false},
|
||||
{"string true", "enabled", "true", true},
|
||||
{"string 1", "enabled", "1", true},
|
||||
{"string t (PG)", "enabled", "t", true},
|
||||
{"string false", "enabled", "false", false},
|
||||
{"string 0", "enabled", "0", false},
|
||||
{"string f (PG)", "enabled", "f", false},
|
||||
{"int 1", "enabled", int(1), true},
|
||||
{"int 0", "enabled", int(0), false},
|
||||
{"int64 1", "enabled", int64(1), true},
|
||||
{"int64 0", "enabled", int64(0), false},
|
||||
{"float64 1", "enabled", float64(1), true},
|
||||
{"float64 0", "enabled", float64(0), false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := maps.MapStr{tt.key: tt.value}
|
||||
job := &Job{}
|
||||
err := mapToStruct(m, job)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, job.Enabled, "field %s with value %v", tt.key, tt.value)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("system field", func(t *testing.T) {
|
||||
m := maps.MapStr{"system": "t"}
|
||||
job := &Job{}
|
||||
err := mapToStruct(m, job)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, job.System)
|
||||
})
|
||||
|
||||
t.Run("readonly field", func(t *testing.T) {
|
||||
m := maps.MapStr{"readonly": "1"}
|
||||
job := &Job{}
|
||||
err := mapToStruct(m, job)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, job.Readonly)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMapToStructTimeFields(t *testing.T) {
|
||||
refTime := time.Date(2024, 6, 15, 10, 30, 45, 0, time.UTC)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
parsed bool
|
||||
}{
|
||||
{"MySQL format", "2024-06-15 10:30:45", true},
|
||||
{"PG fractional", "2024-06-15 10:30:45.123456", true},
|
||||
{"PG with +00", "2024-06-15 10:30:45.123456+00", true},
|
||||
{"PG with -07", "2024-06-15 03:30:45.123456-07", true},
|
||||
{"RFC3339", "2024-06-15T10:30:45Z", true},
|
||||
{"RFC3339Nano", "2024-06-15T10:30:45.123456789Z", true},
|
||||
{"empty string", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := maps.MapStr{
|
||||
"name": "test-job",
|
||||
"job_id": "j-123",
|
||||
"created_at": tt.input,
|
||||
}
|
||||
job := &Job{}
|
||||
err := mapToStruct(m, job)
|
||||
if tt.parsed {
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, job.CreatedAt.IsZero(), "expected non-zero time for input: %s", tt.input)
|
||||
assert.Equal(t, refTime.Unix(), job.CreatedAt.Unix(),
|
||||
"expected %v, got %v for input: %s", refTime, job.CreatedAt, tt.input)
|
||||
} else {
|
||||
assert.True(t, job.CreatedAt.IsZero())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -315,28 +315,30 @@ func (dc *DataCleaner) shouldRunCleanup() bool {
|
|||
return time.Since(dc.lastCleanupTime) >= 24*time.Hour
|
||||
}
|
||||
|
||||
// performCleanup performs the actual data cleanup
|
||||
// performCleanup performs the actual data cleanup.
|
||||
// Deletion order: logs -> executions -> jobs (children first) to avoid
|
||||
// foreign key violations on databases that enforce referential integrity (e.g. PostgreSQL).
|
||||
func (dc *DataCleaner) performCleanup() error {
|
||||
log.Info("Starting daily data cleanup...")
|
||||
|
||||
cutoffTime := time.Now().AddDate(0, 0, -dc.retentionDays)
|
||||
|
||||
// Clean up jobs (excluding running jobs)
|
||||
deletedJobs, err := dc.cleanupJobs(cutoffTime)
|
||||
// Clean up logs first (leaf records)
|
||||
deletedLogs, err := dc.cleanupLogs(cutoffTime)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to cleanup jobs: %w", err)
|
||||
return fmt.Errorf("failed to cleanup logs: %w", err)
|
||||
}
|
||||
|
||||
// Clean up executions
|
||||
// Clean up executions (reference jobs)
|
||||
deletedExecutions, err := dc.cleanupExecutions(cutoffTime)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to cleanup executions: %w", err)
|
||||
}
|
||||
|
||||
// Clean up logs
|
||||
deletedLogs, err := dc.cleanupLogs(cutoffTime)
|
||||
// Clean up jobs last (parent records)
|
||||
deletedJobs, err := dc.cleanupJobs(cutoffTime)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to cleanup logs: %w", err)
|
||||
return fmt.Errorf("failed to cleanup jobs: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Data cleanup completed: %d jobs, %d executions, %d logs deleted",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package types
|
||||
|
||||
import (
|
||||
"time"
|
||||
"github.com/yaoapp/yao/openapi/utils"
|
||||
)
|
||||
|
||||
// Map converts the OIDCUserInfo to a map[string]interface{}, excluding empty values
|
||||
|
|
@ -67,7 +67,7 @@ func (user OIDCUserInfo) Map() map[string]interface{} {
|
|||
}
|
||||
|
||||
// Convert and add UpdatedAt if not nil
|
||||
if converted := unixToMySQL(user.UpdatedAt); converted != nil {
|
||||
if converted := utils.UnixToDBTimestamp(user.UpdatedAt); converted != nil {
|
||||
result["updated_at"] = converted
|
||||
}
|
||||
|
||||
|
|
@ -132,7 +132,7 @@ func (user OIDCUserInfo) Map() map[string]interface{} {
|
|||
if user.YaoTeam.Description != "" {
|
||||
teamMap["description"] = user.YaoTeam.Description
|
||||
}
|
||||
if converted := unixToMySQL(user.YaoTeam.UpdatedAt); converted != nil {
|
||||
if converted := utils.UnixToDBTimestamp(user.YaoTeam.UpdatedAt); converted != nil {
|
||||
teamMap["updated_at"] = converted
|
||||
}
|
||||
if len(teamMap) > 0 {
|
||||
|
|
@ -264,7 +264,7 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo {
|
|||
|
||||
// Updated_at field
|
||||
if updatedAt, ok := user["updated_at"]; ok {
|
||||
if converted := toUnixTimestamp(updatedAt); converted != nil {
|
||||
if converted := utils.ToUnixTimestamp(updatedAt); converted != nil {
|
||||
if unixTime, ok := converted.(int64); ok {
|
||||
userInfo.UpdatedAt = &unixTime
|
||||
}
|
||||
|
|
@ -336,7 +336,7 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo {
|
|||
team.Description = description
|
||||
}
|
||||
if updatedAt, ok := teamData["updated_at"]; ok {
|
||||
if converted := toUnixTimestamp(updatedAt); converted != nil {
|
||||
if converted := utils.ToUnixTimestamp(updatedAt); converted != nil {
|
||||
if unixTime, ok := converted.(int64); ok {
|
||||
team.UpdatedAt = &unixTime
|
||||
}
|
||||
|
|
@ -383,96 +383,3 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo {
|
|||
|
||||
return userInfo
|
||||
}
|
||||
|
||||
// unixToMySQL converts interface{} to MySQL DATETIME string
|
||||
func unixToMySQL(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var unixTime int64
|
||||
switch v := val.(type) {
|
||||
case int64:
|
||||
unixTime = v
|
||||
case *int64:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
unixTime = *v
|
||||
case int:
|
||||
unixTime = int64(v)
|
||||
case float64:
|
||||
unixTime = int64(v)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return time.Unix(unixTime, 0).UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
// mysqlToUnix converts interface{} to Unix timestamp
|
||||
func mysqlToUnix(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var dateTime string
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
dateTime = v
|
||||
case *string:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
dateTime = *v
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
if dateTime == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try MySQL DATETIME format
|
||||
if t, err := time.Parse("2006-01-02 15:04:05", dateTime); err == nil {
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// Try ISO format as fallback
|
||||
if t, err := time.Parse("2006-01-02T15:04:05Z", dateTime); err == nil {
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// toUnixTimestamp converts any interface{} to Unix timestamp
|
||||
func toUnixTimestamp(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch v := val.(type) {
|
||||
case int64:
|
||||
return v
|
||||
case *int64:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return *v
|
||||
case int:
|
||||
return int64(v)
|
||||
case float64:
|
||||
return int64(v)
|
||||
case string:
|
||||
// Handle MySQL DATETIME or ISO format
|
||||
return mysqlToUnix(v)
|
||||
case *string:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return mysqlToUnix(*v)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,22 +178,12 @@ func ToTimeString(v interface{}) string {
|
|||
}
|
||||
return val.Format(time.RFC3339)
|
||||
case string:
|
||||
// Try to parse as RFC3339 first
|
||||
if t, err := time.Parse(time.RFC3339, val); err == nil {
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
// Try to parse as other common formats
|
||||
formats := []string{
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05Z",
|
||||
"2006-01-02T15:04:05.000Z",
|
||||
}
|
||||
for _, format := range formats {
|
||||
for _, format := range DBTimeFormats {
|
||||
if t, err := time.Parse(format, val); err == nil {
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
return val // Return as-is if can't parse
|
||||
return val
|
||||
case int64:
|
||||
// Assume unix timestamp
|
||||
if val > 0 {
|
||||
|
|
@ -224,6 +214,113 @@ func GetTimeFormat(locale string) string {
|
|||
}
|
||||
}
|
||||
|
||||
// DBTimeFormats contains all time formats recognized by database drivers (MySQL, PostgreSQL, SQLite).
|
||||
// Ordered from most specific to least specific for efficient parsing.
|
||||
var DBTimeFormats = []string{
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02 15:04:05.999999",
|
||||
"2006-01-02 15:04:05.999999-07",
|
||||
"2006-01-02 15:04:05.999999+00",
|
||||
"2006-01-02T15:04:05Z",
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02T15:04:05.000Z",
|
||||
time.RFC3339,
|
||||
time.RFC3339Nano,
|
||||
}
|
||||
|
||||
// DBTimestampToUnix converts a database timestamp string (interface{}) to a Unix timestamp (int64).
|
||||
// Supports MySQL DATETIME, PostgreSQL timestamptz (fractional seconds, timezone offsets),
|
||||
// ISO 8601, RFC3339, and RFC3339Nano. Returns nil for nil, empty, or unparseable input.
|
||||
func DBTimestampToUnix(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var dateTime string
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
dateTime = v
|
||||
case *string:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
dateTime = *v
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
if dateTime == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, format := range DBTimeFormats {
|
||||
if t, err := time.Parse(format, dateTime); err == nil {
|
||||
return t.Unix()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnixToDBTimestamp converts a Unix timestamp (int64, *int64, int, float64) to a standard
|
||||
// DATETIME string "2006-01-02 15:04:05" in UTC. Compatible with MySQL, PostgreSQL, and SQLite.
|
||||
// Returns nil for nil or unsupported types.
|
||||
func UnixToDBTimestamp(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var unixTime int64
|
||||
switch v := val.(type) {
|
||||
case int64:
|
||||
unixTime = v
|
||||
case *int64:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
unixTime = *v
|
||||
case int:
|
||||
unixTime = int64(v)
|
||||
case float64:
|
||||
unixTime = int64(v)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return time.Unix(unixTime, 0).UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
// ToUnixTimestamp converts any interface{} value to a Unix timestamp (int64).
|
||||
// Handles numeric types directly and parses string/DB timestamp formats via DBTimestampToUnix.
|
||||
// Returns nil for nil or unsupported types.
|
||||
func ToUnixTimestamp(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch v := val.(type) {
|
||||
case int64:
|
||||
return v
|
||||
case *int64:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return *v
|
||||
case int:
|
||||
return int64(v)
|
||||
case float64:
|
||||
return int64(v)
|
||||
case string:
|
||||
return DBTimestampToUnix(v)
|
||||
case *string:
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return DBTimestampToUnix(*v)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// FormatTimeWithLocale formats a time value (time.Time, *time.Time, or string) using the specified format
|
||||
// If the input is already a string, it will parse it first and then reformat it
|
||||
// Returns empty string if the value cannot be parsed
|
||||
|
|
@ -245,21 +342,13 @@ func FormatTimeWithLocale(v interface{}, targetFormat string) string {
|
|||
return ""
|
||||
}
|
||||
case string:
|
||||
// Try parsing with common formats
|
||||
formats := []string{
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05Z",
|
||||
"2006-01-02T15:04:05",
|
||||
time.RFC3339,
|
||||
}
|
||||
for _, format := range formats {
|
||||
for _, format := range DBTimeFormats {
|
||||
t, err = time.Parse(format, val)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// If all parsing attempts failed, return the original string
|
||||
return val
|
||||
}
|
||||
default:
|
||||
|
|
|
|||
122
openapi/utils/convert_test.go
Normal file
122
openapi/utils/convert_test.go
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDBTimestampToUnix(t *testing.T) {
|
||||
ref := time.Date(2024, 6, 15, 10, 30, 45, 0, time.UTC)
|
||||
refUnix := ref.Unix()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expect interface{}
|
||||
}{
|
||||
{"nil", nil, nil},
|
||||
{"empty string", "", nil},
|
||||
{"MySQL DATETIME", "2024-06-15 10:30:45", refUnix},
|
||||
{"PG with fractional seconds", "2024-06-15 10:30:45.123456", refUnix},
|
||||
{"PG with +00 offset", "2024-06-15 10:30:45.123456+00", refUnix},
|
||||
{"PG with -07 offset", "2024-06-15 03:30:45.123456-07", refUnix},
|
||||
{"ISO Z suffix", "2024-06-15T10:30:45Z", refUnix},
|
||||
{"RFC3339", "2024-06-15T10:30:45+00:00", refUnix},
|
||||
{"RFC3339Nano", "2024-06-15T10:30:45.123456789+00:00", refUnix},
|
||||
{"*string nil", (*string)(nil), nil},
|
||||
{"*string valid", strPtr("2024-06-15 10:30:45"), refUnix},
|
||||
{"*string empty", strPtr(""), nil},
|
||||
{"non-string type", 12345, nil},
|
||||
{"unparseable string", "not-a-date", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := DBTimestampToUnix(tt.input)
|
||||
assert.Equal(t, tt.expect, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixToDBTimestamp(t *testing.T) {
|
||||
refUnix := int64(1718444445)
|
||||
expected := time.Unix(refUnix, 0).UTC().Format("2006-01-02 15:04:05")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expect interface{}
|
||||
}{
|
||||
{"nil", nil, nil},
|
||||
{"int64", refUnix, expected},
|
||||
{"*int64 valid", int64Ptr(refUnix), expected},
|
||||
{"*int64 nil", (*int64)(nil), nil},
|
||||
{"int", int(refUnix), expected},
|
||||
{"float64", float64(refUnix), expected},
|
||||
{"unknown type", "not-a-number", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := UnixToDBTimestamp(tt.input)
|
||||
assert.Equal(t, tt.expect, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToUnixTimestamp(t *testing.T) {
|
||||
refUnix := int64(1718444445)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expect interface{}
|
||||
}{
|
||||
{"nil", nil, nil},
|
||||
{"int64", refUnix, refUnix},
|
||||
{"*int64 valid", int64Ptr(refUnix), refUnix},
|
||||
{"*int64 nil", (*int64)(nil), nil},
|
||||
{"int", int(refUnix), refUnix},
|
||||
{"float64", float64(refUnix), refUnix},
|
||||
{"string MySQL", "2024-06-15 10:00:45", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()},
|
||||
{"string PG fractional", "2024-06-15 10:00:45.123456", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()},
|
||||
{"string RFC3339", "2024-06-15T10:00:45Z", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()},
|
||||
{"*string valid", strPtr("2024-06-15 10:00:45"), time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()},
|
||||
{"*string nil", (*string)(nil), nil},
|
||||
{"string empty", "", nil},
|
||||
{"unknown type", struct{}{}, nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := ToUnixTimestamp(tt.input)
|
||||
assert.Equal(t, tt.expect, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToTimeStringWithPGFormats(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expect string
|
||||
}{
|
||||
{"PG fractional", "2024-06-15 10:30:45.123456", "2024-06-15T10:30:45Z"},
|
||||
{"PG +00 offset", "2024-06-15 10:30:45.123456+00", "2024-06-15T10:30:45Z"},
|
||||
{"PG -07 offset", "2024-06-15 03:30:45.123456-07", "2024-06-15T03:30:45-07:00"},
|
||||
{"MySQL format", "2024-06-15 10:30:45", "2024-06-15T10:30:45Z"},
|
||||
{"RFC3339Nano", "2024-06-15T10:30:45.123456789+00:00", "2024-06-15T10:30:45Z"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := ToTimeString(tt.input)
|
||||
assert.Equal(t, tt.expect, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
func int64Ptr(i int64) *int64 { return &i }
|
||||
|
|
@ -571,6 +571,8 @@ func dbconnect(t *testing.T, cfg config.Config) {
|
|||
switch cfg.DB.Driver {
|
||||
case "sqlite3":
|
||||
capsule.AddConn("primary", "sqlite3", cfg.DB.Primary[0]).SetAsGlobal()
|
||||
case "postgres":
|
||||
capsule.AddConn("primary", "postgres", cfg.DB.Primary[0]).SetAsGlobal()
|
||||
default:
|
||||
capsule.AddConn("primary", "mysql", cfg.DB.Primary[0]).SetAsGlobal()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue