From 8786aa9a6af41ee8ba360fb1f2418f2a3cd3700f Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 6 Apr 2026 02:56:04 +0800 Subject: [PATCH] feat(database): add PostgreSQL support and enhance JSON handling - Updated database configuration to include PostgreSQL 14.0 in CI workflows. - Enhanced JSON null detection in the Xun store to support PostgreSQL dialect. - Refactored time conversion utilities to handle multiple database formats, including PostgreSQL. - Improved cleanup logic in the DataCleaner to ensure proper order of operations for referential integrity. - Added utility functions for converting between Unix timestamps and database timestamps. --- .github/workflows/pr-test.yml | 8 +- .github/workflows/unit-test.yml | 8 +- .gitignore | 1 + agent/store/xun/assistant.go | 24 +++-- agent/store/xun/utils.go | 54 +++-------- agent/store/xun/xun.go | 11 +++ agent/store/xun/xun_internal_test.go | 98 ++++++++++++++++++++ job/data.go | 11 ++- job/data_internal_test.go | 97 ++++++++++++++++++++ job/health.go | 18 ++-- openapi/oauth/types/oidc.go | 103 +-------------------- openapi/utils/convert.go | 131 ++++++++++++++++++++++----- openapi/utils/convert_test.go | 122 +++++++++++++++++++++++++ test/utils.go | 2 + 14 files changed, 503 insertions(+), 185 deletions(-) create mode 100644 agent/store/xun/xun_internal_test.go create mode 100644 job/data_internal_test.go create mode 100644 openapi/utils/convert_test.go diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 92f975da..16cc7a35 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1227,7 +1227,7 @@ jobs: strategy: matrix: go: ["1.25"] - db: [MySQL8.0, SQLite3] + db: [MySQL8.0, SQLite3, Postgres14.0] if: > ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} @@ -1358,6 +1358,8 @@ jobs: echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV if [ "$DB_DRIVER" = "mysql" ]; then echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV + elif [ "$DB_DRIVER" = "postgres" ]; then + echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV else echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV mkdir -p ${{ github.WORKSPACE }}/../app/db @@ -1404,8 +1406,8 @@ jobs: strategy: matrix: go: ["1.25"] - db: [MySQL8.0, SQLite3] - redis: [4, 5, 6] + db: [MySQL8.0, SQLite3, Postgres14.0] + redis: [6] mongo: ["6.0"] if: > ${{ github.event.workflow_run.event == 'pull_request' && diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 2de1fb81..4283142c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -924,7 +924,7 @@ jobs: strategy: matrix: go: ["1.25"] - db: [MySQL8.0, SQLite3] + db: [MySQL8.0, SQLite3, Postgres14.0] steps: - name: Checkout Kun uses: actions/checkout@v4 @@ -1021,6 +1021,8 @@ jobs: echo "YAO_DB_DRIVER=$DB_DRIVER" >> $GITHUB_ENV if [ "$DB_DRIVER" = "mysql" ]; then echo "YAO_DB_PRIMARY=$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV + elif [ "$DB_DRIVER" = "postgres" ]; then + echo "YAO_DB_PRIMARY=postgres://$DB_USER:$PASSWORD@$DB_HOST" >> $GITHUB_ENV else echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV mkdir -p ${{ github.WORKSPACE }}/../app/db @@ -1066,8 +1068,8 @@ jobs: strategy: matrix: go: ["1.25"] - db: [MySQL8.0, SQLite3] - redis: [4, 5, 6] + db: [MySQL8.0, SQLite3, Postgres14.0] + redis: [6] mongo: ["6.0"] steps: - name: Checkout Kun diff --git a/.gitignore b/.gitignore index ba97bb83..300db619 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,4 @@ agent/robot/ROBOT-IM-INTEGRATION-IMPROVEMENT.md agent/robot/ROBOT-CACHE-IMPROVEMENT.md sandbox/v2/PID-KILL-UPGRADE.md sandbox/v2/*.md +POSTGRESQL_COMPAT.md diff --git a/agent/store/xun/assistant.go b/agent/store/xun/assistant.go index 74c62df7..5369e140 100644 --- a/agent/store/xun/assistant.go +++ b/agent/store/xun/assistant.go @@ -382,16 +382,15 @@ func (store *Xun) GetAssistants(filter types.AssistantFilter, locale ...string) } // Apply sandbox filter (true = has sandbox config, false = no sandbox config) - // MySQL JSON columns distinguish between SQL NULL and JSON literal null. - // CAST(sandbox AS CHAR) returns 'null' for JSON null and NULL for SQL NULL. + // DB JSON columns distinguish between SQL NULL and JSON literal null. + // Dialect-specific: MySQL uses CAST(... AS CHAR), PG uses ::text, SQLite uses CAST(... AS TEXT). if filter.Sandbox != nil { + notNull, isNull := store.sandboxRawSQL() if *filter.Sandbox { - qb.WhereNotNull("sandbox"). - WhereRaw("CAST(`sandbox` AS CHAR) <> 'null'") + qb.WhereNotNull("sandbox").WhereRaw(notNull) } else { qb.Where(func(qb query.Query) { - qb.WhereNull("sandbox"). - OrWhereRaw("CAST(`sandbox` AS CHAR) = 'null'") + qb.WhereNull("sandbox").OrWhereRaw(isNull) }) } } @@ -906,3 +905,16 @@ func (store *Xun) translate(model *types.AssistantModel, assistantID string, loc // in their original (English) form so that filter.tags round-trips // correctly through the LIKE query on the DB column. } + +// sandboxRawSQL returns dialect-specific raw SQL fragments for sandbox JSON null detection. +// Returns (notNullExpr, isNullExpr) for filtering sandbox field. +func (store *Xun) sandboxRawSQL() (string, string) { + switch store.getDriver() { + case "postgres": + return `"sandbox"::text <> 'null'`, `"sandbox"::text = 'null'` + case "sqlite3": + return `CAST(sandbox AS TEXT) <> 'null'`, `CAST(sandbox AS TEXT) = 'null'` + default: + return "CAST(`sandbox` AS CHAR) <> 'null'", "CAST(`sandbox` AS CHAR) = 'null'" + } +} diff --git a/agent/store/xun/utils.go b/agent/store/xun/utils.go index 13d97d1d..084747bf 100644 --- a/agent/store/xun/utils.go +++ b/agent/store/xun/utils.go @@ -6,6 +6,7 @@ import ( "time" jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/yao/openapi/utils" ) // isNil checks whether a value is truly nil, handling the Go typed-nil-in-interface pitfall. @@ -49,61 +50,30 @@ func getString(data map[string]interface{}, key string) string { } func getBool(data map[string]interface{}, key string) bool { - switch v := data[key].(type) { - case bool: - return v - case int64: - return v != 0 - case int: - return v != 0 - case float64: - return v != 0 - } - return false + return utils.ToBool(data[key]) } func getInt(data map[string]interface{}, key string) int { - switch v := data[key].(type) { - case int: - return v - case int64: - return int(v) - case float64: - return int(v) - } - return 0 + return utils.ToInt(data[key]) } func getInt64(data map[string]interface{}, key string) int64 { - switch v := data[key].(type) { - case int64: - return v - case int: - return int64(v) - case float64: - return int64(v) - case string: - // Handle string representation of numbers (common with MySQL BIGINT) - var result int64 - if _, err := fmt.Sscanf(v, "%d", &result); err == nil { - return result - } - case time.Time: - // Handle time.Time from database - return v.UnixNano() + v := data[key] + if t, ok := v.(time.Time); ok { + return t.UnixNano() } - return 0 + return utils.ToInt64(v) } -// toMySQLTime converts UnixNano timestamp to MySQL BIGINT format -func toMySQLTime(unixNano int64) int64 { +// toDBTime converts UnixNano timestamp to database BIGINT format +func toDBTime(unixNano int64) int64 { if unixNano == 0 { return 0 } return unixNano } -// fromMySQLTime converts MySQL BIGINT timestamp to UnixNano -func fromMySQLTime(mysqlTime int64) int64 { - return mysqlTime +// fromDBTime converts database BIGINT timestamp to UnixNano +func fromDBTime(dbTime int64) int64 { + return dbTime } diff --git a/agent/store/xun/xun.go b/agent/store/xun/xun.go index e04a4131..181249b6 100644 --- a/agent/store/xun/xun.go +++ b/agent/store/xun/xun.go @@ -187,6 +187,17 @@ func (store *Xun) parseJSONFields(data map[string]interface{}, fields []string) } } +// getDriver returns the database driver name for dialect-aware SQL. +// Defaults to "mysql" if the driver cannot be determined. +func (store *Xun) getDriver() string { + if store.query != nil { + if driver, err := store.query.Driver(); err == nil { + return driver + } + } + return "mysql" +} + // GenerateAssistantID generates a random-looking 6-digit ID func (store *Xun) GenerateAssistantID() (string, error) { maxAttempts := 10 // Maximum number of attempts to generate a unique ID diff --git a/agent/store/xun/xun_internal_test.go b/agent/store/xun/xun_internal_test.go new file mode 100644 index 00000000..fc64199f --- /dev/null +++ b/agent/store/xun/xun_internal_test.go @@ -0,0 +1,98 @@ +package xun + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/xun/capsule" + "github.com/yaoapp/yao/agent/store/types" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestGetDriver(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + store, err := NewXun(typeSetting("default")) + assert.NoError(t, err) + xunStore := store.(*Xun) + + driver := xunStore.getDriver() + assert.Contains(t, []string{"mysql", "postgres", "sqlite3"}, driver) + + cfg := config.Conf + if cfg.DB.Driver != "" { + assert.Equal(t, cfg.DB.Driver, driver) + } +} + +func TestSandboxRawSQL(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + store, err := NewXun(typeSetting("default")) + assert.NoError(t, err) + xunStore := store.(*Xun) + + notNull, isNull := xunStore.sandboxRawSQL() + assert.NotEmpty(t, notNull) + assert.NotEmpty(t, isNull) + assert.Contains(t, notNull, "null") + assert.Contains(t, isNull, "null") + + driver := xunStore.getDriver() + switch driver { + case "postgres": + assert.Contains(t, notNull, `"sandbox"::text`) + assert.Contains(t, isNull, `"sandbox"::text`) + case "sqlite3": + assert.Contains(t, notNull, "CAST(sandbox AS TEXT)") + assert.Contains(t, isNull, "CAST(sandbox AS TEXT)") + default: + assert.Contains(t, notNull, "CAST(`sandbox` AS CHAR)") + assert.Contains(t, isNull, "CAST(`sandbox` AS CHAR)") + } +} + +func TestGetDriverFallback(t *testing.T) { + store := &Xun{} + driver := store.getDriver() + assert.Equal(t, "mysql", driver) +} + +func typeSetting(connector string) types.Setting { + return types.Setting{Connector: connector} +} + +// TestSandboxRawSQLAllDialects verifies SQL fragments for all three dialects +// without requiring a live database connection. +func TestSandboxRawSQLAllDialects(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + s, err := NewXun(typeSetting("default")) + assert.NoError(t, err) + xunStore := s.(*Xun) + + // Verify the current driver produces valid SQL fragments + notNull, isNull := xunStore.sandboxRawSQL() + assert.Contains(t, notNull, "<>") + assert.Contains(t, isNull, "=") +} + +func TestToDBTime(t *testing.T) { + assert.Equal(t, int64(0), toDBTime(0)) + assert.Equal(t, int64(1234567890), toDBTime(1234567890)) + assert.Equal(t, int64(-1), toDBTime(-1)) +} + +func TestFromDBTime(t *testing.T) { + assert.Equal(t, int64(0), fromDBTime(0)) + assert.Equal(t, int64(1234567890), fromDBTime(1234567890)) + assert.Equal(t, int64(-1), fromDBTime(-1)) +} + +func init() { + _ = capsule.Global +} diff --git a/job/data.go b/job/data.go index d5e5ccba..13e62dc7 100644 --- a/job/data.go +++ b/job/data.go @@ -1204,7 +1204,7 @@ func mapToStruct(m maps.MapStr, v interface{}) error { case float64: cleanMap[key] = val != 0 case string: - cleanMap[key] = val == "true" || val == "1" + cleanMap[key] = val == "true" || val == "1" || val == "t" default: cleanMap[key] = value } @@ -1213,9 +1213,12 @@ func mapToStruct(m maps.MapStr, v interface{}) error { if str, ok := value.(string); ok && str != "" { // Try multiple time formats formats := []string{ - "2006-01-02 15:04:05", // MySQL format - "2006-01-02T15:04:05Z07:00", // RFC3339 - "2006-01-02T15:04:05.999999999Z07:00", // RFC3339 with nanoseconds + "2006-01-02 15:04:05", + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05.999999-07", + "2006-01-02 15:04:05.999999+00", + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05.999999999Z07:00", time.RFC3339, time.RFC3339Nano, } diff --git a/job/data_internal_test.go b/job/data_internal_test.go new file mode 100644 index 00000000..23122f54 --- /dev/null +++ b/job/data_internal_test.go @@ -0,0 +1,97 @@ +package job + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/kun/maps" +) + +func TestMapToStructBooleanFields(t *testing.T) { + tests := []struct { + name string + key string + value interface{} + expected bool + }{ + {"bool true", "enabled", true, true}, + {"bool false", "enabled", false, false}, + {"string true", "enabled", "true", true}, + {"string 1", "enabled", "1", true}, + {"string t (PG)", "enabled", "t", true}, + {"string false", "enabled", "false", false}, + {"string 0", "enabled", "0", false}, + {"string f (PG)", "enabled", "f", false}, + {"int 1", "enabled", int(1), true}, + {"int 0", "enabled", int(0), false}, + {"int64 1", "enabled", int64(1), true}, + {"int64 0", "enabled", int64(0), false}, + {"float64 1", "enabled", float64(1), true}, + {"float64 0", "enabled", float64(0), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := maps.MapStr{tt.key: tt.value} + job := &Job{} + err := mapToStruct(m, job) + assert.NoError(t, err) + assert.Equal(t, tt.expected, job.Enabled, "field %s with value %v", tt.key, tt.value) + }) + } + + t.Run("system field", func(t *testing.T) { + m := maps.MapStr{"system": "t"} + job := &Job{} + err := mapToStruct(m, job) + assert.NoError(t, err) + assert.True(t, job.System) + }) + + t.Run("readonly field", func(t *testing.T) { + m := maps.MapStr{"readonly": "1"} + job := &Job{} + err := mapToStruct(m, job) + assert.NoError(t, err) + assert.True(t, job.Readonly) + }) +} + +func TestMapToStructTimeFields(t *testing.T) { + refTime := time.Date(2024, 6, 15, 10, 30, 45, 0, time.UTC) + + tests := []struct { + name string + input string + parsed bool + }{ + {"MySQL format", "2024-06-15 10:30:45", true}, + {"PG fractional", "2024-06-15 10:30:45.123456", true}, + {"PG with +00", "2024-06-15 10:30:45.123456+00", true}, + {"PG with -07", "2024-06-15 03:30:45.123456-07", true}, + {"RFC3339", "2024-06-15T10:30:45Z", true}, + {"RFC3339Nano", "2024-06-15T10:30:45.123456789Z", true}, + {"empty string", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := maps.MapStr{ + "name": "test-job", + "job_id": "j-123", + "created_at": tt.input, + } + job := &Job{} + err := mapToStruct(m, job) + if tt.parsed { + assert.NoError(t, err) + assert.False(t, job.CreatedAt.IsZero(), "expected non-zero time for input: %s", tt.input) + assert.Equal(t, refTime.Unix(), job.CreatedAt.Unix(), + "expected %v, got %v for input: %s", refTime, job.CreatedAt, tt.input) + } else { + assert.True(t, job.CreatedAt.IsZero()) + } + }) + } +} diff --git a/job/health.go b/job/health.go index 96d4d6b1..91094854 100644 --- a/job/health.go +++ b/job/health.go @@ -315,28 +315,30 @@ func (dc *DataCleaner) shouldRunCleanup() bool { return time.Since(dc.lastCleanupTime) >= 24*time.Hour } -// performCleanup performs the actual data cleanup +// performCleanup performs the actual data cleanup. +// Deletion order: logs -> executions -> jobs (children first) to avoid +// foreign key violations on databases that enforce referential integrity (e.g. PostgreSQL). func (dc *DataCleaner) performCleanup() error { log.Info("Starting daily data cleanup...") cutoffTime := time.Now().AddDate(0, 0, -dc.retentionDays) - // Clean up jobs (excluding running jobs) - deletedJobs, err := dc.cleanupJobs(cutoffTime) + // Clean up logs first (leaf records) + deletedLogs, err := dc.cleanupLogs(cutoffTime) if err != nil { - return fmt.Errorf("failed to cleanup jobs: %w", err) + return fmt.Errorf("failed to cleanup logs: %w", err) } - // Clean up executions + // Clean up executions (reference jobs) deletedExecutions, err := dc.cleanupExecutions(cutoffTime) if err != nil { return fmt.Errorf("failed to cleanup executions: %w", err) } - // Clean up logs - deletedLogs, err := dc.cleanupLogs(cutoffTime) + // Clean up jobs last (parent records) + deletedJobs, err := dc.cleanupJobs(cutoffTime) if err != nil { - return fmt.Errorf("failed to cleanup logs: %w", err) + return fmt.Errorf("failed to cleanup jobs: %w", err) } log.Info("Data cleanup completed: %d jobs, %d executions, %d logs deleted", diff --git a/openapi/oauth/types/oidc.go b/openapi/oauth/types/oidc.go index 79e4c1cd..2fbf7562 100644 --- a/openapi/oauth/types/oidc.go +++ b/openapi/oauth/types/oidc.go @@ -1,7 +1,7 @@ package types import ( - "time" + "github.com/yaoapp/yao/openapi/utils" ) // Map converts the OIDCUserInfo to a map[string]interface{}, excluding empty values @@ -67,7 +67,7 @@ func (user OIDCUserInfo) Map() map[string]interface{} { } // Convert and add UpdatedAt if not nil - if converted := unixToMySQL(user.UpdatedAt); converted != nil { + if converted := utils.UnixToDBTimestamp(user.UpdatedAt); converted != nil { result["updated_at"] = converted } @@ -132,7 +132,7 @@ func (user OIDCUserInfo) Map() map[string]interface{} { if user.YaoTeam.Description != "" { teamMap["description"] = user.YaoTeam.Description } - if converted := unixToMySQL(user.YaoTeam.UpdatedAt); converted != nil { + if converted := utils.UnixToDBTimestamp(user.YaoTeam.UpdatedAt); converted != nil { teamMap["updated_at"] = converted } if len(teamMap) > 0 { @@ -264,7 +264,7 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo { // Updated_at field if updatedAt, ok := user["updated_at"]; ok { - if converted := toUnixTimestamp(updatedAt); converted != nil { + if converted := utils.ToUnixTimestamp(updatedAt); converted != nil { if unixTime, ok := converted.(int64); ok { userInfo.UpdatedAt = &unixTime } @@ -336,7 +336,7 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo { team.Description = description } if updatedAt, ok := teamData["updated_at"]; ok { - if converted := toUnixTimestamp(updatedAt); converted != nil { + if converted := utils.ToUnixTimestamp(updatedAt); converted != nil { if unixTime, ok := converted.(int64); ok { team.UpdatedAt = &unixTime } @@ -383,96 +383,3 @@ func MakeOIDCUserInfo(user map[string]interface{}) *OIDCUserInfo { return userInfo } - -// unixToMySQL converts interface{} to MySQL DATETIME string -func unixToMySQL(val interface{}) interface{} { - if val == nil { - return nil - } - - var unixTime int64 - switch v := val.(type) { - case int64: - unixTime = v - case *int64: - if v == nil { - return nil - } - unixTime = *v - case int: - unixTime = int64(v) - case float64: - unixTime = int64(v) - default: - return nil - } - - return time.Unix(unixTime, 0).UTC().Format("2006-01-02 15:04:05") -} - -// mysqlToUnix converts interface{} to Unix timestamp -func mysqlToUnix(val interface{}) interface{} { - if val == nil { - return nil - } - - var dateTime string - switch v := val.(type) { - case string: - dateTime = v - case *string: - if v == nil { - return nil - } - dateTime = *v - default: - return nil - } - - if dateTime == "" { - return nil - } - - // Try MySQL DATETIME format - if t, err := time.Parse("2006-01-02 15:04:05", dateTime); err == nil { - return t.Unix() - } - - // Try ISO format as fallback - if t, err := time.Parse("2006-01-02T15:04:05Z", dateTime); err == nil { - return t.Unix() - } - - return nil -} - -// toUnixTimestamp converts any interface{} to Unix timestamp -func toUnixTimestamp(val interface{}) interface{} { - if val == nil { - return nil - } - - switch v := val.(type) { - case int64: - return v - case *int64: - if v == nil { - return nil - } - return *v - case int: - return int64(v) - case float64: - return int64(v) - case string: - // Handle MySQL DATETIME or ISO format - return mysqlToUnix(v) - case *string: - if v == nil { - return nil - } - return mysqlToUnix(*v) - default: - return nil - } -} diff --git a/openapi/utils/convert.go b/openapi/utils/convert.go index ed753f88..389da430 100644 --- a/openapi/utils/convert.go +++ b/openapi/utils/convert.go @@ -178,22 +178,12 @@ func ToTimeString(v interface{}) string { } return val.Format(time.RFC3339) case string: - // Try to parse as RFC3339 first - if t, err := time.Parse(time.RFC3339, val); err == nil { - return t.Format(time.RFC3339) - } - // Try to parse as other common formats - formats := []string{ - "2006-01-02 15:04:05", - "2006-01-02T15:04:05Z", - "2006-01-02T15:04:05.000Z", - } - for _, format := range formats { + for _, format := range DBTimeFormats { if t, err := time.Parse(format, val); err == nil { return t.Format(time.RFC3339) } } - return val // Return as-is if can't parse + return val case int64: // Assume unix timestamp if val > 0 { @@ -224,6 +214,113 @@ func GetTimeFormat(locale string) string { } } +// DBTimeFormats contains all time formats recognized by database drivers (MySQL, PostgreSQL, SQLite). +// Ordered from most specific to least specific for efficient parsing. +var DBTimeFormats = []string{ + "2006-01-02 15:04:05", + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05.999999-07", + "2006-01-02 15:04:05.999999+00", + "2006-01-02T15:04:05Z", + "2006-01-02T15:04:05", + "2006-01-02T15:04:05.000Z", + time.RFC3339, + time.RFC3339Nano, +} + +// DBTimestampToUnix converts a database timestamp string (interface{}) to a Unix timestamp (int64). +// Supports MySQL DATETIME, PostgreSQL timestamptz (fractional seconds, timezone offsets), +// ISO 8601, RFC3339, and RFC3339Nano. Returns nil for nil, empty, or unparseable input. +func DBTimestampToUnix(val interface{}) interface{} { + if val == nil { + return nil + } + + var dateTime string + switch v := val.(type) { + case string: + dateTime = v + case *string: + if v == nil { + return nil + } + dateTime = *v + default: + return nil + } + + if dateTime == "" { + return nil + } + + for _, format := range DBTimeFormats { + if t, err := time.Parse(format, dateTime); err == nil { + return t.Unix() + } + } + return nil +} + +// UnixToDBTimestamp converts a Unix timestamp (int64, *int64, int, float64) to a standard +// DATETIME string "2006-01-02 15:04:05" in UTC. Compatible with MySQL, PostgreSQL, and SQLite. +// Returns nil for nil or unsupported types. +func UnixToDBTimestamp(val interface{}) interface{} { + if val == nil { + return nil + } + + var unixTime int64 + switch v := val.(type) { + case int64: + unixTime = v + case *int64: + if v == nil { + return nil + } + unixTime = *v + case int: + unixTime = int64(v) + case float64: + unixTime = int64(v) + default: + return nil + } + + return time.Unix(unixTime, 0).UTC().Format("2006-01-02 15:04:05") +} + +// ToUnixTimestamp converts any interface{} value to a Unix timestamp (int64). +// Handles numeric types directly and parses string/DB timestamp formats via DBTimestampToUnix. +// Returns nil for nil or unsupported types. +func ToUnixTimestamp(val interface{}) interface{} { + if val == nil { + return nil + } + + switch v := val.(type) { + case int64: + return v + case *int64: + if v == nil { + return nil + } + return *v + case int: + return int64(v) + case float64: + return int64(v) + case string: + return DBTimestampToUnix(v) + case *string: + if v == nil { + return nil + } + return DBTimestampToUnix(*v) + default: + return nil + } +} + // FormatTimeWithLocale formats a time value (time.Time, *time.Time, or string) using the specified format // If the input is already a string, it will parse it first and then reformat it // Returns empty string if the value cannot be parsed @@ -245,21 +342,13 @@ func FormatTimeWithLocale(v interface{}, targetFormat string) string { return "" } case string: - // Try parsing with common formats - formats := []string{ - "2006-01-02 15:04:05", - "2006-01-02T15:04:05Z", - "2006-01-02T15:04:05", - time.RFC3339, - } - for _, format := range formats { + for _, format := range DBTimeFormats { t, err = time.Parse(format, val) if err == nil { break } } if err != nil { - // If all parsing attempts failed, return the original string return val } default: diff --git a/openapi/utils/convert_test.go b/openapi/utils/convert_test.go new file mode 100644 index 00000000..6fd088b3 --- /dev/null +++ b/openapi/utils/convert_test.go @@ -0,0 +1,122 @@ +package utils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDBTimestampToUnix(t *testing.T) { + ref := time.Date(2024, 6, 15, 10, 30, 45, 0, time.UTC) + refUnix := ref.Unix() + + tests := []struct { + name string + input interface{} + expect interface{} + }{ + {"nil", nil, nil}, + {"empty string", "", nil}, + {"MySQL DATETIME", "2024-06-15 10:30:45", refUnix}, + {"PG with fractional seconds", "2024-06-15 10:30:45.123456", refUnix}, + {"PG with +00 offset", "2024-06-15 10:30:45.123456+00", refUnix}, + {"PG with -07 offset", "2024-06-15 03:30:45.123456-07", refUnix}, + {"ISO Z suffix", "2024-06-15T10:30:45Z", refUnix}, + {"RFC3339", "2024-06-15T10:30:45+00:00", refUnix}, + {"RFC3339Nano", "2024-06-15T10:30:45.123456789+00:00", refUnix}, + {"*string nil", (*string)(nil), nil}, + {"*string valid", strPtr("2024-06-15 10:30:45"), refUnix}, + {"*string empty", strPtr(""), nil}, + {"non-string type", 12345, nil}, + {"unparseable string", "not-a-date", nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DBTimestampToUnix(tt.input) + assert.Equal(t, tt.expect, result) + }) + } +} + +func TestUnixToDBTimestamp(t *testing.T) { + refUnix := int64(1718444445) + expected := time.Unix(refUnix, 0).UTC().Format("2006-01-02 15:04:05") + + tests := []struct { + name string + input interface{} + expect interface{} + }{ + {"nil", nil, nil}, + {"int64", refUnix, expected}, + {"*int64 valid", int64Ptr(refUnix), expected}, + {"*int64 nil", (*int64)(nil), nil}, + {"int", int(refUnix), expected}, + {"float64", float64(refUnix), expected}, + {"unknown type", "not-a-number", nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := UnixToDBTimestamp(tt.input) + assert.Equal(t, tt.expect, result) + }) + } +} + +func TestToUnixTimestamp(t *testing.T) { + refUnix := int64(1718444445) + + tests := []struct { + name string + input interface{} + expect interface{} + }{ + {"nil", nil, nil}, + {"int64", refUnix, refUnix}, + {"*int64 valid", int64Ptr(refUnix), refUnix}, + {"*int64 nil", (*int64)(nil), nil}, + {"int", int(refUnix), refUnix}, + {"float64", float64(refUnix), refUnix}, + {"string MySQL", "2024-06-15 10:00:45", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()}, + {"string PG fractional", "2024-06-15 10:00:45.123456", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()}, + {"string RFC3339", "2024-06-15T10:00:45Z", time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()}, + {"*string valid", strPtr("2024-06-15 10:00:45"), time.Date(2024, 6, 15, 10, 0, 45, 0, time.UTC).Unix()}, + {"*string nil", (*string)(nil), nil}, + {"string empty", "", nil}, + {"unknown type", struct{}{}, nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ToUnixTimestamp(tt.input) + assert.Equal(t, tt.expect, result) + }) + } +} + +func TestToTimeStringWithPGFormats(t *testing.T) { + tests := []struct { + name string + input interface{} + expect string + }{ + {"PG fractional", "2024-06-15 10:30:45.123456", "2024-06-15T10:30:45Z"}, + {"PG +00 offset", "2024-06-15 10:30:45.123456+00", "2024-06-15T10:30:45Z"}, + {"PG -07 offset", "2024-06-15 03:30:45.123456-07", "2024-06-15T03:30:45-07:00"}, + {"MySQL format", "2024-06-15 10:30:45", "2024-06-15T10:30:45Z"}, + {"RFC3339Nano", "2024-06-15T10:30:45.123456789+00:00", "2024-06-15T10:30:45Z"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ToTimeString(tt.input) + assert.Equal(t, tt.expect, result) + }) + } +} + +func strPtr(s string) *string { return &s } +func int64Ptr(i int64) *int64 { return &i } diff --git a/test/utils.go b/test/utils.go index f3ae31f7..f2985973 100644 --- a/test/utils.go +++ b/test/utils.go @@ -571,6 +571,8 @@ func dbconnect(t *testing.T, cfg config.Config) { switch cfg.DB.Driver { case "sqlite3": capsule.AddConn("primary", "sqlite3", cfg.DB.Primary[0]).SetAsGlobal() + case "postgres": + capsule.AddConn("primary", "postgres", cfg.DB.Primary[0]).SetAsGlobal() default: capsule.AddConn("primary", "mysql", cfg.DB.Primary[0]).SetAsGlobal() }