diff --git a/initialize/migrate/database/02126_system_log_idx.down.sql b/initialize/migrate/database/02126_system_log_idx.down.sql new file mode 100644 index 00000000..e605bb27 --- /dev/null +++ b/initialize/migrate/database/02126_system_log_idx.down.sql @@ -0,0 +1 @@ +DROP INDEX idx_type_date ON system_logs; diff --git a/initialize/migrate/database/02126_system_log_idx.up.sql b/initialize/migrate/database/02126_system_log_idx.up.sql new file mode 100644 index 00000000..6b2e0399 --- /dev/null +++ b/initialize/migrate/database/02126_system_log_idx.up.sql @@ -0,0 +1 @@ +CREATE INDEX idx_type_date ON system_logs (type, date); diff --git a/internal/logic/admin/console/queryRevenueStatisticsLogic.go b/internal/logic/admin/console/queryRevenueStatisticsLogic.go index f9dbb373..fac2efa0 100644 --- a/internal/logic/admin/console/queryRevenueStatisticsLogic.go +++ b/internal/logic/admin/console/queryRevenueStatisticsLogic.go @@ -2,6 +2,7 @@ package console import ( "context" + "encoding/json" "os" "strings" "time" @@ -13,6 +14,9 @@ import ( "github.com/pkg/errors" ) +const consoleRevenueStatisticsCacheKey = "console:revenue_statistics" +const consoleRevenueStatisticsCacheTTL = 60 * time.Second + type QueryRevenueStatisticsLogic struct { logger.Logger ctx context.Context @@ -33,6 +37,15 @@ func (l *QueryRevenueStatisticsLogic) QueryRevenueStatistics() (resp *types.Reve return l.mockRevenueStatistics(), nil } + // Try cache first + cached, cacheErr := l.svcCtx.Redis.Get(l.ctx, consoleRevenueStatisticsCacheKey).Result() + if cacheErr == nil && cached != "" { + var result types.RevenueStatisticsResponse + if json.Unmarshal([]byte(cached), &result) == nil { + return &result, nil + } + } + var today, monthly, all types.OrdersStatistics now := time.Now() // Get today's revenue statistics @@ -111,11 +124,18 @@ func (l *QueryRevenueStatisticsLogic) QueryRevenueStatistics() (resp *types.Reve all.List = allList } - return &types.RevenueStatisticsResponse{ + resp = &types.RevenueStatisticsResponse{ Today: today, Monthly: monthly, All: all, - }, nil + } + + // Cache the result + if data, marshalErr := json.Marshal(resp); marshalErr == nil { + l.svcCtx.Redis.Set(l.ctx, consoleRevenueStatisticsCacheKey, data, consoleRevenueStatisticsCacheTTL) + } + + return resp, nil } // mockRevenueStatistics is a mock function to simulate revenue statistics data. diff --git a/internal/logic/admin/console/queryServerTotalDataLogic.go b/internal/logic/admin/console/queryServerTotalDataLogic.go index 5bdafcab..405bca3b 100644 --- a/internal/logic/admin/console/queryServerTotalDataLogic.go +++ b/internal/logic/admin/console/queryServerTotalDataLogic.go @@ -2,8 +2,10 @@ package console import ( "context" + "encoding/json" "os" "strings" + "sync" "time" "github.com/perfect-panel/server/internal/model/log" @@ -17,6 +19,9 @@ import ( "gorm.io/gorm" ) +const consoleServerTotalDataCacheKey = "console:server_total_data" +const consoleServerTotalDataCacheTTL = 60 * time.Second + type QueryServerTotalDataLogic struct { logger.Logger ctx context.Context @@ -38,24 +43,83 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo return l.mockRevenueStatistics(), nil } - now := time.Now() + // Try cache first + cached, cacheErr := l.svcCtx.Redis.Get(l.ctx, consoleServerTotalDataCacheKey).Result() + if cacheErr == nil && cached != "" { + var result types.ServerTotalDataResponse + if json.Unmarshal([]byte(cached), &result) == nil { + return &result, nil + } + } + now := time.Now() todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - todayEnd := todayStart.Add(24 * time.Hour).Add(-time.Second) + todayEnd := todayStart.Add(24 * time.Hour) query := l.svcCtx.DB.WithContext(l.ctx) - var todayTop10User []log.UserTraffic - - err = query.Model(&traffic.TrafficLog{}). - Select("user_id, subscribe_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload"). - Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd). - Group("user_id, subscribe_id"). - Order("total DESC"). - Limit(10). - Scan(&todayTop10User).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - logger.Errorf("[Traffic Stat Queue] Query user traffic failed: %v", err.Error()) - return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), " Query user traffic failed: %v", err.Error()) + + // Parallelize three traffic_log queries to reduce latency + var ( + todayTop10User []log.UserTraffic + todayTop10Server []log.ServerTraffic + todayTraffic struct { + Upload int64 + Download int64 + } + userErr, serverErr, trafficErr error + wg sync.WaitGroup + ) + + wg.Add(3) + + // Query 1: Today's top 10 users by traffic + go func() { + defer wg.Done() + userErr = query.Model(&traffic.TrafficLog{}). + Select("user_id, subscribe_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload"). + Where("timestamp >= ? AND timestamp < ?", todayStart, todayEnd). + Group("user_id, subscribe_id"). + Order("total DESC"). + Limit(10). + Scan(&todayTop10User).Error + }() + + // Query 2: Today's top 10 servers by traffic + go func() { + defer wg.Done() + serverErr = query.Model(&traffic.TrafficLog{}). + Select("server_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload"). + Where("timestamp >= ? AND timestamp < ?", todayStart, todayEnd). + Group("server_id"). + Order("total DESC"). + Limit(10). + Scan(&todayTop10Server).Error + }() + + // Query 3: Today's total upload/download + go func() { + defer wg.Done() + trafficErr = query.Model(&traffic.TrafficLog{}). + Select("COALESCE(SUM(upload), 0) AS upload, COALESCE(SUM(download), 0) AS download"). + Where("timestamp >= ? AND timestamp < ?", todayStart, todayEnd). + Scan(&todayTraffic).Error + }() + + wg.Wait() + + if userErr != nil && !errors.Is(userErr, gorm.ErrRecordNotFound) { + logger.Errorf("[QueryServerTotalData] Query user traffic failed: %v", userErr.Error()) + return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query user traffic failed: %v", userErr.Error()) + } + if serverErr != nil && !errors.Is(serverErr, gorm.ErrRecordNotFound) { + logger.Errorf("[QueryServerTotalData] Query server traffic failed: %v", serverErr.Error()) + return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query server traffic failed: %v", serverErr.Error()) + } + if trafficErr != nil && !errors.Is(trafficErr, gorm.ErrRecordNotFound) { + logger.Errorf("[QueryServerTotalData] Sum today traffic failed: %v", trafficErr.Error()) + return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Sum today traffic failed: %v", trafficErr.Error()) } + + // Build today user traffic ranking var userTodayTrafficRanking []types.UserTrafficData for _, item := range todayTop10User { userTodayTrafficRanking = append(userTodayTrafficRanking, types.UserTrafficData{ @@ -65,7 +129,7 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo }) } - // query yesterday user traffic rank log + // Query yesterday user traffic rank log yesterday := todayStart.Add(-24 * time.Hour).Format(time.DateOnly) var yesterdayLog log.SystemLog @@ -91,35 +155,39 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo } } - // query server traffic rank today - var todayTop10Server []log.ServerTraffic - err = query.Model(&traffic.TrafficLog{}).Select("server_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload"). - Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd). - Group("server_id"). - Order("total DESC"). - Limit(10). - Scan(&todayTop10Server).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - logger.Errorf("[Traffic Stat Queue] Query server traffic failed: %v", err.Error()) - return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), " Query server traffic failed: %v", err.Error()) + // Batch fetch server names for today's ranking + serverIDs := make([]int64, 0, 10) + for _, item := range todayTop10Server { + serverIDs = append(serverIDs, item.ServerId) + } + + serverMap := make(map[int64]*node.Server) + if len(serverIDs) > 0 { + var servers []*node.Server + err = query.Model(&node.Server{}).Where("id IN ?", serverIDs).Find(&servers).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + l.Errorw("[QueryServerTotalDataLogic] Batch fetch servers error", logger.Field("error", err.Error())) + } + for _, s := range servers { + serverMap[s.Id] = s + } } var todayServerRanking []types.ServerTrafficData for _, item := range todayTop10Server { - info, err := l.svcCtx.NodeModel.FindOneServer(l.ctx, item.ServerId) - if err != nil { - l.Errorw("[QueryServerTotalDataLogic] FindOneServer error", logger.Field("error", err.Error()), logger.Field("server_id", item.ServerId)) - continue + name := "" + if s, ok := serverMap[item.ServerId]; ok { + name = s.Name } todayServerRanking = append(todayServerRanking, types.ServerTrafficData{ ServerId: item.ServerId, - Name: info.Name, + Name: name, Upload: item.Upload, Download: item.Download, }) } - // query server traffic rank yesterday + // Query yesterday server traffic rank var yesterdayTop10Server []types.ServerTrafficData var yesterdayServerTrafficLog log.SystemLog err = query.Model(&log.SystemLog{}).Where("`date` = ? AND `type` = ?", yesterday, log.TypeServerTrafficRank).First(&yesterdayServerTrafficLog).Error @@ -134,29 +202,44 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo l.Errorw("[QueryServerTotalDataLogic] Unmarshal yesterday server traffic rank log error", logger.Field("error", err.Error())) } + // Collect yesterday server IDs not already fetched + yesterdayServerIDs := make([]int64, 0, len(rank.Rank)) for _, v := range rank.Rank { - info, err := l.svcCtx.NodeModel.FindOneServer(l.ctx, v.ServerId) - if err != nil { - l.Errorw("[QueryServerTotalDataLogic] FindOneServer error", logger.Field("error", err.Error()), logger.Field("server_id", v.ServerId)) - continue + if _, ok := serverMap[v.ServerId]; !ok { + yesterdayServerIDs = append(yesterdayServerIDs, v.ServerId) + } + } + if len(yesterdayServerIDs) > 0 { + var extraServers []*node.Server + if err := query.Model(&node.Server{}).Where("id IN ?", yesterdayServerIDs).Find(&extraServers).Error; err == nil { + for _, s := range extraServers { + serverMap[s.Id] = s + } + } + } + + for _, v := range rank.Rank { + name := "" + if s, ok := serverMap[v.ServerId]; ok { + name = s.Name } yesterdayTop10Server = append(yesterdayTop10Server, types.ServerTrafficData{ ServerId: v.ServerId, - Name: info.Name, + Name: name, Upload: v.Upload, Download: v.Download, }) } } - // query online user count + // Query online user count onlineUsers, err := l.svcCtx.NodeModel.OnlineUserSubscribeGlobal(l.ctx) if err != nil { l.Errorw("[QueryServerTotalDataLogic] OnlineUserSubscribeGlobal error", logger.Field("error", err.Error())) return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "OnlineUserSubscribeGlobal error: %v", err) } - // query online/offline server count + // Query online/offline server count var onlineServers, offlineServers int64 err = query.Model(&node.Server{}).Where("`last_reported_at` > ?", now.Add(-5*time.Minute)).Count(&onlineServers).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { @@ -169,42 +252,35 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo l.Errorw("[QueryServerTotalDataLogic] Count offline servers error", logger.Field("error", err.Error())) return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Count offline servers error: %v", err) } - // TodayUpload, TodayDownload, MonthlyUpload, MonthlyDownload - var todayUpload, todayDownload, monthlyUpload, monthlyDownload int64 - - type trafficSum struct { - Upload int64 - Download int64 - } - var todayTraffic trafficSum - // Today - err = query.Model(&traffic.TrafficLog{}).Select("SUM(upload) AS upload, SUM(download) AS download"). - Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd). - Scan(&todayTraffic).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - l.Errorw("[QueryServerTotalDataLogic] Sum today traffic error", logger.Field("error", err.Error())) - return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Sum today traffic error: %v", err) - } - todayUpload = todayTraffic.Upload - todayDownload = todayTraffic.Download - // Monthly + // Monthly traffic: today's real-time data + archived daily stats for previous days + todayUpload := todayTraffic.Upload + todayDownload := todayTraffic.Download + var monthlyUpload, monthlyDownload int64 monthlyUpload += todayUpload monthlyDownload += todayDownload - for i := now.Day() - 1; i >= 1; i-- { - var logInfo log.SystemLog - date := time.Date(now.Year(), now.Month(), i, 0, 0, 0, 0, now.Location()).Format(time.DateOnly) - err = query.Model(&log.SystemLog{}).Where("`date` = ? AND `type` = ?", date, log.TypeTrafficStat).First(&logInfo).Error + // Batch query all previous days' traffic stats (eliminates N+1 loop) + if now.Day() > 1 { + dates := make([]string, 0, now.Day()-1) + for i := 1; i < int(now.Day()); i++ { + d := time.Date(now.Year(), now.Month(), i, 0, 0, 0, 0, now.Location()).Format(time.DateOnly) + dates = append(dates, d) + } + + var dailyLogs []log.SystemLog + err = query.Model(&log.SystemLog{}). + Where("`date` IN ? AND `type` = ?", dates, log.TypeTrafficStat). + Find(&dailyLogs).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - l.Errorw("[QueryServerTotalDataLogic] Query daily traffic stat log error", logger.Field("error", err.Error()), logger.Field("date", date)) - return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query daily traffic stat log error: %v", err) + l.Errorw("[QueryServerTotalDataLogic] Batch query daily traffic stats error", logger.Field("error", err.Error())) + return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Batch query daily traffic stats error: %v", err) } - if logInfo.Id > 0 { + + for _, logInfo := range dailyLogs { var stat log.TrafficStat - err = stat.Unmarshal([]byte(logInfo.Content)) - if err != nil { - l.Errorw("[QueryServerTotalDataLogic] Unmarshal daily traffic stat log error", logger.Field("error", err.Error()), logger.Field("date", date)) + if err := stat.Unmarshal([]byte(logInfo.Content)); err != nil { + l.Errorw("[QueryServerTotalDataLogic] Unmarshal daily traffic stat error", logger.Field("error", err.Error()), logger.Field("date", logInfo.Date)) continue } monthlyUpload += stat.Upload @@ -227,6 +303,11 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo UserTrafficRankingYesterday: yesterdayUserRankData, } + // Cache the result + if data, marshalErr := json.Marshal(resp); marshalErr == nil { + l.svcCtx.Redis.Set(l.ctx, consoleServerTotalDataCacheKey, data, consoleServerTotalDataCacheTTL) + } + return resp, nil } @@ -260,42 +341,16 @@ func (l *QueryServerTotalDataLogic) mockRevenueStatistics() *types.ServerTotalDa } } - //// Generate user traffic ranking data for today (top 10) - //userTrafficToday := make([]types.UserTrafficData, 10) - //for i := 0; i < 10; i++ { - // upload := int64(100000000 + (i*20000000) + (i%5)*50000000) // 100MB - 400MB - // download := int64(800000000 + (i*150000000) + (i%3)*300000000) // 800MB - 3GB - // userTrafficToday[i] = types.UserTrafficData{ - // SID: int64(10001 + i), - // Upload: upload, - // Download: download, - // } - //} - - //// Generate user traffic ranking data for yesterday (top 10) - //userTrafficYesterday := make([]types.UserTrafficData, 10) - //for i := 0; i < 10; i++ { - // upload := int64(95000000 + (i*18000000) + (i%5)*45000000) - // download := int64(750000000 + (i*140000000) + (i%3)*280000000) - // userTrafficYesterday[i] = types.UserTrafficData{ - // SID: int64(10001 + i), - // Upload: upload, - // Download: download, - // } - //} - // return &types.ServerTotalDataResponse{ OnlineUsers: 1688, OnlineServers: 8, OfflineServers: 2, - TodayUpload: 8888888888, // ~8.3GB - TodayDownload: 28888888888, // ~26.9GB - MonthlyUpload: 288888888888, // ~269GB - MonthlyDownload: 888888888888, // ~828GB + TodayUpload: 8888888888, + TodayDownload: 28888888888, + MonthlyUpload: 288888888888, + MonthlyDownload: 888888888888, UpdatedAt: now.Unix(), ServerTrafficRankingToday: serverTrafficToday, ServerTrafficRankingYesterday: serverTrafficYesterday, - //UserTrafficRankingToday: userTrafficToday, - //UserTrafficRankingYesterday: userTrafficYesterday, } } diff --git a/internal/logic/admin/console/queryUserStatisticsLogic.go b/internal/logic/admin/console/queryUserStatisticsLogic.go index 746176cc..6154a4fb 100644 --- a/internal/logic/admin/console/queryUserStatisticsLogic.go +++ b/internal/logic/admin/console/queryUserStatisticsLogic.go @@ -2,6 +2,7 @@ package console import ( "context" + "encoding/json" "os" "strings" "time" @@ -11,6 +12,9 @@ import ( "github.com/perfect-panel/server/pkg/logger" ) +const consoleUserStatisticsCacheKey = "console:user_statistics" +const consoleUserStatisticsCacheTTL = 60 * time.Second + type QueryUserStatisticsLogic struct { logger.Logger ctx context.Context @@ -30,6 +34,16 @@ func (l *QueryUserStatisticsLogic) QueryUserStatistics() (resp *types.UserStatis if strings.ToLower(os.Getenv("PPANEL_MODE")) == "demo" { return l.mockRevenueStatistics(), nil } + + // Try cache first + cached, cacheErr := l.svcCtx.Redis.Get(l.ctx, consoleUserStatisticsCacheKey).Result() + if cacheErr == nil && cached != "" { + var result types.UserStatisticsResponse + if json.Unmarshal([]byte(cached), &result) == nil { + return &result, nil + } + } + resp = &types.UserStatisticsResponse{} now := time.Now() // query today user register count @@ -116,6 +130,11 @@ func (l *QueryUserStatisticsLogic) QueryUserStatistics() (resp *types.UserStatis resp.All.List = allList } + // Cache the result + if data, marshalErr := json.Marshal(resp); marshalErr == nil { + l.svcCtx.Redis.Set(l.ctx, consoleUserStatisticsCacheKey, data, consoleUserStatisticsCacheTTL) + } + return }