diff --git a/config.example.yaml b/config.example.yaml index e36d44ca..8ccdacd9 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -30,6 +30,7 @@ oauth2: # DB database: + enabled: true host: "127.0.0.1" port: 3306 username: "root" @@ -40,8 +41,22 @@ database: conn_max_lifetime: 3600 log_level: "info" # error, warn, info, debug, silent +# clickhouse +clickhouse: + enabled: false + hosts: + - "127.0.0.1:9000" + username: "default" + password: "" + database: "linux_do_cdk" + max_idle_conn: 10 + max_open_conn: 100 + conn_max_lifetime: 3600 + dial_timeout: 5 + # Redis配置 redis: + enabled: true host: "127.0.0.1" port: 6379 username: "" @@ -72,4 +87,4 @@ schedule: # Worker worker: - concurrency: 50 \ No newline at end of file + concurrency: 50 diff --git a/frontend/middleware.ts b/frontend/middleware.ts index d5227d8a..ce819759 100644 --- a/frontend/middleware.ts +++ b/frontend/middleware.ts @@ -10,15 +10,9 @@ interface HCaptchaResponse { 'error-codes'?: string[]; } -interface ProjectData { - data?: { - start_time: string; - end_time: string; - }; -} - interface ReceiveRequestBody { captcha_token?: string; + [key: string]: string | number | boolean | null | undefined; } @@ -90,66 +84,6 @@ async function verifyCaptcha(token: string, remoteip?: string): Promise<{ succes } } -/** - * 验证项目时间有效性 - */ -async function validateProjectTime(projectId: string, request: NextRequest): Promise<{ valid: boolean; error?: string }> { - try { - const projectCheckUrl = `${BACKEND_BASE_URL}/api/v1/projects/${projectId}`; - - // 使用 AbortController 实现超时 - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 5000); // 5秒超时 - - const projectResponse = await fetch(projectCheckUrl, { - headers: { - 'Cookie': request.headers.get('cookie') || '', - 'User-Agent': 'CDK-Frontend-Middleware', - }, - credentials: 'include', - signal: controller.signal, - }); - - clearTimeout(timeoutId); - - if (!projectResponse.ok) { - if (projectResponse.status === 404) { - return {valid: false, error: '项目不存在或已被删除'}; - } - if (projectResponse.status === 403) { - return {valid: false, error: '无权访问此项目'}; - } - return {valid: false, error: `获取项目信息失败 (${projectResponse.status})，请稍后重试`}; - } - - const projectData: ProjectData = await projectResponse.json(); - - if (!projectData.data) { - return {valid: false, error: '项目不存在'}; - } - - const now = new Date(); - const startTime = new Date(projectData.data.start_time); - const endTime = new Date(projectData.data.end_time); - - if (now < startTime) { - return {valid: false, error: '项目尚未开始'}; - } - - if (now > endTime) { - return {valid: false, error: '项目已结束'}; - } - - return {valid: true}; - } catch (error) { - if (error instanceof Error && error.name === 'AbortError') { - return {valid: false, error: '验证超时，请稍后重试'}; - } - - return {valid: false, error: formatErrorMessage(error, '服务器暂时无法验证项目信息，请稍后重试')}; - } -} - /** * 处理领取请求 */ @@ -177,20 +111,6 @@ async function handleReceiveRequest(request: NextRequest, pathname: string): Pro ); } - // 提取项目ID并验证时间 - const pathParts = pathname.split('/'); - const projectId = pathParts[4]; // /api/v1/projects/{id}/receive - - if (projectId) { - const timeValidation = await validateProjectTime(projectId, request); - if (!timeValidation.valid) { - return NextResponse.json( - {error_msg: timeValidation.error}, - {status: 400}, - ); - } - } - // 准备转发到后端的请求 // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars const {captcha_token: _captchaToken, ...backendBody} = body; @@ -204,6 +124,8 @@ async function handleReceiveRequest(request: NextRequest, pathname: string): Pro 'X-Original-Host': request.headers.get('host') || '', 'Cookie': request.headers.get('cookie') || '', 'User-Agent': 'CDK-Frontend-Middleware', + 'Referer': request.headers.get('referer') || '', + 'Origin': request.headers.get('origin') || '', }, body: Object.keys(backendBody).length > 0 ? JSON.stringify(backendBody) : undefined, credentials: 'include', diff --git a/go.mod b/go.mod index eec87a23..03ac19d2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/linux-do/cdk go 1.24 require ( + github.com/ClickHouse/clickhouse-go/v2 v2.23.2 github.com/gin-contrib/sessions v1.0.4 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -30,7 +31,6 @@ require ( require ( github.com/ClickHouse/ch-go v0.61.5 // indirect - github.com/ClickHouse/clickhouse-go/v2 v2.23.2 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/boj/redistore v1.4.1 // indirect diff --git a/internal/apps/project/clickhouses.go b/internal/apps/project/clickhouses.go new file mode 100644 index 00000000..ed0663d4 --- /dev/null +++ b/internal/apps/project/clickhouses.go @@ -0,0 +1,76 @@ +/* + * MIT License + * + * Copyright (c) 2025 linux.do + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package project + +import ( + "encoding/json" + "github.com/linux-do/cdk/internal/db" + "github.com/linux-do/cdk/internal/otel_trace" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "time" + + "github.com/gin-gonic/gin" + "github.com/linux-do/cdk/internal/config" +) + +func recordErrProjectReceive(c *gin.Context, reqTime time.Time, userId uint64, userName string, + projectId string, projectStartTime, projectEndTime time.Time, errorMsg string) { + if !config.Config.ClickHouse.Enabled { + return + } + // init trace + ctx, span := otel_trace.Start(c.Request.Context(), "ClickHouse") + defer span.End() + + clientInfo := map[string]interface{}{ + "ip": c.ClientIP(), + "user_agent": c.Request.UserAgent(), + "referer": c.Request.Referer(), + "origin": c.Request.Header.Get("Origin"), + } + + clientInfoJSON, _ := json.Marshal(clientInfo) + traceID := trace.SpanFromContext(c.Request.Context()).SpanContext().TraceID().String() + + if errExec := db.ChConn.AsyncInsert(ctx, ` + INSERT INTO err_receive_logs ( + request_time, trace_id, user_id, user_name, project_id, project_start_time, project_end_time, + error_message, client_info + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + true, + reqTime, + traceID, + userId, + userName, + projectId, + projectStartTime, + projectEndTime, + errorMsg, + string(clientInfoJSON), + ); errExec != nil { + span.SetStatus(codes.Error, errExec.Error()) + } +} diff --git a/internal/apps/project/err.go b/internal/apps/project/err.go index 5ff0f280..280a9dbb 100644 --- a/internal/apps/project/err.go +++ b/internal/apps/project/err.go @@ -31,6 +31,7 @@ const ( TimeTooLate = "已经结束" TrustLevelNotMatch = "需要信任等级 %d" UnknownError = "未知异常" + ScoreNotEnough = "分数未达标" SameIPReceived = "已有相同IP领取" NoStock = "无库存" NotFound = "项目不存在" diff --git a/internal/apps/project/middlewares.go b/internal/apps/project/middlewares.go index 8a4f53a7..680329ef 100644 --- a/internal/apps/project/middlewares.go +++ b/internal/apps/project/middlewares.go @@ -29,6 +29,7 @@ import ( "github.com/linux-do/cdk/internal/apps/oauth" "github.com/linux-do/cdk/internal/db" "net/http" + "time" ) func ProjectCreatorPermMiddleware() gin.HandlerFunc { @@ -59,15 +60,18 @@ func ReceiveProjectMiddleware() gin.HandlerFunc { c.AbortWithStatusJSON(http.StatusNotFound, ProjectResponse{ErrorMsg: err.Error()}) return } + now := time.Now() // load project projectID := c.Param("id") project := &Project{} if err := project.Exact(db.DB(ctx), projectID, true); err != nil { + recordErrProjectReceive(c, now, user.ID, user.Username, projectID, time.Time{}, time.Time{}, NotFound) c.AbortWithStatusJSON(http.StatusNotFound, ProjectResponse{ErrorMsg: err.Error()}) return } // check receivable - if err := project.IsReceivable(ctx, user, c.ClientIP()); err != nil { + if err := project.IsReceivable(ctx, now, user, c.ClientIP()); err != nil { + recordErrProjectReceive(c, now, user.ID, user.Username, project.ID, project.StartTime, project.EndTime, err.Error()) c.AbortWithStatusJSON(http.StatusForbidden, ProjectResponse{ErrorMsg: err.Error()}) return } diff --git a/internal/apps/project/models.go b/internal/apps/project/models.go index 16d4638e..0015ca9d 100644 --- a/internal/apps/project/models.go +++ b/internal/apps/project/models.go @@ -134,7 +134,7 @@ func (p *Project) CreateItemsWithFilter(ctx context.Context, tx *gorm.DB, items if len(items) <= 0 { return nil } - + filteredItems := items if enableFilter { // Get existing items for this project @@ -144,13 +144,13 @@ func (p *Project) CreateItemsWithFilter(ctx context.Context, tx *gorm.DB, items Pluck("content", &existingItems).Error; err != nil { return err } - + // Create a set of existing items for O(1) lookup existingSet := make(map[string]bool) for _, item := range existingItems { existingSet[item] = true } - + // Filter out duplicates filteredItems = make([]string, 0, len(items)) for _, item := range items { @@ -159,7 +159,7 @@ func (p *Project) CreateItemsWithFilter(ctx context.Context, tx *gorm.DB, items } } } - + // Use the original CreateItems method with filtered items return p.CreateItems(ctx, tx, filteredItems) } @@ -169,11 +169,11 @@ func (p *Project) GetFilteredItemsCount(ctx context.Context, tx *gorm.DB, items if len(items) <= 0 { return 0, nil } - + if !enableFilter { return int64(len(items)), nil } - + // Get existing items for this project var existingItems []string if err := tx.Model(&ProjectItem{}). @@ -181,13 +181,13 @@ func (p *Project) GetFilteredItemsCount(ctx context.Context, tx *gorm.DB, items Pluck("content", &existingItems).Error; err != nil { return 0, err } - + // Create a set of existing items for O(1) lookup existingSet := make(map[string]bool) for _, item := range existingItems { existingSet[item] = true } - + // Count unique items uniqueCount := int64(0) for _, item := range items { @@ -195,7 +195,7 @@ func (p *Project) GetFilteredItemsCount(ctx context.Context, tx *gorm.DB, items uniqueCount++ } } - + return uniqueCount, nil } @@ -236,9 +236,8 @@ func (p *Project) HasStock(ctx context.Context) (bool, error) { return stock > 0, nil } -func (p *Project) IsReceivable(ctx context.Context, user *oauth.User, ip string) error { +func (p *Project) IsReceivable(ctx context.Context, now time.Time, user *oauth.User, ip string) error { // check time - now := time.Now() if now.Before(p.StartTime) { return errors.New(TimeTooEarly) } else if p.EndTime.Before(now) { @@ -250,7 +249,7 @@ func (p *Project) IsReceivable(ctx context.Context, user *oauth.User, ip string) } // check risk level if user.RiskLevel() > p.RiskLevel { - return errors.New(UnknownError) + return errors.New(ScoreNotEnough) } // check same ip if sameIPReceived, err := p.CheckSameIPReceived(ctx, ip); err != nil { diff --git a/internal/config/model.go b/internal/config/model.go index 07a4e6f8..3c77679c 100644 --- a/internal/config/model.go +++ b/internal/config/model.go @@ -33,6 +33,7 @@ type configModel struct { Log logConfig `mapstructure:"log"` Schedule scheduleConfig `mapstructure:"schedule"` Worker workerConfig `mapstructure:"worker"` + ClickHouse clickHouseConfig `mapstructure:"clickhouse"` } // appConfig 应用基本配置 @@ -67,6 +68,7 @@ type OAuth2Config struct { // databaseConfig 数据库配置 type databaseConfig struct { + Enabled bool `mapstructure:"enabled"` Host string `mapstructure:"host"` Port int `mapstructure:"port"` Username string `mapstructure:"username"` @@ -78,8 +80,22 @@ type databaseConfig struct { LogLevel string `mapstructure:"log_level"` } +// clickhouse 配置 +type clickHouseConfig struct { + Enabled bool `mapstructure:"enabled"` + Hosts []string `mapstructure:"hosts"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + MaxIdleConn int `mapstructure:"max_idle_conn"` + MaxOpenConn int `mapstructure:"max_open_conn"` + ConnMaxLifetime int `mapstructure:"conn_max_lifetime"` + DialTimeout int `mapstructure:"dial_timeout"` +} + // redisConfig Redis配置 type redisConfig struct { + Enabled bool `mapstructure:"enabled"` Host string `mapstructure:"host"` Port int `mapstructure:"port"` Username string `mapstructure:"username"` diff --git a/internal/db/clickhouse.go b/internal/db/clickhouse.go new file mode 100644 index 00000000..3ea757a4 --- /dev/null +++ b/internal/db/clickhouse.go @@ -0,0 +1,76 @@ +/* + * MIT License + * + * Copyright (c) 2025 linux.do + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package db + +import ( + "context" + "log" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/linux-do/cdk/internal/config" +) + +var ( + ChConn driver.Conn +) + +func init() { + if !config.Config.ClickHouse.Enabled { + return + } + + cfg := config.Config.ClickHouse + var err error + + // 配置 ClickHouse 连接 + ChConn, err = clickhouse.Open(&clickhouse.Options{ + Addr: cfg.Hosts, + Auth: clickhouse.Auth{ + Database: cfg.Database, + Username: cfg.Username, + Password: cfg.Password, + }, + Settings: clickhouse.Settings{ + "max_execution_time": 60, + }, + DialTimeout: time.Duration(cfg.DialTimeout) * time.Second, + MaxOpenConns: cfg.MaxOpenConn, + MaxIdleConns: cfg.MaxIdleConn, + ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second, + }) + + if err != nil { + log.Fatalf("[ClickHouse] init connection failed: %v\n", err) + } + + // 测试连接 + if err = ChConn.Ping(context.Background()); err != nil { + log.Fatalf("[ClickHouse] ping failed: %v\n", err) + } + + log.Println("[ClickHouse] connection established successfully") +} diff --git a/internal/db/mysql.go b/internal/db/mysql.go index 005b150b..ee97d4e5 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -40,6 +40,11 @@ var ( ) func init() { + if !config.Config.Database.Enabled { + log.Println("[MySQL] is disabled, skipping MySQL initialization") + return + } + var err error dsn := fmt.Sprintf( diff --git a/internal/db/redis.go b/internal/db/redis.go index ec169c3c..4603445b 100644 --- a/internal/db/redis.go +++ b/internal/db/redis.go @@ -41,6 +41,11 @@ var ( func init() { redisConfig := config.Config.Redis + if !redisConfig.Enabled { + log.Println("[Redis] is disabled, skipping Redis initialization") + return + } + addr := fmt.Sprintf("%s:%d", redisConfig.Host, redisConfig.Port) Redis = redis.NewClient( diff --git a/support-files/sql/create_clickhouse.sql b/support-files/sql/create_clickhouse.sql new file mode 100644 index 00000000..0ff648c7 --- /dev/null +++ b/support-files/sql/create_clickhouse.sql @@ -0,0 +1,22 @@ +CREATE DATABASE IF NOT EXISTS linux_do_cdk; + +USE linux_do_cdk; + +CREATE TABLE IF NOT EXISTS err_receive_logs +( + request_time DateTime, + trace_id String, + user_id UInt64, + user_name String, + project_id String, + project_start_time DateTime, + project_end_time DateTime, + error_message String DEFAULT '', + client_info String DEFAULT '' +) ENGINE = MergeTree() + PARTITION BY toYYYYMM(request_time) + ORDER BY (request_time, user_id, project_id) + SETTINGS index_granularity = 8192; +ALTER TABLE err_receive_logs + ADD INDEX idx_user_id (user_id) TYPE bloom_filter(0.01) GRANULARITY 1, + ADD INDEX idx_project_id (project_id) TYPE bloom_filter(0.01) GRANULARITY 1;