diff --git a/fetcher/exploitdb.go b/fetcher/exploitdb.go index dfe0057..e46257b 100644 --- a/fetcher/exploitdb.go +++ b/fetcher/exploitdb.go @@ -584,7 +584,13 @@ func joinExploits(toCVEs map[string][]string, toDocument map[string]models.Docum toExploits[eid] = es } - var exploits []models.Exploit + // Calculate total size for preallocation + totalSize := 0 + for _, es := range toExploits { + totalSize += len(es) + } + + exploits := make([]models.Exploit, 0, totalSize) for _, es := range toExploits { exploits = append(exploits, es...) } diff --git a/server/server.go b/server/server.go index 2ba6202..1570e6f 100644 --- a/server/server.go +++ b/server/server.go @@ -1,10 +1,14 @@ package server import ( + "encoding/json" "fmt" + "io" "net/http" "os" "path/filepath" + "strconv" + "time" "github.com/inconshreveable/log15" "github.com/labstack/echo/v4" @@ -21,7 +25,7 @@ func Start(logToFile bool, logDir string, driver db.DB) error { e.Debug = viper.GetBool("debug") // Middleware - e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: os.Stderr})) + e.Use(middleware.RequestLoggerWithConfig(newRequestLoggerConfig(os.Stderr))) e.Use(middleware.Recover()) // setup access logger @@ -32,7 +36,7 @@ func Start(logToFile bool, logDir string, driver db.DB) error { return xerrors.Errorf("Failed to open a log file: %s", err) } defer f.Close() - e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: f})) + e.Use(middleware.RequestLoggerWithConfig(newRequestLoggerConfig(f))) } // Routes @@ -49,6 +53,64 @@ func Start(logToFile bool, logDir string, driver db.DB) error { return e.Start(bindURL) } +func newRequestLoggerConfig(writer io.Writer) middleware.RequestLoggerConfig { + return middleware.RequestLoggerConfig{ + LogLatency: true, + LogRemoteIP: true, + LogHost: true, + LogMethod: true, + LogURI: true, + LogRequestID: true, + LogUserAgent: true, + LogStatus: true, + LogError: true, + LogContentLength: true, + LogResponseSize: true, + + LogValuesFunc: func(_ echo.Context, v middleware.RequestLoggerValues) error { + type logFormat struct { + Time string `json:"time"` + ID string `json:"id"` + RemoteIP string `json:"remote_ip"` + Host string `json:"host"` + Method string `json:"method"` + URI string `json:"uri"` + UserAgent string `json:"user_agent"` + Status int `json:"status"` + Error string `json:"error"` + Latency int64 `json:"latency"` + LatencyHuman string `json:"latency_human"` + BytesIn int64 `json:"bytes_in"` + BytesOut int64 `json:"bytes_out"` + } + + return json.NewEncoder(writer).Encode(logFormat{ + Time: v.StartTime.Format(time.RFC3339Nano), + ID: v.RequestID, + RemoteIP: v.RemoteIP, + Host: v.Host, + Method: v.Method, + URI: v.URI, + UserAgent: v.UserAgent, + Status: v.Status, + Error: func() string { + if v.Error != nil { + return v.Error.Error() + } + return "" + }(), + Latency: v.Latency.Nanoseconds(), + LatencyHuman: v.Latency.String(), + BytesIn: func() int64 { + i, _ := strconv.ParseInt(v.ContentLength, 10, 64) + return i + }(), + BytesOut: v.ResponseSize, + }) + }, + } +} + func health() echo.HandlerFunc { return func(context echo.Context) error { return context.String(http.StatusOK, "")