Skip to content

Commit d3457db

Browse files
scunninghamscunningham
authored
Move timez to logmatch. (#67)
* Move timez to logmatch. Remove side effect of config file creation if non-existent; this should be the responsibility of the caller if desired. Additionally, the user will not pick up new default timestamp formats once the config file is created, so this avoids that confusion. * Make eval an internal package. Change interface to explicitly require a Config object.
1 parent b1de1fa commit d3457db

File tree

12 files changed

+260
-466
lines changed

12 files changed

+260
-466
lines changed

‎cmd/wasm/wasm.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import (
1010
"time"
1111

1212
"github.com/prequel-dev/preq/internal/pkg/config"
13+
"github.com/prequel-dev/preq/internal/pkg/eval"
1314
"github.com/prequel-dev/preq/internal/pkg/ux"
1415
"github.com/prequel-dev/preq/internal/pkg/verz"
15-
"github.com/prequel-dev/preq/pkg/eval"
1616
"github.com/rs/zerolog/log"
1717
)
1818

@@ -66,10 +66,10 @@ func detectWrapper(ctx context.Context) js.Func {
6666
detectFunc := js.FuncOf(func(this js.Value, args []js.Value) any {
6767

6868
var (
69-
cfg, inputData, ruleData string
70-
reportDoc ux.ReportDocT
71-
stats ux.StatsT
72-
err error
69+
inputData, ruleData string
70+
reportDoc ux.ReportDocT
71+
stats ux.StatsT
72+
err error
7373
)
7474

7575
log.Info().
@@ -82,7 +82,7 @@ func detectWrapper(ctx context.Context) js.Func {
8282
ruleData = args[1].String()
8383

8484
// Permit events to arrive out of order within a 1 hour window by default
85-
cfg = config.Marshal(config.WithWindow(time.Hour))
85+
cfg := config.DefaultConfig(config.WithWindow(time.Hour))
8686

8787
if reportDoc, stats, err = eval.Detect(ctx, cfg, inputData, ruleData); err != nil {
8888
return errJson(err)

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ require (
99
github.com/golang-jwt/jwt v3.2.2+incompatible
1010
github.com/jedib0t/go-pretty/v6 v6.7.5
1111
github.com/posener/complete v1.2.3
12-
github.com/prequel-dev/prequel-logmatch v0.0.16
12+
github.com/prequel-dev/prequel-logmatch v0.0.19
1313
github.com/rs/zerolog v1.34.0
1414
github.com/willabides/kongplete v0.4.0
1515
gopkg.in/yaml.v3 v3.0.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,8 @@ github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXq
215215
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
216216
github.com/prequel-dev/prequel-compiler v0.0.20 h1:NTDfvUgmT83on6RDA+GihAgNM4ZvGLo9r34DAdc2tFQ=
217217
github.com/prequel-dev/prequel-compiler v0.0.20/go.mod h1:sqZKM2senjfoqjBqnq6u/r3qoWjTSCxI8rh4sW9zQkQ=
218-
github.com/prequel-dev/prequel-logmatch v0.0.16 h1:7e/ATexC8MmRGDyxpNdy8taDdFWUNRp1y3JQUOwVPfQ=
219-
github.com/prequel-dev/prequel-logmatch v0.0.16/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg=
218+
github.com/prequel-dev/prequel-logmatch v0.0.19 h1:0YslB6BqcEPGv0LUampO6rnfvVhpYGWIVseWgcJlsek=
219+
github.com/prequel-dev/prequel-logmatch v0.0.19/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg=
220220
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
221221
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
222222
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=

‎internal/pkg/cli/cli.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ import (
1313
"github.com/prequel-dev/preq/internal/pkg/resolve"
1414
"github.com/prequel-dev/preq/internal/pkg/rules"
1515
"github.com/prequel-dev/preq/internal/pkg/runbook"
16-
"github.com/prequel-dev/preq/internal/pkg/timez"
1716
"github.com/prequel-dev/preq/internal/pkg/utils"
1817
"github.com/prequel-dev/preq/internal/pkg/ux"
1918
"github.com/prequel-dev/prequel-compiler/pkg/datasrc"
19+
"github.com/prequel-dev/prequel-logmatch/pkg/timez"
2020
"github.com/rs/zerolog/log"
2121
)
2222

‎internal/pkg/config/config.go‎

Lines changed: 38 additions & 197 deletions
Original file line numberDiff line numberDiff line change
@@ -1,180 +1,18 @@
11
package config
22

33
import (
4-
"fmt"
4+
"io"
55
"os"
66
"path/filepath"
77
"strings"
88
"time"
99

1010
"github.com/prequel-dev/preq/internal/pkg/resolve"
11+
"github.com/prequel-dev/prequel-logmatch/pkg/timez"
1112
"github.com/rs/zerolog/log"
1213
"gopkg.in/yaml.v3"
1314
)
1415

15-
var (
16-
defaultConfig = `timestamps:
17-
18-
# Example: {"level":"error","error":"context deadline exceeded","time":1744570895480541,"caller":"server.go:462"}
19-
- format: epochany
20-
pattern: |
21-
"time":(\d{16,19})
22-
23-
# Example: 2006-01-02T15:04:05Z07:00 <log message>
24-
- format: rfc3339
25-
pattern: |
26-
^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+\-]\d{2}:\d{2}))
27-
28-
# Example: 2006/01/02 03:04:05 <log message>
29-
- format: "2006/01/02 03:04:05"
30-
pattern: |
31-
^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})
32-
33-
# Example: 2006-01-02 15:04:05.000 <log message>
34-
# Source: ISO 8601
35-
- format: "2006-01-02 15:04:05.000"
36-
pattern: |
37-
^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})
38-
39-
# Example: Apr 30 23:36:47.715984 WRN <log message>
40-
# Source: RFC 3164 extended
41-
- format: "Jan 2 15:04:05.000000"
42-
pattern: |
43-
^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{6})
44-
45-
# Example: Jan 2 15:04:05 <log message>
46-
# Source: RFC 3164
47-
- format: "Jan 2 15:04:05"
48-
pattern: |
49-
^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2})
50-
51-
# Example: 2006-01-02 15:04:05 <log message>
52-
# Source: w3c, Postgres
53-
- format: "2006-01-02 15:04:05"
54-
pattern: |
55-
^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})
56-
57-
# Example: I0102 15:04:05.000000 <log message>
58-
# Source: go/klog
59-
- format: "0102 15:04:05.000000"
60-
pattern: |
61-
^[IWEF](\d{4} \d{2}:\d{2}:\d{2}\.\d{6})
62-
63-
# Example: [2006-01-02 15:04:05,000] <log message>
64-
- format: "2006-01-02 15:04:05,000"
65-
pattern: |
66-
^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]
67-
68-
# Example: 2006-01-02 15:04:05.000000-0700 <log message>
69-
- format: "2006-01-02 15:04:05.000000-0700"
70-
pattern: |
71-
^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}[+-]\d{4})
72-
73-
# Example: 2006/01/02 15:04:05 <log message>
74-
- format: "2006/01/02 15:04:05"
75-
pattern: |
76-
^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})
77-
78-
# Example: 01/02/2006, 15:04:05 <log message>
79-
# Source: IIS format
80-
- format: "01/02/2006, 15:04:05"
81-
pattern: |
82-
^(\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2})
83-
84-
# Example: 02 Jan 2006 15:04:05.000 <log message>
85-
- format: "02 Jan 2006 15:04:05.000"
86-
pattern: |
87-
^(\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2}\.\d{3})
88-
89-
# Example: 2006 Jan 02 15:04:05.000 <log message>
90-
- format: "2006 Jan 02 15:04:05.000"
91-
pattern: |
92-
^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}\.\d{3})
93-
94-
# Example: 02/Jan/2006:15:04:05.000 <log message>
95-
- format: "02/Jan/2006:15:04:05.000"
96-
pattern: |
97-
^(\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3})
98-
99-
# Example: 01/02/2006 03:04:05 PM <log message>
100-
- format: "01/02/2006 03:04:05 PM"
101-
pattern: |
102-
^(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} (AM|PM))
103-
104-
# Example: 2006 Jan 02 15:04:05 <log message>
105-
- format: "2006 Jan 02 15:04:05"
106-
pattern: |
107-
^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2})
108-
109-
# Example: 2006-01-02 15:04:05.000 <log message>
110-
- format: "2006-01-02 15:04:05.000"
111-
pattern: |
112-
^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})
113-
114-
# Example: {"timestamp":"2025-03-26T14:01:02Z","level":"info", "message":"..."}
115-
# Source: Postgres JSON output
116-
- format: rfc3339
117-
pattern: |
118-
"timestamp"\s*:\s*"([^"]+)"
119-
120-
# Example: {"ts":"2025-03-26T14:01:02Z","level":"info", "message":"..."}
121-
# Source: metallb
122-
- format: rfc3339
123-
pattern: |
124-
"ts"\s*:\s*"([^"]+)"
125-
126-
# Example: [7] 2025/04/25 02:01:04.339092 [ERR] 10.0.6.53:27827 - cid:10110160 - TLS handshake error: EOF
127-
# Source: NATS
128-
- format: "2006/01/02 15:04:05.000000"
129-
pattern: |
130-
^\[\d+\]\s+(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6})
131-
132-
# Example: {"creationTimestamp":"2025-04-23T20:50:35Z","name":"insecure-nginx-conf","namespace":"default","resourceVersion":"825013"}
133-
# Source: Kubernetes events, configmaps
134-
- format: rfc3339
135-
pattern: |
136-
"creationTimestamp":"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)"
137-
138-
# Example: 2025-04-24T21:55:08.535-0500 INFO example-log-entry
139-
# Source: ZAP production
140-
- format: "2006-01-02T15:04:05.000-0700"
141-
pattern: |
142-
^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4})
143-
144-
# Example: {"level":"info","ts":1745549708.5355184,"msg":"example-log-entry"}
145-
# Source: ZAP development
146-
- format: epochany
147-
pattern: |
148-
"ts"\s*:\s*([0-9]+)(?:\.[0-9]+)?
149-
150-
# Example: ts=2025-03-10T13:52:40.623431174Z level=info msg="tail routine: tail channel closed...
151-
# Source: Loki
152-
- format: "2006-01-02T15:04:05.000000000Z"
153-
pattern: |
154-
ts=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9}Z)
155-
156-
# Example: {"event": "DD_API_KEY undefined. Metrics, logs and events will not be reported to DataDog", "timestamp": "2025-02-12T18:12:58.715528Z", "level": "warn...
157-
# Source: DataDog
158-
- format: "2006-01-02T15:04:05.000000Z"
159-
pattern: |
160-
"timestamp"\s*:\s*"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z)"
161-
162-
# Example: {"Id":19,"Version":1,"Opcode":13,"RecordId":1493,"LogName":"System","ProcessId":4324,"ThreadId":10456,"MachineName":"windows","TimeCreated":"\/Date(1743448267142)\/"}
163-
# Source; Windows events via Get-Events w/ JSON output
164-
- format: epochany
165-
pattern: |
166-
/Date\((\d+)\)
167-
168-
# Example: time="2025-02-12T18:12:58.715528Z"
169-
# Source: argocd
170-
- format: rfc3339
171-
pattern: |
172-
time="([^"]+)"
173-
`
174-
175-
windowConfig = `window: %s`
176-
)
177-
17816
type Config struct {
17917
TimestampRegexes []Regex `yaml:"timestamps"`
18018
Rules Rules `yaml:"rules"`
@@ -216,57 +54,60 @@ func parseOpts(opts ...OptT) *optsT {
21654
return o
21755
}
21856

219-
func Marshal(opts ...OptT) string {
220-
o := parseOpts(opts...)
221-
222-
if o.window > 0 {
223-
wcfg := fmt.Sprintf(windowConfig, o.window)
224-
return fmt.Sprintf("%s\n%s\n", defaultConfig, wcfg)
225-
}
226-
227-
return defaultConfig
228-
}
57+
// LoadConfig loads the configuration from the specified directory and file.
58+
// If the file does not exist, it returns a default configuration.
22959

23060
func LoadConfig(dir, file string, opts ...OptT) (*Config, error) {
23161

232-
var config Config
233-
234-
if _, err := os.Stat(dir); os.IsNotExist(err) {
235-
if err := os.MkdirAll(dir, 0755); err != nil {
236-
return nil, err
237-
}
62+
spec := filepath.Join(dir, file)
63+
_, err := os.Stat(spec)
64+
65+
switch {
66+
case err == nil: // NOOP
67+
case os.IsNotExist(err):
68+
log.Info().
69+
Str("file", spec).
70+
Msg("Configuration file does not exist, using default configuration")
71+
return DefaultConfig(opts...), nil
72+
default:
73+
return nil, err
23874
}
23975

240-
if _, err := os.Stat(filepath.Join(dir, file)); os.IsNotExist(err) {
241-
if err := WriteDefaultConfig(filepath.Join(dir, file), opts...); err != nil {
242-
log.Error().Err(err).Msg("Failed to write default config")
243-
return nil, err
244-
}
76+
log.Info().Str("file", spec).Msg("Loading configuration file")
77+
fh, err := os.OpenFile(spec, os.O_RDONLY, 0644)
78+
if err != nil {
79+
return nil, err
24580
}
81+
defer fh.Close()
82+
83+
return ReadConfig(fh)
84+
}
24685

247-
data, err := os.ReadFile(filepath.Join(dir, file))
86+
func ReadConfig(rd io.Reader) (*Config, error) {
87+
data, err := io.ReadAll(rd)
24888
if err != nil {
24989
return nil, err
25090
}
251-
91+
var config Config
25292
if err := yaml.Unmarshal(data, &config); err != nil {
25393
return nil, err
25494
}
255-
25695
return &config, nil
25796
}
25897

259-
func WriteDefaultConfig(path string, opts ...OptT) error {
260-
cfg := Marshal(opts...)
261-
return os.WriteFile(path, []byte(cfg), 0644)
262-
}
98+
func DefaultConfig(opts ...OptT) *Config {
99+
o := parseOpts(opts...)
263100

264-
func LoadConfigFromBytes(data string) (*Config, error) {
265-
var config Config
266-
if err := yaml.Unmarshal([]byte(data), &config); err != nil {
267-
return nil, err
101+
c := &Config{
102+
Window: o.window,
268103
}
269-
return &config, nil
104+
for _, r := range timez.Defaults {
105+
c.TimestampRegexes = append(c.TimestampRegexes, Regex{
106+
Pattern: r.Pattern,
107+
Format: string(r.Format),
108+
})
109+
}
110+
return c
270111
}
271112

272113
func (c *Config) ResolveOpts() (opts []resolve.OptT) {

0 commit comments

Comments
 (0)