Skip to content
This repository was archived by the owner on Apr 7, 2024. It is now read-only.

Commit d0d2e71

Browse files
new grafana notify (#16)
* new grafana notify * working grafana alerts * fix mistakes * blocks update * change image h*w * update orgid feild * fix id remove grafana host * small fixes, remove channel override
1 parent d84e7a4 commit d0d2e71

File tree

6 files changed

+347
-6
lines changed

6 files changed

+347
-6
lines changed

config/notifiers.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,9 @@ type SlackConfigV2 struct {
397397
NotifierConfig `yaml:",inline" json:",inline"`
398398

399399
Token Secret `yaml:"token,omitempty" json:"token,omitempty"`
400+
GrafanaToken Secret `yaml:"grafana_token,omitempty" json:"grafana_token,omitempty"`
401+
UserToken Secret `yaml:"user_token,omitempty" json:"user_token,omitempty"`
402+
GrafanaHost string `yaml:"grafana_host,omitempty" json:"grafana_host,omitempty"`
400403
Channel string `yaml:"channel,omitempty" json:"channel,omitempty"`
401404
Color string `yaml:"color,omitempty" json:"color,omitempty"`
402405
Debug bool `yaml:"debug" json:"debug"`

go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ require (
1515
github.com/go-openapi/validate v0.20.2
1616
github.com/gofrs/uuid v4.0.0+incompatible
1717
github.com/gogo/protobuf v1.3.2
18+
github.com/gorilla/websocket v1.5.0 // indirect
1819
github.com/hashicorp/go-sockaddr v1.0.2
1920
github.com/hashicorp/go-uuid v1.0.1 // indirect
2021
github.com/hashicorp/golang-lru v0.5.4
@@ -30,9 +31,10 @@ require (
3031
github.com/prometheus/common/sigv4 v0.1.0
3132
github.com/prometheus/exporter-toolkit v0.7.0
3233
github.com/rs/cors v1.8.0
34+
github.com/satori/go.uuid v1.2.0
3335
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
3436
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
35-
github.com/slack-go/slack v0.10.0
37+
github.com/slack-go/slack v0.10.4-0.20220503131901-0e14c9d4a15c
3638
github.com/stretchr/testify v1.7.0
3739
github.com/xlab/treeprint v1.1.0
3840
go.uber.org/atomic v1.9.0

go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
273273
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
274274
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
275275
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
276+
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
277+
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
276278
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
277279
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
278280
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -291,6 +293,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
291293
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
292294
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
293295
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
296+
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
297+
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
294298
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
295299
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
296300
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
@@ -435,6 +439,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
435439
github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so=
436440
github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM=
437441
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
442+
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
443+
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
438444
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
439445
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
440446
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -449,6 +455,10 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
449455
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
450456
github.com/slack-go/slack v0.10.0 h1:L16Eqg3QZzRKGXIVsFSZdJdygjOphb2FjRUwH6VrFu8=
451457
github.com/slack-go/slack v0.10.0/go.mod h1:wWL//kk0ho+FcQXcBTmEafUI5dz4qz5f4mMk8oIkioQ=
458+
github.com/slack-go/slack v0.10.3 h1:kKYwlKY73AfSrtAk9UHWCXXfitudkDztNI9GYBviLxw=
459+
github.com/slack-go/slack v0.10.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
460+
github.com/slack-go/slack v0.10.4-0.20220503131901-0e14c9d4a15c h1:fOqgV3BG04rDivs8IKXQATLgQHfsWaOfYu4XZK3XNMM=
461+
github.com/slack-go/slack v0.10.4-0.20220503131901-0e14c9d4a15c/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
452462
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
453463
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
454464
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

notify/slackV2/blocks.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,14 @@ type Field struct {
2121
Type string `json:"type"`
2222
Text string `json:"text"`
2323
}
24+
2425
type Block struct {
2526
Type slack.MessageBlockType `json:"type"`
2627
Text *Text `json:"text,omitempty"`
2728
Fields []*Field `json:"fields,omitempty"`
2829
Elements []*Element `json:"elements,omitempty"`
30+
ImageURL string `json:"image_url,omitempty"`
31+
AltText string `json:"alt_text,omitempty"`
2932
}
3033

3134
func (b Block) BlockType() slack.MessageBlockType {

notify/slackV2/grafana.go

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
package slackV2
2+
3+
import (
4+
"fmt"
5+
"github.com/prometheus/alertmanager/config"
6+
"github.com/prometheus/alertmanager/template"
7+
"github.com/prometheus/common/model"
8+
"github.com/satori/go.uuid"
9+
"github.com/slack-go/slack"
10+
"net/http"
11+
url2 "net/url"
12+
"strconv"
13+
"strings"
14+
"time"
15+
)
16+
17+
func genGrafanaRenderUrl(dash string, panel string, org string, host string) string {
18+
19+
const unixMinute = 1000 * 60
20+
const unixSec = 1000
21+
const imageWidth = "999"
22+
const imageHeight = "333"
23+
const timeZone = "Europe/Moscow"
24+
const urlPath = "/render/d-solo/"
25+
const urlScheme = "https"
26+
27+
url := ""
28+
29+
if u, err := url2.Parse(""); err == nil {
30+
u.Scheme = urlScheme
31+
u.Host = host
32+
u.Path = urlPath + dash + "/"
33+
q := u.Query()
34+
q.Set("orgId", org)
35+
q.Set("from", strconv.FormatInt(time.Now().UnixMilli()-(unixMinute*60), 10))
36+
q.Set("to", strconv.FormatInt(time.Now().UnixMilli()-(unixSec*10), 10))
37+
q.Set("panelId", panel)
38+
q.Set("width", imageWidth)
39+
q.Set("height", imageHeight)
40+
q.Set("tz", timeZone)
41+
u.RawQuery = q.Encode()
42+
url = u.String()
43+
}
44+
return url
45+
}
46+
func genGrafanaUrl(dash string, panel string, org string, host string) string {
47+
48+
const urlScheme = "https"
49+
50+
DashUrl := ""
51+
52+
if u, err := url2.Parse(""); err == nil {
53+
u.Scheme = urlScheme
54+
u.Host = host
55+
u.Path = "d/" + dash
56+
q := u.Query()
57+
q.Set("orgId", org)
58+
if panel != "" {
59+
q.Set("viewPanel", panel)
60+
}
61+
u.RawQuery = q.Encode()
62+
DashUrl = u.String()
63+
}
64+
return DashUrl
65+
}
66+
func urlMerger(kUrl string, pUrl string) string {
67+
imageLink := ""
68+
key := ""
69+
if u, err := url2.Parse(kUrl); err == nil {
70+
key = u.Path
71+
}
72+
73+
trunc := []rune(key)
74+
key = string(trunc[len(trunc)-10:])
75+
76+
if u2, err := url2.Parse(pUrl); err == nil {
77+
q := u2.Query()
78+
q.Set("pub_secret", key)
79+
u2.RawQuery = q.Encode()
80+
imageLink = u2.String()
81+
}
82+
return imageLink
83+
}
84+
85+
func getUploadedImageUrl(url string, token config.Secret, grafanaToken config.Secret) string {
86+
87+
client := &http.Client{}
88+
req, err := http.NewRequest("GET", url, nil)
89+
req.Header.Set("Authorization", "Bearer "+string(grafanaToken))
90+
response, err := client.Do(req)
91+
92+
defer response.Body.Close()
93+
94+
if err != nil {
95+
fmt.Printf("invalid req: %v\n", err)
96+
return ""
97+
}
98+
99+
if response.StatusCode != 200 {
100+
fmt.Printf("received non 200 response code, code: %v\n", response.StatusCode)
101+
return ""
102+
}
103+
104+
uuid := uuid.NewV4()
105+
fileName := strings.Replace(uuid.String(), "-", "", -1)
106+
api := slack.New(string(token))
107+
params := slack.FileUploadParameters{
108+
Reader: response.Body,
109+
Filetype: "jpg",
110+
Filename: fileName + ".jpg",
111+
}
112+
image, err := api.UploadFile(params)
113+
114+
if err != nil {
115+
fmt.Printf("UPLOAD ERROR. Name: %s\n", image.Name)
116+
return ""
117+
}
118+
sharedUrl, _, _, err := api.ShareFilePublicURL(image.ID)
119+
120+
if err != nil {
121+
fmt.Printf("SharedError :%v\n", sharedUrl)
122+
return ""
123+
}
124+
125+
imageUrl := urlMerger(sharedUrl.PermalinkPublic, sharedUrl.URLPrivate)
126+
127+
return imageUrl
128+
129+
}
130+
131+
func (n *Notifier) formatGrafanaMessage(data *template.Data) slack.Blocks {
132+
133+
dashboardUid := ""
134+
panelId := ""
135+
orgId := ""
136+
//grafanaValues := ""
137+
runBook := ""
138+
firing := make([]string, 0)
139+
resolved := make([]string, 0)
140+
severity := make([]string, 0)
141+
envs := make([]string, 0)
142+
blocks := make([]slack.Block, 0)
143+
144+
for _, alert := range data.Alerts {
145+
for _, v := range alert.Labels.SortedPairs() {
146+
switch v.Name {
147+
case "host_name":
148+
switch model.AlertStatus(alert.Status) {
149+
case model.AlertFiring:
150+
firing = append(firing, v.Value)
151+
case model.AlertResolved:
152+
resolved = append(resolved, v.Value)
153+
}
154+
case "severity":
155+
severity = append(severity, v.Value)
156+
case "env":
157+
envs = append(envs, v.Value)
158+
}
159+
}
160+
for _, v := range alert.Annotations.SortedPairs() {
161+
switch v.Name {
162+
163+
case "__dashboardUid__":
164+
dashboardUid = v.Value
165+
case "__panelId__":
166+
panelId = v.Value
167+
case "orgid":
168+
orgId = v.Value
169+
//case "__value_string__":
170+
// grafanaValues = v.Value
171+
case "runbook_url":
172+
runBook = v.Value
173+
}
174+
}
175+
}
176+
177+
severity = UniqStr(severity)
178+
resolved = UniqStr(resolved)
179+
firing = UniqStr(firing)
180+
envs = UniqStr(envs)
181+
182+
grafanaDashUrl := genGrafanaUrl(dashboardUid, "", orgId, n.conf.GrafanaHost)
183+
grafanaPanelUrl := genGrafanaUrl(dashboardUid, panelId, orgId, n.conf.GrafanaHost)
184+
grafanaImageUrl := genGrafanaRenderUrl(dashboardUid, panelId, orgId, n.conf.GrafanaHost)
185+
slackImageUrl := getUploadedImageUrl(grafanaImageUrl, n.conf.UserToken, n.conf.GrafanaToken)
186+
187+
{
188+
url := ""
189+
if urlParsed, err := url2.Parse(data.ExternalURL); err == nil {
190+
urlParsed.Path = "/#/silences/new"
191+
args := urlParsed.Query()
192+
filters := make([]string, 0)
193+
for _, v := range data.CommonLabels.SortedPairs() {
194+
filters = append(filters, fmt.Sprintf("%s=\"%s\"", v.Name, v.Value))
195+
}
196+
args.Add("filter", fmt.Sprintf("{%s}", strings.Join(filters, ",")))
197+
urlParsed.RawQuery = args.Encode()
198+
url = urlParsed.String()
199+
url = strings.Replace(url, "%23", "#", 1)
200+
}
201+
202+
alertEditUrl := ""
203+
for _, alert := range data.Alerts {
204+
if alert.GeneratorURL != "" {
205+
alertEditUrl = alert.GeneratorURL + "?orgId=" + orgId
206+
break
207+
}
208+
}
209+
210+
//Header
211+
blocks = append(blocks, Block{Type: slack.MBTHeader, Text: &Text{Type: slack.PlainTextType, Text: getMapValue(data.CommonLabels, "alertname")}})
212+
213+
//Divider
214+
//blocks = append(blocks, Block{Type: slack.MBTDivider})
215+
216+
//Env and severity
217+
fields := make([]*Field, 0)
218+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Env: %s*", strings.ToUpper(strings.Join(envs, ", ")))})
219+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Severety: %s*", strings.ToUpper(strings.Join(severity, ", ")))})
220+
221+
//Buttons
222+
if grafanaPanelUrl != "" {
223+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|:chart_with_upwards_trend:Panel>*", grafanaPanelUrl)})
224+
} else {
225+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf(":chart_with_upwards_trend:~Panel~")})
226+
}
227+
228+
if url != "" {
229+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|:no_bell:Silence>*", url)})
230+
} else {
231+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|:no_bell:Silence>*", url)})
232+
}
233+
if grafanaDashUrl != "" {
234+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|:dashboard:Dash>*", grafanaDashUrl)})
235+
} else {
236+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf(":dashboard:~Dash~")})
237+
}
238+
if alertEditUrl != "" {
239+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|:gear:Edit>*", alertEditUrl)})
240+
} else {
241+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*:gear:~Edit~")})
242+
}
243+
244+
blocks = append(blocks, Block{Type: slack.MBTSection, Fields: fields})
245+
}
246+
247+
//Firing > Resolved
248+
if len(firing) > 0 && len(resolved) > 0 {
249+
fields := make([]*Field, 0)
250+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Firing:* `%s`", strings.Join(firing, ", "))})
251+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Resolved:* `%s`", strings.Join(resolved, ", "))})
252+
blocks = append(blocks, Block{Type: slack.MBTSection, Fields: fields})
253+
} else {
254+
fields := make([]*Field, 0)
255+
if len(resolved) > 0 {
256+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Resolved: *`%s`", strings.Join(resolved, ", "))})
257+
} else {
258+
fields = append(fields, &Field{Type: slack.MarkdownType, Text: fmt.Sprintf("*Firing: *`%s`", strings.Join(firing, ", "))})
259+
}
260+
blocks = append(blocks, Block{Type: slack.MBTSection, Fields: fields})
261+
}
262+
263+
//GrafanaImage
264+
if slackImageUrl != "" {
265+
blocks = append(blocks, Block{Type: slack.MBTImage, ImageURL: slackImageUrl, AltText: "inspiration"})
266+
}
267+
268+
//Summary and description
269+
{
270+
block := Block{Type: slack.MBTContext, Elements: make([]*Element, 0)}
271+
272+
if val := getMapValue(data.CommonAnnotations, "description"); len(val) > 0 {
273+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*Description:* %s\n\n", val)})
274+
} else {
275+
for _, al := range data.Alerts {
276+
if val, ok := al.Annotations["description"]; ok && len(val) > 0 {
277+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*Description:* %s\n\n", val)})
278+
break
279+
}
280+
}
281+
}
282+
283+
if val := getMapValue(data.CommonAnnotations, "summary"); len(val) > 0 {
284+
if runBook != "" {
285+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|Summary:>* %s", runBook, val)})
286+
} else {
287+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*Summary:* %s", val)})
288+
}
289+
} else {
290+
summary := make([]string, 0)
291+
for _, al := range data.Alerts {
292+
if val, ok := al.Annotations["summary"]; ok && len(val) > 0 {
293+
summary = append(summary, val)
294+
}
295+
}
296+
summary = mergeSameMessages(summary)
297+
if len(summary) > 0 {
298+
if runBook != "" {
299+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*<%s|Summary:>* %s", runBook, cut(strings.Join(summary, ";\n"), 500))})
300+
} else {
301+
block.Elements = append(block.Elements, &Element{Type: slack.MarkdownType, Text: fmt.Sprintf("*Summary:* %s", cut(strings.Join(summary, ";\n"), 500))})
302+
}
303+
}
304+
}
305+
306+
if len(block.Elements) > 0 {
307+
blocks = append(blocks, block)
308+
}
309+
}
310+
311+
result := slack.Blocks{BlockSet: blocks}
312+
return result
313+
}

0 commit comments

Comments
 (0)