root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 265 Packages Projects Releases Wiki Activity

Alerting: Refactor PagerDuty and OpsGenie notifiers to use encoding/json to parse settings (#58925) 

* update pagerduty and opsgenie to deserialize settings using standard JSON library
* update pagerduty truncation to use a function from Alertamanger package
* update opsgenie to use payload model (same as in Alertmanager)
This commit is contained in:
Yuri Tseretyan 2022-12-05 11:38:50 -05:00 committed by GitHub
parent 46adfb596d
commit eeb57cd520
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 251 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
pkg/services/ngalert/notifier/channels/opsgenie.go
View File

@ -13,8 +13,8 @@ import (
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
ptr "github.com/xorcare/pointer"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
@ -35,21 +35,14 @@ var (
// OpsgenieNotifier is responsible for sending alert notifications to Opsgenie.
type OpsgenieNotifier struct {
*Base
APIKey string
APIUrl string
Message string
Description string
AutoClose bool
OverridePriority bool
SendTagsAs string
tmpl *template.Template
log log.Logger
ns notifications.WebhookSender
images ImageStore
settings *opsgenieSettings
}
type OpsgenieConfig struct {
*NotificationChannelConfig
type opsgenieSettings struct {
APIKey string
APIUrl string
Message string
@ -59,62 +52,92 @@ type OpsgenieConfig struct {
SendTagsAs string
}
func buildOpsgenieSettings(fc FactoryConfig) (*opsgenieSettings, error) {
type rawSettings struct {
APIKey string `json:"apiKey,omitempty" yaml:"apiKey,omitempty"`
APIUrl string `json:"apiUrl,omitempty" yaml:"apiUrl,omitempty"`
Message string `json:"message,omitempty" yaml:"message,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
AutoClose *bool `json:"autoClose,omitempty" yaml:"autoClose,omitempty"`
OverridePriority *bool `json:"overridePriority,omitempty" yaml:"overridePriority,omitempty"`
SendTagsAs string `json:"sendTagsAs,omitempty" yaml:"sendTagsAs,omitempty"`
}
raw := rawSettings{}
err := fc.Config.unmarshalSettings(&raw)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal settings: %w", err)
}
raw.APIKey = fc.DecryptFunc(context.Background(), fc.Config.SecureSettings, "apiKey", raw.APIKey)
if raw.APIKey == "" {
return nil, errors.New("could not find api key property in settings")
}
if raw.APIUrl == "" {
raw.APIUrl = OpsgenieAlertURL
}
if strings.TrimSpace(raw.Message) == "" {
raw.Message = DefaultMessageTitleEmbed
}
switch raw.SendTagsAs {
case OpsgenieSendTags, OpsgenieSendDetails, OpsgenieSendBoth:
case "":
raw.SendTagsAs = OpsgenieSendTags
default:
return nil, fmt.Errorf("invalid value for sendTagsAs: %q", raw.SendTagsAs)
}
if raw.AutoClose == nil {
raw.AutoClose = ptr.Bool(true)
}
if raw.OverridePriority == nil {
raw.OverridePriority = ptr.Bool(true)
}
return &opsgenieSettings{
APIKey: raw.APIKey,
APIUrl: raw.APIUrl,
Message: raw.Message,
Description: raw.Description,
AutoClose: *raw.AutoClose,
OverridePriority: *raw.OverridePriority,
SendTagsAs: raw.SendTagsAs,
}, nil
}
func OpsgenieFactory(fc FactoryConfig) (NotificationChannel, error) {
cfg, err := NewOpsgenieConfig(fc.Config, fc.DecryptFunc)
notifier, err := NewOpsgenieNotifier(fc)
if err != nil {
return nil, receiverInitError{
Reason: err.Error(),
Cfg: *fc.Config,
}
}
return NewOpsgenieNotifier(cfg, fc.NotificationService, fc.ImageStore, fc.Template, fc.DecryptFunc), nil
}
func NewOpsgenieConfig(config *NotificationChannelConfig, decryptFunc GetDecryptedValueFn) (*OpsgenieConfig, error) {
apiKey := decryptFunc(context.Background(), config.SecureSettings, "apiKey", config.Settings.Get("apiKey").MustString())
if apiKey == "" {
return nil, errors.New("could not find api key property in settings")
}
sendTagsAs := config.Settings.Get("sendTagsAs").MustString(OpsgenieSendTags)
if sendTagsAs != OpsgenieSendTags &&
sendTagsAs != OpsgenieSendDetails &&
sendTagsAs != OpsgenieSendBoth {
return nil, fmt.Errorf("invalid value for sendTagsAs: %q", sendTagsAs)
}
return &OpsgenieConfig{
NotificationChannelConfig: config,
APIKey: apiKey,
APIUrl: config.Settings.Get("apiUrl").MustString(OpsgenieAlertURL),
AutoClose: config.Settings.Get("autoClose").MustBool(true),
OverridePriority: config.Settings.Get("overridePriority").MustBool(true),
Message: config.Settings.Get("message").MustString(`{{ template "default.title" . }}`),
Description: config.Settings.Get("description").MustString(""),
SendTagsAs: sendTagsAs,
}, nil
return notifier, nil
}
// NewOpsgenieNotifier is the constructor for the Opsgenie notifier
func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender, images ImageStore, t *template.Template, fn GetDecryptedValueFn) *OpsgenieNotifier {
func NewOpsgenieNotifier(fc FactoryConfig) (*OpsgenieNotifier, error) {
settings, err := buildOpsgenieSettings(fc)
if err != nil {
return nil, err
}
return &OpsgenieNotifier{
Base: NewBase(&models.AlertNotification{
Uid: config.UID,
Name: config.Name,
Type: config.Type,
DisableResolveMessage: config.DisableResolveMessage,
Settings: config.Settings,
Uid: fc.Config.UID,
Name: fc.Config.Name,
Type: fc.Config.Type,
DisableResolveMessage: fc.Config.DisableResolveMessage,
Settings: fc.Config.Settings,
}),
APIKey: config.APIKey,
APIUrl: config.APIUrl,
Description: config.Description,
Message: config.Message,
AutoClose: config.AutoClose,
OverridePriority: config.OverridePriority,
SendTagsAs: config.SendTagsAs,
tmpl: t,
log: log.New("alerting.notifier." + config.Name),
ns: ns,
images: images,
}
tmpl: fc.Template,
log: log.New("alerting.notifier.opsgenie"),
ns: fc.NotificationService,
images: fc.ImageStore,
settings: settings,
}, nil
}
// Notify sends an alert notification to Opsgenie
@ -127,7 +150,7 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
return true, nil
}
bodyJSON, url, err := on.buildOpsgenieMessage(ctx, alerts, as)
body, url, err := on.buildOpsgenieMessage(ctx, alerts, as)
if err != nil {
return false, fmt.Errorf("build Opsgenie message: %w", err)
}
@ -138,18 +161,13 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
return true, nil
}
body, err := json.Marshal(bodyJSON)
if err != nil {
return false, fmt.Errorf("marshal json: %w", err)
}
cmd := &models.SendWebhookSync{
Url: url,
Body: string(body),
HttpMethod: http.MethodPost,
HttpHeader: map[string]string{
"Content-Type": "application/json",
"Authorization": fmt.Sprintf("GenieKey %s", on.APIKey),
"Authorization": fmt.Sprintf("GenieKey %s", on.settings.APIKey),
},
}
@ -160,46 +178,37 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
return true, nil
}
func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts model.Alerts, as []*types.Alert) (payload *simplejson.Json, apiURL string, err error) {
func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts model.Alerts, as []*types.Alert) (payload []byte, apiURL string, err error) {
key, err := notify.ExtractGroupKey(ctx)
if err != nil {
return nil, "", err
}
var (
alias = key.Hash()
bodyJSON = simplejson.New()
details = simplejson.New()
)
if alerts.Status() == model.AlertResolved {
// For resolved notification, we only need the source.
// Don't need to run other templates.
if on.AutoClose {
bodyJSON := simplejson.New()
bodyJSON.Set("source", "Grafana")
apiURL = fmt.Sprintf("%s/%s/close?identifierType=alias", on.APIUrl, alias)
return bodyJSON, apiURL, nil
}
if !on.settings.AutoClose { // TODO This should be handled by DisableResolveMessage?
return nil, "", nil
}
msg := opsGenieCloseMessage{
Source: "Grafana",
}
data, err := json.Marshal(msg)
apiURL = fmt.Sprintf("%s/%s/close?identifierType=alias", on.settings.APIUrl, key.Hash())
return data, apiURL, err
}
ruleURL := joinUrlPath(on.tmpl.ExternalURL.String(), "/alerting/list", on.log)
var tmplErr error
tmpl, data := TmplText(ctx, on.tmpl, as, on.log, &tmplErr)
titleTmpl := on.Message
if strings.TrimSpace(titleTmpl) == "" {
titleTmpl = `{{ template "default.title" . }}`
message, truncated := notify.Truncate(tmpl(on.settings.Message), 130)
if truncated {
on.log.Debug("Truncated message", "originalMessage", message)
}
title := tmpl(titleTmpl)
if len(title) > 130 {
title = title[:127] + "..."
}
description := tmpl(on.Description)
description := tmpl(on.settings.Description)
if strings.TrimSpace(description) == "" {
description = fmt.Sprintf(
"%s\n%s\n\n%s",
@ -215,8 +224,7 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
lbls := make(map[string]string, len(data.CommonLabels))
for k, v := range data.CommonLabels {
lbls[k] = tmpl(v)
if k == "og_priority" {
if k == "og_priority" && on.settings.OverridePriority {
if ValidPriorities[v] {
priority = v
}
@ -229,18 +237,13 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
tmplErr = nil
}
bodyJSON.Set("message", title)
bodyJSON.Set("source", "Grafana")
bodyJSON.Set("alias", alias)
bodyJSON.Set("description", description)
details.Set("url", ruleURL)
details := make(map[string]interface{})
details["url"] = ruleURL
if on.sendDetails() {
for k, v := range lbls {
details.Set(k, v)
details[k] = v
}
images := []string{}
var images []string
_ = withStoredImages(ctx, on.log, on.images,
func(_ int, image ngmodels.Image) error {
if len(image.URL) == 0 {
@ -252,7 +255,7 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
as...)
if len(images) != 0 {
details.Set("image_urls", images)
details["image_urls"] = images
}
}
@ -264,19 +267,24 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
}
sort.Strings(tags)
if priority != "" && on.OverridePriority {
bodyJSON.Set("priority", priority)
result := opsGenieCreateMessage{
Alias: key.Hash(),
Description: description,
Tags: tags,
Source: "Grafana",
Message: message,
Details: details,
Priority: priority,
}
bodyJSON.Set("tags", tags)
bodyJSON.Set("details", details)
apiURL = tmpl(on.APIUrl)
apiURL = tmpl(on.settings.APIUrl)
if tmplErr != nil {
on.log.Warn("failed to template Opsgenie URL", "error", tmplErr.Error(), "fallback", on.APIUrl)
apiURL = on.APIUrl
on.log.Warn("failed to template Opsgenie URL", "error", tmplErr.Error(), "fallback", on.settings.APIUrl)
apiURL = on.settings.APIUrl
}
return bodyJSON, apiURL, nil
b, err := json.Marshal(result)
return b, apiURL, err
}
func (on *OpsgenieNotifier) SendResolved() bool {
@ -284,9 +292,34 @@ func (on *OpsgenieNotifier) SendResolved() bool {
}
func (on *OpsgenieNotifier) sendDetails() bool {
return on.SendTagsAs == OpsgenieSendDetails || on.SendTagsAs == OpsgenieSendBoth
return on.settings.SendTagsAs == OpsgenieSendDetails || on.settings.SendTagsAs == OpsgenieSendBoth
}
func (on *OpsgenieNotifier) sendTags() bool {
return on.SendTagsAs == OpsgenieSendTags || on.SendTagsAs == OpsgenieSendBoth
return on.settings.SendTagsAs == OpsgenieSendTags || on.settings.SendTagsAs == OpsgenieSendBoth
}
type opsGenieCreateMessage struct {
Alias string `json:"alias"`
Message string `json:"message"`
Description string `json:"description,omitempty"`
Details map[string]interface{} `json:"details"`
Source string `json:"source"`
Responders []opsGenieCreateMessageResponder `json:"responders,omitempty"`
Tags []string `json:"tags"`
Note string `json:"note,omitempty"`
Priority string `json:"priority,omitempty"`
Entity string `json:"entity,omitempty"`
Actions []string `json:"actions,omitempty"`
}
type opsGenieCreateMessageResponder struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Username string `json:"username,omitempty"`
Type string `json:"type"` // team, user, escalation, schedule etc.
}
type opsGenieCloseMessage struct {
Source string `json:"source"`
}

31
pkg/services/ngalert/notifier/channels/opsgenie_test.go
View File

@ -95,7 +95,7 @@ func TestOpsgenieNotifier(t *testing.T) {
"details": {
"url": "http://localhost/alerting/list"
},
"message": "IyJnsW78xQoiBJ7L7NqASv31JCFf0At3r9KUykqBVxSiC6qkDhvDLDW9VImiFcq0Iw2XwFy5fX4FcbTmlkaZzUzjVwx9VUuokhzqQlJVhWDYFqhj3a5wX0LjyvNQjsq...",
"message": "IyJnsW78xQoiBJ7L7NqASv31JCFf0At3r9KUykqBVxSiC6qkDhvDLDW9VImiFcq0Iw2XwFy5fX4FcbTmlkaZzUzjVwx9VUuokhzqQlJVhWDYFqhj3a5wX0LjyvNQjsqT9…",
"source": "Grafana",
"tags": ["alertname:alert1", "lbl1:val1"]
}`,
@ -234,28 +234,33 @@ func TestOpsgenieNotifier(t *testing.T) {
require.NoError(t, err)
secureSettings := make(map[string][]byte)
m := &NotificationChannelConfig{
Name: "opsgenie_testing",
Type: "opsgenie",
Settings: settingsJSON,
SecureSettings: secureSettings,
}
webhookSender := mockNotificationService()
webhookSender.Webhook.Body = "<not-sent>"
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
cfg, err := NewOpsgenieConfig(m, decryptFn)
fc := FactoryConfig{
Config: &NotificationChannelConfig{
Name: "opsgenie_testing",
Type: "opsgenie",
Settings: settingsJSON,
SecureSettings: secureSettings,
},
NotificationService: webhookSender,
DecryptFunc: decryptFn,
ImageStore: &UnavailableImageStore{},
Template: tmpl,
}
ctx := notify.WithGroupKey(context.Background(), "alertname")
ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
pn, err := NewOpsgenieNotifier(fc)
if c.expInitError != "" {
require.Error(t, err)
require.Equal(t, c.expInitError, err.Error())
return
}
require.NoError(t, err)
ctx := notify.WithGroupKey(context.Background(), "alertname")
ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
pn := NewOpsgenieNotifier(cfg, webhookSender, &UnavailableImageStore{}, tmpl, decryptFn)
ok, err := pn.Notify(ctx, c.alerts...)
if c.expMsgError != nil {
require.False(t, ok)

69
pkg/services/ngalert/notifier/channels/pagerduty.go
View File

@ -24,6 +24,8 @@ const (
pagerDutyEventResolve = "resolve"
defaultSeverity = "critical"
defaultClass = "default"
defaultGroup = "default"
)
var (
@ -39,7 +41,7 @@ type PagerdutyNotifier struct {
log log.Logger
ns notifications.WebhookSender
images ImageStore
settings pagerdutySettings
settings *pagerdutySettings
}
type pagerdutySettings struct {
@ -52,6 +54,44 @@ type pagerdutySettings struct {
Summary string `json:"summary,omitempty" yaml:"summary,omitempty"`
}
func buildPagerdutySettings(fc FactoryConfig) (*pagerdutySettings, error) {
settings := pagerdutySettings{}
err := fc.Config.unmarshalSettings(&settings)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal settings: %w", err)
}
settings.Key = fc.DecryptFunc(context.Background(), fc.Config.SecureSettings, "integrationKey", settings.Key)
if settings.Key == "" {
return nil, errors.New("could not find integration key property in settings")
}
settings.customDetails = map[string]string{
"firing": `{{ template "__text_alert_list" .Alerts.Firing }}`,
"resolved": `{{ template "__text_alert_list" .Alerts.Resolved }}`,
"num_firing": `{{ .Alerts.Firing | len }}`,
"num_resolved": `{{ .Alerts.Resolved | len }}`,
}
if settings.Severity == "" {
settings.Severity = defaultSeverity
}
if settings.Class == "" {
settings.Class = defaultClass
}
if settings.Component == "" {
settings.Component = "Grafana"
}
if settings.Group == "" {
settings.Group = defaultGroup
}
if settings.Summary == "" {
settings.Summary = DefaultMessageTitleEmbed
}
return &settings, nil
}
func PagerdutyFactory(fc FactoryConfig) (NotificationChannel, error) {
pdn, err := newPagerdutyNotifier(fc)
if err != nil {
@ -65,9 +105,9 @@ func PagerdutyFactory(fc FactoryConfig) (NotificationChannel, error) {
// NewPagerdutyNotifier is the constructor for the PagerDuty notifier
func newPagerdutyNotifier(fc FactoryConfig) (*PagerdutyNotifier, error) {
key := fc.DecryptFunc(context.Background(), fc.Config.SecureSettings, "integrationKey", fc.Config.Settings.Get("integrationKey").MustString())
if key == "" {
return nil, errors.New("could not find integration key property in settings")
settings, err := buildPagerdutySettings(fc)
if err != nil {
return nil, err
}
return &PagerdutyNotifier{
@ -82,20 +122,7 @@ func newPagerdutyNotifier(fc FactoryConfig) (*PagerdutyNotifier, error) {
log: log.New("alerting.notifier." + fc.Config.Name),
ns: fc.NotificationService,
images: fc.ImageStore,
settings: pagerdutySettings{
Key: key,
Severity: fc.Config.Settings.Get("severity").MustString(defaultSeverity),
customDetails: map[string]string{
"firing": `{{ template "__text_alert_list" .Alerts.Firing }}`,
"resolved": `{{ template "__text_alert_list" .Alerts.Resolved }}`,
"num_firing": `{{ .Alerts.Firing | len }}`,
"num_resolved": `{{ .Alerts.Resolved | len }}`,
},
Class: fc.Config.Settings.Get("class").MustString("default"),
Component: fc.Config.Settings.Get("component").MustString("Grafana"),
Group: fc.Config.Settings.Get("group").MustString("default"),
Summary: fc.Config.Settings.Get("summary").MustString(DefaultMessageTitleEmbed),
},
settings: settings,
}, nil
}
@ -192,9 +219,9 @@ func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts m
},
as...)
if len(msg.Payload.Summary) > 1024 {
// This is the Pagerduty limit.
msg.Payload.Summary = msg.Payload.Summary[:1021] + "..."
if summary, truncated := notify.Truncate(msg.Payload.Summary, 1024); truncated {
pn.log.Debug("Truncated summary", "original", msg.Payload.Summary)
msg.Payload.Summary = summary
}
if hostname, err := os.Hostname(); err == nil {

41
pkg/services/ngalert/notifier/channels/pagerduty_test.go
View File

@ -3,8 +3,11 @@ package channels
import (
"context"
"encoding/json"
"fmt"
"math/rand"
"net/url"
"os"
"strings"
"testing"
"github.com/prometheus/alertmanager/notify"
@ -184,7 +187,43 @@ func TestPagerdutyNotifier(t *testing.T) {
Links: []pagerDutyLink{{HRef: "http://localhost", Text: "External URL"}},
},
expMsgError: nil,
}, {
},
{
name: "should truncate long summary",
settings: fmt.Sprintf(`{"integrationKey": "abcdefgh0123456789", "summary": "%s"}`, strings.Repeat("1", rand.Intn(100)+1025)),
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh"},
},
},
},
expMsg: &pagerDutyMessage{
RoutingKey: "abcdefgh0123456789",
DedupKey: "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
EventAction: "trigger",
Payload: pagerDutyPayload{
Summary: fmt.Sprintf("%s…", strings.Repeat("1", 1023)),
Source: hostname,
Severity: "critical",
Class: "default",
Component: "Grafana",
Group: "default",
CustomDetails: map[string]string{
"firing": "\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
"num_firing": "1",
"num_resolved": "0",
"resolved": "",
},
},
Client: "Grafana",
ClientURL: "http://localhost",
Links: []pagerDutyLink{{HRef: "http://localhost", Text: "External URL"}},
},
expMsgError: nil,
},
{
name: "Error in initing",
settings: `{}`,
expInitError: `could not find integration key property in settings`,