diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index ae78167fb45..90afb80be5a 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -118,10 +118,16 @@ To set up PagerDuty, all you have to do is to provide an integration key. Setting | Description ---------- | ----------- Integration Key | Integration key for PagerDuty. -Severity | Level for dynamic notifications, default is `critical` +Severity | Level for dynamic notifications, default is `critical` (1) Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok +Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2) -**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. +>**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. + +>Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event. +This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`. +Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`. +This behavior will become the default in a future version of Grafana. ### Webhook diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index e32590901ea..fc919ef5b47 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -44,6 +44,15 @@ func init() { tooltip="Resolve incidents in pagerduty once the alert goes back to ok."> +
+ + +
`, }) } @@ -57,16 +66,18 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e severity := model.Settings.Get("severity").MustString("critical") autoResolve := model.Settings.Get("autoResolve").MustBool(false) key := model.Settings.Get("integrationKey").MustString() + messageInDetails := model.Settings.Get("messageInDetails").MustBool(false) if key == "" { return nil, alerting.ValidationError{Reason: "Could not find integration key property in settings"} } return &PagerdutyNotifier{ - NotifierBase: NewNotifierBase(model), - Key: key, - Severity: severity, - AutoResolve: autoResolve, - log: log.New("alerting.notifier.pagerduty"), + NotifierBase: NewNotifierBase(model), + Key: key, + Severity: severity, + AutoResolve: autoResolve, + MessageInDetails: messageInDetails, + log: log.New("alerting.notifier.pagerduty"), }, nil } @@ -74,10 +85,11 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e // alert notifications to pagerduty type PagerdutyNotifier struct { NotifierBase - Key string - Severity string - AutoResolve bool - log log.Logger + Key string + Severity string + AutoResolve bool + MessageInDetails bool + log log.Logger } // buildEventPayload is responsible for building the event payload body for sending to Pagerduty v2 API @@ -88,8 +100,17 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext eventType = "resolve" } customData := simplejson.New() - for _, evt := range evalContext.EvalMatches { - customData.Set(evt.Metric, evt.Value) + if pn.MessageInDetails { + queries := make(map[string]interface{}) + for _, evt := range evalContext.EvalMatches { + queries[evt.Metric] = evt.Value + } + customData.Set("queries", queries) + customData.Set("message", evalContext.Rule.Message) + } else { + for _, evt := range evalContext.EvalMatches { + customData.Set(evt.Metric, evt.Value) + } } pn.log.Info("Notifying Pagerduty", "event_type", eventType) @@ -129,7 +150,12 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext } } - summary := evalContext.Rule.Name + " - " + evalContext.Rule.Message + var summary string + if pn.MessageInDetails { + summary = evalContext.Rule.Name + } else { + summary = evalContext.Rule.Name + " - " + evalContext.Rule.Message + } if len(summary) > 1024 { summary = summary[0:1024] } diff --git a/pkg/services/alerting/notifiers/pagerduty_test.go b/pkg/services/alerting/notifiers/pagerduty_test.go index b6f2e2e20df..a5d94287beb 100644 --- a/pkg/services/alerting/notifiers/pagerduty_test.go +++ b/pkg/services/alerting/notifiers/pagerduty_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" @@ -167,6 +168,76 @@ func TestPagerdutyNotifier(t *testing.T) { So(diff, ShouldBeEmpty) }) + Convey("should return properly formatted payload with message moved to details", func() { + json := `{ + "integrationKey": "abcdefgh0123456789", + "autoResolve": false, + "messageInDetails": true + }` + + settingsJSON, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + model := &models.AlertNotification{ + Name: "pagerduty_testing", + Type: "pagerduty", + Settings: settingsJSON, + } + + not, err := NewPagerdutyNotifier(model) + So(err, ShouldBeNil) + + pagerdutyNotifier := not.(*PagerdutyNotifier) + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ + ID: 0, + Name: "someRule", + Message: "someMessage", + State: models.AlertStateAlerting, + }) + evalContext.IsTestRun = true + evalContext.EvalMatches = []*alerting.EvalMatch{ + { + // nil is a terrible value to test with, but the cmp.Diff doesn't + // like comparing actual floats. So this is roughly the equivalent + // of <> + Value: null.FloatFromPtr(nil), + Metric: "someMetric", + }, + } + + payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext) + So(err, ShouldBeNil) + payload, err := simplejson.NewJson(payloadJSON) + So(err, ShouldBeNil) + + diff := cmp.Diff(map[string]interface{}{ + "client": "Grafana", + "client_url": "", + "dedup_key": "alertId-0", + "event_action": "trigger", + "links": []interface{}{ + map[string]interface{}{ + "href": "", + }, + }, + "payload": map[string]interface{}{ + "component": "Grafana", + "source": "<>", + "custom_details": map[string]interface{}{ + "message": "someMessage", + "queries": map[string]interface{}{ + "someMetric": nil, + }, + }, + "severity": "critical", + "summary": "someRule", + "timestamp": "<>", + }, + "routing_key": "abcdefgh0123456789", + }, payload.Interface(), cmp.Comparer(presenceComparer)) + So(diff, ShouldBeEmpty) + }) + Convey("should return properly formatted v2 event payload when using override tags", func() { json := `{ "integrationKey": "abcdefgh0123456789",