Alerting: Remove message from PagerDuty summary field (#24813)

closes #16930

docs/sources/alerting/notifications.md

@@ -118,10 +118,16 @@ To set up PagerDuty, all you have to do is to provide an integration key.
 Setting | Description
 ---------- | -----------
 Integration Key | Integration key for PagerDuty.
-Severity | Level for dynamic notifications, default is `critical`
+Severity | Level for dynamic notifications, default is `critical` (1)
 Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
+Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2)
 
-**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
+>**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
+
+>Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event.
+This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`.
+Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`.
+This behavior will become the default in a future version of Grafana.
 
 ### Webhook
 

pkg/services/alerting/notifiers/pagerduty.go

@@ -44,6 +44,15 @@ func init() {
            tooltip="Resolve incidents in pagerduty once the alert goes back to ok.">
         </gf-form-switch>
       </div>
+      <div class="gf-form">
+        <gf-form-switch
+           class="gf-form"
+           label="Include message in details"
+           label-class="width-14"
+           checked="ctrl.model.settings.messageInDetails"
+           tooltip="Move the alert message from the PD summary into the custom details. This changes the custom details object and may break event rules you have configured">
+        </gf-form-switch>
+      </div>
     `,
 	})
 }
@@ -57,16 +66,18 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e
 	severity := model.Settings.Get("severity").MustString("critical")
 	autoResolve := model.Settings.Get("autoResolve").MustBool(false)
 	key := model.Settings.Get("integrationKey").MustString()
+	messageInDetails := model.Settings.Get("messageInDetails").MustBool(false)
 	if key == "" {
 		return nil, alerting.ValidationError{Reason: "Could not find integration key property in settings"}
 	}
 
 	return &PagerdutyNotifier{
-		NotifierBase: NewNotifierBase(model),
-		Key:          key,
-		Severity:     severity,
-		AutoResolve:  autoResolve,
-		log:          log.New("alerting.notifier.pagerduty"),
+		NotifierBase:     NewNotifierBase(model),
+		Key:              key,
+		Severity:         severity,
+		AutoResolve:      autoResolve,
+		MessageInDetails: messageInDetails,
+		log:              log.New("alerting.notifier.pagerduty"),
 	}, nil
 }
 
@@ -74,10 +85,11 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e
 // alert notifications to pagerduty
 type PagerdutyNotifier struct {
 	NotifierBase
-	Key         string
-	Severity    string
-	AutoResolve bool
-	log         log.Logger
+	Key              string
+	Severity         string
+	AutoResolve      bool
+	MessageInDetails bool
+	log              log.Logger
 }
 
 // buildEventPayload is responsible for building the event payload body for sending to Pagerduty v2 API
@@ -88,8 +100,17 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 		eventType = "resolve"
 	}
 	customData := simplejson.New()
-	for _, evt := range evalContext.EvalMatches {
-		customData.Set(evt.Metric, evt.Value)
+	if pn.MessageInDetails {
+		queries := make(map[string]interface{})
+		for _, evt := range evalContext.EvalMatches {
+			queries[evt.Metric] = evt.Value
+		}
+		customData.Set("queries", queries)
+		customData.Set("message", evalContext.Rule.Message)
+	} else {
+		for _, evt := range evalContext.EvalMatches {
+			customData.Set(evt.Metric, evt.Value)
+		}
 	}
 
 	pn.log.Info("Notifying Pagerduty", "event_type", eventType)
@@ -129,7 +150,12 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 		}
 	}
 
-	summary := evalContext.Rule.Name + " - " + evalContext.Rule.Message
+	var summary string
+	if pn.MessageInDetails {
+		summary = evalContext.Rule.Name
+	} else {
+		summary = evalContext.Rule.Name + " - " + evalContext.Rule.Message
+	}
 	if len(summary) > 1024 {
 		summary = summary[0:1024]
 	}

pkg/services/alerting/notifiers/pagerduty_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/grafana/grafana/pkg/components/null"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
@@ -167,6 +168,76 @@ func TestPagerdutyNotifier(t *testing.T) {
 				So(diff, ShouldBeEmpty)
 			})
 
+			Convey("should return properly formatted payload with message moved to details", func() {
+				json := `{
+					"integrationKey": "abcdefgh0123456789",
+					"autoResolve": false,
+					"messageInDetails": true
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "pagerduty_testing",
+					Type:     "pagerduty",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewPagerdutyNotifier(model)
+				So(err, ShouldBeNil)
+
+				pagerdutyNotifier := not.(*PagerdutyNotifier)
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateAlerting,
+				})
+				evalContext.IsTestRun = true
+				evalContext.EvalMatches = []*alerting.EvalMatch{
+					{
+						// nil is a terrible value to test with, but the cmp.Diff doesn't
+						// like comparing actual floats. So this is roughly the equivalent
+						// of <<PRESENCE>>
+						Value:  null.FloatFromPtr(nil),
+						Metric: "someMetric",
+					},
+				}
+
+				payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+				payload, err := simplejson.NewJson(payloadJSON)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"client":       "Grafana",
+					"client_url":   "",
+					"dedup_key":    "alertId-0",
+					"event_action": "trigger",
+					"links": []interface{}{
+						map[string]interface{}{
+							"href": "",
+						},
+					},
+					"payload": map[string]interface{}{
+						"component": "Grafana",
+						"source":    "<<PRESENCE>>",
+						"custom_details": map[string]interface{}{
+							"message": "someMessage",
+							"queries": map[string]interface{}{
+								"someMetric": nil,
+							},
+						},
+						"severity":  "critical",
+						"summary":   "someRule",
+						"timestamp": "<<PRESENCE>>",
+					},
+					"routing_key": "abcdefgh0123456789",
+				}, payload.Interface(), cmp.Comparer(presenceComparer))
+				So(diff, ShouldBeEmpty)
+			})
+
 			Convey("should return properly formatted v2 event payload when using override tags", func() {
 				json := `{
 					"integrationKey": "abcdefgh0123456789",