root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 243 Packages Projects Releases Wiki Activity

Alerting: Enable Alert rule `severity` tag to override VictorOps Severity setting (#29392) 

* add severity to victorops

* Update docs, add test

* Fix spelling and lint

* fix resolving not working

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* remove table, update supported alert notifiers

* add docs

* 7.4->7.5

* fix

* Update docs/sources/alerting/notifications.md

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
This commit is contained in:
Ottavio M. Hartman 2021-03-18 09:50:07 -04:00 committed by GitHub
parent 61521216ed
commit 7a9a52c317
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 155 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/sources/alerting/notifications.md
View File

@ -68,7 +68,7 @@ Sensu | `sensu` | yes, external only | no
[Slack](#slack) | `slack` | yes | no
Telegram | `telegram` | yes | no
Threema | `threema` | yes, external only | no
VictorOps | `victorops` | yes, external only | no
VictorOps | `victorops` | yes, external only | yes
[Webhook](#webhook) | `webhook` | yes, external only | yes
[Zenduty](#zenduty) | `webhook` | yes, external only | yes
@ -133,6 +133,11 @@ This behavior will become the default in a future version of Grafana.
> **Note:** The `state` tag overrides the current alert state inside the `custom_details` payload.
### VictorOps
To configure VictorOps, provide the URL from the Grafana Integration and substitute `$routing_key` with a valid key.
> **Note:** The tag `Severity` has special meaning in the [VictorOps Incident Fields](https://help.victorops.com/knowledge-base/incident-fields-glossary/). If an alert panel defines this key, then it replaces the `message_type` in the root of the event sent to VictorOps.
### Pushover
To set up Pushover, you must provide a user key and an API token. Refer to [What is Pushover and how do I use it](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) for instructions on how to generate them.

6
docs/sources/installation/upgrading.md
View File

@ -313,4 +313,10 @@ NOTE: Only snapshots created on Grafana 7.3 or later will use this column to sto
The Grafana Docker images use the `root` group instead of the `grafana` group. This change can cause builds to break for users who extend the Grafana Docker image. Learn more about this change in the [Docker migration instructions]({{< relref "docker/#migrate-to-v73-or-later">}})
## Upgrading to v7.5
### VictorOps Alert Notifier
The VictorOps alert notifier now accepts a `severity` tag, in a similar vein to the PagerDuty alert notifier. The possible values are outlined in the [VictorOps docs](https://help.victorops.com/knowledge-base/incident-fields-glossary/).
For example, if you want an alert to be `INFO`-level in VictorOps, create a tag `severity=info` (case-insensitive) in your alert.

38
pkg/services/alerting/notifiers/victorops.go
View File

@ -1,6 +1,7 @@
package notifiers
import (
"strings"
"time"
"github.com/grafana/grafana/pkg/bus"
@ -74,22 +75,35 @@ type VictoropsNotifier struct {
log log.Logger
}
// Notify sends notification to Victorops via POST to URL endpoint
func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
func (vn *VictoropsNotifier) buildEventPayload(evalContext *alerting.EvalContext) (*simplejson.Json, error) {
ruleURL, err := evalContext.GetRuleURL()
if err != nil {
vn.log.Error("Failed get rule link", "error", err)
return err
return nil, err
}
if evalContext.Rule.State == models.AlertStateOK && !vn.AutoResolve {
vn.log.Info("Not alerting VictorOps", "state", evalContext.Rule.State, "auto resolve", vn.AutoResolve)
return nil
return nil, nil
}
messageType := AlertStateCritical // Default to alerting and change based on state checks (Ensures string type)
for _, tag := range evalContext.Rule.AlertRuleTags {
if strings.ToLower(tag.Key) == "severity" {
// Only set severity if it's one of the PD supported enum values
// Info, Warning, Error, or Critical (case insensitive)
switch sev := strings.ToUpper(tag.Value); sev {
case "INFO":
fallthrough
case "WARNING":
fallthrough
case "CRITICAL":
messageType = sev
default:
vn.log.Warn("Ignoring invalid severity tag", "severity", sev)
}
}
}
if evalContext.Rule.State == models.AlertStateNoData { // translate 'NODATA' to set alert
messageType = vn.NoDataAlertType
@ -127,6 +141,18 @@ func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
bodyJSON.Set("image_url", evalContext.ImagePublicURL)
}
return bodyJSON, nil
}
// Notify sends notification to Victorops via POST to URL endpoint
func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
bodyJSON, err := vn.buildEventPayload(evalContext)
if err != nil {
return err
}
data, _ := bodyJSON.MarshalJSON()
cmd := &models.SendWebhookSync{Url: vn.URL, Body: string(data)}

111
pkg/services/alerting/notifiers/victorops_test.go
View File

@ -1,13 +1,27 @@
package notifiers
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/services/validations"
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
. "github.com/smartystreets/goconvey/convey"
)
func presenceComparerInt(a, b int64) bool {
if a == -1 {
return b != 0
}
if b == -1 {
return a != 0
}
return a == b
}
func TestVictoropsNotifier(t *testing.T) {
Convey("Victorops notifier tests", t, func() {
Convey("Parsing alert notification from settings", func() {
@ -46,6 +60,103 @@ func TestVictoropsNotifier(t *testing.T) {
So(victoropsNotifier.Type, ShouldEqual, "victorops")
So(victoropsNotifier.URL, ShouldEqual, "http://google.com")
})
Convey("should return properly formatted event payload when using severity override tag", func() {
json := `
{
"url": "http://google.com"
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "victorops_testing",
Type: "victorops",
Settings: settingsJSON,
}
not, err := NewVictoropsNotifier(model)
So(err, ShouldBeNil)
victoropsNotifier := not.(*VictoropsNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateAlerting,
AlertRuleTags: []*models.Tag{
{Key: "keyOnly"},
{Key: "severity", Value: "warning"},
},
}, &validations.OSSPluginRequestValidator{})
evalContext.IsTestRun = true
payload, err := victoropsNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"alert_url": "",
"entity_display_name": "[Alerting] someRule",
"entity_id": "someRule",
"message_type": "WARNING",
"metrics": map[string]interface{}{},
"monitoring_tool": "Grafana v",
"state_message": "someMessage",
"state_start_time": int64(-1),
"timestamp": int64(-1),
}, payload.Interface(), cmp.Comparer(presenceComparerInt))
So(diff, ShouldBeEmpty)
})
Convey("resolving with severity works properly", func() {
json := `
{
"url": "http://google.com"
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "victorops_testing",
Type: "victorops",
Settings: settingsJSON,
}
not, err := NewVictoropsNotifier(model)
So(err, ShouldBeNil)
victoropsNotifier := not.(*VictoropsNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateOK,
AlertRuleTags: []*models.Tag{
{Key: "keyOnly"},
{Key: "severity", Value: "warning"},
},
}, &validations.OSSPluginRequestValidator{})
evalContext.IsTestRun = true
payload, err := victoropsNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"alert_url": "",
"entity_display_name": "[OK] someRule",
"entity_id": "someRule",
"message_type": "RECOVERY",
"metrics": map[string]interface{}{},
"monitoring_tool": "Grafana v",
"state_message": "someMessage",
"state_start_time": int64(-1),
"timestamp": int64(-1),
}, payload.Interface(), cmp.Comparer(presenceComparerInt))
So(diff, ShouldBeEmpty)
})
})
})
}