root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 73 Packages Projects Releases Wiki Activity

Alerting: Add "Keep Last State" backend functionality (#83940) 

* Implement keep last state for state transitions

* Respect For duration when keeping state

* Only keep transition from recording an annotation

* Add keep last state option for nodata/error in UI
This commit is contained in:
William Wernert 2024-03-12 10:00:43 -04:00 committed by GitHub
parent fe1ed0a9e1
commit 10dc6c6d75
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 988 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/services/ngalert/migration/alert_rule.go
View File

@ -235,7 +235,7 @@ func transNoData(l log.Logger, s string) ngmodels.NoDataState {
case legacymodels.NoDataSetAlerting:
return ngmodels.Alerting
case legacymodels.NoDataKeepState:
return ngmodels.NoData // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert.
return ngmodels.KeepLast
default:
l.Warn("Unable to translate execution of NoData state. Using default execution", "old", s, "new", ngmodels.NoData)
return ngmodels.NoData
@ -247,9 +247,7 @@ func transExecErr(l log.Logger, s string) ngmodels.ExecutionErrorState {
case "", legacymodels.ExecutionErrorSetAlerting:
return ngmodels.AlertingErrState
case legacymodels.ExecutionErrorKeepState:
// Keep last state is translated to error as we now emit a
// DatasourceError alert when the state is error
return ngmodels.ErrorErrState
return ngmodels.KeepLastErrState
case legacymodels.ExecutionErrorSetOk:
return ngmodels.OkErrState
default:

12
pkg/services/ngalert/models/alert_rule.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/google/go-cmp/cmp"
@ -53,6 +54,8 @@ func NoDataStateFromString(state string) (NoDataState, error) {
return NoData, nil
case string(OK):
return OK, nil
case string(KeepLast):
return KeepLast, nil
default:
return "", fmt.Errorf("unknown NoData state option %s", state)
}
@ -62,6 +65,7 @@ const (
Alerting NoDataState = "Alerting"
NoData NoDataState = "NoData"
OK NoDataState = "OK"
KeepLast NoDataState = "KeepLast"
)
// swagger:enum ExecutionErrorState
@ -79,6 +83,8 @@ func ErrStateFromString(opt string) (ExecutionErrorState, error) {
return ErrorErrState, nil
case string(OkErrState):
return OkErrState, nil
case string(KeepLastErrState):
return KeepLastErrState, nil
default:
return "", fmt.Errorf("unknown Error state option %s", opt)
}
@ -88,6 +94,7 @@ const (
AlertingErrState ExecutionErrorState = "Alerting"
ErrorErrState ExecutionErrorState = "Error"
OkErrState ExecutionErrorState = "OK"
KeepLastErrState ExecutionErrorState = "KeepLast"
)
const (
@ -137,8 +144,13 @@ const (
StateReasonPaused = "Paused"
StateReasonUpdated = "Updated"
StateReasonRuleDeleted = "RuleDeleted"
StateReasonKeepLast = "KeepLast"
)
func ConcatReasons(reasons ...string) string {
return strings.Join(reasons, ", ")
}
var (
// InternalLabelNameSet are labels that grafana automatically include as part of the labelset.
InternalLabelNameSet = map[string]struct{}{

6
pkg/services/ngalert/state/historian/core.go
View File

@ -41,6 +41,12 @@ func ShouldRecordAnnotation(t state.StateTransition) bool {
return false
}
// Do not log transitions when keeping last state
toKeepLast := strings.Contains(t.StateReason, models.StateReasonKeepLast) && !strings.Contains(t.PreviousStateReason, models.StateReasonKeepLast)
if toKeepLast {
return false
}
// Do not record transitions between Normal and Normal (NoData)
if t.State.State == eval.Normal && t.PreviousState == eval.Normal {
if (t.State.StateReason == "" && t.PreviousStateReason == models.StateReasonNoData) ||

19
pkg/services/ngalert/state/manager.go
View File

@ -316,14 +316,14 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time
}
func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition {
if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK) { // If it is no data, check the mapping and switch all results to the new state
if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state
// TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation
transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger)
if len(transitions) > 0 {
return transitions // if there are no current states for the rule. Create ones for each result
}
}
if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState) {
if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) {
// TODO squash all errors into one, and provide as annotation
transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger)
if len(transitions) > 0 {
@ -352,6 +352,7 @@ func (st *Manager) setNextStateForAll(ctx context.Context, alertRule *ngModels.A
// Set the current state based on evaluation results
func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, currentState *State, result eval.Result, logger log.Logger) StateTransition {
start := st.clock.Now()
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = result.EvaluationDuration
currentState.Results = append(currentState.Results, Evaluation{
@ -392,10 +393,10 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
switch result.State {
case eval.Normal:
logger.Debug("Setting next state", "handler", "resultNormal")
resultNormal(currentState, alertRule, result, logger)
resultNormal(currentState, alertRule, result, logger, "")
case eval.Alerting:
logger.Debug("Setting next state", "handler", "resultAlerting")
resultAlerting(currentState, alertRule, result, logger)
resultAlerting(currentState, alertRule, result, logger, "")
case eval.Error:
logger.Debug("Setting next state", "handler", "resultError")
resultError(currentState, alertRule, result, logger)
@ -412,7 +413,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
if currentState.State != result.State &&
result.State != eval.Normal &&
result.State != eval.Alerting {
currentState.StateReason = result.State.String()
currentState.StateReason = resultStateReason(result, alertRule)
}
// Set Resolved property so the scheduler knows to send a postable alert
@ -446,6 +447,14 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
return nextState
}
func resultStateReason(result eval.Result, rule *ngModels.AlertRule) string {
if rule.ExecErrState == ngModels.KeepLastErrState || rule.NoDataState == ngModels.KeepLast {
return ngModels.ConcatReasons(result.State.String(), ngModels.StateReasonKeepLast)
}
return result.State.String()
}
func (st *Manager) GetAll(orgID int64) []*State {
allStates := st.cache.getAll(orgID, st.doNotSaveNormalState)
return allStates

740
pkg/services/ngalert/state/manager_private_test.go
View File

@ -812,6 +812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
ngmodels.NoData: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.NoData)),
ngmodels.Alerting: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.Alerting)),
ngmodels.OK: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.OK)),
ngmodels.KeepLast: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.KeepLast)),
}
type noDataTestCase struct {
@ -919,6 +920,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t1: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
},
},
},
},
},
{
@ -985,6 +1004,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -1025,6 +1062,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
},
},
},
{
@ -1202,6 +1258,55 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
Resolved: true,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -1345,6 +1450,76 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -1533,6 +1708,53 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -1663,6 +1885,70 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Alerting,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -1734,6 +2020,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -1774,6 +2078,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -1889,6 +2212,39 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Results: []Evaluation{
newEvaluation(t2, eval.Normal),
newEvaluation(t3, eval.Normal),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.NoData),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -1955,6 +2311,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -1995,10 +2369,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
},
},
},
{
desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t3",
desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t2,t3",
results: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Alerting)),
@ -2129,6 +2522,41 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
Resolved: true,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + no-data"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -2208,6 +2636,44 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Alerting,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -2294,6 +2760,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
@ -2334,6 +2818,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLast: {
t3: {
{
PreviousState: eval.Pending,
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
Results: []Evaluation{
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
}
@ -2350,6 +2853,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
ngmodels.ErrorErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.ErrorErrState)),
ngmodels.AlertingErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.AlertingErrState)),
ngmodels.OkErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.OkErrState)),
ngmodels.KeepLastErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.KeepLastErrState)),
}
cacheID := func(lbls data.Labels) string {
@ -2472,6 +2976,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t1: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
},
},
},
},
},
{
@ -2540,6 +3062,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t1: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
},
},
},
},
},
{
@ -2613,6 +3153,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
ngmodels.AlertingErrState: {
@ -2652,6 +3210,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
},
},
},
{
@ -2724,6 +3300,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
},
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
ngmodels.AlertingErrState: {
@ -2765,6 +3359,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
},
},
},
{
@ -2886,6 +3499,39 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t3: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Results: []Evaluation{
newEvaluation(t2, eval.Normal),
newEvaluation(t3, eval.Normal),
},
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t3,
},
},
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.StateReasonMissingSeries,
Results: []Evaluation{
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t3,
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -2961,6 +3607,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
},
},
},
{
@ -3049,6 +3714,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
},
},
},
{
@ -3178,6 +3861,42 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
State: eval.Pending,
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
Results: []Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Pending,
PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Alerting,
Results: []Evaluation{
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
},
},
},
},
{
@ -3248,6 +3967,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
},
ngmodels.KeepLastErrState: {
t2: {
{
PreviousState: eval.Normal,
PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
State: eval.Normal,
Results: []Evaluation{
newEvaluation(t1, eval.Error),
newEvaluation(t2, eval.Normal),
},
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
},
},
},
},
},
},
}

140
pkg/services/ngalert/state/manager_test.go
View File

@ -875,6 +875,76 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> normal (NoData, KeepLastState) -> alerting -> alerting (NoData, KeepLastState) - keeps last state when result is NoData and NoDataState is KeepLast",
alertRule: baseRuleWith(models.WithForNTimes(0), models.WithNoDataExecAs(models.KeepLast)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
},
{
desc: "normal -> pending -> pending (NoData, KeepLastState) -> alerting (NoData, KeepLastState) - keep last state respects For when result is NoData",
alertRule: baseRuleWith(models.WithForNTimes(2), models.WithNoDataExecAs(models.KeepLast)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
tn(4): {
newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
},
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is ok",
alertRule: baseRuleWith(models.WithNoDataExecAs(models.OK)),
@ -1012,6 +1082,76 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> normal (Error, KeepLastState) -> alerting -> alerting (Error, KeepLastState) - keeps last state when result is Error and ExecErrState is KeepLast",
alertRule: baseRuleWith(models.WithForNTimes(0), models.WithErrorExecAs(models.KeepLastErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
t3: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
tn(4): {
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Error),
},
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
},
{
desc: "normal -> pending -> pending (Error, KeepLastState) -> alerting (Error, KeepLastState) - keep last state respects For when result is Error",
alertRule: baseRuleWith(models.WithForNTimes(2), models.WithErrorExecAs(models.KeepLastErrState)),
evalResults: map[time.Time]eval.Results{
t1: {
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
},
t2: {
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
},
t3: {
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
tn(4): {
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
},
},
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
},
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
},
{
desc: "normal -> normal when result is Error and ExecErrState is OK",
alertRule: baseRuleWith(models.WithForNTimes(6), models.WithErrorExecAs(models.OkErrState)),

88
pkg/services/ngalert/state/state.go
View File

@ -191,7 +191,7 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float
return result
}
func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger log.Logger) {
func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger log.Logger, reason string) {
if state.State == eval.Normal {
logger.Debug("Keeping state", "state", state.State)
} else {
@ -206,11 +206,11 @@ func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger
"next_ends_at",
nextEndsAt)
// Normal states have the same start and end timestamps
state.SetNormal("", nextEndsAt, nextEndsAt)
state.SetNormal(reason, nextEndsAt, nextEndsAt)
}
}
func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) {
func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger, reason string) {
switch state.State {
case eval.Alerting:
prevEndsAt := state.EndsAt
@ -235,7 +235,7 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo
state.EndsAt,
"next_ends_at",
nextEndsAt)
state.SetAlerting("", result.EvaluatedAt, nextEndsAt)
state.SetAlerting(reason, result.EvaluatedAt, nextEndsAt)
}
default:
nextEndsAt := nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)
@ -250,7 +250,7 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo
state.EndsAt,
"next_ends_at",
nextEndsAt)
state.SetPending("", result.EvaluatedAt, nextEndsAt)
state.SetPending(reason, result.EvaluatedAt, nextEndsAt)
} else {
logger.Debug("Changing state",
"previous_state",
@ -261,18 +261,20 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo
state.EndsAt,
"next_ends_at",
nextEndsAt)
state.SetAlerting("", result.EvaluatedAt, nextEndsAt)
state.SetAlerting(reason, result.EvaluatedAt, nextEndsAt)
}
}
}
func resultError(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) {
handlerStr := "resultError"
switch rule.ExecErrState {
case models.AlertingErrState:
logger.Debug("Execution error state is Alerting", "handler", "resultAlerting", "previous_handler", "resultError")
resultAlerting(state, rule, result, logger)
logger.Debug("Execution error state is Alerting", "handler", "resultAlerting", "previous_handler", handlerStr)
resultAlerting(state, rule, result, logger, models.StateReasonError)
// This is a special case where Alerting and Pending should also have an error and reason
state.Error = result.Error
state.StateReason = models.StateReasonError
case models.ErrorErrState:
if state.State == eval.Error {
prevEndsAt := state.EndsAt
@ -316,8 +318,11 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge
}
}
case models.OkErrState:
logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", "resultError")
resultNormal(state, rule, result, logger)
logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", handlerStr)
resultNormal(state, rule, result, logger, "") // TODO: Should we add a reason?
case models.KeepLastErrState:
logger := logger.New("previous_handler", handlerStr)
resultKeepLast(state, rule, result, logger)
default:
err := fmt.Errorf("unsupported execution error state: %s", rule.ExecErrState)
state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt))
@ -326,11 +331,12 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge
}
func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) {
handlerStr := "resultNoData"
switch rule.NoDataState {
case models.Alerting:
logger.Debug("Execution no data state is Alerting", "handler", "resultAlerting", "previous_handler", "resultNoData")
resultAlerting(state, rule, result, logger)
state.StateReason = models.StateReasonNoData
logger.Debug("Execution no data state is Alerting", "handler", "resultAlerting", "previous_handler", handlerStr)
resultAlerting(state, rule, result, logger, models.StateReasonNoData)
case models.NoData:
if state.State == eval.NoData {
prevEndsAt := state.EndsAt
@ -357,9 +363,11 @@ func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logg
state.SetNoData("", result.EvaluatedAt, nextEndsAt)
}
case models.OK:
logger.Debug("Execution no data state is Normal", "handler", "resultNormal", "previous_handler", "resultNoData")
resultNormal(state, rule, result, logger)
state.StateReason = models.StateReasonNoData
logger.Debug("Execution no data state is Normal", "handler", "resultNormal", "previous_handler", handlerStr)
resultNormal(state, rule, result, logger, models.StateReasonNoData)
case models.KeepLast:
logger := logger.New("previous_handler", handlerStr)
resultKeepLast(state, rule, result, logger)
default:
err := fmt.Errorf("unsupported no data state: %s", rule.NoDataState)
state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt))
@ -367,6 +375,31 @@ func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logg
}
}
func resultKeepLast(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) {
reason := models.ConcatReasons(result.State.String(), models.StateReasonKeepLast)
switch state.State {
case eval.Alerting:
logger.Debug("Execution keep last state is Alerting", "handler", "resultAlerting")
resultAlerting(state, rule, result, logger, reason)
case eval.Pending:
// respect 'for' setting on rule
if result.EvaluatedAt.Sub(state.StartsAt) >= rule.For {
logger.Debug("Execution keep last state is Pending", "handler", "resultAlerting")
resultAlerting(state, rule, result, logger, reason)
} else {
logger.Debug("Ignoring set next state to pending")
}
case eval.Normal:
logger.Debug("Execution keep last state is Normal", "handler", "resultNormal")
resultNormal(state, rule, result, logger, reason)
default:
// this should not happen, add as failsafe
logger.Debug("Reverting invalid state to normal", "handler", "resultNormal")
resultNormal(state, rule, result, logger, reason)
}
}
func (a *State) NeedsSending(resendDelay time.Duration) bool {
switch a.State {
case eval.Pending:
@ -486,21 +519,28 @@ func FormatStateAndReason(state eval.State, reason string) string {
// ParseFormattedState parses a state string in the format "state (reason)"
// and returns the state and reason separately.
func ParseFormattedState(stateStr string) (eval.State, string, error) {
split := strings.Split(stateStr, " ")
if len(split) == 0 {
return -1, "", errors.New("invalid state format")
p := 0
// walk string until we find a space
for i, c := range stateStr {
if c == ' ' {
p = i
break
}
}
if p == 0 {
p = len(stateStr)
}
state, err := eval.ParseStateString(split[0])
state, err := eval.ParseStateString(stateStr[:p])
if err != nil {
return -1, "", err
}
var reason string
if len(split) > 1 {
reason = strings.Trim(split[1], "()")
if p == len(stateStr) {
return state, "", nil
}
reason := strings.Trim(stateStr[p+1:], "()")
return state, reason, nil
}

9
pkg/services/ngalert/state/state_test.go
View File

@ -681,6 +681,15 @@ func TestParseFormattedState(t *testing.T) {
require.Equal(t, ngmodels.StateReasonMissingSeries, reason)
})
t.Run("should parse formatted state with concatenated reasons", func(t *testing.T) {
stateStr := "Normal (Error, KeepLast)"
s, reason, err := ParseFormattedState(stateStr)
require.NoError(t, err)
require.Equal(t, eval.Normal, s)
require.Equal(t, ngmodels.ConcatReasons(ngmodels.StateReasonError, ngmodels.StateReasonKeepLast), reason)
})
t.Run("should error on empty string", func(t *testing.T) {
stateStr := ""
_, _, err := ParseFormattedState(stateStr)

1
public/app/features/alerting/unified/components/rule-editor/GrafanaAlertStatePicker.tsx
View File

@ -15,6 +15,7 @@ const options: SelectableValue[] = [
{ value: GrafanaAlertStateDecision.NoData, label: 'No Data' },
{ value: GrafanaAlertStateDecision.OK, label: 'OK' },
{ value: GrafanaAlertStateDecision.Error, label: 'Error' },
{ value: GrafanaAlertStateDecision.KeepLast, label: 'Keep Last State' },
];
export const GrafanaAlertStatePicker = ({ includeNoData, includeError, ...props }: Props) => {

2
public/app/types/unified-alerting-dto.ts
View File

@ -178,7 +178,7 @@ export interface RulerAlertingRuleDTO extends RulerRuleBaseDTO {
export enum GrafanaAlertStateDecision {
Alerting = 'Alerting',
NoData = 'NoData',
KeepLastState = 'KeepLastState',
KeepLast = 'KeepLast',
OK = 'OK',
Error = 'Error',
}