diff --git a/pkg/services/ngalert/migration/alert_rule.go b/pkg/services/ngalert/migration/alert_rule.go index a47ddb1f4c3..393b8b1fbaa 100644 --- a/pkg/services/ngalert/migration/alert_rule.go +++ b/pkg/services/ngalert/migration/alert_rule.go @@ -235,7 +235,7 @@ func transNoData(l log.Logger, s string) ngmodels.NoDataState { case legacymodels.NoDataSetAlerting: return ngmodels.Alerting case legacymodels.NoDataKeepState: - return ngmodels.NoData // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert. + return ngmodels.KeepLast default: l.Warn("Unable to translate execution of NoData state. Using default execution", "old", s, "new", ngmodels.NoData) return ngmodels.NoData @@ -247,9 +247,7 @@ func transExecErr(l log.Logger, s string) ngmodels.ExecutionErrorState { case "", legacymodels.ExecutionErrorSetAlerting: return ngmodels.AlertingErrState case legacymodels.ExecutionErrorKeepState: - // Keep last state is translated to error as we now emit a - // DatasourceError alert when the state is error - return ngmodels.ErrorErrState + return ngmodels.KeepLastErrState case legacymodels.ExecutionErrorSetOk: return ngmodels.OkErrState default: diff --git a/pkg/services/ngalert/models/alert_rule.go b/pkg/services/ngalert/models/alert_rule.go index c73c21b04bf..320d29386b5 100644 --- a/pkg/services/ngalert/models/alert_rule.go +++ b/pkg/services/ngalert/models/alert_rule.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "github.com/google/go-cmp/cmp" @@ -53,6 +54,8 @@ func NoDataStateFromString(state string) (NoDataState, error) { return NoData, nil case string(OK): return OK, nil + case string(KeepLast): + return KeepLast, nil default: return "", fmt.Errorf("unknown NoData state option %s", state) } @@ -62,6 +65,7 @@ const ( Alerting NoDataState = "Alerting" NoData NoDataState = "NoData" OK NoDataState = "OK" + KeepLast NoDataState = "KeepLast" ) // swagger:enum ExecutionErrorState @@ -79,6 +83,8 @@ func ErrStateFromString(opt string) (ExecutionErrorState, error) { return ErrorErrState, nil case string(OkErrState): return OkErrState, nil + case string(KeepLastErrState): + return KeepLastErrState, nil default: return "", fmt.Errorf("unknown Error state option %s", opt) } @@ -88,6 +94,7 @@ const ( AlertingErrState ExecutionErrorState = "Alerting" ErrorErrState ExecutionErrorState = "Error" OkErrState ExecutionErrorState = "OK" + KeepLastErrState ExecutionErrorState = "KeepLast" ) const ( @@ -137,8 +144,13 @@ const ( StateReasonPaused = "Paused" StateReasonUpdated = "Updated" StateReasonRuleDeleted = "RuleDeleted" + StateReasonKeepLast = "KeepLast" ) +func ConcatReasons(reasons ...string) string { + return strings.Join(reasons, ", ") +} + var ( // InternalLabelNameSet are labels that grafana automatically include as part of the labelset. InternalLabelNameSet = map[string]struct{}{ diff --git a/pkg/services/ngalert/state/historian/core.go b/pkg/services/ngalert/state/historian/core.go index 24d477c21f0..cbeb4f45205 100644 --- a/pkg/services/ngalert/state/historian/core.go +++ b/pkg/services/ngalert/state/historian/core.go @@ -41,6 +41,12 @@ func ShouldRecordAnnotation(t state.StateTransition) bool { return false } + // Do not log transitions when keeping last state + toKeepLast := strings.Contains(t.StateReason, models.StateReasonKeepLast) && !strings.Contains(t.PreviousStateReason, models.StateReasonKeepLast) + if toKeepLast { + return false + } + // Do not record transitions between Normal and Normal (NoData) if t.State.State == eval.Normal && t.PreviousState == eval.Normal { if (t.State.StateReason == "" && t.PreviousStateReason == models.StateReasonNoData) || diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 4b0f11e397e..da6d614ff5e 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -316,14 +316,14 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time } func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition { - if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK) { // If it is no data, check the mapping and switch all results to the new state + if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state // TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger) if len(transitions) > 0 { return transitions // if there are no current states for the rule. Create ones for each result } } - if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState) { + if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState || alertRule.ExecErrState == ngModels.KeepLastErrState) { // TODO squash all errors into one, and provide as annotation transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger) if len(transitions) > 0 { @@ -352,6 +352,7 @@ func (st *Manager) setNextStateForAll(ctx context.Context, alertRule *ngModels.A // Set the current state based on evaluation results func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, currentState *State, result eval.Result, logger log.Logger) StateTransition { start := st.clock.Now() + currentState.LastEvaluationTime = result.EvaluatedAt currentState.EvaluationDuration = result.EvaluationDuration currentState.Results = append(currentState.Results, Evaluation{ @@ -392,10 +393,10 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu switch result.State { case eval.Normal: logger.Debug("Setting next state", "handler", "resultNormal") - resultNormal(currentState, alertRule, result, logger) + resultNormal(currentState, alertRule, result, logger, "") case eval.Alerting: logger.Debug("Setting next state", "handler", "resultAlerting") - resultAlerting(currentState, alertRule, result, logger) + resultAlerting(currentState, alertRule, result, logger, "") case eval.Error: logger.Debug("Setting next state", "handler", "resultError") resultError(currentState, alertRule, result, logger) @@ -412,7 +413,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu if currentState.State != result.State && result.State != eval.Normal && result.State != eval.Alerting { - currentState.StateReason = result.State.String() + currentState.StateReason = resultStateReason(result, alertRule) } // Set Resolved property so the scheduler knows to send a postable alert @@ -446,6 +447,14 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu return nextState } +func resultStateReason(result eval.Result, rule *ngModels.AlertRule) string { + if rule.ExecErrState == ngModels.KeepLastErrState || rule.NoDataState == ngModels.KeepLast { + return ngModels.ConcatReasons(result.State.String(), ngModels.StateReasonKeepLast) + } + + return result.State.String() +} + func (st *Manager) GetAll(orgID int64) []*State { allStates := st.cache.getAll(orgID, st.doNotSaveNormalState) return allStates diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 681e44a38f1..798b37a0720 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -812,6 +812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ngmodels.NoData: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.NoData)), ngmodels.Alerting: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.Alerting)), ngmodels.OK: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.OK)), + ngmodels.KeepLast: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.KeepLast)), } type noDataTestCase struct { @@ -919,6 +920,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t1: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + }, + }, + }, + }, }, }, { @@ -985,6 +1004,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -1025,6 +1062,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { @@ -1202,6 +1258,55 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + Resolved: true, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -1345,6 +1450,76 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Alerting, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -1533,6 +1708,53 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -1663,6 +1885,70 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Alerting, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -1734,6 +2020,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -1774,6 +2078,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -1889,6 +2212,39 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + Results: []Evaluation{ + newEvaluation(t2, eval.Normal), + newEvaluation(t3, eval.Normal), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.NoData), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -1955,6 +2311,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -1995,10 +2369,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { - desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t3", + desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t2,t3", results: map[time.Time]eval.Results{ t1: { newResult(eval.WithState(eval.Alerting)), @@ -2129,6 +2522,41 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + Resolved: true, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -2208,6 +2636,44 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Alerting, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -2294,6 +2760,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.Alerting: { @@ -2334,6 +2818,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLast: { + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, } @@ -2350,6 +2853,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ngmodels.ErrorErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.ErrorErrState)), ngmodels.AlertingErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.AlertingErrState)), ngmodels.OkErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.OkErrState)), + ngmodels.KeepLastErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.KeepLastErrState)), } cacheID := func(lbls data.Labels) string { @@ -2472,6 +2976,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t1: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Error), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + }, + }, + }, + }, }, }, { @@ -2540,6 +3062,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t1: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Error), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t1, + }, + }, + }, + }, }, }, { @@ -2613,6 +3153,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.AlertingErrState: { @@ -2652,6 +3210,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { @@ -2724,6 +3300,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + }, }, expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ ngmodels.AlertingErrState: { @@ -2765,6 +3359,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.Error), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { @@ -2886,6 +3499,39 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t3: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + Results: []Evaluation{ + newEvaluation(t2, eval.Normal), + newEvaluation(t3, eval.Normal), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.StateReasonMissingSeries, + Results: []Evaluation{ + newEvaluation(t1, eval.Error), + }, + StartsAt: t1, + EndsAt: t3, + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -2961,6 +3607,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.Error), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { @@ -3049,6 +3714,24 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, { @@ -3178,6 +3861,42 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule"], + State: eval.Pending, + StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.Error), + }, + StartsAt: t1, + EndsAt: t1.Add(ResendDelay * 4), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 4), + LastEvaluationTime: t3, + }, + }, + }, + }, }, }, { @@ -3248,6 +3967,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + ngmodels.KeepLastErrState: { + t2: { + { + PreviousState: eval.Normal, + PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast), + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + Results: []Evaluation{ + newEvaluation(t1, eval.Error), + newEvaluation(t2, eval.Normal), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, }, }, } diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index cd856fb66a9..38a7519f846 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -875,6 +875,76 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, + { + desc: "normal -> normal (NoData, KeepLastState) -> alerting -> alerting (NoData, KeepLastState) - keeps last state when result is NoData and NoDataState is KeepLast", + alertRule: baseRuleWith(models.WithForNTimes(0), models.WithNoDataExecAs(models.KeepLast)), + evalResults: map[time.Time]eval.Results{ + t1: { + newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), + }, + t2: { + newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels + }, + t3: { + newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), + }, + tn(4): { + newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels + }, + }, + expectedAnnotations: 1, + expectedStates: []*state.State{ + { + Labels: labels["system + rule + labels1"], + ResultFingerprint: labels1.Fingerprint(), + State: eval.Alerting, + StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), + Results: []state.Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + newEvaluation(tn(4), eval.NoData), + }, + StartsAt: t3, + EndsAt: tn(4).Add(state.ResendDelay * 4), + LastEvaluationTime: tn(4), + }, + }, + }, + { + desc: "normal -> pending -> pending (NoData, KeepLastState) -> alerting (NoData, KeepLastState) - keep last state respects For when result is NoData", + alertRule: baseRuleWith(models.WithForNTimes(2), models.WithNoDataExecAs(models.KeepLast)), + evalResults: map[time.Time]eval.Results{ + t1: { + newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), + }, + t2: { + newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), + }, + t3: { + newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels + }, + tn(4): { + newResult(eval.WithState(eval.NoData), eval.WithLabels(labels1)), // TODO fix it because NoData does not have same labels + }, + }, + expectedAnnotations: 2, + expectedStates: []*state.State{ + { + Labels: labels["system + rule + labels1"], + ResultFingerprint: labels1.Fingerprint(), + State: eval.Alerting, + StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), + Results: []state.Evaluation{ + newEvaluation(t3, eval.NoData), + newEvaluation(tn(4), eval.NoData), + }, + StartsAt: tn(4), + EndsAt: tn(4).Add(state.ResendDelay * 4), + LastEvaluationTime: tn(4), + }, + }, + }, { desc: "normal -> normal when result is NoData and NoDataState is ok", alertRule: baseRuleWith(models.WithNoDataExecAs(models.OK)), @@ -1012,6 +1082,76 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, + { + desc: "normal -> normal (Error, KeepLastState) -> alerting -> alerting (Error, KeepLastState) - keeps last state when result is Error and ExecErrState is KeepLast", + alertRule: baseRuleWith(models.WithForNTimes(0), models.WithErrorExecAs(models.KeepLastErrState)), + evalResults: map[time.Time]eval.Results{ + t1: { + newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), + }, + t2: { + newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + }, + t3: { + newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), + }, + tn(4): { + newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + }, + }, + expectedAnnotations: 1, + expectedStates: []*state.State{ + { + Labels: labels["system + rule + labels1"], + ResultFingerprint: labels1.Fingerprint(), + State: eval.Alerting, + StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), + Results: []state.Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.Error), + newEvaluation(t3, eval.Alerting), + newEvaluation(tn(4), eval.Error), + }, + StartsAt: t3, + EndsAt: tn(4).Add(state.ResendDelay * 4), + LastEvaluationTime: tn(4), + }, + }, + }, + { + desc: "normal -> pending -> pending (Error, KeepLastState) -> alerting (Error, KeepLastState) - keep last state respects For when result is Error", + alertRule: baseRuleWith(models.WithForNTimes(2), models.WithErrorExecAs(models.KeepLastErrState)), + evalResults: map[time.Time]eval.Results{ + t1: { + newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), + }, + t2: { + newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)), + }, + t3: { + newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + }, + tn(4): { + newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different + }, + }, + expectedAnnotations: 2, + expectedStates: []*state.State{ + { + Labels: labels["system + rule + labels1"], + ResultFingerprint: labels1.Fingerprint(), + State: eval.Alerting, + StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), + Results: []state.Evaluation{ + newEvaluation(t3, eval.Error), + newEvaluation(tn(4), eval.Error), + }, + StartsAt: tn(4), + EndsAt: tn(4).Add(state.ResendDelay * 4), + LastEvaluationTime: tn(4), + }, + }, + }, { desc: "normal -> normal when result is Error and ExecErrState is OK", alertRule: baseRuleWith(models.WithForNTimes(6), models.WithErrorExecAs(models.OkErrState)), diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 6a1f3b2bab1..11c86f6b10e 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -191,7 +191,7 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float return result } -func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger log.Logger) { +func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger log.Logger, reason string) { if state.State == eval.Normal { logger.Debug("Keeping state", "state", state.State) } else { @@ -206,11 +206,11 @@ func resultNormal(state *State, _ *models.AlertRule, result eval.Result, logger "next_ends_at", nextEndsAt) // Normal states have the same start and end timestamps - state.SetNormal("", nextEndsAt, nextEndsAt) + state.SetNormal(reason, nextEndsAt, nextEndsAt) } } -func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) { +func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger, reason string) { switch state.State { case eval.Alerting: prevEndsAt := state.EndsAt @@ -235,7 +235,7 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo state.EndsAt, "next_ends_at", nextEndsAt) - state.SetAlerting("", result.EvaluatedAt, nextEndsAt) + state.SetAlerting(reason, result.EvaluatedAt, nextEndsAt) } default: nextEndsAt := nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt) @@ -250,7 +250,7 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo state.EndsAt, "next_ends_at", nextEndsAt) - state.SetPending("", result.EvaluatedAt, nextEndsAt) + state.SetPending(reason, result.EvaluatedAt, nextEndsAt) } else { logger.Debug("Changing state", "previous_state", @@ -261,18 +261,20 @@ func resultAlerting(state *State, rule *models.AlertRule, result eval.Result, lo state.EndsAt, "next_ends_at", nextEndsAt) - state.SetAlerting("", result.EvaluatedAt, nextEndsAt) + state.SetAlerting(reason, result.EvaluatedAt, nextEndsAt) } } } + func resultError(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) { + handlerStr := "resultError" + switch rule.ExecErrState { case models.AlertingErrState: - logger.Debug("Execution error state is Alerting", "handler", "resultAlerting", "previous_handler", "resultError") - resultAlerting(state, rule, result, logger) + logger.Debug("Execution error state is Alerting", "handler", "resultAlerting", "previous_handler", handlerStr) + resultAlerting(state, rule, result, logger, models.StateReasonError) // This is a special case where Alerting and Pending should also have an error and reason state.Error = result.Error - state.StateReason = models.StateReasonError case models.ErrorErrState: if state.State == eval.Error { prevEndsAt := state.EndsAt @@ -316,8 +318,11 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge } } case models.OkErrState: - logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", "resultError") - resultNormal(state, rule, result, logger) + logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", handlerStr) + resultNormal(state, rule, result, logger, "") // TODO: Should we add a reason? + case models.KeepLastErrState: + logger := logger.New("previous_handler", handlerStr) + resultKeepLast(state, rule, result, logger) default: err := fmt.Errorf("unsupported execution error state: %s", rule.ExecErrState) state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)) @@ -326,11 +331,12 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge } func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) { + handlerStr := "resultNoData" + switch rule.NoDataState { case models.Alerting: - logger.Debug("Execution no data state is Alerting", "handler", "resultAlerting", "previous_handler", "resultNoData") - resultAlerting(state, rule, result, logger) - state.StateReason = models.StateReasonNoData + logger.Debug("Execution no data state is Alerting", "handler", "resultAlerting", "previous_handler", handlerStr) + resultAlerting(state, rule, result, logger, models.StateReasonNoData) case models.NoData: if state.State == eval.NoData { prevEndsAt := state.EndsAt @@ -357,9 +363,11 @@ func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logg state.SetNoData("", result.EvaluatedAt, nextEndsAt) } case models.OK: - logger.Debug("Execution no data state is Normal", "handler", "resultNormal", "previous_handler", "resultNoData") - resultNormal(state, rule, result, logger) - state.StateReason = models.StateReasonNoData + logger.Debug("Execution no data state is Normal", "handler", "resultNormal", "previous_handler", handlerStr) + resultNormal(state, rule, result, logger, models.StateReasonNoData) + case models.KeepLast: + logger := logger.New("previous_handler", handlerStr) + resultKeepLast(state, rule, result, logger) default: err := fmt.Errorf("unsupported no data state: %s", rule.NoDataState) state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt)) @@ -367,6 +375,31 @@ func resultNoData(state *State, rule *models.AlertRule, result eval.Result, logg } } +func resultKeepLast(state *State, rule *models.AlertRule, result eval.Result, logger log.Logger) { + reason := models.ConcatReasons(result.State.String(), models.StateReasonKeepLast) + + switch state.State { + case eval.Alerting: + logger.Debug("Execution keep last state is Alerting", "handler", "resultAlerting") + resultAlerting(state, rule, result, logger, reason) + case eval.Pending: + // respect 'for' setting on rule + if result.EvaluatedAt.Sub(state.StartsAt) >= rule.For { + logger.Debug("Execution keep last state is Pending", "handler", "resultAlerting") + resultAlerting(state, rule, result, logger, reason) + } else { + logger.Debug("Ignoring set next state to pending") + } + case eval.Normal: + logger.Debug("Execution keep last state is Normal", "handler", "resultNormal") + resultNormal(state, rule, result, logger, reason) + default: + // this should not happen, add as failsafe + logger.Debug("Reverting invalid state to normal", "handler", "resultNormal") + resultNormal(state, rule, result, logger, reason) + } +} + func (a *State) NeedsSending(resendDelay time.Duration) bool { switch a.State { case eval.Pending: @@ -486,21 +519,28 @@ func FormatStateAndReason(state eval.State, reason string) string { // ParseFormattedState parses a state string in the format "state (reason)" // and returns the state and reason separately. func ParseFormattedState(stateStr string) (eval.State, string, error) { - split := strings.Split(stateStr, " ") - if len(split) == 0 { - return -1, "", errors.New("invalid state format") + p := 0 + // walk string until we find a space + for i, c := range stateStr { + if c == ' ' { + p = i + break + } + } + if p == 0 { + p = len(stateStr) } - state, err := eval.ParseStateString(split[0]) + state, err := eval.ParseStateString(stateStr[:p]) if err != nil { return -1, "", err } - var reason string - if len(split) > 1 { - reason = strings.Trim(split[1], "()") + if p == len(stateStr) { + return state, "", nil } + reason := strings.Trim(stateStr[p+1:], "()") return state, reason, nil } diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index 870f0b1d87d..1684b7036c1 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -681,6 +681,15 @@ func TestParseFormattedState(t *testing.T) { require.Equal(t, ngmodels.StateReasonMissingSeries, reason) }) + t.Run("should parse formatted state with concatenated reasons", func(t *testing.T) { + stateStr := "Normal (Error, KeepLast)" + s, reason, err := ParseFormattedState(stateStr) + require.NoError(t, err) + + require.Equal(t, eval.Normal, s) + require.Equal(t, ngmodels.ConcatReasons(ngmodels.StateReasonError, ngmodels.StateReasonKeepLast), reason) + }) + t.Run("should error on empty string", func(t *testing.T) { stateStr := "" _, _, err := ParseFormattedState(stateStr) diff --git a/public/app/features/alerting/unified/components/rule-editor/GrafanaAlertStatePicker.tsx b/public/app/features/alerting/unified/components/rule-editor/GrafanaAlertStatePicker.tsx index ee732387de3..1494cc9a135 100644 --- a/public/app/features/alerting/unified/components/rule-editor/GrafanaAlertStatePicker.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/GrafanaAlertStatePicker.tsx @@ -15,6 +15,7 @@ const options: SelectableValue[] = [ { value: GrafanaAlertStateDecision.NoData, label: 'No Data' }, { value: GrafanaAlertStateDecision.OK, label: 'OK' }, { value: GrafanaAlertStateDecision.Error, label: 'Error' }, + { value: GrafanaAlertStateDecision.KeepLast, label: 'Keep Last State' }, ]; export const GrafanaAlertStatePicker = ({ includeNoData, includeError, ...props }: Props) => { diff --git a/public/app/types/unified-alerting-dto.ts b/public/app/types/unified-alerting-dto.ts index 923339e22f2..c7d8251c10c 100644 --- a/public/app/types/unified-alerting-dto.ts +++ b/public/app/types/unified-alerting-dto.ts @@ -178,7 +178,7 @@ export interface RulerAlertingRuleDTO extends RulerRuleBaseDTO { export enum GrafanaAlertStateDecision { Alerting = 'Alerting', NoData = 'NoData', - KeepLastState = 'KeepLastState', + KeepLast = 'KeepLast', OK = 'OK', Error = 'Error', }