mirror of https://github.com/grafana/grafana.git
Alerting: Attach hash of instance labels to state history log lines (#65968)
* Add instanceID which is hash of labels * Rename field to fingerprint * Move to prometheus style signature * Appease linter
This commit is contained in:
parent
c96b704af3
commit
3634079b8f
|
|
@ -1,10 +1,12 @@
|
||||||
package historian
|
package historian
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||||
|
prometheus "github.com/prometheus/common/model"
|
||||||
|
|
||||||
"github.com/grafana/grafana/pkg/infra/log"
|
"github.com/grafana/grafana/pkg/infra/log"
|
||||||
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
||||||
|
|
@ -42,6 +44,12 @@ func removePrivateLabels(labels data.Labels) data.Labels {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// labelFingerprint calculates a stable Prometheus-style signature for a label set.
|
||||||
|
func labelFingerprint(labels data.Labels) string {
|
||||||
|
sig := prometheus.LabelsToSignature(labels)
|
||||||
|
return fmt.Sprintf("%016x", sig)
|
||||||
|
}
|
||||||
|
|
||||||
// panelKey uniquely identifies a panel.
|
// panelKey uniquely identifies a panel.
|
||||||
type panelKey struct {
|
type panelKey struct {
|
||||||
orgID int64
|
orgID int64
|
||||||
|
|
|
||||||
|
|
@ -255,6 +255,7 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sanitizedLabels := removePrivateLabels(state.Labels)
|
||||||
entry := lokiEntry{
|
entry := lokiEntry{
|
||||||
SchemaVersion: 1,
|
SchemaVersion: 1,
|
||||||
Previous: state.PreviousFormatted(),
|
Previous: state.PreviousFormatted(),
|
||||||
|
|
@ -263,7 +264,8 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
|
||||||
Condition: rule.Condition,
|
Condition: rule.Condition,
|
||||||
DashboardUID: rule.DashboardUID,
|
DashboardUID: rule.DashboardUID,
|
||||||
PanelID: rule.PanelID,
|
PanelID: rule.PanelID,
|
||||||
InstanceLabels: removePrivateLabels(state.Labels),
|
Fingerprint: labelFingerprint(sanitizedLabels),
|
||||||
|
InstanceLabels: sanitizedLabels,
|
||||||
}
|
}
|
||||||
if state.State.State == eval.Error {
|
if state.State.State == eval.Error {
|
||||||
entry.Error = state.Error.Error()
|
entry.Error = state.Error.Error()
|
||||||
|
|
@ -306,6 +308,7 @@ type lokiEntry struct {
|
||||||
Condition string `json:"condition"`
|
Condition string `json:"condition"`
|
||||||
DashboardUID string `json:"dashboardUID"`
|
DashboardUID string `json:"dashboardUID"`
|
||||||
PanelID int64 `json:"panelID"`
|
PanelID int64 `json:"panelID"`
|
||||||
|
Fingerprint string `json:"fingerprint"`
|
||||||
// InstanceLabels is exactly the set of labels associated with the alert instance in Alertmanager.
|
// InstanceLabels is exactly the set of labels associated with the alert instance in Alertmanager.
|
||||||
// These should not be conflated with labels associated with log streams.
|
// These should not be conflated with labels associated with log streams.
|
||||||
InstanceLabels map[string]string `json:"labels"`
|
InstanceLabels map[string]string `json:"labels"`
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,25 @@ func TestRemoteLokiBackend(t *testing.T) {
|
||||||
entry := requireSingleEntry(t, res)
|
entry := requireSingleEntry(t, res)
|
||||||
require.Equal(t, rule.Condition, entry.Condition)
|
require.Equal(t, rule.Condition, entry.Condition)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("stores fingerprint of instance labels", func(t *testing.T) {
|
||||||
|
rule := createTestRule()
|
||||||
|
l := log.NewNopLogger()
|
||||||
|
states := singleFromNormal(&state.State{
|
||||||
|
State: eval.Alerting,
|
||||||
|
Labels: data.Labels{
|
||||||
|
"statelabel": "labelvalue",
|
||||||
|
"labeltwo": "labelvalue",
|
||||||
|
"labelthree": "labelvalue",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
res := statesToStream(rule, states, nil, l)
|
||||||
|
|
||||||
|
entry := requireSingleEntry(t, res)
|
||||||
|
exp := labelFingerprint(states[0].Labels)
|
||||||
|
require.Equal(t, exp, entry.Fingerprint)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("selector string", func(t *testing.T) {
|
t.Run("selector string", func(t *testing.T) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue