From 30bd4e7dba1567c41cf15834b5bd336953688d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ida=20=C5=A0tambuk?= Date: Wed, 29 Oct 2025 18:47:33 +0100 Subject: [PATCH] CloudWatch Logs: Support Log Anomalies query type (#113067) --- .../x/CloudWatchDataQuery_types.gen.ts | 36 +++ pkg/tsdb/cloudwatch/cloudwatch.go | 19 +- .../kinds/dataquery/types_dataquery_gen.go | 45 ++- pkg/tsdb/cloudwatch/log_anomalies_query.go | 146 +++++++++ .../cloudwatch/log_anomalies_query_test.go | 212 +++++++++++++ pkg/tsdb/cloudwatch/log_sync_query_test.go | 2 +- pkg/tsdb/cloudwatch/mocks/logs.go | 6 + pkg/tsdb/cloudwatch/models/api.go | 1 + pkg/tsdb/cloudwatch/test_utils.go | 16 + .../LogsAnomaliesQueryEditor.tsx | 41 +++ .../LogsQueryEditor/LogsQueryEditor.tsx | 61 +++- .../datasource/cloudwatch/dataquery.cue | 22 +- .../datasource/cloudwatch/dataquery.gen.ts | 36 +++ .../datasource/cloudwatch/datasource.test.ts | 4 +- .../datasource/cloudwatch/datasource.ts | 17 +- .../plugins/datasource/cloudwatch/guards.ts | 18 +- .../CloudWatchLogsQueryRunner.test.ts | 300 +++++++++++++++++- .../query-runner/CloudWatchLogsQueryRunner.ts | 132 +++++++- .../plugins/datasource/cloudwatch/types.ts | 1 + 19 files changed, 1080 insertions(+), 35 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/log_anomalies_query.go create mode 100644 pkg/tsdb/cloudwatch/log_anomalies_query_test.go create mode 100644 public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsAnomaliesQueryEditor.tsx diff --git a/packages/grafana-schema/src/raw/composable/cloudwatch/dataquery/x/CloudWatchDataQuery_types.gen.ts b/packages/grafana-schema/src/raw/composable/cloudwatch/dataquery/x/CloudWatchDataQuery_types.gen.ts index 2334d1d1efd..61cb1e34254 100644 --- a/packages/grafana-schema/src/raw/composable/cloudwatch/dataquery/x/CloudWatchDataQuery_types.gen.ts +++ b/packages/grafana-schema/src/raw/composable/cloudwatch/dataquery/x/CloudWatchDataQuery_types.gen.ts @@ -218,6 +218,11 @@ export interface QueryEditorArrayExpression { export type QueryEditorExpression = (QueryEditorArrayExpression | QueryEditorPropertyExpression | QueryEditorGroupByExpression | QueryEditorFunctionExpression | QueryEditorFunctionParameterExpression | QueryEditorOperatorExpression); +export enum LogsMode { + Anomalies = 'Anomalies', + Insights = 'Insights', +} + export enum LogsQueryLanguage { CWLI = 'CWLI', PPL = 'PPL', @@ -241,6 +246,10 @@ export interface CloudWatchLogsQuery extends common.DataQuery { * Log groups to query */ logGroups?: Array; + /** + * Whether a query is a Logs Insights or Logs Anomalies query + */ + logsMode?: LogsMode; /** * Language used for querying logs, can be CWLI, SQL, or PPL. If empty, the default language is CWLI. */ @@ -265,6 +274,33 @@ export const defaultCloudWatchLogsQuery: Partial = { statsGroups: [], }; +/** + * Shape of a Cloudwatch Logs Anomalies query + */ +export interface CloudWatchLogsAnomaliesQuery extends common.DataQuery { + /** + * Used to filter only the anomalies found by a certain anomaly detector + */ + anomalyDetectionARN?: string; + id: string; + /** + * Whether a query is a Logs Insights or Logs Anomalies query + */ + logsMode?: LogsMode; + /** + * Whether a query is a Metrics, Logs or Annotations query + */ + queryMode?: CloudWatchQueryMode; + /** + * AWS region to query for the logs + */ + region: string; + /** + * Filter to return only anomalies that are 'SUPPRESSED', 'UNSUPPRESSED', or 'ALL' (default) + */ + suppressionState?: string; +} + export interface LogGroup { /** * AccountId of the log group diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 1015393477b..3ebe95515c0 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -36,7 +36,7 @@ const ( headerFromAlert = "FromAlert" defaultRegion = "default" - logsQueryMode = "Logs" + queryModeLogs = "Logs" // QueryTypes annotationQuery = "annotationQuery" logAction = "logAction" @@ -45,7 +45,8 @@ const ( type DataQueryJson struct { dataquery.CloudWatchAnnotationQuery - Type string `json:"type,omitempty"` + Type string `json:"type,omitempty"` + LogsMode dataquery.LogsMode `json:"logsMode,omitempty"` } type DataSource struct { @@ -147,12 +148,22 @@ func (ds *DataSource) QueryData(ctx context.Context, req *backend.QueryDataReque if model.QueryMode != "" { queryMode = string(model.QueryMode) } - fromPublicDashboard := model.Type == "" && queryMode == logsQueryMode - isSyncLogQuery := ((fromAlert || fromExpression) && queryMode == logsQueryMode) || fromPublicDashboard + + fromPublicDashboard := model.Type == "" + + isLogInsightsQuery := queryMode == queryModeLogs && (model.LogsMode == "" || model.LogsMode == dataquery.LogsModeInsights) + + isSyncLogQuery := isLogInsightsQuery && ((fromAlert || fromExpression) || fromPublicDashboard) + if isSyncLogQuery { return executeSyncLogQuery(ctx, ds, req) } + isLogsAnomaliesQuery := model.QueryMode == dataquery.CloudWatchQueryModeLogs && model.LogsMode == dataquery.LogsModeAnomalies + if isLogsAnomaliesQuery { + return executeLogAnomaliesQuery(ctx, ds, req) + } + var result *backend.QueryDataResponse switch model.Type { case annotationQuery: diff --git a/pkg/tsdb/cloudwatch/kinds/dataquery/types_dataquery_gen.go b/pkg/tsdb/cloudwatch/kinds/dataquery/types_dataquery_gen.go index 76c706094a6..ce8bc54c6c5 100644 --- a/pkg/tsdb/cloudwatch/kinds/dataquery/types_dataquery_gen.go +++ b/pkg/tsdb/cloudwatch/kinds/dataquery/types_dataquery_gen.go @@ -285,6 +285,13 @@ func NewQueryEditorOperatorValueType() *QueryEditorOperatorValueType { return NewStringOrBoolOrInt64OrArrayOfQueryEditorOperatorType() } +type LogsMode string + +const ( + LogsModeInsights LogsMode = "Insights" + LogsModeAnomalies LogsMode = "Anomalies" +) + type LogsQueryLanguage string const ( @@ -297,7 +304,9 @@ const ( type CloudWatchLogsQuery struct { // Whether a query is a Metrics, Logs, or Annotations query QueryMode CloudWatchQueryMode `json:"queryMode"` - Id string `json:"id"` + // Whether a query is a Logs Insights or Logs Anomalies query + LogsMode *LogsMode `json:"logsMode,omitempty"` + Id string `json:"id"` // AWS region to query for the logs Region string `json:"region"` // The CloudWatch Logs Insights query to execute @@ -347,6 +356,40 @@ func NewLogGroup() *LogGroup { return &LogGroup{} } +// Shape of a Cloudwatch Logs Anomalies query +type CloudWatchLogsAnomaliesQuery struct { + Id string `json:"id"` + // AWS region to query for the logs + Region string `json:"region"` + // Whether a query is a Metrics, Logs or Annotations query + QueryMode *CloudWatchQueryMode `json:"queryMode,omitempty"` + // Whether a query is a Logs Insights or Logs Anomalies query + LogsMode *LogsMode `json:"logsMode,omitempty"` + // Filter to return only anomalies that are 'SUPPRESSED', 'UNSUPPRESSED', or 'ALL' (default) + SuppressionState *string `json:"suppressionState,omitempty"` + // A unique identifier for the query within the list of targets. + // In server side expressions, the refId is used as a variable name to identify results. + // By default, the UI will assign A->Z; however setting meaningful names may be useful. + RefId string `json:"refId"` + // If hide is set to true, Grafana will filter out the response(s) associated with this query before returning it to the panel. + Hide *bool `json:"hide,omitempty"` + // Specify the query flavor + // TODO make this required and give it a default + QueryType *string `json:"queryType,omitempty"` + // Used to filter only the anomalies found by a certain anomaly detector + AnomalyDetectionARN *string `json:"anomalyDetectionARN,omitempty"` + // For mixed data sources the selected datasource is on the query level. + // For non mixed scenarios this is undefined. + // TODO find a better way to do this ^ that's friendly to schema + // TODO this shouldn't be unknown but DataSourceRef | null + Datasource any `json:"datasource,omitempty"` +} + +// NewCloudWatchLogsAnomaliesQuery creates a new CloudWatchLogsAnomaliesQuery object. +func NewCloudWatchLogsAnomaliesQuery() *CloudWatchLogsAnomaliesQuery { + return &CloudWatchLogsAnomaliesQuery{} +} + // Shape of a CloudWatch Annotation query // TS type is CloudWatchDefaultQuery = Omit & CloudWatchMetricsQuery, declared in veneer // #CloudWatchDefaultQuery: #CloudWatchLogsQuery & #CloudWatchMetricsQuery @cuetsy(kind="type") diff --git a/pkg/tsdb/cloudwatch/log_anomalies_query.go b/pkg/tsdb/cloudwatch/log_anomalies_query.go new file mode 100644 index 00000000000..f39b92eeffa --- /dev/null +++ b/pkg/tsdb/cloudwatch/log_anomalies_query.go @@ -0,0 +1,146 @@ +package cloudwatch + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + + cloudwatchLogsTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/kinds/dataquery" +) + +var executeLogAnomaliesQuery = func(ctx context.Context, ds *DataSource, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + resp := backend.NewQueryDataResponse() + + for _, q := range req.Queries { + var anomaliesQuery dataquery.CloudWatchLogsAnomaliesQuery + err := json.Unmarshal(q.JSON, &anomaliesQuery) + if err != nil { + continue + } + + region := anomaliesQuery.Region + if region == "" || region == defaultRegion { + anomaliesQuery.Region = ds.Settings.Region + } + + logsClient, err := ds.getCWLogsClient(ctx, region) + if err != nil { + return nil, err + } + + listAnomaliesInput := &cloudwatchlogs.ListAnomaliesInput{} + if anomaliesQuery.SuppressionState != nil { + listAnomaliesInput.SuppressionState = getSuppressionState(*anomaliesQuery.SuppressionState) + } + + if anomaliesQuery.AnomalyDetectionARN == nil || *anomaliesQuery.AnomalyDetectionARN != "" { + listAnomaliesInput.AnomalyDetectorArn = anomaliesQuery.AnomalyDetectionARN + } + + response, err := logsClient.ListAnomalies(ctx, listAnomaliesInput) + + if err != nil { + result := backend.NewQueryDataResponse() + result.Responses[q.RefID] = backend.ErrorResponseWithErrorSource(backend.DownstreamError(fmt.Errorf("%v: %w", "failed to call cloudwatch:ListAnomalies", err))) + return result, nil + } + + dataframe, err := logsAnomaliesResultsToDataframes(response) + if err != nil { + return nil, err + } + + respD := resp.Responses[q.RefID] + respD.Frames = data.Frames{dataframe} + resp.Responses[q.RefID] = respD + } + + return resp, nil + +} + +func logsAnomaliesResultsToDataframes(response *cloudwatchlogs.ListAnomaliesOutput) (*data.Frame, error) { + frame := data.NewFrame("Log anomalies") + + if len(response.Anomalies) == 0 { + return frame, nil + } + + n := len(response.Anomalies) + anomalyArns := make([]string, n) + descriptions := make([]string, n) + suppressedStatus := make([]bool, n) + + priorities := make([]string, n) + patterns := make([]string, n) + statuses := make([]string, n) + logGroupArnLists := make([]string, n) + firstSeens := make([]time.Time, n) + lastSeens := make([]time.Time, n) + logTrends := make([]*json.RawMessage, n) + + for i, anomaly := range response.Anomalies { + anomalyArns[i] = *anomaly.AnomalyDetectorArn + descriptions[i] = *anomaly.Description + suppressedStatus[i] = *anomaly.Suppressed + priorities[i] = *anomaly.Priority + if anomaly.PatternString != nil { + patterns[i] = *anomaly.PatternString + } + statuses[i] = string(anomaly.State) + logGroupArnLists[i] = strings.Join(anomaly.LogGroupArnList, ",") + + firstSeens[i] = time.UnixMilli(anomaly.FirstSeen) + + lastSeens[i] = time.UnixMilli(anomaly.LastSeen) + + // data.Frame returned from the backend cannot contain fields of type data.Frames + // so histogram is kept as json.RawMessageto be built as sparkline table cell on the FE + histogramField := anomaly.Histogram + histogramJSON, err := json.Marshal(histogramField) + if err != nil { + logTrends[i] = nil + } else { + rawMsg := json.RawMessage(histogramJSON) + logTrends[i] = &rawMsg + } + } + + newFields := make([]*data.Field, 0, len(response.Anomalies)) + + newFields = append(newFields, data.NewField("state", nil, statuses).SetConfig(&data.FieldConfig{DisplayName: "State"})) + newFields = append(newFields, data.NewField("description", nil, descriptions).SetConfig(&data.FieldConfig{DisplayName: "Anomaly"})) + newFields = append(newFields, data.NewField("priority", nil, priorities).SetConfig(&data.FieldConfig{DisplayName: "Priority"})) + newFields = append(newFields, data.NewField("patternString", nil, patterns).SetConfig(&data.FieldConfig{DisplayName: "Log Pattern"})) + // FE expects the field name to be logTrend in order to identify histogram field for sparkline rendering + newFields = append(newFields, data.NewField("logTrend", nil, logTrends).SetConfig(&data.FieldConfig{DisplayName: "Log Trend"})) + newFields = append(newFields, data.NewField("firstSeen", nil, firstSeens).SetConfig(&data.FieldConfig{DisplayName: "First seen"})) + newFields = append(newFields, data.NewField("lastSeen", nil, lastSeens).SetConfig(&data.FieldConfig{DisplayName: "Last seen"})) + newFields = append(newFields, data.NewField("suppressed", nil, suppressedStatus).SetConfig(&data.FieldConfig{DisplayName: "Suppressed?"})) + newFields = append(newFields, data.NewField("logGroupArnList", nil, logGroupArnLists).SetConfig(&data.FieldConfig{DisplayName: "Log Groups"})) + newFields = append(newFields, data.NewField("anomalyArn", nil, anomalyArns).SetConfig(&data.FieldConfig{DisplayName: "Anomaly Arn"})) + + frame.Fields = newFields + setPreferredVisType(frame, data.VisTypeTable) + return frame, nil +} + +func getSuppressionState(suppressionState string) cloudwatchLogsTypes.SuppressionState { + switch suppressionState { + case "suppressed": + return cloudwatchLogsTypes.SuppressionStateSuppressed + case "unsuppressed": + return cloudwatchLogsTypes.SuppressionStateUnsuppressed + case "all": + return "" + default: + return "" + } +} diff --git a/pkg/tsdb/cloudwatch/log_anomalies_query_test.go b/pkg/tsdb/cloudwatch/log_anomalies_query_test.go new file mode 100644 index 00000000000..d9b3b85314d --- /dev/null +++ b/pkg/tsdb/cloudwatch/log_anomalies_query_test.go @@ -0,0 +1,212 @@ +package cloudwatch + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + cloudwatchLogsTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/assert" +) + +func Test_executeLogAnomaliesQuery(t *testing.T) { + origNewCWClient := NewCWClient + t.Cleanup(func() { + NewCWClient = origNewCWClient + }) + + var cli fakeCWLogsClient + NewCWLogsClient = func(aws.Config) models.CWLogsClient { + return &cli + } + + t.Run("getCWLogsClient is called with correct suppression state", func(t *testing.T) { + testcases := []struct { + name string + suppressionStateInQuery string + result cloudwatchLogsTypes.SuppressionState + }{ + { + "suppressed state", + "suppressed", + cloudwatchLogsTypes.SuppressionStateSuppressed, + }, + { + "unsuppressed state", + "unsuppressed", + cloudwatchLogsTypes.SuppressionStateUnsuppressed, + }, + { + "empty state", + "", + "", + }, + { + "all state", + "all", + "", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + cli = fakeCWLogsClient{anomalies: []cloudwatchLogsTypes.Anomaly{}} + ds := newTestDatasource() + + _, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + Headers: map[string]string{headerFromAlert: ""}, + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "queryMode": "Logs", + "logsMode": "Anomalies", + "suppressionState": "` + tc.suppressionStateInQuery + `", + "region": "us-east-1" + }`), + }, + }, + }) + assert.NoError(t, err) + assert.Equal(t, tc.result, cli.calls.listAnomalies[0].SuppressionState) + }) + } + }) +} + +func Test_executeLogAnomaliesQuery_returns_data_frames(t *testing.T) { + origNewCWClient := NewCWClient + t.Cleanup(func() { + NewCWClient = origNewCWClient + }) + + var cli fakeCWLogsClient + NewCWLogsClient = func(aws.Config) models.CWLogsClient { + return &cli + } + t.Run("returns log anomalies data frames", func(t *testing.T) { + cli = fakeCWLogsClient{anomalies: []cloudwatchLogsTypes.Anomaly{ + { + AnomalyId: aws.String("anomaly-1"), + AnomalyDetectorArn: aws.String("arn:aws:logs:us-east-1:123456789012:anomaly-detector:anomaly-detector-1"), + FirstSeen: 1622505600000, // June 1, 2021 00:00:00 GMT + LastSeen: 1622592000000, // June 2, 2021 00:00:00 GMT + LogGroupArnList: []string{"arn:aws:logs:us-east-1:1234567:log-group-1:id-1", "arn:aws:logs:us-east-1:1234567:log-group-2:id-2"}, + Description: aws.String("Description 1"), + State: cloudwatchLogsTypes.StateActive, + Priority: aws.String("high"), + PatternString: aws.String(`{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite",,"instance":"instance"-5:Token-6,"job":"kubernetes-service-endpoints","pod_name":"pod_name"-9,"prom_metric_type":"counter"}`), + Suppressed: aws.Bool(false), + Histogram: map[string]int64{ + "1622505600000": 5, + "1622519200000": 10, + "1622532800000": 7, + }, + }, + { + AnomalyId: aws.String("anomaly-2"), + AnomalyDetectorArn: aws.String("arn:aws:logs:us-east-1:123456789012:anomaly-detector:anomaly-detector-2"), + FirstSeen: 1622592000000, // June 2, 2021 00:00:00 GMT + LastSeen: 1622678400000, // June 3, 2021 00:00:00 GMT + LogGroupArnList: []string{"arn:aws:logs:us-east-1:1234567:log-group-1:id-3", "arn:aws:logs:us-east-1:1234567:log-group-2:id-4"}, + Description: aws.String("Description 2"), + State: cloudwatchLogsTypes.StateSuppressed, + Priority: aws.String("low"), + PatternString: aws.String(`{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite","dotnet_collection_count_total":"dotnet_collection_count_total"-3}`), + Suppressed: aws.Bool(true), + Histogram: map[string]int64{ + "1622592000000": 3, + }, + }, + }} + + ds := newTestDatasource() + + resp, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + Headers: map[string]string{headerFromAlert: ""}, + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "queryMode": "Logs", + "logsMode": "Anomalies", + "suppressionState": "all", + "region": "us-east-1" + }`), + }, + }, + }) + + assert.NoError(t, err) + assert.Len(t, resp.Responses, 1) + for _, r := range resp.Responses { + assert.Len(t, r.Frames, 1) + frame := r.Frames[0] + assert.Equal(t, "Log anomalies", frame.Name) + assert.Len(t, frame.Fields, 10) + + stateField := frame.Fields[0] + assert.Equal(t, "state", stateField.Name) + assert.Equal(t, "Active", stateField.At(0)) + assert.Equal(t, "Suppressed", stateField.At(1)) + + descriptionField := frame.Fields[1] + assert.Equal(t, "description", descriptionField.Name) + assert.Equal(t, "Description 1", descriptionField.At(0)) + assert.Equal(t, "Description 2", descriptionField.At(1)) + + priorityField := frame.Fields[2] + assert.Equal(t, "priority", priorityField.Name) + assert.Equal(t, "high", priorityField.At(0)) + assert.Equal(t, "low", priorityField.At(1)) + + patternStringField := frame.Fields[3] + assert.Equal(t, "patternString", patternStringField.Name) + assert.Equal(t, `{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite",,"instance":"instance"-5:Token-6,"job":"kubernetes-service-endpoints","pod_name":"pod_name"-9,"prom_metric_type":"counter"}`, patternStringField.At(0)) + assert.Equal(t, `{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite","dotnet_collection_count_total":"dotnet_collection_count_total"-3}`, patternStringField.At(1)) + + histogramField := frame.Fields[4] + assert.Equal(t, "logTrend", histogramField.Name) + + histogram0 := histogramField.At(0).(*json.RawMessage) + var histData0 map[string]int64 + err = json.Unmarshal(*histogram0, &histData0) + assert.NoError(t, err) + assert.Equal(t, int64(5), histData0["1622505600000"]) + assert.Equal(t, int64(10), histData0["1622519200000"]) + assert.Equal(t, int64(7), histData0["1622532800000"]) + + firstSeenField := frame.Fields[5] + assert.Equal(t, "firstSeen", firstSeenField.Name) + assert.Equal(t, time.Unix(1622505600, 0), firstSeenField.At(0)) + assert.Equal(t, time.Unix(1622592000, 0), firstSeenField.At(1)) + + lastSeenField := frame.Fields[6] + assert.Equal(t, "lastSeen", lastSeenField.Name) + assert.Equal(t, time.Unix(1622592000, 0), lastSeenField.At(0)) + assert.Equal(t, time.Unix(1622678400, 0), lastSeenField.At(1)) + + suppressedField := frame.Fields[7] + assert.Equal(t, "suppressed", suppressedField.Name) + assert.Equal(t, false, suppressedField.At(0)) + assert.Equal(t, true, suppressedField.At(1)) + + logGroupArnListField := frame.Fields[8] + assert.Equal(t, "logGroupArnList", logGroupArnListField.Name) + assert.Equal(t, "arn:aws:logs:us-east-1:1234567:log-group-1:id-1,arn:aws:logs:us-east-1:1234567:log-group-2:id-2", logGroupArnListField.At(0)) + assert.Equal(t, "arn:aws:logs:us-east-1:1234567:log-group-1:id-3,arn:aws:logs:us-east-1:1234567:log-group-2:id-4", logGroupArnListField.At(1)) + + anomalyDetectorArnField := frame.Fields[9] + assert.Equal(t, "anomalyArn", anomalyDetectorArnField.Name) + assert.Equal(t, "arn:aws:logs:us-east-1:123456789012:anomaly-detector:anomaly-detector-1", anomalyDetectorArnField.At(0)) + assert.Equal(t, "arn:aws:logs:us-east-1:123456789012:anomaly-detector:anomaly-detector-2", anomalyDetectorArnField.At(1)) + } + }) +} diff --git a/pkg/tsdb/cloudwatch/log_sync_query_test.go b/pkg/tsdb/cloudwatch/log_sync_query_test.go index b1c0b99175e..376464d9340 100644 --- a/pkg/tsdb/cloudwatch/log_sync_query_test.go +++ b/pkg/tsdb/cloudwatch/log_sync_query_test.go @@ -137,7 +137,7 @@ func Test_executeSyncLogQuery(t *testing.T) { executeSyncLogQuery = origExecuteSyncLogQuery }) - t.Run("when query mode is 'Logs' and does not include type or subtype", func(t *testing.T) { + t.Run("when query mode is 'Logs Insights' and does not include type or subtype", func(t *testing.T) { origExecuteSyncLogQuery := executeSyncLogQuery syncCalled := false executeSyncLogQuery = func(ctx context.Context, e *DataSource, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { diff --git a/pkg/tsdb/cloudwatch/mocks/logs.go b/pkg/tsdb/cloudwatch/mocks/logs.go index 7843fdaacd1..da8dd2cc47c 100644 --- a/pkg/tsdb/cloudwatch/mocks/logs.go +++ b/pkg/tsdb/cloudwatch/mocks/logs.go @@ -66,3 +66,9 @@ func (m *MockLogEvents) GetLogEvents(ctx context.Context, input *cloudwatchlogs. return args.Get(0).(*cloudwatchlogs.GetLogEventsOutput), args.Error(1) } + +func (m *MockLogEvents) ListAnomalies(ctx context.Context, input *cloudwatchlogs.ListAnomaliesInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListAnomaliesOutput, error) { + args := m.Called(ctx, input, optFns) + + return args.Get(0).(*cloudwatchlogs.ListAnomaliesOutput), args.Error(1) +} diff --git a/pkg/tsdb/cloudwatch/models/api.go b/pkg/tsdb/cloudwatch/models/api.go index e4b487288f1..eed0243b69d 100644 --- a/pkg/tsdb/cloudwatch/models/api.go +++ b/pkg/tsdb/cloudwatch/models/api.go @@ -76,6 +76,7 @@ type CWLogsClient interface { cloudwatchlogs.GetLogEventsAPIClient cloudwatchlogs.DescribeLogGroupsAPIClient + cloudwatchlogs.ListAnomaliesAPIClient } type CWClient interface { diff --git a/pkg/tsdb/cloudwatch/test_utils.go b/pkg/tsdb/cloudwatch/test_utils.go index 82ce33099ee..3f4c797dea4 100644 --- a/pkg/tsdb/cloudwatch/test_utils.go +++ b/pkg/tsdb/cloudwatch/test_utils.go @@ -29,12 +29,15 @@ type fakeCWLogsClient struct { queryResults cloudwatchlogs.GetQueryResultsOutput logGroupsIndex int + + anomalies []cloudwatchlogstypes.Anomaly } type logsQueryCalls struct { startQuery []*cloudwatchlogs.StartQueryInput getEvents []*cloudwatchlogs.GetLogEventsInput describeLogGroups []*cloudwatchlogs.DescribeLogGroupsInput + listAnomalies []*cloudwatchlogs.ListAnomaliesInput } func (m *fakeCWLogsClient) GetQueryResults(_ context.Context, _ *cloudwatchlogs.GetQueryResultsInput, _ ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetQueryResultsOutput, error) { @@ -55,6 +58,14 @@ func (m *fakeCWLogsClient) StopQuery(_ context.Context, _ *cloudwatchlogs.StopQu }, nil } +func (m *fakeCWLogsClient) ListAnomalies(_ context.Context, input *cloudwatchlogs.ListAnomaliesInput, _ ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListAnomaliesOutput, error) { + m.calls.listAnomalies = append(m.calls.listAnomalies, input) + + return &cloudwatchlogs.ListAnomaliesOutput{ + Anomalies: m.anomalies, + }, nil +} + type mockLogsSyncClient struct { mock.Mock } @@ -80,6 +91,11 @@ func (m *mockLogsSyncClient) StartQuery(ctx context.Context, input *cloudwatchlo return args.Get(0).(*cloudwatchlogs.StartQueryOutput), args.Error(1) } +func (m *mockLogsSyncClient) ListAnomalies(ctx context.Context, input *cloudwatchlogs.ListAnomaliesInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListAnomaliesOutput, error) { + args := m.Called(ctx, input, optFns) + return args.Get(0).(*cloudwatchlogs.ListAnomaliesOutput), args.Error(1) +} + func (m *fakeCWLogsClient) DescribeLogGroups(_ context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, _ ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { m.calls.describeLogGroups = append(m.calls.describeLogGroups, input) output := &m.logGroups[m.logGroupsIndex] diff --git a/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsAnomaliesQueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsAnomaliesQueryEditor.tsx new file mode 100644 index 00000000000..218d16e6b98 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsAnomaliesQueryEditor.tsx @@ -0,0 +1,41 @@ +import { EditorField, EditorRow } from '@grafana/plugin-ui'; +import { Combobox, Input } from '@grafana/ui'; + +import { CloudWatchLogsAnomaliesQuery } from '../../../dataquery.gen'; + +interface Props { + query: CloudWatchLogsAnomaliesQuery; + onChange: (value: CloudWatchLogsAnomaliesQuery) => void; +} + +const supressionStateOptions = [ + { label: 'All', value: 'all' }, + { label: 'Suppressed', value: 'suppressed' }, + { label: 'Unsuppressed', value: 'unsuppressed' }, +]; + +export const LogsAnomaliesQueryEditor = (props: Props) => { + return ( + <> + + + { + props.onChange({ ...props.query, anomalyDetectionARN: e.currentTarget.value }); + }} + /> + + + { + props.onChange({ ...props.query, suppressionState: e.value }); + }} + /> + + + + ); +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx index 6be68db99cb..bf59c959066 100644 --- a/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/QueryEditor/LogsQueryEditor/LogsQueryEditor.tsx @@ -6,9 +6,10 @@ import { InlineSelect } from '@grafana/plugin-ui'; import { CloudWatchDatasource } from '../../../datasource'; import { DEFAULT_CWLI_QUERY_STRING, DEFAULT_PPL_QUERY_STRING, DEFAULT_SQL_QUERY_STRING } from '../../../defaultQueries'; -import { CloudWatchJsonData, CloudWatchLogsQuery, CloudWatchQuery, LogsQueryLanguage } from '../../../types'; +import { CloudWatchJsonData, CloudWatchLogsQuery, CloudWatchQuery, LogsMode, LogsQueryLanguage } from '../../../types'; import { CloudWatchLink } from './CloudWatchLink'; +import { LogsAnomaliesQueryEditor } from './LogsAnomaliesQueryEditor'; import { CloudWatchLogsQueryField } from './LogsQueryField'; type Props = QueryEditorProps & { @@ -22,6 +23,11 @@ const logsQueryLanguageOptions: Array> = [ { label: 'OpenSearch PPL', value: LogsQueryLanguage.PPL }, ]; +const logsModeOptions: Array> = [ + { label: 'Logs Insights', value: LogsMode.Insights }, + { label: 'Logs Anomalies', value: LogsMode.Anomalies }, +]; + export const CloudWatchLogsQueryEditor = memo(function CloudWatchLogsQueryEditor(props: Props) { const { query, data, datasource, onChange, extraHeaderElementLeft } = props; @@ -42,6 +48,13 @@ export const CloudWatchLogsQueryEditor = memo(function CloudWatchLogsQueryEditor [isQueryNew, onChange, query] ); + const onLogsModeChange = useCallback( + (logsMode: LogsMode | undefined) => { + onChange({ ...query, logsMode }); + }, + [query, onChange] + ); + // if the query has already been saved from before, we shouldn't replace it with a default one useEffectOnce(() => { if (query.expression) { @@ -58,20 +71,32 @@ export const CloudWatchLogsQueryEditor = memo(function CloudWatchLogsQueryEditor useEffect(() => { extraHeaderElementLeft?.( - { - onQueryLanguageChange(value); - }} - /> + <> + { + onLogsModeChange(value); + }} + /> + {query.logsMode !== LogsMode.Anomalies && ( + { + onQueryLanguageChange(value); + }} + /> + )} + ); return () => { extraHeaderElementLeft?.(undefined); }; - }, [extraHeaderElementLeft, onChange, onQueryLanguageChange, query]); + }, [extraHeaderElementLeft, onChange, onQueryLanguageChange, query, onLogsModeChange]); const onQueryStringChange = (query: CloudWatchQuery) => { onChange(query); @@ -79,11 +104,17 @@ export const CloudWatchLogsQueryEditor = memo(function CloudWatchLogsQueryEditor }; return ( - } - /> + <> + {query.logsMode === LogsMode.Anomalies ? ( + + ) : ( + } + /> + )} + ); }); diff --git a/public/app/plugins/datasource/cloudwatch/dataquery.cue b/public/app/plugins/datasource/cloudwatch/dataquery.cue index a0afcfb729f..4144b423419 100644 --- a/public/app/plugins/datasource/cloudwatch/dataquery.cue +++ b/public/app/plugins/datasource/cloudwatch/dataquery.cue @@ -146,7 +146,7 @@ composableKinds: DataQuery: { } @cuetsy(kind="interface") #QueryEditorExpression: #QueryEditorArrayExpression | #QueryEditorPropertyExpression | #QueryEditorGroupByExpression | #QueryEditorFunctionExpression | #QueryEditorFunctionParameterExpression | #QueryEditorOperatorExpression @cuetsy(kind="type") - + #LogsMode: "Insights" | "Anomalies" @cuetsy(kind="enum") #LogsQueryLanguage: "CWLI" | "SQL" | "PPL" @cuetsy(kind="enum") // Shape of a CloudWatch Logs query @@ -155,6 +155,8 @@ composableKinds: DataQuery: { // Whether a query is a Metrics, Logs, or Annotations query queryMode: #CloudWatchQueryMode + // Whether a query is a Logs Insights or Logs Anomalies query + logsMode?: #LogsMode id: string // AWS region to query for the logs region: string @@ -166,9 +168,27 @@ composableKinds: DataQuery: { logGroups?: [...#LogGroup] // @deprecated use logGroups logGroupNames?: [...string] + // Language used for querying logs, can be CWLI, SQL, or PPL. If empty, the default language is CWLI. queryLanguage?: #LogsQueryLanguage } @cuetsy(kind="interface") + + // Shape of a Cloudwatch Logs Anomalies query + #CloudWatchLogsAnomaliesQuery: { + common.DataQuery + id: string + // AWS region to query for the logs + region: string + // Whether a query is a Metrics, Logs or Annotations query + queryMode?: #CloudWatchQueryMode + // Whether a query is a Logs Insights or Logs Anomalies query + logsMode?: #LogsMode + // Filter to return only anomalies that are 'SUPPRESSED', 'UNSUPPRESSED', or 'ALL' (default) + suppressionState?: string + // Used to filter only the anomalies found by a certain anomaly detector + anomalyDetectionARN?: string + } @cuetsy(kind="interface") + #LogGroup: { // ARN of the log group arn: string diff --git a/public/app/plugins/datasource/cloudwatch/dataquery.gen.ts b/public/app/plugins/datasource/cloudwatch/dataquery.gen.ts index 461ca94247e..ba0a1134d2d 100644 --- a/public/app/plugins/datasource/cloudwatch/dataquery.gen.ts +++ b/public/app/plugins/datasource/cloudwatch/dataquery.gen.ts @@ -216,6 +216,11 @@ export interface QueryEditorArrayExpression { export type QueryEditorExpression = (QueryEditorArrayExpression | QueryEditorPropertyExpression | QueryEditorGroupByExpression | QueryEditorFunctionExpression | QueryEditorFunctionParameterExpression | QueryEditorOperatorExpression); +export enum LogsMode { + Anomalies = 'Anomalies', + Insights = 'Insights', +} + export enum LogsQueryLanguage { CWLI = 'CWLI', PPL = 'PPL', @@ -239,6 +244,10 @@ export interface CloudWatchLogsQuery extends common.DataQuery { * Log groups to query */ logGroups?: Array; + /** + * Whether a query is a Logs Insights or Logs Anomalies query + */ + logsMode?: LogsMode; /** * Language used for querying logs, can be CWLI, SQL, or PPL. If empty, the default language is CWLI. */ @@ -263,6 +272,33 @@ export const defaultCloudWatchLogsQuery: Partial = { statsGroups: [], }; +/** + * Shape of a Cloudwatch Logs Anomalies query + */ +export interface CloudWatchLogsAnomaliesQuery extends common.DataQuery { + /** + * Used to filter only the anomalies found by a certain anomaly detector + */ + anomalyDetectionARN?: string; + id: string; + /** + * Whether a query is a Logs Insights or Logs Anomalies query + */ + logsMode?: LogsMode; + /** + * Whether a query is a Metrics, Logs or Annotations query + */ + queryMode?: CloudWatchQueryMode; + /** + * AWS region to query for the logs + */ + region: string; + /** + * Filter to return only anomalies that are 'SUPPRESSED', 'UNSUPPRESSED', or 'ALL' (default) + */ + suppressionState?: string; +} + export interface LogGroup { /** * AccountId of the log group diff --git a/public/app/plugins/datasource/cloudwatch/datasource.test.ts b/public/app/plugins/datasource/cloudwatch/datasource.test.ts index dc15557ee8a..3186c92121c 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.test.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.test.ts @@ -306,7 +306,7 @@ describe('datasource', () => { }); }); - it('should add a data link field to log queries', async () => { + it('should add links to log insights queries', async () => { const { datasource } = setupForLogs(); const observable = datasource.query({ @@ -439,7 +439,7 @@ describe('datasource', () => { const { datasource } = setupMockedDataSource(); expect(datasource.getDefaultQuery(CoreApp.PanelEditor).queryMode).toEqual('Metrics'); }); - it('should set default log groups in default query', () => { + it('should set default log groups in default logs insights query', () => { const { datasource } = setupMockedDataSource({ customInstanceSettings: { ...CloudWatchSettings, diff --git a/public/app/plugins/datasource/cloudwatch/datasource.ts b/public/app/plugins/datasource/cloudwatch/datasource.ts index 685dce3c15d..27c240dd6cd 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.ts @@ -16,7 +16,12 @@ import { DataSourceWithBackend, TemplateSrv, getTemplateSrv } from '@grafana/run import { CloudWatchAnnotationSupport } from './annotationSupport'; import { DEFAULT_METRICS_QUERY, getDefaultLogsQuery } from './defaultQueries'; -import { isCloudWatchAnnotationQuery, isCloudWatchLogsQuery, isCloudWatchMetricsQuery } from './guards'; +import { + isCloudWatchAnnotationQuery, + isCloudWatchLogsQuery, + isCloudWatchMetricsQuery, + isLogsAnomaliesQuery, +} from './guards'; import { CloudWatchLogsLanguageProvider } from './language/cloudwatch-logs/CloudWatchLogsLanguageProvider'; import { LogsSQLCompletionItemProvider, @@ -40,6 +45,7 @@ import { ResourcesAPI } from './resources/ResourcesAPI'; import { CloudWatchAnnotationQuery, CloudWatchJsonData, + CloudWatchLogsAnomaliesQuery, CloudWatchLogsQuery, CloudWatchMetricsQuery, CloudWatchQuery, @@ -104,11 +110,14 @@ export class CloudWatchDatasource const logQueries: CloudWatchLogsQuery[] = []; const metricsQueries: CloudWatchMetricsQuery[] = []; + const logsAnomaliesQueries: CloudWatchLogsAnomaliesQuery[] = []; const annotationQueries: CloudWatchAnnotationQuery[] = []; queries.forEach((query) => { if (isCloudWatchAnnotationQuery(query)) { annotationQueries.push(query); + } else if (isLogsAnomaliesQuery(query)) { + logsAnomaliesQueries.push(query); } else if (isCloudWatchLogsQuery(query)) { logQueries.push(query); } else { @@ -127,6 +136,12 @@ export class CloudWatchDatasource ); } + if (logsAnomaliesQueries.length) { + dataQueryResponses.push( + this.logsQueryRunner.handleLogAnomaliesQueries(logsAnomaliesQueries, options, super.query.bind(this)) + ); + } + if (annotationQueries.length) { dataQueryResponses.push( this.annotationQueryRunner.handleAnnotationQuery(annotationQueries, options, super.query.bind(this)) diff --git a/public/app/plugins/datasource/cloudwatch/guards.ts b/public/app/plugins/datasource/cloudwatch/guards.ts index b5346df3c25..2a54dd9719c 100644 --- a/public/app/plugins/datasource/cloudwatch/guards.ts +++ b/public/app/plugins/datasource/cloudwatch/guards.ts @@ -1,10 +1,26 @@ import { AnnotationQuery } from '@grafana/data'; -import { CloudWatchAnnotationQuery, CloudWatchLogsQuery, CloudWatchMetricsQuery, CloudWatchQuery } from './types'; +import { + CloudWatchAnnotationQuery, + CloudWatchLogsAnomaliesQuery, + CloudWatchLogsQuery, + CloudWatchMetricsQuery, + CloudWatchQuery, + LogsMode, +} from './types'; export const isCloudWatchLogsQuery = (cloudwatchQuery: CloudWatchQuery): cloudwatchQuery is CloudWatchLogsQuery => cloudwatchQuery.queryMode === 'Logs'; +export const isLogsAnomaliesQuery = ( + cloudwatchQuery: CloudWatchQuery +): cloudwatchQuery is CloudWatchLogsAnomaliesQuery => { + if (isCloudWatchLogsQuery(cloudwatchQuery)) { + return cloudwatchQuery.logsMode === LogsMode.Anomalies; + } + return false; +}; + export const isCloudWatchMetricsQuery = (cloudwatchQuery: CloudWatchQuery): cloudwatchQuery is CloudWatchMetricsQuery => cloudwatchQuery.queryMode === 'Metrics' || !cloudwatchQuery.hasOwnProperty('queryMode'); // in early versions of this plugin, queryMode wasn't defined in a CloudWatchMetricsQuery diff --git a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts index b38fd288633..ec7715dc4df 100644 --- a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts +++ b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts @@ -2,20 +2,26 @@ import { lastValueFrom, of } from 'rxjs'; import { DataQueryRequest, + DataQueryResponse, + Field, FieldType, LogLevel, LogRowContextQueryDirection, LogRowModel, - MutableDataFrame, } from '@grafana/data'; import { regionVariable } from '../mocks/CloudWatchDataSource'; import { setupMockedLogsQueryRunner } from '../mocks/LogsQueryRunner'; import { LogsRequestMock } from '../mocks/Request'; import { validLogsQuery } from '../mocks/queries'; -import { CloudWatchLogsQuery } from '../types'; // Add this import statement +import { TimeRangeMock } from '../mocks/timeRange'; +import { CloudWatchLogsAnomaliesQuery, CloudWatchLogsQuery, LogsMode } from '../types'; // Add this import statement -import { LOGSTREAM_IDENTIFIER_INTERNAL, LOG_IDENTIFIER_INTERNAL } from './CloudWatchLogsQueryRunner'; +import { + LOGSTREAM_IDENTIFIER_INTERNAL, + LOG_IDENTIFIER_INTERNAL, + convertTrendHistogramToSparkline, +} from './CloudWatchLogsQueryRunner'; describe('CloudWatchLogsQueryRunner', () => { beforeEach(() => { @@ -28,14 +34,15 @@ describe('CloudWatchLogsQueryRunner', () => { const row: LogRowModel = { entryFieldIndex: 0, rowIndex: 0, - dataFrame: new MutableDataFrame({ + dataFrame: { refId: 'B', + length: 1, fields: [ - { name: 'ts', type: FieldType.time, values: [1] }, - { name: LOG_IDENTIFIER_INTERNAL, type: FieldType.string, values: ['foo'], labels: {} }, - { name: LOGSTREAM_IDENTIFIER_INTERNAL, type: FieldType.string, values: ['bar'], labels: {} }, + { name: 'ts', type: FieldType.time, values: [1], config: {} }, + { name: LOG_IDENTIFIER_INTERNAL, type: FieldType.string, values: ['foo'], labels: {}, config: {} }, + { name: LOGSTREAM_IDENTIFIER_INTERNAL, type: FieldType.string, values: ['bar'], labels: {}, config: {} }, ], - }), + }, entry: '4', labels: {}, hasAnsi: false, @@ -481,6 +488,120 @@ describe('CloudWatchLogsQueryRunner', () => { }); }); }); + + describe('handleLogAnomaliesQueries', () => { + it('appends -anomalies to the requestId', async () => { + const { runner, queryMock } = setupMockedLogsQueryRunner(); + const logsAnomaliesRequestMock: DataQueryRequest = { + requestId: 'mockId', + range: TimeRangeMock, + rangeRaw: { from: TimeRangeMock.from, to: TimeRangeMock.to }, + targets: [ + { + id: '1', + logsMode: LogsMode.Anomalies, + queryMode: 'Logs', + refId: 'A', + region: 'us-east-1', + }, + ], + interval: '', + intervalMs: 0, + scopedVars: { __interval: { value: '20s' } }, + timezone: '', + app: '', + startTime: 0, + }; + await expect( + runner.handleLogAnomaliesQueries(LogsRequestMock.targets, logsAnomaliesRequestMock, queryMock) + ).toEmitValuesWith(() => { + expect(queryMock.mock.calls[0][0].requestId).toEqual('mockId-logsAnomalies'); + }); + }); + + it('processes log trend histogram data correctly', async () => { + const response = structuredClone(anomaliesQueryResponse); + + convertTrendHistogramToSparkline(response); + + expect(response.data[0].fields.find((field: Field) => field.name === 'Log trend')).toEqual({ + name: 'Log trend', + type: 'frame', + config: { + custom: { + drawStyle: 'bars', + cellOptions: { + type: 'sparkline', + hideValue: true, + }, + }, + }, + values: [ + { + name: 'Trend_row_0', + length: 8, + fields: [ + { + name: 'time', + type: 'time', + values: [ + 1760454000000, 1760544000000, 1760724000000, 1761282000000, 1761300000000, 1761354000000, + 1761372000000, 1761390000000, + ], + config: {}, + }, + { + name: 'value', + type: 'number', + values: [81, 35, 35, 36, 36, 36, 72, 36], + config: {}, + }, + ], + }, + { + name: 'Trend_row_1', + length: 2, + fields: [ + { + name: 'time', + type: 'time', + values: [1760687665000, 1760687670000], + config: {}, + }, + { + name: 'value', + type: 'number', + values: [3, 3], + config: {}, + }, + ], + }, + ], + }); + }); + + it('replaces log trend histogram field at the same index in the frame', () => { + const response = structuredClone(anomaliesQueryResponse); + convertTrendHistogramToSparkline(response); + expect(response.data[0].fields[4].name).toEqual('Log trend'); + }); + + it('ignore invalid timestamps in log trend histogram', () => { + const response = structuredClone(anomaliesQueryResponse); + + response.data[0].fields[4].values[1] = { + invalidTimestamp: 3, + '1760687670000': 3, + anotherInvalidTimestamp: 2, + '1760687670010': 3, + }; + + convertTrendHistogramToSparkline(response); + + expect(response.data[0].fields[4].values[1].fields[0].values.length).toEqual(2); + expect(response.data[0].fields[4].values[1].fields[1].values.length).toEqual(2); + }); + }); }); const rawLogQueriesStub: CloudWatchLogsQuery[] = [ @@ -641,3 +762,166 @@ const getQueryErrorResponseStub = { const stopQueryResponseStub = { state: 'Done', }; + +const anomaliesQueryResponse: DataQueryResponse = { + data: [ + { + name: 'Logs anomalies', + refId: 'A', + meta: { + preferredVisualisationType: 'table', + }, + fields: [ + { + name: 'state', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'State', + }, + values: ['Active', 'Active'], + entities: {}, + }, + { + name: 'description', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'Anomaly', + }, + values: [ + '50.0% increase in count of value "405" for "code"-3', + '151.3% increase in count of value 1 for "dotnet_collection_count_total"-3', + ], + entities: {}, + }, + { + name: 'priority', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'Priority', + }, + values: ['MEDIUM', 'MEDIUM'], + entities: {}, + }, + { + name: 'patternString', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'Log Pattern', + }, + values: [ + '{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite","Timestamp":<*>,"Version":<*>,"code":<*>,"container_name":"petsite","http_requests_received_total":<*>,"instance":<*>:<*>,"job":"kubernetes-service-endpoints","kubernetes_node":<*>,"method":<*>,"pod_name":<*>,"prom_metric_type":"counter"}', + '{"ClusterName":"PetSite","Namespace":"default","Service":"service-petsite","Timestamp":<*>,"Version":<*>,"container_name":"petsite","dotnet_collection_count_total":<*>,"generation":<*>,"instance":<*>:<*>,"job":"kubernetes-service-endpoints","kubernetes_node":<*>,"pod_name":<*>,"prom_metric_type":"counter"}', + ], + entities: {}, + }, + { + name: 'logTrend', + type: 'other', + typeInfo: { + frame: 'json.RawMessage', + nullable: true, + }, + config: { + displayName: 'Log Trend', + }, + values: [ + { + '1760454000000': 81, + '1760544000000': 35, + '1760724000000': 35, + '1761282000000': 36, + '1761300000000': 36, + '1761354000000': 36, + '1761372000000': 72, + '1761390000000': 36, + }, + { + '1760687665000': 3, + '1760687670000': 3, + }, + ], + entities: {}, + }, + { + name: 'firstSeen', + type: 'time', + typeInfo: { + frame: 'time.Time', + }, + config: { + displayName: 'First seen', + }, + values: [1760462460000, 1760687640000], + entities: {}, + }, + { + name: 'lastSeen', + type: 'time', + typeInfo: { + frame: 'time.Time', + }, + config: { + displayName: 'Last seen', + }, + values: [1761393660000, 1760687940000], + entities: {}, + }, + { + name: 'suppressed', + type: 'boolean', + typeInfo: { + frame: 'bool', + }, + config: { + displayName: 'Suppressed?', + }, + values: [false, false], + entities: {}, + }, + { + name: 'logGroupArnList', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'Log Groups', + }, + values: [ + 'arn:aws:logs:us-east-2:569069006612:log-group:/aws/containerinsights/PetSite/prometheus', + 'arn:aws:logs:us-east-2:569069006612:log-group:/aws/containerinsights/PetSite/prometheus', + ], + entities: {}, + }, + { + name: 'anomalyArn', + type: 'string', + typeInfo: { + frame: 'string', + }, + config: { + displayName: 'Anomaly Arn', + }, + values: [ + 'arn:aws:logs:us-east-2:569069006612:anomaly-detector:dca8b129-d09d-4167-86e9-7bf62ede2f95', + 'arn:aws:logs:us-east-2:569069006612:anomaly-detector:dca8b129-d09d-4167-86e9-7bf62ede2f95', + ], + entities: {}, + }, + ], + length: 2, + }, + ], +}; diff --git a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts index 0ccc22029c1..7f1ba927f47 100644 --- a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts +++ b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts @@ -23,6 +23,8 @@ import { DataQueryRequest, DataQueryResponse, DataSourceInstanceSettings, + Field, + FieldType, LoadingState, LogRowContextOptions, LogRowContextQueryDirection, @@ -33,15 +35,19 @@ import { } from '@grafana/data'; import { TemplateSrv } from '@grafana/runtime'; import { type CustomFormatterVariable } from '@grafana/scenes'; +import { GraphDrawStyle } from '@grafana/schema/dist/esm/index'; +import { TableCellDisplayMode } from '@grafana/ui'; import { CloudWatchJsonData, + CloudWatchLogsAnomaliesQuery, CloudWatchLogsQuery, CloudWatchLogsQueryStatus, CloudWatchLogsRequest, CloudWatchQuery, GetLogEventsRequest, LogAction, + LogsMode, LogsQueryLanguage, QueryParam, StartQueryRequest, @@ -101,6 +107,7 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest { logGroups, logGroupNames, queryLanguage: target.queryLanguage, + logsMode: target.logsMode ?? LogsMode.Insights, }; }); @@ -136,6 +143,63 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest { ); }; + public handleLogAnomaliesQueries = ( + logAnomaliesQueries: CloudWatchLogsAnomaliesQuery[], + options: DataQueryRequest, + queryFn: (request: DataQueryRequest) => Observable + ): Observable => { + const logAnomalyTargets: StartQueryRequest[] = logAnomaliesQueries.map((target: CloudWatchLogsAnomaliesQuery) => { + return { + refId: target.refId, + region: this.templateSrv.replace(this.getActualRegion(target.region)), + queryString: '', + logGroups: [], + logsMode: LogsMode.Anomalies, + suppressionState: target.suppressionState || 'all', + anomalyDetectionARN: target.anomalyDetectionARN || '', + }; + }); + + const range = options?.range || getDefaultTimeRange(); + // append -logsAnomalies to prevent requestId from matching metric or logs queries from the same panel + const requestId = options?.requestId ? `${options?.requestId}-logsAnomalies` : ''; + + const requestParams: DataQueryRequest = { + ...options, + range, + skipQueryCache: true, + requestId, + interval: options?.interval || '', // dummy + intervalMs: options?.intervalMs || 1, // dummy + scopedVars: options?.scopedVars || {}, // dummy + timezone: options?.timezone || '', // dummy + app: options?.app || '', // dummy + startTime: options?.startTime || 0, // dummy + targets: logAnomalyTargets.map((t) => ({ + ...t, + id: '', + queryMode: 'Logs', + refId: t.refId || 'A', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasource: this.ref, + type: 'logAction', + logsMode: LogsMode.Anomalies, + })), + }; + + return queryFn(requestParams).pipe( + mergeMap((dataQueryResponse) => { + return from( + (async () => { + convertTrendHistogramToSparkline(dataQueryResponse); + return dataQueryResponse; + })() + ); + }) + ); + }; + /** * Called by datasource.ts, invoked when user clicks on a log row in the logs visualization and the "show context button" */ @@ -450,7 +514,7 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest { const hasMissingLogGroups = !query.logGroups?.length; const hasMissingQueryString = !query.expression?.length; - // log groups are not mandatory if language is SQL + // log groups are not mandatory if language is SQL and LogsMode is not Insights const isInvalidCWLIQuery = query.queryLanguage !== 'SQL' && hasMissingLogGroups && hasMissingLegacyLogGroupNames; if (isInvalidCWLIQuery || hasMissingQueryString) { return false; @@ -479,3 +543,69 @@ function parseLogGroupName(logIdentifier: string): string { const colonIndex = logIdentifier.lastIndexOf(':'); return logIdentifier.slice(colonIndex + 1); } + +const LOG_TREND_FIELD_NAME = 'logTrend'; + +/** + * Takes DataQueryResponse and converts any "log trend" fields (that are in JSON.rawMessage form) + * into data frame fields that the table vis will be able to display + */ +export function convertTrendHistogramToSparkline(dataQueryResponse: DataQueryResponse): void { + dataQueryResponse.data.forEach((frame) => { + let fieldIndexToReplace = null; + // log trend histogram field from CW API is of shape Record + const sparklineRawData: Field> = frame.fields.find((field: Field, index: number) => { + if (field.name === LOG_TREND_FIELD_NAME && field.type === FieldType.other) { + fieldIndexToReplace = index; + return true; + } + return false; + }); + + if (sparklineRawData) { + const sparklineField: Field = { + name: 'Log trend', + type: FieldType.frame, + config: { + custom: { + drawStyle: GraphDrawStyle.Bars, + cellOptions: { + type: TableCellDisplayMode.Sparkline, + // hiding the value here as it's not useful or clear on what it represents for log trend + hideValue: true, + }, + }, + }, + values: [], + }; + + sparklineRawData.values.forEach((sparklineValue, rowIndex) => { + const timestamps: number[] = []; + const values: number[] = []; + Object.keys(sparklineValue).map((t, i) => { + let n = Number(t); + if (!isNaN(n)) { + timestamps.push(n); + values.push(sparklineValue[t]); + } + }); + + const sparklineFieldFrame: DataFrame = { + name: `Trend_row_${rowIndex}`, + length: timestamps.length, + fields: [ + { name: 'time', type: FieldType.time, values: timestamps, config: {} }, + { name: 'value', type: FieldType.number, values, config: {} }, + ], + }; + + sparklineField.values.push(sparklineFieldFrame); + }); + + if (fieldIndexToReplace) { + // Make sure sparkline field is placed in the same order as coming from BE + frame.fields[fieldIndexToReplace] = sparklineField; + } + } + }); +} diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index 97db6242aaf..b774bfade40 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -10,6 +10,7 @@ export type CloudWatchQuery = | raw.CloudWatchMetricsQuery | raw.CloudWatchLogsQuery | raw.CloudWatchAnnotationQuery + | raw.CloudWatchLogsAnomaliesQuery | CloudWatchDefaultQuery; // We want to allow setting defaults for both Logs and Metrics queries