From 89b3663a23b4f1692ae5541337cbfab5b5253996 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:28:43 +0100 Subject: [PATCH] Elasticsearch: Add processing for raw data to backend (#63208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP * WIP * Refactor * Add tests * Cleanup * Fix whitespace * Fix test and lint * In snapshot tests update counter to be number * Add boolean value for snapshot testing * Update pkg/tsdb/elasticsearch/response_parser.go Co-authored-by: Gábor Farkas * Update pkg/tsdb/elasticsearch/response_parser.go Co-authored-by: Gábor Farkas * Use generic to reuse logic when creating fields * Use nullable fields * Fix lint * WIP (#63272) wip * Fix snapshot test after we changed field types to nullable --------- Co-authored-by: Gábor Farkas --- pkg/tsdb/elasticsearch/response_parser.go | 204 ++++++- .../elasticsearch/response_parser_test.go | 172 +++++- pkg/tsdb/elasticsearch/snapshot_test.go | 1 + .../testdata_response/raw_data.a.golden.jsonc | 521 ++++++++++++++++++ .../testdata_response/raw_data.queries.json | 23 + .../testdata_response/raw_data.response.json | 208 +++++++ pkg/tsdb/elasticsearch/time_series_query.go | 2 +- 7 files changed, 1120 insertions(+), 11 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc create mode 100644 pkg/tsdb/elasticsearch/testdata_response/raw_data.queries.json create mode 100644 pkg/tsdb/elasticsearch/testdata_response/raw_data.response.json diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 0fb4a1216ba..35108a2c6bf 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -1,6 +1,7 @@ package elasticsearch import ( + "encoding/json" "errors" "regexp" "sort" @@ -35,7 +36,7 @@ const ( logsType = "logs" ) -func parseResponse(responses []*es.SearchResponse, targets []*Query) (*backend.QueryDataResponse, error) { +func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField string) (*backend.QueryDataResponse, error) { result := backend.QueryDataResponse{ Responses: backend.Responses{}, } @@ -56,19 +57,123 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query) (*backend.Q queryRes := backend.DataResponse{} - props := make(map[string]string) - err := processBuckets(res.Aggregations, target, &queryRes, props, 0) - if err != nil { - return &backend.QueryDataResponse{}, err - } - nameFields(queryRes, target) - trimDatapoints(queryRes, target) + if isDocumentQuery(target) { + err := processDocumentResponse(res, target, timeField, &queryRes) + if err != nil { + return &backend.QueryDataResponse{}, err + } + result.Responses[target.RefID] = queryRes + } else { + props := make(map[string]string) + err := processBuckets(res.Aggregations, target, &queryRes, props, 0) + if err != nil { + return &backend.QueryDataResponse{}, err + } + nameFields(queryRes, target) + trimDatapoints(queryRes, target) - result.Responses[target.RefID] = queryRes + result.Responses[target.RefID] = queryRes + } } return &result, nil } +func processDocumentResponse(res *es.SearchResponse, target *Query, timeField string, queryRes *backend.DataResponse) error { + docs := make([]map[string]interface{}, len(res.Hits.Hits)) + propNames := make(map[string]bool) + + for hitIdx, hit := range res.Hits.Hits { + var flattened map[string]interface{} + if hit["_source"] != nil { + flattened = flatten(hit["_source"].(map[string]interface{})) + } + + doc := map[string]interface{}{ + "_id": hit["_id"], + "_type": hit["_type"], + "_index": hit["_index"], + "sort": hit["sort"], + "highlight": hit["highlight"], + "_source": flattened, + } + + for k, v := range flattened { + doc[k] = v + } + + for key := range doc { + propNames[key] = true + } + + docs[hitIdx] = doc + } + + size := len(docs) + isFilterable := true + allFields := make([]*data.Field, len(propNames)) + + sortedPropNames := sortPropNames(propNames, timeField) + + for propNameIdx, propName := range sortedPropNames { + // Special handling for time field + if propName == timeField { + timeVector := make([]*time.Time, size) + for i, doc := range docs { + timeString, ok := doc[timeField].(string) + if !ok { + continue + } + timeValue, err := time.Parse(time.RFC3339Nano, timeString) + if err != nil { + // We skip time values that cannot be parsed + continue + } else { + timeVector[i] = &timeValue + } + } + field := data.NewField(timeField, nil, timeVector) + field.Config = &data.FieldConfig{Filterable: &isFilterable} + allFields[propNameIdx] = field + continue + } + + propNameValue := findTheFirstNonNilDocValueForPropName(docs, propName) + switch propNameValue.(type) { + // We are checking for default data types values (float64, int, bool, string) + // and default to json.RawMessage if we cannot find any of them + case float64: + allFields[propNameIdx] = createFieldOfType[float64](docs, propName, size, isFilterable) + case int: + allFields[propNameIdx] = createFieldOfType[int](docs, propName, size, isFilterable) + case string: + allFields[propNameIdx] = createFieldOfType[string](docs, propName, size, isFilterable) + case bool: + allFields[propNameIdx] = createFieldOfType[bool](docs, propName, size, isFilterable) + default: + fieldVector := make([]*json.RawMessage, size) + for i, doc := range docs { + bytes, err := json.Marshal(doc[propName]) + if err != nil { + // We skip values that cannot be marshalled + continue + } + value := json.RawMessage(bytes) + fieldVector[i] = &value + } + field := data.NewField(propName, nil, fieldVector) + field.Config = &data.FieldConfig{Filterable: &isFilterable} + allFields[propNameIdx] = field + } + } + + frames := data.Frames{} + frame := data.NewFrame("", allFields...) + frames = append(frames, frame) + + queryRes.Frames = frames + return nil +} + func processBuckets(aggs map[string]interface{}, target *Query, queryResult *backend.DataResponse, props map[string]string, depth int) error { var err error @@ -753,3 +858,84 @@ func getErrorFromElasticResponse(response *es.SearchResponse) string { return errorString } + +// flatten flattens multi-level objects to single level objects. It uses dot notation to join keys. +func flatten(target map[string]interface{}) map[string]interface{} { + // On frontend maxDepth wasn't used but as we are processing on backend + // let's put a limit to avoid infinite loop. 10 was chosen arbitrary. + maxDepth := 10 + currentDepth := 0 + delimiter := "" + output := make(map[string]interface{}) + + var step func(object map[string]interface{}, prev string) + + step = func(object map[string]interface{}, prev string) { + for key, value := range object { + if prev == "" { + delimiter = "" + } else { + delimiter = "." + } + newKey := prev + delimiter + key + + v, ok := value.(map[string]interface{}) + shouldStepInside := ok && len(v) > 0 && currentDepth < maxDepth + if shouldStepInside { + currentDepth++ + step(v, newKey) + } else { + output[newKey] = value + } + } + } + + step(target, "") + return output +} + +// sortPropNames orders propNames so that timeField is first (if it exists) and rest of propNames are ordered alphabetically +func sortPropNames(propNames map[string]bool, timeField string) []string { + hasTimeField := false + + var sortedPropNames []string + for k := range propNames { + if k == timeField { + hasTimeField = true + } else { + sortedPropNames = append(sortedPropNames, k) + } + } + + sort.Strings(sortedPropNames) + + if hasTimeField { + sortedPropNames = append([]string{timeField}, sortedPropNames...) + } + + return sortedPropNames +} + +// findTheFirstNonNilDocValueForPropName finds the first non-nil value for propName in docs. If none of the values are non-nil, it returns the value of propName in the first doc. +func findTheFirstNonNilDocValueForPropName(docs []map[string]interface{}, propName string) interface{} { + for _, doc := range docs { + if doc[propName] != nil { + return doc[propName] + } + } + return docs[0][propName] +} + +func createFieldOfType[T int | float64 | bool | string](docs []map[string]interface{}, propName string, size int, isFilterable bool) *data.Field { + fieldVector := make([]*T, size) + for i, doc := range docs { + value, ok := doc[propName].(T) + if !ok { + continue + } + fieldVector[i] = &value + } + field := data.NewField(propName, nil, fieldVector) + field.Config = &data.FieldConfig{Filterable: &isFilterable} + return field +} diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index 9b0dd2aca4c..ed90d9c5a55 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/experimental" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1041,6 +1042,122 @@ func TestResponseParser(t *testing.T) { require.Equal(t, frame.Fields[4].Len(), 2) require.Nil(t, frame.Fields[1].Config) }) + + t.Run("Raw data query", func(t *testing.T) { + targets := map[string]string{ + "A": `{ + "metrics": [{ "type": "raw_data" }] + }`, + } + + response := `{ + "responses":[ + { + "hits":{ + "total":{ + "value":109, + "relation":"eq" + }, + "max_score":null, + "hits":[ + { + "_index":"logs-2023.02.08", + "_id":"GB2UMYYBfCQ-FCMjayJa", + "_score":null, + "_source":{ + "@timestamp":"2023-02-08T15:10:55.830Z", + "line":"log text [479231733]", + "counter":"109", + "float":58.253758485091, + "label":"val1", + "level":"info", + "location":"17.089705232090438, 41.62861966340297", + "nested": { + "field": { + "double_nested": "value" + } + }, + "shapes":[ + { + "type":"triangle" + }, + { + "type":"square" + } + ], + "xyz": null + }, + "sort":[ + 1675869055830, + 4 + ] + }, + { + "_index":"logs-2023.02.08", + "_id":"Fx2UMYYBfCQ-FCMjZyJ_", + "_score":null, + "_source":{ + "@timestamp":"2023-02-08T15:10:54.835Z", + "line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]", + "counter":"108", + "float":54.5977098233944, + "label":"val1", + "level":"info", + "location":"19.766305918490463, 40.42639175509792", + "nested": { + "field": { + "double_nested": "value" + } + }, + "shapes":[ + { + "type":"triangle" + }, + { + "type":"square" + } + ], + "xyz": "def" + }, + "sort":[ + 1675869054835, + 7 + ] + } + ] + }, + "status":200 + } + ] + }` + + result, err := parseTestResponse(targets, response) + require.NoError(t, err) + require.Len(t, result.Responses, 1) + + queryRes := result.Responses["A"] + require.NotNil(t, queryRes) + dataframes := queryRes.Frames + require.Len(t, dataframes, 1) + frame := dataframes[0] + + require.Equal(t, 16, len(frame.Fields)) + // Fields have the correct length + require.Equal(t, 2, frame.Fields[0].Len()) + // First field is timeField + require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type()) + // Correctly uses string types + require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type()) + // Correctly detects float64 types + require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[6].Type()) + // Correctly detects json types + require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[7].Type()) + // Correctly flattens fields + require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name) + require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type()) + // Correctly detects type even if first value is null + require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type()) + }) }) t.Run("With top_metrics", func(t *testing.T) { @@ -1135,6 +1252,7 @@ func TestResponseParser(t *testing.T) { func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) { from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + timeField := "@timestamp" timeRange := backend.TimeRange{ From: from, To: to, @@ -1162,7 +1280,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac return nil, err } - return parseResponse(response.Responses, queries) + return parseResponse(response.Responses, queries, timeField) } func TestLabelOrderInFieldName(t *testing.T) { @@ -1255,3 +1373,55 @@ func TestLabelOrderInFieldName(t *testing.T) { requireTimeSeriesName(t, "val1 info", frames[4]) requireTimeSeriesName(t, "val1 error", frames[5]) } + +func TestFlatten(t *testing.T) { + t.Run("Flattens simple object", func(t *testing.T) { + obj := map[string]interface{}{ + "foo": "bar", + "nested": map[string]interface{}{ + "bax": map[string]interface{}{ + "baz": "qux", + }, + }, + } + + flattened := flatten(obj) + require.Len(t, flattened, 2) + require.Equal(t, "bar", flattened["foo"]) + require.Equal(t, "qux", flattened["nested.bax.baz"]) + }) + + t.Run("Flattens object to max 10 nested levels", func(t *testing.T) { + obj := map[string]interface{}{ + "nested0": map[string]interface{}{ + "nested1": map[string]interface{}{ + "nested2": map[string]interface{}{ + "nested3": map[string]interface{}{ + "nested4": map[string]interface{}{ + "nested5": map[string]interface{}{ + "nested6": map[string]interface{}{ + "nested7": map[string]interface{}{ + "nested8": map[string]interface{}{ + "nested9": map[string]interface{}{ + "nested10": map[string]interface{}{ + "nested11": map[string]interface{}{ + "nested12": "abc", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + flattened := flatten(obj) + require.Len(t, flattened, 1) + require.Equal(t, map[string]interface{}{"nested11": map[string]interface{}{"nested12": "abc"}}, flattened["nested0.nested1.nested2.nested3.nested4.nested5.nested6.nested7.nested8.nested9.nested10"]) + }) +} diff --git a/pkg/tsdb/elasticsearch/snapshot_test.go b/pkg/tsdb/elasticsearch/snapshot_test.go index 0eaec69ab3e..f3a8800de54 100644 --- a/pkg/tsdb/elasticsearch/snapshot_test.go +++ b/pkg/tsdb/elasticsearch/snapshot_test.go @@ -131,6 +131,7 @@ func TestResponseSnapshots(t *testing.T) { {name: "simple metric test", path: "metric_simple"}, {name: "complex metric test", path: "metric_complex"}, {name: "multi metric test", path: "metric_multi"}, + {name: "raw data test", path: "raw_data"}, } snapshotCount := findResponseSnapshotCounts(t, "testdata_response") diff --git a/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc new file mode 100644 index 00000000000..423d4771def --- /dev/null +++ b/pkg/tsdb/elasticsearch/testdata_response/raw_data.a.golden.jsonc @@ -0,0 +1,521 @@ +// 🌟 This was machine generated. Do not edit. 🌟 +// +// Frame[0] +// Name: +// Dimensions: 17 Fields by 5 Rows +// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// | Name: @timestamp | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: line | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort | +// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | +// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// | 2023-02-09T14:40:01.475Z | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:01.475Z","abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","level":"info","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | log text [106619125] | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] | +// | 2023-02-09T14:40:00.513Z | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:00.513Z","abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","level":"error","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}]} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | log text with [781660944] | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] | +// | 2023-02-09T14:39:59.556Z | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:59.556Z","abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","level":"info","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}]} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | log text [894867430] | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] | +// | 2023-02-09T14:39:58.608Z | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:58.608Z","abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","level":"info","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | log text [478598889] | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] | +// | 2023-02-09T14:39:57.665Z | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:57.665Z","abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","level":"info","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | log text [526995818] | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] | +// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ +// +// +// 🌟 This was machine generated. Do not edit. 🌟 +{ + "status": 200, + "frames": [ + { + "schema": { + "fields": [ + { + "name": "@timestamp", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "_id", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "_index", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "_source", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "_type", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "abc", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "counter", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "float", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "highlight", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "is_true", + "type": "boolean", + "typeInfo": { + "frame": "bool", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "label", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "level", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "line", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "location", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "nested_field.internal.nested", + "type": "string", + "typeInfo": { + "frame": "string", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "shapes", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + }, + "config": { + "filterable": true + } + }, + { + "name": "sort", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage", + "nullable": true + }, + "config": { + "filterable": true + } + } + ] + }, + "data": { + "values": [ + [ + "2023-02-09T14:40:01.475Z", + "2023-02-09T14:40:00.513Z", + "2023-02-09T14:39:59.556Z", + "2023-02-09T14:39:58.608Z", + "2023-02-09T14:39:57.665Z" + ], + [ + "g2aeNoYB7vaC3bq-ezfK", + "gmaeNoYB7vaC3bq-eDcN", + "gWaeNoYB7vaC3bq-dDdL", + "gGaeNoYB7vaC3bq-cDeY", + "f2aeNoYB7vaC3bq-bDf7" + ], + [ + "logs-2023.02.09", + "logs-2023.02.09", + "logs-2023.02.09", + "logs-2023.02.09", + "logs-2023.02.09" + ], + [ + { + "@timestamp": "2023-02-09T14:40:01.475Z", + "abc": null, + "counter": 81, + "float": 10.911972180833306, + "is_true": true, + "label": "val3", + "level": "info", + "line": "log text [106619125]", + "location": "-42.73465234425797, -14.097854057104112", + "nested_field.internal.nested": "value1", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ] + }, + { + "@timestamp": "2023-02-09T14:40:00.513Z", + "abc": null, + "counter": 80, + "float": 62.94120607636795, + "is_true": false, + "label": "val3", + "level": "error", + "line": "log text with [781660944]", + "location": "42.07571917624318, 15.95725088484611", + "nested_field.internal.nested": "value2", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "square" + } + ] + }, + { + "@timestamp": "2023-02-09T14:39:59.556Z", + "abc": "def", + "counter": 79, + "float": 53.323706427230455, + "is_true": true, + "label": "val1", + "level": "info", + "line": "log text [894867430]", + "location": "-38.27341566189766, -23.66739642570781", + "nested_field.internal.nested": "value3", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "square" + } + ] + }, + { + "@timestamp": "2023-02-09T14:39:58.608Z", + "abc": "def", + "counter": 78, + "float": 82.72012623471589, + "is_true": false, + "label": "val1", + "level": "info", + "line": "log text [478598889]", + "location": "12.373240290451287, 43.265493464362024", + "nested_field.internal.nested": "value4", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ] + }, + { + "@timestamp": "2023-02-09T14:39:57.665Z", + "abc": "def", + "counter": 77, + "float": 35.05784443331803, + "is_true": false, + "label": "val3", + "level": "info", + "line": "log text [526995818]", + "location": "-31.524344042228194, -32.11254790120572", + "nested_field.internal.nested": "value5", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ] + } + ], + [ + null, + null, + null, + null, + null + ], + [ + null, + null, + "def", + "def", + "def" + ], + [ + 81, + 80, + 79, + 78, + 77 + ], + [ + 10.911972180833306, + 62.94120607636795, + 53.323706427230455, + 82.72012623471589, + 35.05784443331803 + ], + [ + null, + null, + null, + null, + null + ], + [ + true, + false, + true, + false, + false + ], + [ + "val3", + "val3", + "val1", + "val1", + "val3" + ], + [ + "info", + "error", + "info", + "info", + "info" + ], + [ + "log text [106619125]", + "log text with [781660944]", + "log text [894867430]", + "log text [478598889]", + "log text [526995818]" + ], + [ + "-42.73465234425797, -14.097854057104112", + "42.07571917624318, 15.95725088484611", + "-38.27341566189766, -23.66739642570781", + "12.373240290451287, 43.265493464362024", + "-31.524344042228194, -32.11254790120572" + ], + [ + "value1", + "value2", + "value3", + "value4", + "value5" + ], + [ + [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ], + [ + { + "type": "triangle" + }, + { + "type": "square" + } + ], + [ + { + "type": "triangle" + }, + { + "type": "square" + } + ], + [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ], + [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ] + ], + [ + [ + 1675953601475, + 4 + ], + [ + 1675953600513, + 7 + ], + [ + 1675953599556, + 10 + ], + [ + 1675953598608, + 15 + ], + [ + 1675953597665, + 20 + ] + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/tsdb/elasticsearch/testdata_response/raw_data.queries.json b/pkg/tsdb/elasticsearch/testdata_response/raw_data.queries.json new file mode 100644 index 00000000000..95b65503e7e --- /dev/null +++ b/pkg/tsdb/elasticsearch/testdata_response/raw_data.queries.json @@ -0,0 +1,23 @@ +[ + { + "alias": "", + "datasource": { + "type": "elasticsearch", + "uid": "haha" + }, + "datasourceId": 42, + "expression": "", + "hide": true, + "intervalMs": 200, + "maxDataPoints": 1248, + "metrics": [ + { + "id": "1", + "type": "raw_data" + } + ], + "query": "", + "refId": "a", + "window": "" + } +] diff --git a/pkg/tsdb/elasticsearch/testdata_response/raw_data.response.json b/pkg/tsdb/elasticsearch/testdata_response/raw_data.response.json new file mode 100644 index 00000000000..8429a69dad0 --- /dev/null +++ b/pkg/tsdb/elasticsearch/testdata_response/raw_data.response.json @@ -0,0 +1,208 @@ +{ + "took": 6, + "responses": [ + { + "took": 6, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 81, + "relation": "eq" + }, + "max_score": null, + "hits": [ + { + "_index": "logs-2023.02.09", + "_id": "g2aeNoYB7vaC3bq-ezfK", + "_score": null, + "_source": { + "abc": null, + "is_true": true, + "@timestamp": "2023-02-09T14:40:01.475Z", + "line": "log text [106619125]", + "counter": 81, + "float": 10.911972180833306, + "label": "val3", + "level": "info", + "location": "-42.73465234425797, -14.097854057104112", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ], + "nested_field": { + "internal": { + "nested": "value1" + } + } + }, + "sort": [ + 1675953601475, + 4 + ] + }, + { + "_index": "logs-2023.02.09", + "_id": "gmaeNoYB7vaC3bq-eDcN", + "_score": null, + "_source": { + "abc": null, + "is_true": false, + "@timestamp": "2023-02-09T14:40:00.513Z", + "line": "log text with [781660944]", + "counter": 80, + "float": 62.94120607636795, + "label": "val3", + "level": "error", + "location": "42.07571917624318, 15.95725088484611", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "square" + } + ], + "nested_field": { + "internal": { + "nested": "value2" + } + } + }, + "sort": [ + 1675953600513, + 7 + ] + }, + { + "_index": "logs-2023.02.09", + "_id": "gWaeNoYB7vaC3bq-dDdL", + "_score": null, + "_source": { + "abc": "def", + "is_true": true, + "@timestamp": "2023-02-09T14:39:59.556Z", + "line": "log text [894867430]", + "counter": 79, + "float": 53.323706427230455, + "label": "val1", + "level": "info", + "location": "-38.27341566189766, -23.66739642570781", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "square" + } + ], + "nested_field": { + "internal": { + "nested": "value3" + } + } + }, + "sort": [ + 1675953599556, + 10 + ] + }, + { + "_index": "logs-2023.02.09", + "_id": "gGaeNoYB7vaC3bq-cDeY", + "_score": null, + "_source": { + "abc": "def", + "is_true": false, + "@timestamp": "2023-02-09T14:39:58.608Z", + "line": "log text [478598889]", + "counter": 78, + "float": 82.72012623471589, + "label": "val1", + "level": "info", + "location": "12.373240290451287, 43.265493464362024", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ], + "nested_field": { + "internal": { + "nested": "value4" + } + } + }, + "sort": [ + 1675953598608, + 15 + ] + }, + { + "_index": "logs-2023.02.09", + "_id": "f2aeNoYB7vaC3bq-bDf7", + "_score": null, + "_source": { + "abc": "def", + "is_true": false, + "@timestamp": "2023-02-09T14:39:57.665Z", + "line": "log text [526995818]", + "counter": 77, + "float": 35.05784443331803, + "label": "val3", + "level": "info", + "location": "-31.524344042228194, -32.11254790120572", + "shapes": [ + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "triangle" + }, + { + "type": "square" + } + ], + "nested_field": { + "internal": { + "nested": "value5" + } + } + }, + "sort": [ + 1675953597665, + 20 + ] + } + ] + }, + "status": 200 + } + ] +} diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index f8fc378a7e3..22828567ab1 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -54,7 +54,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) { return &backend.QueryDataResponse{}, err } - return parseResponse(res.Responses, queries) + return parseResponse(res.Responses, queries, e.client.GetTimeField()) } func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64) error {