root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 76 Packages Projects Releases Wiki Activity

Elasticsearch: Fix handling of inline scripts in different ES versions (#34070) 

* Devenv: add block for es 5.0, provisioned datasource & dashboard

* Trasnsform script property based on running ES version

* Handle different scripts format in BE
This commit is contained in:
Giordano Ricci 2021-05-14 11:50:15 +01:00 committed by GitHub
parent ddb2fc1ae6
commit 8ec87250c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1339 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
devenv/datasources.yaml
View File

@ -124,6 +124,26 @@ datasources:
timeField: "@timestamp"
esVersion: 5
- name: gdev-elasticsearch-v56-metrics
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://localhost:13200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 56
- name: gdev-elasticsearch-v56-logs
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://localhost:13200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 56
- name: gdev-elasticsearch-v6-metrics
type: elasticsearch
access: proxy

22
devenv/datasources_docker.yaml
View File

@ -85,7 +85,7 @@ datasources:
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://elasticsearch5:10200
url: http://elasticsearch5:9200
jsonData:
interval: Daily
timeField: "@timestamp"
@ -101,6 +101,26 @@ datasources:
timeField: "@timestamp"
esVersion: 5
- name: gdev-elasticsearch-v56-metrics
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://elasticsearch5:9200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 56
- name: gdev-elasticsearch-v56-logs
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://elasticsearch5:9200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 56
- name: gdev-elasticsearch-v6-metrics
type: elasticsearch
access: proxy

731
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_compare.json
View File

File diff suppressed because it is too large Load Diff

737
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v56.json Normal file
View File

@ -0,0 +1,737 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
},
{
"datasource": "Elastic 5 Logs",
"enable": false,
"iconColor": "rgba(255, 96, 96, 1)",
"limit": 100,
"name": "test",
"query": "",
"showIn": 0,
"textField": "description",
"type": "alert"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"iteration": 1591027589702,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v5-metrics",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 1,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "1",
"size": "5"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "max"
}
],
"query": "*",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Top 5 servers",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v56-metrics",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 7
},
"hiddenSeries": false,
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {
"percents": [25, 50, 75, 95, 99]
},
"type": "percentiles"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Percentiles & Metric filter",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v56-metrics",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 7
},
"hiddenSeries": false,
"id": 3,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {
"std_deviation_bounds_lower": true,
"std_deviation_bounds_upper": true
},
"settings": {},
"type": "extended_stats"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Standard dev",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@hostname",
"value": "@hostname"
},
{
"text": "Average",
"value": "Average"
},
{
"text": "Max",
"value": "Max"
},
{
"text": "Sum",
"value": "Sum"
}
],
"datasource": "gdev-elasticsearch-v56-metrics",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 13
},
"id": 6,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"align": "auto",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"align": "auto",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "_term",
"size": "0"
},
"type": "terms"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
},
{
"field": "@value",
"id": "3",
"meta": {},
"settings": {},
"type": "max"
},
{
"field": "@value",
"id": "4",
"meta": {},
"settings": {},
"type": "sum"
}
],
"refId": "B",
"timeField": "@timestamp"
}
],
"title": "ES Metrics",
"transform": "table",
"type": "table-old"
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "@message",
"value": "@message"
},
{
"text": "tags",
"value": "tags"
},
{
"text": "description",
"value": "description"
}
],
"datasource": "gdev-elasticsearch-v56-logs",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 20
},
"id": 5,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"align": "auto",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
}
],
"targets": [
{
"bucketAggs": [],
"dsType": "elasticsearch",
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"title": "ES Log query",
"transform": "json",
"type": "table-old"
},
{
"circleMaxSize": 30,
"circleMinSize": 2,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"datasource": "gdev-elasticsearch-v56-metrics",
"decimals": 0,
"esGeoPoint": "@location",
"esLocationName": "",
"esMetric": "Average",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"hideEmpty": false,
"hideZero": false,
"id": 8,
"initialZoom": 1,
"links": [],
"locationData": "geohash",
"mapCenter": "(0°, 0°)",
"mapCenterLatitude": 0,
"mapCenterLongitude": 0,
"maxDataPoints": 1,
"mouseWheelZoom": false,
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "@location",
"id": "3",
"settings": {
"precision": 2
},
"type": "geohash_grid"
}
],
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": "0,10",
"title": "World map panel",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"valueName": "total"
}
],
"schemaVersion": 25,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v56-metrics",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"status": "Stable",
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"],
"type": "timepicker"
},
"timezone": "browser",
"title": "Datasource tests - Elasticsearch v56",
"uid": "8HjT32BmO",
"version": 1
}

2
devenv/docker/blocks/elastic5/docker-compose.yaml
View File

@ -1,5 +1,5 @@
elasticsearch5:
image: elasticsearch:5
image: elasticsearch:5.0.2
command: elasticsearch
ports:
- '10200:9200'

2
devenv/docker/blocks/elastic5/elasticsearch.yml
View File

@ -1,2 +0,0 @@
script.inline: on
script.indexed: on

15
devenv/docker/blocks/elastic56/docker-compose.yaml Normal file
View File

@ -0,0 +1,15 @@
elasticsearch56:
image: elasticsearch:5.6.16
command: elasticsearch
ports:
- '13200:9200'
- '13300:9300'
fake-elastic56-data:
image: grafana/fake-data-gen
links:
- elasticsearch56
environment:
FD_SERVER: elasticsearch56
FD_DATASOURCE: elasticsearch
FD_PORT: 9200

17
pkg/tsdb/elasticsearch/models.go
View File

@ -73,6 +73,16 @@ var pipelineAggType = map[string]string{
"bucket_script": "bucket_script",
}
var scriptableAggType = map[string]string{
"avg": "avg",
"sum": "sum",
"max": "max",
"min": "min",
"extended_stats": "extended_stats",
"percentiles": "percentiles",
"bucket_script": "bucket_script",
}
var pipelineAggWithMultipleBucketPathsType = map[string]string{
"bucket_script": "bucket_script",
}
@ -84,6 +94,13 @@ func isPipelineAgg(metricType string) bool {
return false
}
func isMetricAggregationWithInlineScriptSupport(metricType string) bool {
if _, ok := scriptableAggType[metricType]; ok {
return true
}
return false
}
func isPipelineAggWithMultipleBucketPaths(metricType string) bool {
if _, ok := pipelineAggWithMultipleBucketPathsType[metricType]; ok {
return true

27
pkg/tsdb/elasticsearch/time_series_query.go
View File

@ -5,6 +5,7 @@ import (
"regexp"
"strconv"
"github.com/Masterminds/semver"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/plugins"
es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
@ -143,7 +144,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
}
aggBuilder.Pipeline(m.ID, m.Type, bucketPaths, func(a *es.PipelineAggregation) {
a.Settings = m.Settings.MustMap()
a.Settings = m.generateSettingsForDSL(e.client.GetVersion())
})
} else {
continue
@ -164,7 +165,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
}
aggBuilder.Pipeline(m.ID, m.Type, bucketPath, func(a *es.PipelineAggregation) {
a.Settings = m.generateSettingsForDSL()
a.Settings = m.generateSettingsForDSL(e.client.GetVersion())
})
}
} else {
@ -173,7 +174,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
}
} else {
aggBuilder.Metric(m.ID, m.Type, m.Field, func(a *es.MetricAggregation) {
a.Settings = m.Settings.MustMap()
a.Settings = m.generateSettingsForDSL(e.client.GetVersion())
})
}
}
@ -182,7 +183,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
}
// Casts values to int when required by Elastic's query DSL
func (metricAggregation MetricAgg) generateSettingsForDSL() map[string]interface{} {
func (metricAggregation MetricAgg) generateSettingsForDSL(version *semver.Version) map[string]interface{} {
setFloatPath := func(path ...string) {
if stringValue, err := metricAggregation.Settings.GetPath(path...).String(); err == nil {
if value, err := strconv.ParseFloat(stringValue, 64); err == nil {
@ -203,6 +204,24 @@ func (metricAggregation MetricAgg) generateSettingsForDSL() map[string]interface
setFloatPath("lag")
}
if isMetricAggregationWithInlineScriptSupport(metricAggregation.Type) {
scriptValue, err := metricAggregation.Settings.GetPath("script").String()
if err != nil {
// the script is stored using the old format : `script:{inline: "value"}` or is not set
scriptValue, err = metricAggregation.Settings.GetPath("script", "inline").String()
}
constraint, _ := semver.NewConstraint(">=5.6.0")
if err == nil {
if constraint.Check(version) {
metricAggregation.Settings.SetPath([]string{"script"}, scriptValue)
} else {
metricAggregation.Settings.SetPath([]string{"script"}, map[string]interface{}{"inline": scriptValue})
}
}
}
return metricAggregation.Settings.MustMap()
}

87
pkg/tsdb/elasticsearch/time_series_query_test.go
View File

@ -933,6 +933,93 @@ func TestSettingsCasting(t *testing.T) {
assert.Equal(t, 1., serialDiffSettings["lag"])
})
t.Run("Inline Script", func(t *testing.T) {
t.Run("Correctly handles scripts for ES < 5.6", func(t *testing.T) {
c := newFakeClient("5.0.0")
for key := range scriptableAggType {
t.Run("Inline Script", func(t *testing.T) {
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{ "type": "date_histogram", "field": "@timestamp", "id": "2" }
],
"metrics": [
{
"id": "1",
"type": "`+key+`",
"settings": {
"script": "my_script"
}
},
{
"id": "3",
"type": "`+key+`",
"settings": {
"script": {
"inline": "my_script"
}
}
}
]
}`, from, to, 15*time.Second)
assert.Nil(t, err)
sr := c.multisearchRequests[0].Requests[0]
newFormatAggSettings := sr.Aggs[0].Aggregation.Aggs[0].Aggregation.Aggregation.(*es.MetricAggregation).Settings
oldFormatAggSettings := sr.Aggs[0].Aggregation.Aggs[1].Aggregation.Aggregation.(*es.MetricAggregation).Settings
assert.Equal(t, map[string]interface{}{"inline": "my_script"}, newFormatAggSettings["script"])
assert.Equal(t, map[string]interface{}{"inline": "my_script"}, oldFormatAggSettings["script"])
})
}
})
t.Run("Correctly handles scripts for ES >= 5.6", func(t *testing.T) {
c := newFakeClient("5.6.0")
for key := range scriptableAggType {
fmt.Println(key)
t.Run("Inline Script", func(t *testing.T) {
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{ "type": "date_histogram", "field": "@timestamp", "id": "2" }
],
"metrics": [
{
"id": "1",
"type": "`+key+`",
"settings": {
"script": "my_script"
}
},
{
"id": "3",
"type": "`+key+`",
"settings": {
"script": {
"inline": "my_script"
}
}
}
]
}`, from, to, 15*time.Second)
assert.Nil(t, err)
sr := c.multisearchRequests[0].Requests[0]
newFormatAggSettings := sr.Aggs[0].Aggregation.Aggs[0].Aggregation.Aggregation.(*es.MetricAggregation).Settings
oldFormatAggSettings := sr.Aggs[0].Aggregation.Aggs[1].Aggregation.Aggregation.(*es.MetricAggregation).Settings
assert.Equal(t, "my_script", newFormatAggSettings["script"])
assert.Equal(t, "my_script", oldFormatAggSettings["script"])
})
}
})
})
}
type fakeClient struct {

13
public/app/plugins/datasource/elasticsearch/query_builder.ts
View File

@ -346,7 +346,8 @@ export class ElasticQueryBuilder {
Object.entries(metric.settings || {})
.filter(([_, v]) => v !== null)
.forEach(([k, v]) => {
metricAgg[k] = k === 'script' ? getScriptValue(metric as MetricAggregationWithInlineScript) : v;
metricAgg[k] =
k === 'script' ? this.buildScript(getScriptValue(metric as MetricAggregationWithInlineScript)) : v;
});
// Elasticsearch isn't generally too picky about the data types in the request body,
@ -388,6 +389,16 @@ export class ElasticQueryBuilder {
return query;
}
private buildScript(script: string) {
if (gte(this.esVersion, '5.6.0')) {
return script;
}
return {
inline: script,
};
}
private toNumber(stringValue: unknown): unknown | number {
const parsedValue = parseFloat(`${stringValue}`);
if (isNaN(parsedValue)) {