root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 83 Packages Projects Releases Wiki Activity

Elasticsearch: Support extended stats and percentiles in terms order by (#28910) 

Adds support to the terms aggregation for ordering by percentiles and extended stats. 

Closes #5148

Co-authored-by: Giordano Ricci <grdnricci@gmail.com>
Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
This commit is contained in:
Chris Cowan 2021-01-15 04:10:16 -07:00 committed by GitHub
parent b32c4f34cd
commit 5088e2044a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 4516 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4329
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_compare.json
View File

File diff suppressed because it is too large Load Diff

21
pkg/tsdb/elasticsearch/time_series_query.go
View File

@ -2,6 +2,7 @@ package elasticsearch
import (
"fmt"
"regexp"
"strconv"
"github.com/grafana/grafana/pkg/components/simplejson"
@ -240,15 +241,27 @@ func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*Metr
}
if orderBy, err := bucketAgg.Settings.Get("orderBy").String(); err == nil {
a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
/*
The format for extended stats and percentiles is {metricId}[bucket_path]
for everything else it's just {metricId}, _count, _term, or _key
*/
metricIdRegex := regexp.MustCompile(`^(\d+)`)
metricId := metricIdRegex.FindString(orderBy)
if _, err := strconv.Atoi(orderBy); err == nil {
if len(metricId) > 0 {
for _, m := range metrics {
if m.ID == orderBy {
b.Metric(m.ID, m.Type, m.Field, nil)
if m.ID == metricId {
if m.Type == "count" {
a.Order["_count"] = bucketAgg.Settings.Get("order").MustString("desc")
} else {
a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
b.Metric(m.ID, m.Type, m.Field, nil)
}
break
}
}
} else {
a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
}
}

74
pkg/tsdb/elasticsearch/time_series_query_test.go
View File

@ -127,6 +127,80 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
So(avgAgg.Aggregation.Type, ShouldEqual, "avg")
})
Convey("With term agg and order by count metric agg", func() {
c := newFakeClient(5)
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{
"type": "terms",
"field": "@host",
"id": "2",
"settings": { "size": "5", "order": "asc", "orderBy": "1" }
},
{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
],
"metrics": [
{"type": "count", "id": "1" }
]
}`, from, to, 15*time.Second)
So(err, ShouldBeNil)
sr := c.multisearchRequests[0].Requests[0]
termsAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.TermsAggregation)
So(termsAgg.Order["_count"], ShouldEqual, "asc")
})
Convey("With term agg and order by percentiles agg", func() {
c := newFakeClient(5)
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{
"type": "terms",
"field": "@host",
"id": "2",
"settings": { "size": "5", "order": "asc", "orderBy": "1[95.0]" }
},
{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
],
"metrics": [
{"type": "percentiles", "field": "@value", "id": "1", "settings": { "percents": ["95","99"] } }
]
}`, from, to, 15*time.Second)
So(err, ShouldBeNil)
sr := c.multisearchRequests[0].Requests[0]
orderByAgg := sr.Aggs[0].Aggregation.Aggs[0]
So(orderByAgg.Key, ShouldEqual, "1")
So(orderByAgg.Aggregation.Type, ShouldEqual, "percentiles")
})
Convey("With term agg and order by extended stats agg", func() {
c := newFakeClient(5)
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{
"type": "terms",
"field": "@host",
"id": "2",
"settings": { "size": "5", "order": "asc", "orderBy": "1[std_deviation]" }
},
{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
],
"metrics": [
{"type": "extended_stats", "field": "@value", "id": "1", "meta": { "std_deviation": true } }
]
}`, from, to, 15*time.Second)
So(err, ShouldBeNil)
sr := c.multisearchRequests[0].Requests[0]
orderByAgg := sr.Aggs[0].Aggregation.Aggs[0]
So(orderByAgg.Key, ShouldEqual, "1")
So(orderByAgg.Aggregation.Type, ShouldEqual, "extended_stats")
})
Convey("With term agg and order by term", func() {
c := newFakeClient(5)
_, err := executeTsdbQuery(c, `{

12
public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/index.tsx
View File

@ -4,11 +4,16 @@ import { useDispatch } from '../../../../hooks/useStatelessReducer';
import { SettingsEditorContainer } from '../../SettingsEditorContainer';
import { changeBucketAggregationSetting } from '../state/actions';
import { BucketAggregation } from '../aggregations';
import { bucketAggregationConfig, intervalOptions, orderByOptions, orderOptions, sizeOptions } from '../utils';
import {
bucketAggregationConfig,
createOrderByOptionsFromMetrics,
intervalOptions,
orderOptions,
sizeOptions,
} from '../utils';
import { FiltersSettingsEditor } from './FiltersSettingsEditor';
import { useDescription } from './useDescription';
import { useQuery } from '../../ElasticsearchQueryContext';
import { describeMetric } from '../../../../utils';
const inlineFieldProps: Partial<ComponentProps<typeof InlineField>> = {
labelWidth: 16,
@ -22,8 +27,7 @@ export const SettingsEditor: FunctionComponent<Props> = ({ bucketAgg }) => {
const dispatch = useDispatch();
const { metrics } = useQuery();
const settingsDescription = useDescription(bucketAgg);
const orderBy = [...orderByOptions, ...(metrics || []).map(m => ({ label: describeMetric(m), value: m.id }))];
const orderBy = createOrderByOptionsFromMetrics(metrics);
return (
<SettingsEditorContainer label={settingsDescription}>

4
public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/useDescription.ts
View File

@ -1,4 +1,4 @@
import { describeMetric } from '../../../../utils';
import { describeMetric, convertOrderByToMetricId } from '../../../../utils';
import { useQuery } from '../../ElasticsearchQueryContext';
import { BucketAggregation } from '../aggregations';
import { bucketAggregationConfig, orderByOptions, orderOptions } from '../utils';
@ -34,7 +34,7 @@ export const useDescription = (bucketAgg: BucketAggregation): string => {
if (orderByOption) {
description += orderByOption.label;
} else {
const metric = metrics?.find(m => m.id === orderBy);
const metric = metrics?.find(m => m.id === convertOrderByToMetricId(orderBy));
if (metric) {
description += describeMetric(metric);
} else {

66
public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/utils.ts
View File

@ -1,5 +1,13 @@
import { BucketsConfiguration } from '../../../types';
import { defaultFilter } from './SettingsEditor/FiltersSettingsEditor/utils';
import { describeMetric } from '../../../utils';
import {
ExtendedStatMetaType,
ExtendedStats,
MetricAggregation,
Percentiles,
} from '../MetricAggregationsEditor/aggregations';
import { SelectableValue } from '@grafana/data';
export const bucketAggregationConfig: BucketsConfiguration = {
terms: {
@ -46,7 +54,8 @@ export const bucketAggregationConfig: BucketsConfiguration = {
};
// TODO: Define better types for the following
export const orderOptions = [
type OrderByOption = SelectableValue<string>;
export const orderOptions: OrderByOption[] = [
{ label: 'Top', value: 'desc' },
{ label: 'Bottom', value: 'asc' },
];
@ -77,3 +86,58 @@ export const intervalOptions = [
{ label: '1h', value: '1h' },
{ label: '1d', value: '1d' },
];
/**
* This returns the valid options for each of the enabled extended stat
*/
function createOrderByOptionsForExtendedStats(metric: ExtendedStats): OrderByOption[] {
if (!metric.meta) {
return [];
}
const metaKeys = Object.keys(metric.meta) as ExtendedStatMetaType[];
return metaKeys
.filter(key => metric.meta?.[key])
.map(key => {
let method = key as string;
// The bucket path for std_deviation_bounds.lower and std_deviation_bounds.upper
// is accessed via std_lower and std_upper, respectively.
if (key === 'std_deviation_bounds_lower') {
method = 'std_lower';
}
if (key === 'std_deviation_bounds_upper') {
method = 'std_upper';
}
return { label: `${describeMetric(metric)} (${method})`, value: `${metric.id}[${method}]` };
});
}
/**
* This returns the valid options for each of the percents listed in the percentile settings
*/
function createOrderByOptionsForPercentiles(metric: Percentiles): OrderByOption[] {
if (!metric.settings?.percents) {
return [];
}
return metric.settings.percents.map(percent => {
// The bucket path for percentile numbers is appended with a `.0` if the number is whole
// otherwise you have to use the actual value.
const percentString = /^\d+\.\d+/.test(`${percent}`) ? percent : `${percent}.0`;
return { label: `${describeMetric(metric)} (${percent})`, value: `${metric.id}[${percentString}]` };
});
}
/**
* This creates all the valid order by options based on the metrics
*/
export const createOrderByOptionsFromMetrics = (metrics: MetricAggregation[] = []): OrderByOption[] => {
const metricOptions = metrics.flatMap(metric => {
if (metric.type === 'extended_stats') {
return createOrderByOptionsForExtendedStats(metric);
} else if (metric.type === 'percentiles') {
return createOrderByOptionsForPercentiles(metric);
} else {
return { label: describeMetric(metric), value: metric.id };
}
});
return [...orderByOptions, ...metricOptions];
};

2
public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/aggregations.ts
View File

@ -114,7 +114,7 @@ export interface ExtendedStats extends MetricAggregationWithField, MetricAggrega
};
}
interface Percentiles extends MetricAggregationWithField, MetricAggregationWithInlineScript {
export interface Percentiles extends MetricAggregationWithField, MetricAggregationWithInlineScript {
type: 'percentiles';
settings?: {
percents?: string[];

19
public/app/plugins/datasource/elasticsearch/query_builder.ts
View File

@ -12,6 +12,7 @@ import {
} from './components/QueryEditor/MetricAggregationsEditor/aggregations';
import { defaultBucketAgg, defaultMetricAgg, findMetricById } from './query_def';
import { ElasticsearchQuery } from './types';
import { convertOrderByToMetricId } from './utils';
export class ElasticQueryBuilder {
timeField: string;
@ -34,7 +35,6 @@ export class ElasticQueryBuilder {
}
buildTermsAgg(aggDef: Terms, queryNode: { terms?: any; aggs?: any }, target: ElasticsearchQuery) {
let metricRef;
queryNode.terms = { field: aggDef.field };
if (!aggDef.settings) {
@ -54,14 +54,17 @@ export class ElasticQueryBuilder {
}
// if metric ref, look it up and add it to this agg level
metricRef = parseInt(aggDef.settings.orderBy, 10);
if (!isNaN(metricRef)) {
const metricId = convertOrderByToMetricId(aggDef.settings.orderBy);
if (metricId) {
for (let metric of target.metrics || []) {
if (metric.id === aggDef.settings.orderBy) {
queryNode.aggs = {};
queryNode.aggs[metric.id] = {};
if (isMetricAggregationWithField(metric)) {
queryNode.aggs[metric.id][metric.type] = { field: metric.field };
if (metric.id === metricId) {
if (metric.type === 'count') {
queryNode.terms.order = { _count: aggDef.settings.order };
} else if (isMetricAggregationWithField(metric)) {
queryNode.aggs = {};
queryNode.aggs[metric.id] = {
[metric.type]: { field: metric.field },
};
}
break;
}

78
public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts
View File

@ -127,6 +127,84 @@ describe('ElasticQueryBuilder', () => {
expect(secondLevel.aggs['5'].avg.field).toBe('@value');
});
it('with term agg and order by count agg', () => {
const query = builder.build(
{
refId: 'A',
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
'1000'
);
expect(query.aggs['2'].terms.order._count).toEqual('asc');
expect(query.aggs['2'].aggs).not.toHaveProperty('1');
});
it('with term agg and order by extended_stats agg', () => {
const query = builder.build(
{
refId: 'A',
metrics: [{ type: 'extended_stats', id: '1', field: '@value', meta: { std_deviation: true } }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1[std_deviation]' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
'1000'
);
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['1'].extended_stats.field).toBe('@value');
expect(secondLevel.aggs['1'].extended_stats.field).toBe('@value');
});
it('with term agg and order by percentiles agg', () => {
const query = builder.build(
{
refId: 'A',
metrics: [{ type: 'percentiles', id: '1', field: '@value', settings: { percents: ['95', '99'] } }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1[95.0]' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
'1000'
);
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['1'].percentiles.field).toBe('@value');
expect(secondLevel.aggs['1'].percentiles.field).toBe('@value');
});
it('with term agg and valid min_doc_count', () => {
const query = builder.build(
{

10
public/app/plugins/datasource/elasticsearch/utils.ts
View File

@ -52,3 +52,13 @@ export const removeEmpty = <T>(obj: T): Partial<T> =>
[key]: value,
};
}, {});
/**
* This function converts an order by string to the correct metric id For example,
* if the user uses the standard deviation extended stat for the order by,
* the value would be "1[std_deviation]" and this would return "1"
*/
export const convertOrderByToMetricId = (orderBy: string): string | undefined => {
const metricIdMatches = orderBy.match(/^(\d+)/);
return metricIdMatches ? metricIdMatches[1] : void 0;
};