root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 142 Packages Projects Releases Wiki Activity

Graphite: Migrate query endpoint entirely to backend (#111138) 

* Update query type

* Support metric tank queries

- Update tests
- Appropriately set URL parameter

* Support queries via the backend

- Add the filterQuery and applyTemplateVariables methods
- Separate the frontend query path into its own function
- Ensure format is always json
- Add method for building backend query objects (maintain the existing template replacement logic)
- Fix a bug in metric find queries

* Update tests

* Fix lint

* Update types
This commit is contained in:
Andreas Christou 2025-09-18 12:14:24 +02:00 committed by GitHub
parent 8f9d8f1154
commit 51d3624bf9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 814 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
eslint-suppressions.json
View File

@ -4088,14 +4088,9 @@
"count": 3
}
},
"public/app/plugins/datasource/graphite/datasource.test.ts": {
"@typescript-eslint/no-explicit-any": {
"count": 3
}
},
"public/app/plugins/datasource/graphite/datasource.ts": {
"@typescript-eslint/consistent-type-assertions": {
"count": 4
"count": 3
},
"@typescript-eslint/no-explicit-any": {
"count": 8

24
pkg/tsdb/graphite/query.go
View File

@ -123,11 +123,11 @@ func (s *Service) RunQuery(ctx context.Context, req *backend.QueryDataRequest, d
// processQuery converts a Graphite data source query to a Graphite query target. It returns the target,
// and the model if the target is invalid
func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, error) {
func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, bool, error) {
queryJSON := GraphiteQuery{}
err := json.Unmarshal(query.JSON, &queryJSON)
if err != nil {
return "", &queryJSON, fmt.Errorf("failed to decode the Graphite query: %w", err)
return "", &queryJSON, false, fmt.Errorf("failed to decode the Graphite query: %w", err)
}
s.logger.Debug("Graphite", "query", queryJSON)
currTarget := queryJSON.TargetFull
@ -137,11 +137,11 @@ func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery,
}
if currTarget == "" {
s.logger.Debug("Graphite", "empty query target", queryJSON)
return "", &queryJSON, nil
return "", &queryJSON, false, nil
}
target := fixIntervalFormat(currTarget)
return target, nil, nil
return target, nil, queryJSON.IsMetricTank, nil
}
func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQuery, dsInfo *datasourceInfo) (*http.Request, url.Values, *GraphiteQuery, error) {
@ -158,7 +158,7 @@ func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQ
"target": []string{},
}
target, emptyQuery, err := s.processQuery(query)
target, emptyQuery, isMetricTank, err := s.processQuery(query)
if err != nil {
return nil, formData, nil, err
}
@ -172,11 +172,17 @@ func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQ
s.logger.Debug("Graphite request", "params", formData)
params := map[string][]string{}
if isMetricTank {
params["meta"] = []string{"true"}
}
graphiteReq, err := s.createRequest(ctx, dsInfo, URLParams{
SubPath: "render",
Method: http.MethodPost,
Body: strings.NewReader(formData.Encode()),
Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},
SubPath: "render",
Method: http.MethodPost,
Body: strings.NewReader(formData.Encode()),
Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},
QueryParams: params,
})
if err != nil {
return nil, formData, nil, err

21
pkg/tsdb/graphite/query_test.go
View File

@ -32,10 +32,11 @@ func TestProcessQuery(t *testing.T) {
}`),
},
}
target, jsonModel, err := service.processQuery(queries[0])
target, jsonModel, isMetricTank, err := service.processQuery(queries[0])
assert.NoError(t, err)
assert.Nil(t, jsonModel)
assert.Equal(t, "app.grafana.*.dashboards.views.1M.count", target)
assert.False(t, isMetricTank)
})
t.Run("Returns if target is empty", func(t *testing.T) {
@ -48,10 +49,26 @@ func TestProcessQuery(t *testing.T) {
},
}
emptyQuery := GraphiteQuery{Target: ""}
target, jsonModel, err := service.processQuery(queries[0])
target, jsonModel, isMetricTank, err := service.processQuery(queries[0])
assert.NoError(t, err)
assert.Equal(t, &emptyQuery, jsonModel)
assert.Equal(t, "", target)
assert.False(t, isMetricTank)
})
t.Run("Returns isMetricTank value", func(t *testing.T) {
queries := []backend.DataQuery{
{
RefID: "A",
JSON: []byte(`{
"target": "app.grafana.*.dashboards.views.1M.count",
"isMetricTank": true
}`),
},
}
_, _, isMetricTank, err := service.processQuery(queries[0])
assert.NoError(t, err)
assert.True(t, isMetricTank)
})
t.Run("QueryData with no valid queries returns bad request response", func(t *testing.T) {

1
pkg/tsdb/graphite/types.go
View File

@ -27,6 +27,7 @@ type GraphiteQuery struct {
TargetFull string `json:"targetFull,omitempty"`
Tags []string `json:"tags,omitempty"`
FromAnnotations *bool `json:"fromAnnotations,omitempty"`
IsMetricTank bool `json:"isMetricTank,omitempty"`
}
type GraphiteEventsRequest struct {

693
public/app/plugins/datasource/graphite/datasource.test.ts
View File

@ -1,25 +1,37 @@
import { isArray } from 'lodash';
import moment from 'moment';
import { of } from 'rxjs';
import { Observable, of } from 'rxjs';
import {
AbstractLabelMatcher,
AbstractLabelOperator,
CoreApp,
DataQueryRequest,
DataQueryResponse,
dateMath,
dateTime,
getFrameDisplayName,
MetricFindValue,
PluginType,
ScopedVars,
} from '@grafana/data';
import { BackendSrvRequest, FetchResponse, getTemplateSrv, TemplateSrv, VariableInterpolation } from '@grafana/runtime';
import {
BackendSrvRequest,
config,
FetchResponse,
getTemplateSrv,
TemplateSrv,
VariableInterpolation,
} from '@grafana/runtime';
import { fromString } from './configuration/parseLokiLabelMappings';
import { GraphiteDatasource } from './datasource';
import { GraphiteQuery, GraphiteQueryType } from './types';
import { GraphiteQuery, GraphiteQueryType, GraphiteType } from './types';
import { DEFAULT_GRAPHITE_VERSION } from './versions';
const fetchMock = jest.fn();
const postResourceMock = jest.fn();
const getResourceMock = jest.fn();
jest.mock('@grafana/runtime', () => ({
...(jest.requireActual('@grafana/runtime') as unknown as object),
@ -50,21 +62,53 @@ const createFetchResponse = <T>(data: T): FetchResponse<T> => ({
ok: true,
});
const instanceSettings = {
id: 1,
uid: 'graphiteUid',
type: 'graphite',
readOnly: false,
access: 'proxy' as 'proxy',
meta: {
id: '1',
name: 'graphite',
type: PluginType.datasource,
info: {
description: 'Graphite datasource',
author: {
name: 'Grafana Labs',
url: 'https://grafana.com',
},
keywords: ['graphite', 'datasource'],
links: [],
logos: {
large: '',
small: '',
},
screenshots: [],
updated: '',
version: '',
},
module: '',
baseUrl: '',
},
url: '/api/datasources/proxy/1',
name: 'graphiteProd',
jsonData: {
rollupIndicatorEnabled: true,
graphiteType: GraphiteType.Default,
},
};
describe('graphiteDatasource', () => {
let ctx = {} as Context;
beforeEach(() => {
jest.clearAllMocks();
const instanceSettings = {
url: '/api/datasources/proxy/1',
name: 'graphiteProd',
jsonData: {
rollupIndicatorEnabled: true,
},
};
const templateSrv = getTemplateSrv();
const ds = new GraphiteDatasource(instanceSettings, templateSrv);
ds.postResource = postResourceMock;
ds.getResource = getResourceMock;
ctx = { templateSrv, ds };
});
@ -186,19 +230,33 @@ describe('graphiteDatasource', () => {
});
describe('When querying graphite with one target using query editor target spec', () => {
let response: unknown;
let response: Observable<DataQueryResponse>;
let requestOptions: BackendSrvRequest;
beforeEach(() => {
const query = {
const query: DataQueryRequest<GraphiteQuery> = {
panelId: 3,
dashboardId: 5,
range: { from: dateTime('2022-04-01T00:00:00'), to: dateTime('2022-07-01T00:00:00') },
dashboardUID: '5',
range: {
from: dateTime('2022-04-01T00:00:00'),
to: dateTime('2022-07-01T00:00:00'),
raw: {
from: '1648789200',
to: '1656655200',
},
},
targets: [
{ target: 'prod1.count', refId: 'A' },
{ target: 'prod2.count', refId: 'B' },
],
maxDataPoints: 500,
requestId: '',
interval: '1m',
intervalMs: 60000,
scopedVars: {},
timezone: 'utc',
app: CoreApp.Dashboard,
startTime: 0,
};
fetchMock.mockImplementation((options) => {
requestOptions = options;
@ -215,11 +273,11 @@ describe('graphiteDatasource', () => {
);
});
response = ctx.ds.query(query as unknown as DataQueryRequest<GraphiteQuery>);
response = ctx.ds.query(query);
});
it('X-Dashboard and X-Panel headers to be set!', () => {
expect(requestOptions.headers?.['X-Dashboard-Id']).toBe(5);
expect(requestOptions.headers?.['X-Dashboard-Id']).toBe('5');
expect(requestOptions.headers?.['X-Panel-Id']).toBe(3);
});
@ -245,7 +303,7 @@ describe('graphiteDatasource', () => {
});
it('should return series list', async () => {
await expect(response).toEmitValuesWith((values: any) => {
await expect(response).toEmitValuesWith((values: DataQueryResponse[]) => {
const results = values[0];
expect(results.data.length).toBe(1);
expect(results.data[0].name).toBe('prod1.count');
@ -253,7 +311,7 @@ describe('graphiteDatasource', () => {
});
it('should convert to millisecond resolution', async () => {
await expect(response).toEmitValuesWith((values: any) => {
await expect(response).toEmitValuesWith((values: DataQueryResponse[]) => {
const results = values[0];
expect(results.data[0].fields[1].values[0]).toBe(10);
});
@ -261,7 +319,12 @@ describe('graphiteDatasource', () => {
});
describe('when fetching Graphite Events as annotations', () => {
let results: any;
let results: Array<{
annotation: string;
time: string;
title: string;
tags?: string[];
}>;
let errorSpy: jest.SpyInstance;
beforeEach(() => {
@ -278,7 +341,8 @@ describe('graphiteDatasource', () => {
fromAnnotations: true,
tags: ['tag1'],
queryType: 'tags',
} as GraphiteQuery,
refId: 'Anno',
},
],
range: {
@ -313,6 +377,9 @@ describe('graphiteDatasource', () => {
it('should parse the tags string into an array', () => {
expect(isArray(results[0].tags)).toEqual(true);
if (!results[0].tags) {
throw new Error('tags not defined');
}
expect(results[0].tags.length).toEqual(2);
expect(results[0].tags[0]).toEqual('tag1');
expect(results[0].tags[1]).toEqual('tag2');
@ -342,6 +409,9 @@ describe('graphiteDatasource', () => {
it('should parse the tags string into an array', () => {
expect(isArray(results[0].tags)).toEqual(true);
if (!results[0].tags) {
throw new Error('tags not defined');
}
expect(results[0].tags.length).toEqual(2);
expect(results[0].tags[0]).toEqual('tag1');
expect(results[0].tags[1]).toEqual('tag2');
@ -360,6 +430,120 @@ describe('graphiteDatasource', () => {
});
});
describe('when fetching Graphite Events as annotations (backend)', () => {
let results: Array<{
annotation: string;
time: string;
title: string;
tags?: string[];
}>;
let errorSpy: jest.SpyInstance;
beforeEach(() => {
errorSpy = jest.spyOn(console, 'error').mockImplementation();
config.featureToggles.graphiteBackendMode = true;
});
afterEach(() => {
errorSpy.mockRestore();
config.featureToggles.graphiteBackendMode = false;
});
const options = {
targets: [
{
fromAnnotations: true,
tags: ['tag1'],
queryType: 'tags',
refId: 'Anno',
},
],
range: {
from: dateTime('2022-06-06T07:03:03.109Z'),
to: dateTime('2022-06-07T07:03:03.109Z'),
raw: {
from: '2022-06-06T07:03:03.109Z',
to: '2022-06-07T07:03:03.109Z',
},
},
};
describe('and tags are returned as string', () => {
const response = [
{
when: 1507222850,
tags: 'tag1 tag2',
data: 'some text',
id: 2,
what: 'Event - deploy',
},
];
beforeEach(async () => {
postResourceMock.mockImplementation((options) => {
return createFetchResponse(response);
});
await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => {
results = data;
});
});
it('should parse the tags string into an array', () => {
expect(isArray(results[0].tags)).toEqual(true);
if (!results[0].tags) {
throw new Error('tags not defined');
}
expect(results[0].tags.length).toEqual(2);
expect(results[0].tags[0]).toEqual('tag1');
expect(results[0].tags[1]).toEqual('tag2');
});
});
describe('and tags are returned as an array', () => {
const response = [
{
when: 1507222850,
tags: ['tag1', 'tag2'],
data: 'some text',
id: 2,
what: 'Event - deploy',
},
];
beforeEach(async () => {
fetchMock.mockImplementation((options) => {
return of(createFetchResponse(response));
});
await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => {
results = data;
});
});
it('should parse the tags string into an array', () => {
expect(isArray(results[0].tags)).toEqual(true);
if (!results[0].tags) {
throw new Error('tags not defined');
}
expect(results[0].tags.length).toEqual(2);
expect(results[0].tags[0]).toEqual('tag1');
expect(results[0].tags[1]).toEqual('tag2');
});
});
it('and tags response is invalid', async () => {
postResourceMock.mockImplementation((options) => {
return createFetchResponse('zzzzzzz');
});
await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => {
results = data;
});
expect(results).toEqual([]);
expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/Unable to get annotations/));
});
});
describe('when fetching Graphite function descriptions', () => {
// `"default": Infinity` (invalid JSON) in params passed by Graphite API in 1.1.7
const INVALID_JSON =
@ -392,11 +576,17 @@ describe('graphiteDatasource', () => {
});
describe('building graphite params', () => {
const defaultQueryProperties = {
from: 'now-1h',
until: 'now',
format: 'json',
};
it('should return empty array if no targets', () => {
const originalTargetMap = { A: '' };
const results = ctx.ds.buildGraphiteParams(
{
targets: [{}],
targets: [],
...defaultQueryProperties,
},
originalTargetMap
);
@ -410,7 +600,11 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'prod1.{test,test2}' }, { target: 'prod2.count' }],
targets: [
{ target: 'prod1.{test,test2}', refId: 'A' },
{ target: 'prod2.count', refId: 'B' },
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -425,7 +619,12 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'series1' }, { target: 'series2' }, { target: 'asPercent(#A,#B)' }],
targets: [
{ target: 'series1', refId: 'A' },
{ target: 'series2', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C ' },
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -441,10 +640,11 @@ describe('graphiteDatasource', () => {
const results = ctx.ds.buildGraphiteParams(
{
targets: [
{ target: 'series1', hide: true },
{ target: 'sumSeries(#A)', hide: true },
{ target: 'asPercent(#A,#B)' },
{ target: 'series1', hide: true, refId: 'A' },
{ target: 'sumSeries(#A)', hide: true, refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -459,7 +659,12 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'series1' }, { target: 'sumSeries(#A)' }, { target: 'asPercent(#A,#B)' }],
targets: [
{ target: 'series1', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -490,7 +695,12 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: '[[metric]]' }, { target: 'sumSeries(#A)' }, { target: 'asPercent(#A,#B)' }],
targets: [
{ target: '[[metric]]', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -526,7 +736,11 @@ describe('graphiteDatasource', () => {
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: '$metric' }, { target: 'sumSeries(#A)' }],
targets: [
{ target: '$metric', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
],
...defaultQueryProperties,
},
originalTargetMap,
scopedVars
@ -564,7 +778,11 @@ describe('graphiteDatasource', () => {
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: '$server.cpu', hide: true }, { target: 'avg(#A)' }],
targets: [
{ target: '$server.cpu', hide: true, refId: 'A' },
{ target: 'avg(#A)', refId: 'B' },
],
...defaultQueryProperties,
},
originalTargetMap,
scopedVars
@ -579,7 +797,8 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)' }],
targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)', refId: 'A' }],
...defaultQueryProperties,
},
originalTargetMap
);
@ -599,11 +818,14 @@ describe('graphiteDatasource', () => {
targets: [
{
target: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)',
refId: 'A',
},
{
target: 'add(carbon.test.test-host.cpuUsage, 1.5)',
refId: 'B',
},
],
...defaultQueryProperties,
},
originalTargetMap
);
@ -621,7 +843,8 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: "summarize(prod.25m.count, '25m', 'sum')" }],
targets: [{ target: "summarize(prod.25m.count, '25m', 'sum')", refId: 'A' }],
...defaultQueryProperties,
},
originalTargetMap
);
@ -634,7 +857,8 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: "summarize(prod.5M.count, '5M', 'sum')" }],
targets: [{ target: "summarize(prod.5M.count, '5M', 'sum')", refId: 'A' }],
...defaultQueryProperties,
},
originalTargetMap
);
@ -648,11 +872,15 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'series1' }, { target: '' }],
targets: [
{ target: 'series1', refId: 'A' },
{ target: '', refId: 'B' },
],
...defaultQueryProperties,
},
originalTargetMap
);
expect(results.length).toBe(2);
expect(results.length).toBe(4);
});
describe('when formatting targets', () => {
@ -679,11 +907,12 @@ describe('graphiteDatasource', () => {
};
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'my.$metric.*' }],
targets: [{ target: 'my.$metric.*', refId: 'A' }],
...defaultQueryProperties,
},
originalTargetMap
);
expect(results).toStrictEqual(['target=my.b.*', 'format=json']);
expect(results).toStrictEqual(['target=my.b.*', 'from=now-1h', 'until=now', 'format=json']);
});
it('globs for more than one variable', () => {
@ -707,12 +936,345 @@ describe('graphiteDatasource', () => {
const originalTargetMap = { A: 'my.[[metric]].*' };
const results = ctx.ds.buildGraphiteParams(
{
targets: [{ target: 'my.[[metric]].*' }],
targets: [{ target: 'my.[[metric]].*', refId: 'A' }],
...defaultQueryProperties,
},
originalTargetMap
);
expect(results).toStrictEqual(['target=my.%7Ba%2Cb%7D.*', 'format=json']);
expect(results).toEqual(['target=my.%7Ba%2Cb%7D.*', 'from=now-1h', 'until=now', 'format=json']);
});
});
});
describe('building graphite queries (backend)', () => {
const defaultQueryProperties = {
requestId: 'reqId',
interval: '1s',
intervalMs: 1000,
maxDataPoints: 100,
range: {
from: dateTime('2022-06-06T07:03:03.109Z'),
to: dateTime('2022-06-07T07:03:03.109Z'),
raw: {
from: '2022-06-06T07:03:03.109Z',
to: '2022-06-07T07:03:03.109Z',
},
},
scopedVars: {},
timezone: '',
app: 'dashboards',
startTime: 0,
};
it('should return empty array if no targets', () => {
const originalTargetMap = { A: '' };
const results = ctx.ds.backendBuildGraphiteQueries(
{
targets: [],
...defaultQueryProperties,
},
originalTargetMap
);
expect(results.length).toBe(0);
});
it('should replace target placeholder', () => {
const originalTargetMap = {
A: 'series1',
B: 'series2',
C: 'asPercent(#A,#B)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: 'series1', refId: 'A' },
{ target: 'series2', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
},
originalTargetMap
);
expect(results[2].target).toBe('asPercent(series1,series2)');
});
it('should replace target placeholder for hidden series', () => {
const originalTargetMap = {
A: 'series1',
B: 'sumSeries(#A)',
C: 'asPercent(#A,#B)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: 'series1', hide: true, refId: 'A' },
{ target: 'sumSeries(#A)', hide: true, refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
},
originalTargetMap
);
expect(results[0].target).toBe('asPercent(series1,sumSeries(series1))');
});
it('should replace target placeholder when nesting query references', () => {
const originalTargetMap = {
A: 'series1',
B: 'sumSeries(#A)',
C: 'asPercent(#A,#B)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: 'series1', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
},
originalTargetMap
);
expect(results[2].target).toBe('asPercent(series1,sumSeries(series1))');
});
it('should replace target placeholder when nesting query references with template variables', () => {
const originalReplace = ctx.templateSrv.replace;
ctx.templateSrv.replace = jest
.fn()
.mockImplementation(
(
target?: string | undefined,
scopedVars?: ScopedVars | undefined,
format?: string | Function,
interpolations?: VariableInterpolation[]
): string => {
if (target?.includes('[[metric]]')) {
return target.replaceAll('[[metric]]', 'aMetricName');
}
return originalReplace(target, scopedVars, format, interpolations);
}
);
const originalTargetMap = {
A: '[[metric]]',
B: 'sumSeries(#A)',
C: 'asPercent(#A,#B)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: '[[metric]]', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
{ target: 'asPercent(#A,#B)', refId: 'C' },
],
},
originalTargetMap
);
expect(results[2].target).toBe('asPercent(aMetricName,sumSeries(aMetricName))');
});
it('should use scoped variables when nesting query references', () => {
const originalReplace = ctx.templateSrv.replace;
ctx.templateSrv.replace = jest
.fn()
.mockImplementation(
(
target?: string | undefined,
scopedVars?: ScopedVars | undefined,
format?: string | Function,
interpolations?: VariableInterpolation[]
): string => {
if (target?.includes('$metric')) {
return target.replaceAll('$metric', 'scopedValue');
}
return originalReplace(target, scopedVars, format, interpolations);
}
);
const originalTargetMap = {
A: '$metric',
B: 'sumSeries(#A)',
};
const scopedVars = {
metric: { text: 'scopedValue', value: 'scopedValue' },
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: '$metric', refId: 'A' },
{ target: 'sumSeries(#A)', refId: 'B' },
],
scopedVars,
},
originalTargetMap
);
expect(results[1].target).toBe('sumSeries(scopedValue)');
});
it('should apply scoped variables to nested references with hidden targets', () => {
const originalReplace = ctx.templateSrv.replace;
ctx.templateSrv.replace = jest
.fn()
.mockImplementation(
(
target?: string | undefined,
scopedVars?: ScopedVars | undefined,
format?: string | Function,
interpolations?: VariableInterpolation[]
): string => {
if (target?.includes('$server')) {
return target.replaceAll('$server', scopedVars?.server?.value);
}
return originalReplace(target, scopedVars, format, interpolations);
}
);
const originalTargetMap = {
A: '$server.cpu',
B: 'avg(#A)',
};
const scopedVars = {
server: { text: 'web01', value: 'web01' },
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: '$server.cpu', hide: true, refId: 'A' },
{ target: 'avg(#A)', refId: 'B' },
],
scopedVars,
},
originalTargetMap
);
expect(results[0].target).toBe('avg(web01.cpu)');
});
it('should not recursively replace queries that reference themselves', () => {
const originalTargetMap = {
A: 'sumSeries(carbon.test.test-host.cpuUsage, #A)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)', refId: 'A' }],
},
originalTargetMap
);
expect(results[0].target).toBe(
'sumSeries(carbon.test.test-host.cpuUsage, sumSeries(carbon.test.test-host.cpuUsage, #A))'
);
});
it('should not recursively replace queries that reference themselves, but will replace nested references', () => {
const originalTargetMap = {
A: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)',
B: 'add(carbon.test.test-host.cpuUsage, 1.5)',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{
target: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)',
refId: 'A',
},
{
target: 'add(carbon.test.test-host.cpuUsage, 1.5)',
refId: 'B',
},
],
},
originalTargetMap
);
expect(results[0].target).toBe(
'sumSeries(carbon.test.test-host.cpuUsage, sumSeries(carbon.test.test-host.cpuUsage, #A, #B), add(carbon.test.test-host.cpuUsage, 1.5))'
);
});
it('should ignore empty targets', () => {
const originalTargetMap = {
A: 'series1',
B: '',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [
{ target: 'series1', refId: 'A' },
{ target: '', refId: 'B' },
],
},
originalTargetMap
);
expect(results.length).toBe(1);
});
describe('when formatting targets', () => {
it('does not attempt to glob for one variable', () => {
const originalReplace = ctx.templateSrv.replace;
ctx.templateSrv.replace = jest
.fn()
.mockImplementation(
(
target?: string | undefined,
scopedVars?: ScopedVars | undefined,
format?: string | Function,
interpolations?: VariableInterpolation[]
): string => {
if (target?.includes('$metric')) {
return target.replaceAll('$metric', 'b');
}
return originalReplace(target, scopedVars, format, interpolations);
}
);
const originalTargetMap = {
A: 'my.$metric.*',
};
const results = ctx.ds.backendBuildGraphiteQueries(
{ ...defaultQueryProperties, targets: [{ target: 'my.$metric.*', refId: 'A' }] },
originalTargetMap
);
expect(results[0].target).toEqual('my.b.*');
});
it('globs for more than one variable', () => {
const originalReplace = ctx.templateSrv.replace;
ctx.templateSrv.replace = jest
.fn()
.mockImplementation(
(
target?: string | undefined,
scopedVars?: ScopedVars | undefined,
format?: string | Function,
interpolations?: VariableInterpolation[]
): string => {
if (target?.includes('[[metric]]')) {
return target.replaceAll('[[metric]]', '{a,b}');
}
return originalReplace(target, scopedVars, format, interpolations);
}
);
const originalTargetMap = { A: 'my.[[metric]].*' };
const results = ctx.ds.backendBuildGraphiteQueries(
{
...defaultQueryProperties,
targets: [{ target: 'my.[[metric]].*', refId: 'A' }],
},
originalTargetMap
);
expect(results[0].target).toEqual('my.{a,b}.*');
});
});
});
@ -1108,37 +1670,58 @@ describe('graphiteDatasource', () => {
});
});
function accessScenario(name: string, url: string, fn: ({ headers }: { headers: Record<string, unknown> }) => void) {
describe('access scenario ' + name, () => {
describe('access scenarios', () => {
describe('access scenario with proxy access', () => {
const ctx = {
templateSrv: getTemplateSrv(),
instanceSettings: { url: 'url', name: 'graphiteProd', jsonData: {} },
instanceSettings: { ...instanceSettings, url: 'url', name: 'graphiteProd' },
};
const httpOptions = {
const httpOptions: BackendSrvRequest = {
headers: {},
url,
url: '/api/datasources/proxy/1',
};
describe('when using proxy mode', () => {
const options = { dashboardId: 1, panelId: 2 };
const options = { dashboardUID: '1', panelId: 2 };
it('tracing headers should be added', () => {
ctx.instanceSettings.url = url;
ctx.instanceSettings.url = '/api/datasources/proxy/1';
const ds = new GraphiteDatasource(ctx.instanceSettings, ctx.templateSrv);
ds.addTracingHeaders(httpOptions, options);
fn(httpOptions);
if (!httpOptions.headers) {
throw new Error('headers not set');
}
expect(httpOptions.headers['X-Dashboard-Id']).toBe('1');
expect(httpOptions.headers['X-Panel-Id']).toBe(2);
});
});
});
}
accessScenario('with proxy access', '/api/datasources/proxy/1', (httpOptions) => {
expect(httpOptions.headers['X-Dashboard-Id']).toBe(1);
expect(httpOptions.headers['X-Panel-Id']).toBe(2);
});
describe('access scenario with direct access', () => {
const ctx = {
templateSrv: getTemplateSrv(),
instanceSettings: { ...instanceSettings, url: 'url', name: 'graphiteProd' },
};
accessScenario('with direct access', 'http://localhost:8080', (httpOptions) => {
expect(httpOptions.headers['X-Dashboard-Id']).toBe(undefined);
expect(httpOptions.headers['X-Panel-Id']).toBe(undefined);
const httpOptions: BackendSrvRequest = {
headers: {},
url: 'http://localhost:8080',
};
describe('when using proxy mode', () => {
const options = { dashboardUid: '1', panelId: 2 };
it('tracing headers should be added', () => {
ctx.instanceSettings.url = 'http://localhost:8080';
const ds = new GraphiteDatasource(ctx.instanceSettings, ctx.templateSrv);
ds.addTracingHeaders(httpOptions, options);
if (!httpOptions.headers) {
throw new Error('headers not set');
}
expect(httpOptions.headers['X-Dashboard-Id']).toBe(undefined);
expect(httpOptions.headers['X-Panel-Id']).toBe(undefined);
});
});
});
});

182
public/app/plugins/datasource/graphite/datasource.ts
View File

@ -47,7 +47,6 @@ import {
GraphiteOptions,
GraphiteQuery,
GraphiteQueryImportConfiguration,
GraphiteQueryRequest,
GraphiteQueryType,
GraphiteType,
MetricTankRequestMeta,
@ -201,48 +200,35 @@ export class GraphiteDatasource
};
}
query(options: DataQueryRequest<GraphiteQuery>): Observable<DataQueryResponse> {
if (options.targets.some((target: GraphiteQuery) => target.fromAnnotations)) {
const streams: Array<Observable<DataQueryResponse>> = [];
for (const target of options.targets) {
streams.push(
new Observable((subscriber) => {
this.annotationEvents(options.range, target)
.then((events) => subscriber.next({ data: [toDataFrame(events)] }))
.catch((ex) => subscriber.error(new Error(ex)))
.finally(() => subscriber.complete());
})
);
}
return merge(...streams);
filterQuery(item: GraphiteQuery): boolean {
if (!item.target && !item.fromAnnotations) {
return false;
}
// Use this object to map the sanitised refID to the original
const formattedRefIdsMap: { [key: string]: string } = {};
// Use this object to map the original refID to the original target
const originalTargetMap: { [key: string]: string } = {};
for (const target of options.targets) {
// Sanitise the refID otherwise the Graphite query will fail
const formattedRefId = target.refId.replaceAll(' ', '_');
formattedRefIdsMap[formattedRefId] = target.refId;
// Track the original target to ensure if we need to interpolate a series, we interpolate using the original target
// rather than the target wrapped in aliasSub e.g.:
// Suppose a query has three targets: A: metric1 B: sumSeries(#A) and C: asPercent(#A, #B)
// We want the targets to be interpolated to: A: aliasSub(metric1, "(^.*$)", "\\1 A"), B: aliasSub(sumSeries(metric1), "(^.*$)", "\\1 B") and C: asPercent(metric1, sumSeries(metric1))
originalTargetMap[target.refId] = target.target || '';
// Use aliasSub to include the refID in the response series name. This allows us to set the refID on the frame.
const updatedTarget = `aliasSub(${target.target}, "(^.*$)", "\\1 ${formattedRefId}")`;
target.target = updatedTarget;
}
return true;
}
// Note that we do not omit queries with the hide flag set to true to avoid breaking nested series replacement
// e.g. sumSeries(#A) where A is hidden
applyTemplateVariables(target: GraphiteQuery, scopedVars: ScopedVars) {
return {
...target,
target: this.templateSrv.replace(target.target ?? '', scopedVars),
targetFull: this.templateSrv.replace(target.targetFull ?? '', scopedVars),
};
}
frontendQuery(
options: DataQueryRequest<GraphiteQuery>,
originalTargetMap: { [key: string]: string },
formattedRefIdsMap: { [key: string]: string }
): Observable<DataQueryResponse> {
// handle the queries here
const graphOptions = {
from: this.translateTime(options.range.from, false, options.timezone),
until: this.translateTime(options.range.to, true, options.timezone),
targets: options.targets,
format: (options as GraphiteQueryRequest).format,
format: 'json',
cacheTimeout: options.cacheTimeout || this.cacheTimeout,
maxDataPoints: options.maxDataPoints,
};
@ -276,17 +262,123 @@ export class GraphiteDatasource
);
}
addTracingHeaders(
httpOptions: BackendSrvRequest,
options: { dashboardId?: number; panelId?: number; panelPluginId?: string }
) {
const proxyMode = !this.url.match(/^http/);
backendBuildGraphiteQueries(
options: DataQueryRequest<GraphiteQuery>,
originalTargetMap: { [key: string]: string }
): GraphiteQuery[] {
const referenceTargets: Record<string, string> = {};
const finalTargets: GraphiteQuery[] = [];
let target: GraphiteQuery, targetValue, i;
for (i = 0; i < options.targets.length; i++) {
target = options.targets[i];
if (!target.target) {
continue;
}
if (!target.refId) {
target.refId = this._seriesRefLetters[i];
}
referenceTargets[target.refId] = target.target;
}
const seriesReferenceRegex = /\#([A-Z])/g;
function nestedSeriesRegexReplacer(match: string, g1: string) {
// Handle the case where a query references itself to prevent infinite recursion
if (target.refId === g1) {
return referenceTargets[g1] || match;
}
// Recursively replace all nested series references
return originalTargetMap[g1].replace(seriesReferenceRegex, nestedSeriesRegexReplacer) || match;
}
for (i = 0; i < options.targets.length; i++) {
const targetClone = { ...options.targets[i] };
target = options.targets[i];
if (!targetClone.target) {
continue;
}
targetValue = this.templateSrv.replace(
referenceTargets[target.refId].replace(seriesReferenceRegex, nestedSeriesRegexReplacer),
options.scopedVars
);
targetClone.target = targetValue;
if (this.isMetricTank) {
targetClone.isMetricTank = true;
}
if (!targetClone.hide) {
finalTargets.push(targetClone);
}
}
return finalTargets;
}
query(options: DataQueryRequest<GraphiteQuery>): Observable<DataQueryResponse> {
if (options.targets.some((target: GraphiteQuery) => target.fromAnnotations)) {
const streams: Array<Observable<DataQueryResponse>> = [];
for (const target of options.targets) {
streams.push(
new Observable((subscriber) => {
this.annotationEvents(options.range, target)
.then((events) => subscriber.next({ data: [toDataFrame(events)] }))
.catch((ex) => subscriber.error(new Error(ex)))
.finally(() => subscriber.complete());
})
);
}
return merge(...streams);
}
// Use this object to map the sanitised refID to the original
const formattedRefIdsMap: { [key: string]: string } = {};
// Use this object to map the original refID to the original target
const originalTargetMap: { [key: string]: string } = {};
for (const target of options.targets) {
// Sanitise the refID otherwise the Graphite query will fail
const formattedRefId = target.refId.replaceAll(' ', '_');
formattedRefIdsMap[formattedRefId] = target.refId;
// Track the original target to ensure if we need to interpolate a series, we interpolate using the original target
// rather than the target wrapped in aliasSub e.g.:
// Suppose a query has three targets: A: metric1 B: sumSeries(#A) and C: asPercent(#A, #B)
// We want the targets to be interpolated to: A: aliasSub(metric1, "(^.*$)", "\\1 A"), B: aliasSub(sumSeries(metric1), "(^.*$)", "\\1 B") and C: asPercent(metric1, sumSeries(metric1))
originalTargetMap[target.refId] = target.target || '';
// We only need to alias queries in frontend mode
if (!config.featureToggles.graphiteBackendMode) {
// Use aliasSub to include the refID in the response series name. This allows us to set the refID on the frame.
const updatedTarget = `aliasSub(${target.target}, "(^.*$)", "\\1 ${formattedRefId}")`;
target.target = updatedTarget;
}
}
if (config.featureToggles.graphiteBackendMode) {
const graphiteQueries = this.backendBuildGraphiteQueries(options, originalTargetMap);
options.targets = graphiteQueries;
return super.query(options);
}
return this.frontendQuery(options, originalTargetMap, formattedRefIdsMap);
}
addTracingHeaders(httpOptions: BackendSrvRequest, options: Partial<DataQueryRequest<GraphiteQuery>>) {
const proxyMode = !this.url?.match(/^http/);
if (!httpOptions.headers) {
httpOptions.headers = {};
}
if (proxyMode) {
if (options.dashboardId) {
httpOptions.headers['X-Dashboard-Id'] = options.dashboardId;
if (options.dashboardUID) {
httpOptions.headers['X-Dashboard-Id'] = options.dashboardUID;
}
if (options.panelId) {
httpOptions.headers['X-Panel-Id'] = options.panelId;
@ -660,6 +752,9 @@ export class GraphiteDatasource
let result: MetricFindValue[];
if (queryType === GraphiteQueryType.Value) {
if (!data.data || data.data.length === 0) {
return Promise.resolve([]);
}
result = data.data[0].fields[1].values
.filter((f?: number) => !!f)
.map((v: number) => ({
@ -1014,6 +1109,7 @@ export class GraphiteDatasource
);
}
// Can be removed when the frontend query path is removed
buildGraphiteParams(options: any, originalTargetMap: { [key: string]: string }, scopedVars?: ScopedVars): string[] {
const graphiteOptions = ['from', 'until', 'rawData', 'format', 'maxDataPoints', 'cacheTimeout'];
const cleanOptions = [],
@ -1045,7 +1141,7 @@ export class GraphiteDatasource
const regex = /\#([A-Z])/g;
function nestedSeriesRegexReplacer(match: string, g1: string | number) {
function nestedSeriesRegexReplacer(match: string, g1: string) {
// Handle the case where a query references itself to prevent infinite recursion
if (target.refId === g1) {
return targets[g1] || match;

1
public/app/plugins/datasource/graphite/types.ts
View File

@ -17,6 +17,7 @@ export interface GraphiteQuery extends DataQuery {
targetFull?: string;
tags?: string[];
fromAnnotations?: boolean;
isMetricTank?: boolean;
}
export interface GraphiteOptions extends DataSourceJsonData {