root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 73 Packages Projects Releases Wiki Activity

Transformations: Add gazetteer transformation (#40967)

This commit is contained in:
nikki-kiga 2021-10-27 20:20:56 -07:00 committed by GitHub
parent 45e1765733
commit 963544ffe6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 293 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/grafana-data/src/transformations/transformers/ids.ts
View File

@ -27,4 +27,5 @@ export enum DataTransformerID {
rowsToFields = 'rowsToFields',
prepareTimeSeries = 'prepareTimeSeries',
convertFieldType = 'convertFieldType',
fieldLookup = 'fieldLookup',
}

91
public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx Normal file
View File

@ -0,0 +1,91 @@
import React, { useCallback } from 'react';
import {
DataTransformerID,
FieldNamePickerConfigSettings,
PluginState,
StandardEditorsRegistryItem,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { InlineField, InlineFieldRow } from '@grafana/ui';
import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
import { GazetteerPathEditor } from 'app/plugins/panel/geomap/editor/GazetteerPathEditor';
import { GazetteerPathEditorConfigSettings } from 'app/plugins/panel/geomap/types';
import { FieldLookupOptions, fieldLookupTransformer } from './fieldLookup';
import { FieldType } from '../../../../../../packages/grafana-data/src';
const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
settings: {
width: 30,
filter: (f) => f.type === FieldType.string,
placeholderText: 'Select text field',
noFieldsMessage: 'No text fields found',
},
name: '',
id: '',
editor: () => null,
};
const fieldLookupSettings: StandardEditorsRegistryItem<string, GazetteerPathEditorConfigSettings> = {
settings: {},
} as any;
export const FieldLookupTransformerEditor: React.FC<TransformerUIProps<FieldLookupOptions>> = ({
input,
options,
onChange,
}) => {
const onPickLookupField = useCallback(
(value: string | undefined) => {
onChange({
...options,
lookupField: value,
});
},
[onChange, options]
);
const onPickGazetteer = useCallback(
(value: string | undefined) => {
onChange({
...options,
gazetteer: value,
});
},
[onChange, options]
);
return (
<div>
<InlineFieldRow>
<InlineField label={'Field'} labelWidth={12}>
<FieldNamePicker
context={{ data: input }}
value={options?.lookupField ?? ''}
onChange={onPickLookupField}
item={fieldNamePickerSettings as any}
/>
</InlineField>
</InlineFieldRow>
<InlineFieldRow>
<InlineField label={'Lookup'} labelWidth={12}>
<GazetteerPathEditor
value={options?.gazetteer ?? ''}
context={{ data: input }}
item={fieldLookupSettings}
onChange={onPickGazetteer}
/>
</InlineField>
</InlineFieldRow>
</div>
);
};
export const fieldLookupTransformRegistryItem: TransformerRegistryItem<FieldLookupOptions> = {
id: DataTransformerID.fieldLookup,
editor: FieldLookupTransformerEditor,
transformation: fieldLookupTransformer,
name: 'Field lookup',
description: `Use a field value to lookup additional fields from an external source. This current supports spatial data, but will eventuall support more formats`,
state: PluginState.alpha,
};

115
public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts Normal file
View File

@ -0,0 +1,115 @@
import { FieldMatcherID, fieldMatchers, FieldType } from '@grafana/data';
import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
import { DataTransformerID } from '@grafana/data/src/transformations/transformers/ids';
import { Gazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
import { addFieldsFromGazetteer } from './fieldLookup';
describe('Lookup gazetteer', () => {
it('adds lat/lon based on string field', async () => {
const cfg = {
id: DataTransformerID.fieldLookup,
options: {
lookupField: 'location',
gazetteer: 'public/gazetteer/usa-states.json',
},
};
const data = toDataFrame({
name: 'locations',
fields: [
{ name: 'location', type: FieldType.string, values: ['AL', 'AK', 'Arizona', 'Arkansas', 'Somewhere'] },
{ name: 'values', type: FieldType.number, values: [0, 10, 5, 1, 5] },
],
});
const matcher = fieldMatchers.get(FieldMatcherID.byName).get(cfg.options?.lookupField);
const values = new Map()
.set('AL', { name: 'Alabama', id: 'AL', coords: [-80.891064, 12.448457] })
.set('AK', { name: 'Arkansas', id: 'AK', coords: [-100.891064, 24.448457] })
.set('AZ', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] })
.set('Arizona', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] });
const gaz: Gazetteer = {
count: 3,
examples: () => ['AL', 'AK', 'AZ'],
find: (k) => {
let v = values.get(k);
if (!v && typeof k === 'string') {
v = values.get(k.toUpperCase());
}
return v;
},
path: 'public/gazetteer/usa-states.json',
};
expect(await addFieldsFromGazetteer([data], gaz, matcher)).toMatchInlineSnapshot(`
Array [
Object {
"creator": [Function],
"fields": Array [
Object {
"config": Object {},
"name": "location",
"type": "string",
"values": Array [
"AL",
"AK",
"Arizona",
"Arkansas",
"Somewhere",
],
},
Object {
"config": Object {},
"name": "lon",
"type": "number",
"values": Array [
-80.891064,
-100.891064,
-111.891064,
undefined,
undefined,
],
},
Object {
"config": Object {},
"name": "lat",
"type": "number",
"values": Array [
12.448457,
24.448457,
33.448457,
undefined,
undefined,
],
},
Object {
"config": Object {},
"name": "values",
"state": Object {
"displayName": "values",
},
"type": "number",
"values": Array [
0,
10,
5,
1,
5,
],
},
],
"first": Array [
"AL",
"AK",
"Arizona",
"Arkansas",
"Somewhere",
],
"length": 5,
"name": "locations",
},
]
`);
});
});

69
public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts Normal file
View File

@ -0,0 +1,69 @@
import {
ArrayVector,
DataFrame,
DataTransformerID,
Field,
FieldMatcher,
FieldMatcherID,
fieldMatchers,
FieldType,
DataTransformerInfo,
} from '@grafana/data';
import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer';
import { mergeMap, from } from 'rxjs';
export interface FieldLookupOptions {
lookupField?: string;
gazetteer?: string;
}
export const fieldLookupTransformer: DataTransformerInfo<FieldLookupOptions> = {
id: DataTransformerID.fieldLookup,
name: 'Lookup fields from resource',
description: 'Retrieve matching data based on specified field',
defaultOptions: {},
operator: (options) => (source) => source.pipe(mergeMap((data) => from(doGazetteerXform(data, options)))),
};
async function doGazetteerXform(frames: DataFrame[], options: FieldLookupOptions): Promise<DataFrame[]> {
const fieldMatches = fieldMatchers.get(FieldMatcherID.byName).get(options?.lookupField);
const gaz = await getGazetteer(options?.gazetteer ?? COUNTRIES_GAZETTEER_PATH);
return addFieldsFromGazetteer(frames, gaz, fieldMatches);
}
export function addFieldsFromGazetteer(frames: DataFrame[], gaz: Gazetteer, matcher: FieldMatcher): DataFrame[] {
return frames.map((frame) => {
const fields: Field[] = [];
for (const field of frame.fields) {
fields.push(field);
//if the field matches
if (matcher(field, frame, frames)) {
const values = field.values.toArray();
const lat = new Array<Number>(values.length);
const lon = new Array<Number>(values.length);
//for each value find the corresponding value in the gazetteer
for (let v = 0; v < values.length; v++) {
const foundMatchingValue = gaz.find(values[v]);
//for now start by adding lat and lon
if (foundMatchingValue && foundMatchingValue?.coords.length) {
lon[v] = foundMatchingValue.coords[0];
lat[v] = foundMatchingValue.coords[1];
}
}
fields.push({ name: 'lon', type: FieldType.number, values: new ArrayVector(lon), config: {} });
fields.push({ name: 'lat', type: FieldType.number, values: new ArrayVector(lat), config: {} });
}
}
return {
...frame,
fields,
};
});
}

2
public/app/core/utils/standardTransformers.ts
View File

@ -18,6 +18,7 @@ import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/
import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor';
import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor';
import { fieldLookupTransformRegistryItem } from '../components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor';
export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
return [
@ -40,5 +41,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
configFromQueryTransformRegistryItem,
prepareTimeseriesTransformerRegistryItem,
convertFieldTypeTransformRegistryItem,
fieldLookupTransformRegistryItem,
];
};

15
public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx
View File

@ -3,8 +3,9 @@ import { StandardEditorProps, SelectableValue, GrafanaTheme2 } from '@grafana/da
import { Alert, Select, stylesFactory, useTheme2 } from '@grafana/ui';
import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from '../gazetteer/gazetteer';
import { css } from '@emotion/css';
import { GazetteerPathEditorConfigSettings } from '../types';
const paths: Array<SelectableValue<string>> = [
const defaultPaths: Array<SelectableValue<string>> = [
{
label: 'Countries',
description: 'Lookup countries by name, two letter code, or three letter code',
@ -22,9 +23,15 @@ const paths: Array<SelectableValue<string>> = [
},
];
export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({ value, onChange, context }) => {
export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any, GazetteerPathEditorConfigSettings>> = ({
value,
onChange,
context,
item,
}) => {
const styles = getStyles(useTheme2());
const [gaz, setGaz] = useState<Gazetteer>();
const settings = item.settings as any;
useEffect(() => {
async function fetchData() {
@ -35,7 +42,7 @@ export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({
}, [value, setGaz]);
const { current, options } = useMemo(() => {
let options = [...paths];
let options = settings?.options ? [...settings.options] : [...defaultPaths];
let current = options.find((f) => f.value === gaz?.path);
if (!current && gaz) {
current = {
@ -45,7 +52,7 @@ export const GazetteerPathEditor: FC<StandardEditorProps<string, any, any>> = ({
options.push(current);
}
return { options, current };
}, [gaz]);
}, [gaz, settings.options]);
return (
<>

5
public/app/plugins/panel/geomap/types.ts
View File

@ -1,4 +1,4 @@
import { MapLayerHandler, MapLayerOptions } from '@grafana/data';
import { MapLayerHandler, MapLayerOptions, SelectableValue } from '@grafana/data';
import BaseLayer from 'ol/layer/Base';
import { Units } from 'ol/proj/Units';
import { Style } from 'ol/style';
@ -64,6 +64,9 @@ export enum ComparisonOperation {
GTE = 'gte',
}
export interface GazetteerPathEditorConfigSettings {
options?: Array<SelectableValue<string>>;
}
//-------------------
// Runtime model
//-------------------