Transformations: Extended support for variables in filter by name (#75734)

* Extend support for variables in filter by name

* Simlpify help and include variable support

* Simplify regexp

* Remove id that was left from an erlier implementation attempt

* Update docs/sources/panels-visualizations/query-transform-data/transform-data/index.md

Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com>

* Improve variable name and fix react warning

---------

Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com>

.betterer.results

@@ -5265,9 +5265,6 @@ exports[`better eslint`] = {
     "public/app/features/transformers/editors/ConvertFieldTypeTransformerEditor.tsx:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"]
     ],
-    "public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx:5381": [
-      [0, 0, 0, "Styles should be written using objects.", "0"]
-    ],
     "public/app/features/transformers/editors/GroupByTransformerEditor.tsx:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Styles should be written using objects.", "1"],

docs/sources/panels-visualizations/query-transform-data/transform-data/index.md

@@ -281,28 +281,41 @@ You'll get the following output:
 
 ### Filter by name
 
-Use this transformation to remove portions of the query results.
+Use this transformation to remove parts of the query results.
 
-Grafana displays the **Identifier** field, followed by the fields returned by your query.
+You can filter field names in three different ways:
 
-You can apply filters in one of two ways:
+- [Using a regular expression](#use-a-regular-expression)
+- [Manually selecting included fields](#manually-select-included-fields)
+- [Using a dashboard variable](#use-a-dashboard-variable)
 
-- Enter a regex expression.
+#### Use a regular expression
-- Click a field to toggle filtering on that field. Filtered fields are displayed with dark gray text, unfiltered fields have white text.
 
-In the example below, I removed the Min field from the results.
+When you filter using a regular expression, field names that match the regular expression are included.
 
-Here is the original query table. (This is streaming data, so numbers change over time and between screenshots.)
+From the input data:
 
-{{< figure src="/static/img/docs/transformations/filter-name-table-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+| Time                | dev-eu-west | dev-eu-north | prod-eu-west | prod-eu-north |
+| ------------------- | ----------- | ------------ | ------------ | ------------- |
+| 2023-03-04 23:56:23 | 23.5        | 24.5         | 22.2         | 20.2          |
+| 2023-03-04 23:56:23 | 23.6        | 24.4         | 22.1         | 20.1          |
 
-Here is the table after I applied the transformation to remove the Min field.
+The result from using the regular expression `prod.*` would be:
 
-{{< figure src="/static/img/docs/transformations/filter-name-table-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+| Time                | prod-eu-west | prod-eu-north |
+| ------------------- | ------------ | ------------- |
+| 2023-03-04 23:56:23 | 22.2         | 20.2          |
+| 2023-03-04 23:56:23 | 22.1         | 20.1          |
 
-Here is the same query using a Stat visualization.
+The regular expression can include an interpolated dashboard variable by using the `${[variable name]}` syntax.
 
-{{< figure src="/static/img/docs/transformations/filter-name-stat-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+#### Manually select included fields
+
+Click and uncheck the field names to remove them from the result. Fields that are matched by the regular expression are still included, even if they're unchecked.
+
+#### Use a dashboard variable
+
+Enable `From variable` to let you select a dashboard variable that's used to include fields. By setting up a [dashboard variable][] with multiple choices, the same fields can be displayed across multiple visualizations.
 
 ### Filter data by query
 
@@ -1010,4 +1023,8 @@ Use this transformation to format the output of a time field. Output can be form
 
 [feature toggle]: "/docs/grafana/ -> /docs/grafana/<GRAFANA VERSION>/setup-grafana/configure-grafana#feature_toggles"
 [feature toggle]: "/docs/grafana-cloud/ -> /docs/grafana/<GRAFANA VERSION>/setup-grafana/configure-grafana#feature_toggles"
+
+[dashboard variable]: "/docs/grafana/ -> docs/grafana/<GRAFANA VERSION>/dashboards/variables"
+[dashboard variable]: "/docs/grafana-cloud/ -> docs/grafana/<GRAFANA VERSION>/dashboards/variables"
+
 {{% /docs/reference %}}

packages/grafana-data/src/transformations/matchers/nameMatcher.ts

@@ -8,6 +8,7 @@ import { FieldMatcherID, FrameMatcherID } from './ids';
 export interface RegexpOrNamesMatcherOptions {
   pattern?: string;
   names?: string[];
+  variable?: string;
 }
 
 /**

packages/grafana-data/src/transformations/transformers/filterByName.test.ts

@@ -196,8 +196,83 @@ describe('filterByName transformer', () => {
         expect(filtered.fields[0].name).toBe('B');
       });
     });
+    it('it can use a variable with multiple comma separated', async () => {
+      const cfg = {
+        id: DataTransformerID.filterFieldsByName,
+        options: {
+          include: {
+            variable: '$var',
+          },
+          byVariable: true,
+        },
+      };
 
-    it('uses template variable substituion', async () => {
+      const ctx = {
+        interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => {
+          if (!target) {
+            return '';
+          }
+          const variables: ScopedVars = {
+            var: {
+              value: 'B,D',
+              text: 'Test',
+            },
+          };
+          for (const key of Object.keys(variables)) {
+            return target.replace(`$${key}`, variables[key]!.value);
+          }
+          return target;
+        },
+      };
+
+      await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => {
+        const data = received[0];
+        const filtered = data[0];
+        expect(filtered.fields.length).toBe(2);
+        expect(filtered.fields[0].name).toBe('B');
+        expect(filtered.fields[1].name).toBe('D');
+      });
+    });
+
+    it('it can use a variable with multiple comma separated values in {}', async () => {
+      const cfg = {
+        id: DataTransformerID.filterFieldsByName,
+        options: {
+          include: {
+            variable: '$var',
+          },
+          byVariable: true,
+        },
+      };
+
+      const ctx = {
+        interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => {
+          if (!target) {
+            return '';
+          }
+          const variables: ScopedVars = {
+            var: {
+              value: '{B,D}',
+              text: 'Test',
+            },
+          };
+          for (const key of Object.keys(variables)) {
+            return target.replace(`$${key}`, variables[key]!.value);
+          }
+          return target;
+        },
+      };
+
+      await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => {
+        const data = received[0];
+        const filtered = data[0];
+        expect(filtered.fields.length).toBe(2);
+        expect(filtered.fields[0].name).toBe('B');
+        expect(filtered.fields[1].name).toBe('D');
+      });
+    });
+
+    it('uses template variable substitution', async () => {
       const cfg = {
         id: DataTransformerID.filterFieldsByName,
         options: {

packages/grafana-data/src/transformations/transformers/filterByName.ts

@@ -1,4 +1,4 @@
-import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
+import { DataTransformContext, DataTransformerInfo, MatcherConfig } from '../../types/transformations';
 import { FieldMatcherID } from '../matchers/ids';
 import { RegexpOrNamesMatcherOptions } from '../matchers/nameMatcher';
 
@@ -8,6 +8,7 @@ import { DataTransformerID } from './ids';
 export interface FilterFieldsByNameTransformerOptions {
   include?: RegexpOrNamesMatcherOptions;
   exclude?: RegexpOrNamesMatcherOptions;
+  byVariable?: boolean;
 }
 
 export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNameTransformerOptions> = {
@@ -20,25 +21,38 @@ export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNa
    * Return a modified copy of the series. If the transform is not or should not
    * be applied, just return the input series
    */
-  operator: (options, replace) => (source) =>
+  operator: (options, ctx) => (source) =>
     source.pipe(
       filterFieldsTransformer.operator(
         {
-          include: getMatcherConfig(options.include),
+          include: getMatcherConfig(ctx, options.include, options.byVariable),
-          exclude: getMatcherConfig(options.exclude),
+          exclude: getMatcherConfig(ctx, options.exclude, options.byVariable),
         },
-        replace
+        ctx
       )
     ),
 };
 
 // Exported to share with other implementations, but not exported to `@grafana/data`
-export const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => {
+export const getMatcherConfig = (
+  ctx: DataTransformContext,
+  options?: RegexpOrNamesMatcherOptions,
+  byVariable?: boolean
+): MatcherConfig | undefined => {
   if (!options) {
     return undefined;
   }
 
-  const { names, pattern } = options;
+  const { names, pattern, variable } = options;
+
+  if (byVariable && variable) {
+    const stringOfNames = ctx.interpolate(variable);
+    if (/\{.*\}/.test(stringOfNames)) {
+      const namesFromString = stringOfNames.slice(1).slice(0, -1).split(',');
+      return { id: FieldMatcherID.byNames, options: { names: namesFromString } };
+    }
+    return { id: FieldMatcherID.byNames, options: { names: stringOfNames.split(',') } };
+  }
 
   if ((!Array.isArray(names) || names.length === 0) && !pattern) {
     return undefined;

public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx

@@ -1,4 +1,3 @@
-import { css } from '@emotion/css';
 import React from 'react';
 
 import {
@@ -10,9 +9,11 @@ import {
   getFieldDisplayName,
   stringToJsRegex,
   TransformerCategory,
+  SelectableValue,
 } from '@grafana/data';
 import { FilterFieldsByNameTransformerOptions } from '@grafana/data/src/transformations/transformers/filterByName';
-import { Field, Input, FilterPill, HorizontalGroup } from '@grafana/ui';
+import { getTemplateSrv } from '@grafana/runtime/src/services';
+import { Input, FilterPill, InlineFieldRow, InlineField, InlineSwitch, Select } from '@grafana/ui';
 
 interface FilterByNameTransformerEditorProps extends TransformerUIProps<FilterFieldsByNameTransformerOptions> {}
 
@@ -21,6 +22,9 @@ interface FilterByNameTransformerEditorState {
   options: FieldNameInfo[];
   selected: string[];
   regex?: string;
+  variable?: string;
+  variables: SelectableValue[];
+  byVariable: boolean;
   isRegexValid?: boolean;
 }
 
@@ -37,7 +41,10 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
     this.state = {
       include: props.options.include?.names || [],
       regex: props.options.include?.pattern,
+      variable: props.options.include?.variable,
+      byVariable: props.options.byVariable || false,
       options: [],
+      variables: [],
       selected: [],
       isRegexValid: true,
     };
@@ -57,6 +64,9 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
     const { input, options } = this.props;
     const configuredOptions = Array.from(options.include?.names ?? []);
 
+    const variables = getTemplateSrv()
+      .getVariables()
+      .map((v) => ({ label: '$' + v.name, value: '$' + v.name }));
     const allNames: FieldNameInfo[] = [];
     const byName: KeyValue<FieldNameInfo> = {};
 
@@ -97,12 +107,18 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
       this.setState({
         options: allNames,
         selected: selected.map((s) => s.name),
+        variables: variables,
+        byVariable: options.byVariable || false,
+        variable: options.include?.variable,
         regex: options.include?.pattern,
       });
     } else {
       this.setState({
         options: allNames,
         selected: allNames.map((n) => n.name),
+        variables: variables,
+        byVariable: options.byVariable || false,
+        variable: options.include?.variable,
         regex: options.include?.pattern,
       });
     }
@@ -161,19 +177,46 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
     this.setState({ isRegexValid });
   };
 
+  onVariableChange = (selected: SelectableValue) => {
+    this.props.onChange({
+      ...this.props.options,
+      include: { variable: selected.value },
+    });
+
+    this.setState({ variable: selected.value });
+  };
+
+  onFromVariableChange = (e: React.FormEvent<HTMLInputElement>) => {
+    const val = e.currentTarget.checked;
+    this.props.onChange({ ...this.props.options, byVariable: val });
+    this.setState({ byVariable: val });
+  };
+
   render() {
     const { options, selected, isRegexValid } = this.state;
     return (
-      <div className="gf-form-inline">
+      <div>
-        <div className="gf-form gf-form--grow">
+        <InlineFieldRow label="Use variable">
-          <div className="gf-form-label width-8">Identifier</div>
+          <InlineField label="From variable">
-          <HorizontalGroup spacing="xs" align="flex-start" wrap>
+            <InlineSwitch value={this.state.byVariable} onChange={this.onFromVariableChange}></InlineSwitch>
-            <Field
+          </InlineField>
+        </InlineFieldRow>
+        {this.state.byVariable ? (
+          <InlineFieldRow>
+            <InlineField label="Variable">
+              <Select
+                value={this.state.variable}
+                onChange={this.onVariableChange}
+                options={this.state.variables || []}
+              ></Select>
+            </InlineField>
+          </InlineFieldRow>
+        ) : (
+          <InlineFieldRow label="Identifier">
+            <InlineField
+              label="Identifier"
               invalid={!isRegexValid}
               error={!isRegexValid ? 'Invalid pattern' : undefined}
-              className={css`
-                margin-bottom: 0;
-              `}
             >
               <Input
                 placeholder="Regular expression pattern"
@@ -182,7 +225,7 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
                 onBlur={this.onInputBlur}
                 width={25}
               />
-            </Field>
+            </InlineField>
             {options.map((o, i) => {
               const label = `${o.name}${o.count > 1 ? ' (' + o.count + ')' : ''}`;
               const isSelected = selected.indexOf(o.name) > -1;
@@ -197,8 +240,8 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
                 />
               );
             })}
-          </HorizontalGroup>
+          </InlineFieldRow>
-        </div>
+        )}
       </div>
     );
   }

public/app/features/transformers/partitionByValues/partitionByValues.ts

@@ -73,7 +73,7 @@ export const partitionByValuesTransformer: SynchronousDataTransformerInfo<Partit
     source.pipe(map((data) => partitionByValuesTransformer.transformer(options, ctx)(data))),
 
   transformer: (options: PartitionByValuesTransformerOptions, ctx: DataTransformContext) => {
-    const matcherConfig = getMatcherConfig({ names: options.fields });
+    const matcherConfig = getMatcherConfig(ctx, { names: options.fields });
 
     if (!matcherConfig) {
       return noopTransformer.transformer({}, ctx);