[Scopes]: Pass formatted scope filters to adhoc (#101217)

* pass formatted scope filters to adhoc

* fix

* fix

* fix scenario where we have equals and not-equals filters with the same key

* add canary packages for testing

* WIP

* refactor to pass all filter values

* rename property

* refactor

* update canary scenes

* update scenes version

* fix tests

* fix arg startProfile bug that arised with scenes update

.betterer.results

@@ -285,6 +285,9 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
     ],
+    "packages/grafana-data/src/types/scopes.ts:5381": [
+      [0, 0, 0, "Do not use any type assertions.", "0"]
+    ],
     "packages/grafana-data/src/types/select.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]

packages/grafana-data/src/index.ts

@@ -576,6 +576,8 @@ export {
   type ScopeNodeSpec,
   type ScopeNode,
   scopeFilterOperatorMap,
+  reverseScopeFilterOperatorMap,
+  isEqualityOrMultiOperator,
 } from './types/scopes';
 export {
   PluginState,

packages/grafana-data/src/types/scopes.ts

@@ -18,6 +18,12 @@ export interface ScopeDashboardBinding {
 }
 
 export type ScopeFilterOperator = 'equals' | 'not-equals' | 'regex-match' | 'regex-not-match' | 'one-of' | 'not-one-of';
+export type EqualityOrMultiOperator = Extract<ScopeFilterOperator, 'equals' | 'not-equals' | 'one-of' | 'not-one-of'>;
+
+export function isEqualityOrMultiOperator(value: string): value is EqualityOrMultiOperator {
+  const operators = new Set(['equals', 'not-equals', 'one-of', 'not-one-of']);
+  return operators.has(value);
+}
 
 export const scopeFilterOperatorMap: Record<string, ScopeFilterOperator> = {
   '=': 'equals',
@@ -28,6 +34,11 @@ export const scopeFilterOperatorMap: Record<string, ScopeFilterOperator> = {
   '!=|': 'not-one-of',
 };
 
+export const reverseScopeFilterOperatorMap: Record<ScopeFilterOperator, string> = Object.fromEntries(
+  Object.entries(scopeFilterOperatorMap).map(([symbol, operator]) => [operator, symbol])
+  // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+) as Record<ScopeFilterOperator, string>;
+
 export interface ScopeSpecFilter {
   key: string;
   value: string;

public/app/features/dashboard-scene/scene/DashboardScopesFacade.ts

@@ -1,6 +1,10 @@
-import { sceneGraph } from '@grafana/scenes';
+import { AdHocFiltersVariable, sceneGraph } from '@grafana/scenes';
 import { ScopesFacade } from 'app/features/scopes';
 
+import { getDashboardSceneFor } from '../utils/utils';
+
+import { convertScopesToAdHocFilters } from './convertScopesToAdHocFilters';
+
 export interface DashboardScopesFacadeState {
   reloadOnParamsChange?: boolean;
   uid?: string;
@@ -13,7 +17,32 @@ export class DashboardScopesFacade extends ScopesFacade {
         if (!reloadOnParamsChange || !uid) {
           sceneGraph.getTimeRange(facade).onRefresh();
         }
+
+        // push filters as soon as they come
+        this.pushScopeFiltersToAdHocVariable();
       },
     });
+
+    this.addActivationHandler(() => {
+      // also try to push filters on activation, for
+      // when the dashboard is changed
+      this.pushScopeFiltersToAdHocVariable();
+    });
+  }
+
+  private pushScopeFiltersToAdHocVariable() {
+    const dashboard = getDashboardSceneFor(this);
+
+    const adhoc = dashboard.state.$variables?.state.variables.find((v) => v instanceof AdHocFiltersVariable);
+
+    if (!adhoc) {
+      return;
+    }
+
+    const filters = convertScopesToAdHocFilters(this.value);
+
+    adhoc.setState({
+      baseFilters: filters,
+    });
   }
 }

public/app/features/dashboard-scene/scene/convertScopesToAdHocFilters.test.ts

@@ -0,0 +1,288 @@
+import { Scope, ScopeSpecFilter } from '@grafana/data';
+import { FilterOrigin } from '@grafana/scenes';
+
+import { convertScopesToAdHocFilters } from './convertScopesToAdHocFilters';
+
+describe('convertScopesToAdHocFilters', () => {
+  it('should return empty filters when no scopes are provided', () => {
+    let scopes = generateScopes([]);
+
+    expect(scopes).toEqual([]);
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([]);
+
+    scopes = generateScopes([[], []]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([]);
+  });
+
+  it('should return filters formatted for adHoc from a single scope', () => {
+    let scopes = generateScopes([
+      [
+        { key: 'key1', value: 'value1', operator: 'equals' },
+        { key: 'key2', value: 'value2', operator: 'not-equals' },
+        { key: 'key3', value: 'value3', operator: 'regex-not-match' },
+      ],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      { key: 'key1', value: 'value1', operator: '=', origin: FilterOrigin.Scopes, values: ['value1'] },
+      { key: 'key2', value: 'value2', operator: '!=', origin: FilterOrigin.Scopes, values: ['value2'] },
+      { key: 'key3', value: 'value3', operator: '!~', origin: FilterOrigin.Scopes, values: ['value3'] },
+    ]);
+
+    scopes = generateScopes([[{ key: 'key3', value: 'value3', operator: 'regex-match' }]]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      { key: 'key3', value: 'value3', operator: '=~', origin: FilterOrigin.Scopes, values: ['value3'] },
+    ]);
+  });
+
+  it('should return filters formatted for adHoc from multiple scopes with single values', () => {
+    let scopes = generateScopes([
+      [{ key: 'key1', value: 'value1', operator: 'equals' }],
+      [{ key: 'key2', value: 'value2', operator: 'regex-match' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      { key: 'key1', value: 'value1', operator: '=', origin: FilterOrigin.Scopes, values: ['value1'] },
+      { key: 'key2', value: 'value2', operator: '=~', origin: FilterOrigin.Scopes, values: ['value2'] },
+    ]);
+  });
+
+  it('should return filters formatted for adHoc from multiple scopes with multiple values', () => {
+    let scopes = generateScopes([
+      [
+        { key: 'key1', value: 'value1', operator: 'equals' },
+        { key: 'key2', value: 'value2', operator: 'not-equals' },
+      ],
+      [
+        { key: 'key3', value: 'value3', operator: 'regex-match' },
+        { key: 'key4', value: 'value4', operator: 'regex-match' },
+      ],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      { key: 'key1', value: 'value1', operator: '=', origin: FilterOrigin.Scopes, values: ['value1'] },
+      { key: 'key2', value: 'value2', operator: '!=', origin: FilterOrigin.Scopes, values: ['value2'] },
+      { key: 'key3', value: 'value3', operator: '=~', origin: FilterOrigin.Scopes, values: ['value3'] },
+      { key: 'key4', value: 'value4', operator: '=~', origin: FilterOrigin.Scopes, values: ['value4'] },
+    ]);
+  });
+
+  it('should return formatted filters and concat values of the same key, coming from different scopes, if operator supports multi-value', () => {
+    let scopes = generateScopes([
+      [
+        { key: 'key1', value: 'value1', operator: 'equals' },
+        { key: 'key2', value: 'value2', operator: 'not-equals' },
+      ],
+      [
+        { key: 'key1', value: 'value3', operator: 'equals' },
+        { key: 'key2', value: 'value4', operator: 'not-equals' },
+      ],
+      [{ key: 'key1', value: 'value5', operator: 'equals' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=|',
+        origin: FilterOrigin.Scopes,
+        values: ['value1', 'value3', 'value5'],
+      },
+      { key: 'key2', value: 'value2', operator: '!=|', origin: FilterOrigin.Scopes, values: ['value2', 'value4'] },
+    ]);
+  });
+
+  it('should ignore the rest of the duplicate filters, if they are a combination of equals and not-equals', () => {
+    let scopes = generateScopes([
+      [{ key: 'key1', value: 'value1', operator: 'equals' }],
+      [{ key: 'key1', value: 'value2', operator: 'not-equals' }],
+      [{ key: 'key1', value: 'value3', operator: 'equals' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=|',
+        origin: FilterOrigin.Scopes,
+        values: ['value1', 'value3'],
+      },
+      {
+        key: 'key1',
+        value: 'value2',
+        operator: '!=',
+        origin: FilterOrigin.Scopes,
+        values: ['value2'],
+      },
+    ]);
+  });
+
+  it('should return formatted filters and keep only the first filter of the same key if operator is not multi-value', () => {
+    let scopes = generateScopes([
+      [
+        { key: 'key1', value: 'value1', operator: 'regex-match' },
+        { key: 'key2', value: 'value2', operator: 'not-equals' },
+      ],
+      [
+        { key: 'key1', value: 'value3', operator: 'regex-match' },
+        { key: 'key2', value: 'value4', operator: 'not-equals' },
+      ],
+      [{ key: 'key1', value: 'value5', operator: 'equals' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value1'],
+      },
+      { key: 'key2', value: 'value2', operator: '!=|', origin: FilterOrigin.Scopes, values: ['value2', 'value4'] },
+      {
+        key: 'key1',
+        value: 'value3',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value3'],
+      },
+      {
+        key: 'key1',
+        value: 'value5',
+        operator: '=',
+        origin: FilterOrigin.Scopes,
+        values: ['value5'],
+      },
+    ]);
+
+    scopes = generateScopes([
+      [{ key: 'key1', value: 'value1', operator: 'regex-match' }],
+      [{ key: 'key1', value: 'value5', operator: 'equals' }],
+      [{ key: 'key1', value: 'value3', operator: 'regex-match' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value1'],
+      },
+      {
+        key: 'key1',
+        value: 'value5',
+        operator: '=',
+        origin: FilterOrigin.Scopes,
+        values: ['value5'],
+      },
+      {
+        key: 'key1',
+        value: 'value3',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value3'],
+      },
+    ]);
+  });
+
+  it('should return formatted filters and concat values that are multi-value and drop duplicates with non multi-value operator', () => {
+    let scopes = generateScopes([
+      [{ key: 'key1', value: 'value1', operator: 'equals' }],
+      [{ key: 'key1', value: 'value2', operator: 'regex-match' }],
+      [{ key: 'key1', value: 'value3', operator: 'equals' }],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=|',
+        origin: FilterOrigin.Scopes,
+        values: ['value1', 'value3'],
+      },
+      {
+        key: 'key1',
+        value: 'value2',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value2'],
+      },
+    ]);
+
+    scopes = generateScopes([
+      [
+        { key: 'key1', value: 'value1', operator: 'equals' },
+        { key: 'key2', value: 'value2', operator: 'equals' },
+      ],
+      [
+        { key: 'key1', value: 'value3', operator: 'equals' },
+        { key: 'key2', value: 'value4', operator: 'equals' },
+      ],
+      [
+        { key: 'key1', value: 'value5', operator: 'regex-match' },
+        { key: 'key2', value: 'value6', operator: 'equals' },
+      ],
+      [
+        { key: 'key1', value: 'value7', operator: 'equals' },
+        { key: 'key2', value: 'value8', operator: 'regex-match' },
+      ],
+      [
+        { key: 'key1', value: 'value9', operator: 'equals' },
+        { key: 'key2', value: 'value10', operator: 'equals' },
+      ],
+    ]);
+
+    expect(convertScopesToAdHocFilters(scopes)).toEqual([
+      {
+        key: 'key1',
+        value: 'value1',
+        operator: '=|',
+        origin: FilterOrigin.Scopes,
+        values: ['value1', 'value3', 'value7', 'value9'],
+      },
+      {
+        key: 'key2',
+        value: 'value2',
+        operator: '=|',
+        origin: FilterOrigin.Scopes,
+        values: ['value2', 'value4', 'value6', 'value10'],
+      },
+      {
+        key: 'key1',
+        value: 'value5',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value5'],
+      },
+      {
+        key: 'key2',
+        value: 'value8',
+        operator: '=~',
+        origin: FilterOrigin.Scopes,
+        values: ['value8'],
+      },
+    ]);
+  });
+});
+
+function generateScopes(filtersSpec: ScopeSpecFilter[][]) {
+  const scopes: Scope[] = [];
+
+  for (let i = 0; i < filtersSpec.length; i++) {
+    scopes.push({
+      metadata: { name: `name-${i}` },
+      spec: {
+        title: `scope-${i}`,
+        type: '',
+        description: 'desc',
+        category: '',
+        filters: filtersSpec[i],
+      },
+    });
+  }
+
+  return scopes;
+}

public/app/features/dashboard-scene/scene/convertScopesToAdHocFilters.ts

@@ -0,0 +1,92 @@
+import {
+  Scope,
+  ScopeSpecFilter,
+  isEqualityOrMultiOperator,
+  reverseScopeFilterOperatorMap,
+  scopeFilterOperatorMap,
+} from '@grafana/data';
+import { AdHocFilterWithLabels, FilterOrigin } from '@grafana/scenes';
+
+export function convertScopesToAdHocFilters(scopes: Scope[]): AdHocFilterWithLabels[] {
+  const formattedFilters: Map<string, AdHocFilterWithLabels> = new Map();
+  // duplicated filters that could not be processed in any way are just appended to the list
+  const duplicatedFilters: AdHocFilterWithLabels[] = [];
+  const allFilters = scopes.flatMap((scope) => scope.spec.filters);
+
+  for (const filter of allFilters) {
+    processFilter(formattedFilters, duplicatedFilters, filter);
+  }
+
+  return [...formattedFilters.values(), ...duplicatedFilters];
+}
+
+function processFilter(
+  formattedFilters: Map<string, AdHocFilterWithLabels>,
+  duplicatedFilters: AdHocFilterWithLabels[],
+  filter: ScopeSpecFilter
+) {
+  const existingFilter = formattedFilters.get(filter.key);
+
+  if (existingFilter && canValueBeMerged(existingFilter.operator, filter.operator)) {
+    mergeFilterValues(existingFilter, filter);
+  } else if (!existingFilter) {
+    // Add filter to map either only if it is new.
+    // Otherwise it is an existing filter that cannot be converted to multi-value
+    // and thus will be moved to the duplicatedFilters list
+    formattedFilters.set(filter.key, {
+      key: filter.key,
+      operator: reverseScopeFilterOperatorMap[filter.operator],
+      value: filter.value,
+      values: filter.values ?? [filter.value],
+      origin: FilterOrigin.Scopes,
+    });
+  } else {
+    duplicatedFilters.push({
+      key: filter.key,
+      operator: reverseScopeFilterOperatorMap[filter.operator],
+      value: filter.value,
+      values: filter.values ?? [filter.value],
+      origin: FilterOrigin.Scopes,
+    });
+  }
+}
+
+function mergeFilterValues(adHocFilter: AdHocFilterWithLabels, filter: ScopeSpecFilter) {
+  const values = filter.values ?? [filter.value];
+
+  for (const value of values) {
+    if (!adHocFilter.values?.includes(value)) {
+      adHocFilter.values?.push(value);
+    }
+  }
+
+  // If there's only one value, there's no need to update the
+  // operator to its multi-value equivalent
+  if (adHocFilter.values?.length === 1) {
+    return;
+  }
+
+  // Otherwise update it to the equivalent multi-value operator
+  if (filter.operator === 'equals' && adHocFilter.operator === reverseScopeFilterOperatorMap['equals']) {
+    adHocFilter.operator = reverseScopeFilterOperatorMap['one-of'];
+  } else if (filter.operator === 'not-equals' && adHocFilter.operator === reverseScopeFilterOperatorMap['not-equals']) {
+    adHocFilter.operator = reverseScopeFilterOperatorMap['not-one-of'];
+  }
+}
+
+function canValueBeMerged(adHocFilterOperator: string, filterOperator: string) {
+  const scopeConvertedOperator = scopeFilterOperatorMap[adHocFilterOperator];
+
+  if (!isEqualityOrMultiOperator(scopeConvertedOperator) || !isEqualityOrMultiOperator(filterOperator)) {
+    return false;
+  }
+
+  if (
+    (scopeConvertedOperator.includes('not') && !filterOperator.includes('not')) ||
+    (!scopeConvertedOperator.includes('not') && filterOperator.includes('not'))
+  ) {
+    return false;
+  }
+
+  return true;
+}