Chore: Move to Cypress 12 and decouple cypress from `@grafana/e2e` (#74084)

* update drone to use cypress 12 image

* upgrade cypress to 12 in core

* cypress config actually valid

* update @grafana/e2e imports and add lint rule

* ignore grafana-e2e from betterer now it's deprecated

* fix remaining type errors

* fix failing tests

* remove unnecessary tsconfig

* remove unnecessary comment

* update enterprise suite commands to work

* add cypress config to CODEOWNERS

* export setTimeRange in utils

* remove @grafana/e2e from core deps

* try running the command through yarn

* move CMD to scripts

* Update cloud-data-sources e2e image

* Update paths

---------

Co-authored-by: Andreas Christou <andreas.christou@grafana.com>

.betterer.results

@@ -5,6 +5,61 @@
 //
 exports[`better eslint`] = {
   value: `{
+    "e2e/cypress/support/index.d.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/addDashboard.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Do not use any type assertions.", "1"]
+    ],
+    "e2e/utils/flows/addDataSource.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Do not use any type assertions.", "1"]
+    ],
+    "e2e/utils/flows/addPanel.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/configurePanel.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/deleteDashboard.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/deleteDataSource.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/openDashboard.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/flows/revertAllChanges.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
+    ],
+    "e2e/utils/flows/selectOption.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
+    ],
+    "e2e/utils/support/localStorage.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
+      [0, 0, 0, "Do not use any type assertions.", "5"]
+    ],
+    "e2e/utils/support/scenarioContext.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
+    ],
+    "e2e/utils/support/types.ts:5381": [
+      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
+      [0, 0, 0, "Do not use any type assertions.", "3"],
+      [0, 0, 0, "Do not use any type assertions.", "4"],
+      [0, 0, 0, "Do not use any type assertions.", "5"],
+      [0, 0, 0, "Do not use any type assertions.", "6"],
+      [0, 0, 0, "Do not use any type assertions.", "7"]
+    ],
     "packages/grafana-data/src/dataframe/ArrayDataFrame.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
@@ -718,73 +773,6 @@ exports[`better eslint`] = {
     "packages/grafana-data/test/__mocks__/pluginMocks.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
     ],
-    "packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts:5381": [
-      [0, 0, 0, "Do not use any type assertions.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Do not use any type assertions.", "3"],
-      [0, 0, 0, "Do not use any type assertions.", "4"],
-      [0, 0, 0, "Do not use any type assertions.", "5"],
-      [0, 0, 0, "Do not use any type assertions.", "6"]
-    ],
-    "packages/grafana-e2e/cypress/support/commands.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/cypress/support/index.d.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/addDashboard.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Do not use any type assertions.", "1"]
-    ],
-    "packages/grafana-e2e/src/flows/addDataSource.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Do not use any type assertions.", "1"]
-    ],
-    "packages/grafana-e2e/src/flows/addPanel.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/configurePanel.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/deleteDashboard.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/deleteDataSource.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/openDashboard.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/flows/revertAllChanges.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
-    ],
-    "packages/grafana-e2e/src/flows/selectOption.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
-    ],
-    "packages/grafana-e2e/src/support/localStorage.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
-      [0, 0, 0, "Do not use any type assertions.", "5"]
-    ],
-    "packages/grafana-e2e/src/support/scenarioContext.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
-    ],
-    "packages/grafana-e2e/src/support/types.ts:5381": [
-      [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
-      [0, 0, 0, "Do not use any type assertions.", "3"],
-      [0, 0, 0, "Do not use any type assertions.", "4"],
-      [0, 0, 0, "Do not use any type assertions.", "5"],
-      [0, 0, 0, "Do not use any type assertions.", "6"],
-      [0, 0, 0, "Do not use any type assertions.", "7"]
-    ],
     "packages/grafana-runtime/src/analytics/types.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
     ],

.betterer.ts

@@ -8,7 +8,7 @@ export default {
   'better eslint': () =>
     countEslintErrors()
       .include('**/*.{ts,tsx}')
-      .exclude(/public\/app\/angular/),
+      .exclude(/public\/app\/angular|packages\/grafana-e2e/),
   'no undocumented stories': () => countUndocumentedStories().include('**/!(*.internal).story.tsx'),
 };
 

.drone.yml

@@ -627,40 +627,36 @@ steps:
   image: grafana/build-container:1.7.5
   name: grafana-server
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite dashboards-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-dashboards-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-smoke-tests-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite panels-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-panels-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite various-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-various-suite
 - commands:
   - cd /
@@ -678,7 +674,7 @@ steps:
     GITHUB_TOKEN:
       from_secret: github_token
     HOST: grafana-server
-  image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
+  image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:2.0.0
   name: end-to-end-tests-cloud-plugins-suite-azure
   when:
     paths:
@@ -1786,40 +1782,36 @@ steps:
   image: grafana/build-container:1.7.5
   name: grafana-server
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite dashboards-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-dashboards-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-smoke-tests-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite panels-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-panels-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite various-suite
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-various-suite
 - commands:
   - cd /
@@ -1837,7 +1829,7 @@ steps:
     GITHUB_TOKEN:
       from_secret: github_token
     HOST: grafana-server
-  image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest
+  image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:2.0.0
   name: end-to-end-tests-cloud-plugins-suite-azure
   when:
     paths:
@@ -3387,40 +3379,36 @@ steps:
   image: grafana/build-container:1.7.5
   name: grafana-server
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-dashboards-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-smoke-tests-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-panels-suite
 - commands:
-  - apt-get install -y netcat
   - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3
   depends_on:
   - grafana-server
   environment:
     HOST: grafana-server
-  image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  image: cypress/included:12.15.0
   name: end-to-end-tests-various-suite
 - commands:
   - apt-get update
@@ -4456,7 +4444,7 @@ steps:
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/drone-downstream
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:dbd975af06
-  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:12.15.0
   depends_on:
   - authenticate-gcr
   image: aquasec/trivy:0.21.0
@@ -4484,7 +4472,7 @@ steps:
   - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/drone-downstream
   - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0
   - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:dbd975af06
-  - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97
+  - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:12.15.0
   depends_on:
   - authenticate-gcr
   environment:
@@ -4735,6 +4723,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: fa64513236ee2677770f4e09ac6ddb06d1b85db22916efb44c1d332c92edf99b
+hmac: e7a7b8ebee80baceff6c915e84e76f8f5e54565741f923e9e1e17f3f8880dce9
 
 ...

.eslintrc

@@ -35,6 +35,10 @@
             "name": "react-i18next",
             "importNames": ["Trans", "t"],
             "message": "Please import from app/core/internationalization instead"
+          },
+          {
+            "name": "@grafana/e2e",
+            "message": "@grafana/e2e is deprecated. Please import from ./e2e/utils instead"
           }
         ]
       }

.github/CODEOWNERS

@@ -353,6 +353,7 @@ lerna.json @grafana/frontend-ops
 /lefthook.yml @grafana/frontend-ops
 /lefthook.rc @grafana/frontend-ops
 .husky/pre-commit @grafana/frontend-ops
+cypress.config.js @grafana/grafana-frontend-platform
 .levignore.js @grafana/plugins-platform-frontend
 
 

cypress.config.js

@@ -0,0 +1,84 @@
+const { defineConfig } = require('cypress');
+const fs = require('fs');
+const path = require('path');
+
+const benchmarkPlugin = require('./e2e/cypress/plugins/benchmark/index');
+const compareScreenshots = require('./e2e/cypress/plugins/compareScreenshots');
+const readProvisions = require('./e2e/cypress/plugins/readProvisions');
+const typescriptPreprocessor = require('./e2e/cypress/plugins/typescriptPreprocessor');
+
+module.exports = defineConfig({
+  projectId: 'zb7k1c',
+  videoCompression: 20,
+  viewportWidth: 1920,
+  viewportHeight: 1080,
+
+  e2e: {
+    supportFile: './e2e/cypress/support/e2e.js',
+    setupNodeEvents(on, config) {
+      on('file:preprocessor', typescriptPreprocessor);
+      on('task', {
+        log({ message, optional }) {
+          optional ? console.log(message, optional) : console.log(message);
+          return null;
+        },
+      });
+
+      if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) {
+        benchmarkPlugin.initialize(on, config);
+      }
+
+      on('task', {
+        compareScreenshots,
+        readProvisions: (filePaths) => readProvisions({ CWD: process.cwd(), filePaths }),
+      });
+
+      on('task', {
+        getJSONFilesFromDir: async (relativePath) => {
+          // CWD is set for plugins in the cli but not for the main grafana repo: https://github.com/grafana/grafana/blob/main/packages/grafana-e2e/cli.js#L12
+          const projectPath = config.env.CWD || config.fileServerFolder || process.cwd();
+          const directoryPath = path.join(projectPath, relativePath);
+          const jsonFiles = fs.readdirSync(directoryPath);
+          return jsonFiles
+            .filter((fileName) => /.json$/i.test(fileName))
+            .map((fileName) => {
+              const fileBuffer = fs.readFileSync(path.join(directoryPath, fileName));
+              return JSON.parse(fileBuffer);
+            });
+        },
+      });
+
+      on('before:browser:launch', (browser = {}, launchOptions) => {
+        console.log('launching browser %s is headless? %s', browser.name, browser.isHeadless);
+
+        // the browser width and height we want to get
+        // our screenshots and videos will be of that resolution
+        const width = 1920;
+        const height = 1080;
+
+        console.log('setting the browser window size to %d x %d', width, height);
+
+        if (browser.name === 'chrome' && browser.isHeadless) {
+          launchOptions.args.push(`--window-size=${width},${height}`);
+
+          // force screen to be non-retina and just use our given resolution
+          launchOptions.args.push('--force-device-scale-factor=1');
+        }
+
+        if (browser.name === 'electron' && browser.isHeadless) {
+          // might not work on CI for some reason
+          launchOptions.preferences.width = width;
+          launchOptions.preferences.height = height;
+        }
+
+        if (browser.name === 'firefox' && browser.isHeadless) {
+          launchOptions.args.push(`--width=${width}`);
+          launchOptions.args.push(`--height=${height}`);
+        }
+
+        // IMPORTANT: return the updated browser launch options
+        return launchOptions;
+      });
+    },
+  },
+});

e2e/benchmarks/live/4-20hz-panels.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../../utils';
 
 type WithGrafanaRuntime<T> = T & {
   grafanaRuntime: {

e2e/benchmarks/tsconfig.json

@@ -1,5 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "include": ["**/*.ts", "../../packages/grafana-e2e/cypress/support/index.d.ts"],
-  "resolveJsonModule": true
-}

e2e/cloud-plugins-suite/azure-monitor.spec.ts

@@ -2,16 +2,15 @@ import { Interception } from 'cypress/types/net-stubbing';
 import { load } from 'js-yaml';
 import { v4 as uuidv4 } from 'uuid';
 
-import { e2e } from '@grafana/e2e';
-
 import { selectors } from '../../public/app/plugins/datasource/azuremonitor/e2e/selectors';
 import {
   AzureDataSourceJsonData,
   AzureDataSourceSecureJsonData,
   AzureQueryType,
 } from '../../public/app/plugins/datasource/azuremonitor/types';
+import { e2e } from '../utils';
 
-const provisioningPath = `../../provisioning/datasources/azmonitor-ds.yaml`;
+const provisioningPath = `provisioning/datasources/azmonitor-ds.yaml`;
 const e2eSelectors = e2e.getSelectors(selectors.components);
 
 type AzureMonitorConfig = {
@@ -166,7 +165,7 @@ e2e.scenario({
     const CI = e2e.env('CI');
     if (CI) {
       e2e()
-        .readFile('../../outputs.json')
+        .readFile('outputs.json')
         .then((outputs) => {
           provisionAzureMonitorDatasources([
             {

e2e/cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

e2e/cypress/fixtures/exemplars-query-response.json

@@ -0,0 +1,323 @@
+{
+  "results": {
+    "A": {
+      "frames": [
+        {
+          "schema": {
+            "name": "histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))",
+            "refId": "A",
+            "meta": { "custom": { "resultType": "matrix" } },
+            "fields": [
+              { "name": "Time", "type": "time", "typeInfo": { "frame": "time.Time" } },
+              {
+                "name": "Value",
+                "type": "number",
+                "typeInfo": { "frame": "float64" },
+                "labels": {},
+                "config": {
+                  "displayNameFromDS": "histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))"
+                }
+              }
+            ]
+          },
+          "data": {
+            "values": [
+              [
+                1633619595000, 1633619610000, 1633619625000, 1633619640000, 1633619655000, 1633619670000, 1633619685000,
+                1633619700000, 1633619715000, 1633619730000, 1633619745000, 1633619760000, 1633619775000, 1633619790000,
+                1633619805000, 1633619820000, 1633619835000, 1633619850000, 1633619865000, 1633619880000, 1633619895000
+              ],
+              [
+                0.07245212135073513, 0.07253198890830721, 0.07247862573797707, 0.07238248338231042, 0.07221687487740913,
+                0.07223291298743946, 0.07225427016727755, 0.024531677091864545, 0.02317081920915543,
+                0.07548902139580993, 0.0777721702857508, 0.07768649905047344, 0.07782257603228229, 0.07788810213200052,
+                0.07791835055437593, 0.07798387201529966, 0.07790826751849372, 0.07794858648610933, 0.07778729925797964,
+                0.07769657495236215, 0.077550401329267
+              ]
+            ]
+          }
+        },
+        {
+          "schema": {
+            "name": "histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))",
+            "refId": "A",
+            "meta": { "custom": { "resultType": "vector" } },
+            "fields": [
+              { "name": "Time", "type": "time", "typeInfo": { "frame": "time.Time" } },
+              {
+                "name": "Value",
+                "type": "number",
+                "typeInfo": { "frame": "float64" },
+                "labels": {},
+                "config": {
+                  "displayNameFromDS": "histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))"
+                }
+              }
+            ]
+          },
+          "data": { "values": [[1633619900000], [0.0775504013292671]] }
+        },
+        {
+          "schema": {
+            "name": "exemplar",
+            "refId": "A",
+            "meta": { "custom": { "resultType": "exemplar" } },
+            "fields": [
+              { "name": "Time", "type": "time", "typeInfo": { "frame": "time.Time" } },
+              { "name": "Value", "type": "number", "typeInfo": { "frame": "float64" } },
+              { "name": "instance", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "__name__", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "job", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "status_code", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "method", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "traceID", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "route", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "ws", "type": "string", "typeInfo": { "frame": "string" } },
+              { "name": "le", "type": "string", "typeInfo": { "frame": "string" } }
+            ]
+          },
+          "data": {
+            "values": [
+              [
+                1633619598000, 1633619622000, 1633619625000, 1633619646000, 1633619658000, 1633619682000, 1633619695000,
+                1633619712000, 1633619712000, 1633619724000, 1633619717000, 1633619742000, 1633619757000, 1633619771000,
+                1633619784000, 1633619801000, 1633619806000, 1633619833000, 1633619833000, 1633619845000, 1633619862000,
+                1633619877000, 1633619889000
+              ],
+              [
+                0.0146153, 0.0118506, 0.0473847, 0.026997, 0.0164318, 0.0113532, 0.0105197, 0.162789, 0.0556026,
+                0.148856, 0.0433809, 0.0117758, 0.0114496, 0.0114099, 0.0421927, 0.0134148, 0.0152827, 0.6975967,
+                0.0394788, 0.0137441, 0.0110939, 0.0104496, 0.0101284
+              ],
+              [
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "db:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80",
+                "app:80"
+              ],
+              [
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket",
+                "tns_request_duration_seconds_bucket"
+              ],
+              [
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/db",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app",
+                "tns/app"
+              ],
+              [
+                "302",
+                "200",
+                "200",
+                "200",
+                "200",
+                "200",
+                "200",
+                "500",
+                "200",
+                "302",
+                "208",
+                "200",
+                "200",
+                "200",
+                "200",
+                "200",
+                "302",
+                "200",
+                "200",
+                "200",
+                "200",
+                "200",
+                "200"
+              ],
+              [
+                "POST",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "POST",
+                "POST",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "POST",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET",
+                "GET"
+              ],
+              [
+                "6a3cf561ef6c32a0",
+                "396bcdf29601a149",
+                "57c04ef608f11158",
+                "77c757dab83c665f",
+                "3d1069567e873f5e",
+                "b337949f6213efd",
+                "21b20cbe533cf099",
+                "2c10b3aa30fabd66",
+                "42ac6088a757636b",
+                "2f81158008cd4dcc",
+                "320b803ad7323b37",
+                "7f15fd82aeb8b361",
+                "11c79266da8a74cd",
+                "5a8571bdcc04c990",
+                "3de3f4f42ccb93ae",
+                "23343ac91cc0638",
+                "5cea3aad17ab11c8",
+                "5d334e2843d3405a",
+                "3cf6834596d4b6b6",
+                "1ab6cff012959723",
+                "2f78bc2c398b8b20",
+                "6d5862a70c3abd42",
+                "f5421be4054f501"
+              ],
+              [
+                "post",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root",
+                "post",
+                "post",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root",
+                "post",
+                "metrics",
+                "root",
+                "root",
+                "root",
+                "root",
+                "root"
+              ],
+              [
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false",
+                "false"
+              ],
+              [
+                "0.025",
+                "0.025",
+                "0.05",
+                "0.05",
+                "0.025",
+                "0.025",
+                "0.025",
+                "0.25",
+                "0.1",
+                "0.25",
+                "0.05",
+                "0.025",
+                "0.025",
+                "0.025",
+                "0.05",
+                "0.025",
+                "0.025",
+                "1.0",
+                "0.05",
+                "0.025",
+                "0.025",
+                "0.025",
+                "0.025"
+              ]
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

e2e/cypress/fixtures/prometheus-query-range-response.json

@@ -0,0 +1,81 @@
+{
+  "status": "success",
+  "data": {
+    "resultType": "matrix",
+    "result": [
+      {
+        "metric": {},
+        "values": [
+          [1620758235, "0.07554431352019486"],
+          [1620758250, "0.0756695553961457"],
+          [1620758265, "0.0757369945411682"],
+          [1620758280, "0.07560212035898113"],
+          [1620758295, "0.07556358506832812"],
+          [1620758310, "0.07558766859344893"],
+          [1620758325, "0.07552022996976834"],
+          [1620758340, "0.07553949807996531"],
+          [1620758355, "0.07554913414209416"],
+          [1620758370, "0.07539017545449077"],
+          [1620758385, "0.07524566527721041"],
+          [1620758400, "0.06631294924007665"],
+          [1620758415, "0.020769530989205368"],
+          [1620758430, "0.05720168751283235"],
+          [1620758445, "0.07271760187022697"],
+          [1620758460, "0.07282398348834057"],
+          [1620758475, "0.07272243619599422"],
+          [1620758490, "0.0727659581600079"],
+          [1620758505, "0.07290135207155769"],
+          [1620758520, "0.07293036876672591"],
+          [1620758535, "0.0727901374111541"],
+          [1620758550, "0.07272727333735175"],
+          [1620758565, "0.07264506733699574"],
+          [1620758580, "0.07272243607717656"],
+          [1620758595, "0.0728288184987238"],
+          [1620758610, "0.07298839709448537"],
+          [1620758625, "0.07301257421338406"],
+          [1620758640, "0.07304158515671498"],
+          [1620758655, "0.07311895518980911"],
+          [1620758670, "0.07325918868870857"],
+          [1620758685, "0.07340909025275498"],
+          [1620758700, "0.06640878600261439"],
+          [1620758715, "0.016943481796378928"],
+          [1620758730, "0.009846410786372045"],
+          [1620758745, "0.009846533933076818"],
+          [1620758760, "0.009865643995544734"],
+          [1620758775, "0.009877495333796778"],
+          [1620758790, "0.009894557340703772"],
+          [1620758805, "0.0098843910341446"],
+          [1620758820, "0.00990408341969324"],
+          [1620758835, "0.00989844441243741"],
+          [1620758850, "0.009889907575638773"],
+          [1620758865, "0.009918898761738633"],
+          [1620758880, "0.009937127911002756"],
+          [1620758895, "0.009940908363410796"],
+          [1620758910, "0.00998103477604732"],
+          [1620758925, "0.009972785096318881"],
+          [1620758940, "0.012851280416358784"],
+          [1620758955, "0.016073228821362785"],
+          [1620758970, "0.020414802032173343"],
+          [1620761580, "0.007599075245347286"],
+          [1620761595, "0.008931710803442608"],
+          [1620761610, "0.008726716914241494"],
+          [1620761625, "0.008200081743024097"],
+          [1620761640, "0.00855242238708798"],
+          [1620761655, "0.008286349295644651"],
+          [1620761670, "0.008226278261449314"],
+          [1620761685, "0.008195191146355274"],
+          [1620761700, "0.008187372718523614"],
+          [1620761715, "0.008513095070485845"],
+          [1620761730, "0.08239661322810221"],
+          [1620761745, "0.0859446307478243"],
+          [1620761760, "0.08307358128715034"],
+          [1620761775, "0.08068720480328369"],
+          [1620761790, "0.07619009806120529"],
+          [1620761805, "0.0750613052160521"],
+          [1620761820, "0.07146092807229597"],
+          [1620761835, "0.06898128960085806"]
+        ]
+      }
+    ]
+  }
+}

e2e/cypress/fixtures/prometheus-query-response.json

@@ -0,0 +1,4 @@
+{
+  "status": "success",
+  "data": { "resultType": "vector", "result": [{ "metric": {}, "value": [1620761849, "0.06765848222986065"] }] }
+}

e2e/cypress/plugins/benchmark/CDPDataCollector.js

@@ -0,0 +1,120 @@
+const CDP = require('chrome-remote-interface');
+const { countBy, mean } = require('lodash');
+const Tracelib = require('tracelib');
+
+class CDPDataCollector {
+  tracingCategories;
+  state;
+
+  constructor(deps) {
+    this.state = this.getDefaultState();
+    this.tracingCategories = [
+      'disabled-by-default-v8.cpu_profile',
+      'disabled-by-default-v8.cpu_profiler',
+      'disabled-by-default-v8.cpu_profiler.hires',
+      'disabled-by-default-devtools.timeline.frame',
+      'disabled-by-default-devtools.timeline',
+      'disabled-by-default-devtools.timeline.inputs',
+      'disabled-by-default-devtools.timeline.stack',
+      'disabled-by-default-devtools.timeline.invalidationTracking',
+      'disabled-by-default-layout_shift.debug',
+      'disabled-by-default-cc.debug.scheduler.frames',
+      'disabled-by-default-blink.debug.display_lock',
+    ];
+  }
+
+  getName = () => DataCollectorName.CDP;
+
+  resetState = async () => {
+    if (this.state.client) {
+      await this.state.client.close();
+    }
+    this.state = this.getDefaultState();
+  };
+
+  getDefaultState = () => ({
+    traceEvents: [],
+  });
+
+  // workaround for type declaration issues in cdp lib
+  asApis = (client) => client;
+
+  getClientApis = async () => this.asApis(await this.getClient());
+
+  getClient = async () => {
+    if (this.state.client) {
+      return this.state.client;
+    }
+
+    const client = await CDP({ port: this.deps.port });
+
+    const { Profiler, Page } = this.asApis(client);
+    await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]);
+
+    this.state.client = client;
+
+    return client;
+  };
+
+  start = async ({ id }) => {
+    if (this.state.tracingPromise) {
+      throw new Error(`collection in progress - can't start another one! ${id}`);
+    }
+
+    const { Tracing, Profiler } = await this.getClientApis();
+
+    await Promise.all([
+      Tracing.start({
+        bufferUsageReportingInterval: 1000,
+        traceConfig: {
+          includedCategories: this.tracingCategories,
+        },
+      }),
+      Profiler.start(),
+    ]);
+
+    Tracing.on('dataCollected', ({ value: events }) => {
+      this.state.traceEvents.push(...events);
+    });
+
+    let resolveFn;
+    this.state.tracingPromise = new Promise((resolve) => {
+      resolveFn = resolve;
+    });
+    Tracing.on('tracingComplete', ({ dataLossOccurred }) => {
+      const t = new Tracelib(this.state.traceEvents);
+
+      const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name);
+
+      const fps = t.getFPS();
+
+      resolveFn({
+        eventCounts,
+        fps: mean(fps.values),
+        tracingDataLoss: dataLossOccurred ? 1 : 0,
+        warnings: t.getWarningCounts(),
+      });
+    });
+  };
+
+  stop = async (req) => {
+    if (!this.state.tracingPromise) {
+      throw new Error(`collection was never started - there is nothing to stop!`);
+    }
+
+    const { Tracing, Profiler } = await this.getClientApis();
+
+    // TODO: capture profiler data
+    const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]);
+
+    await this.resetState();
+
+    return traceData;
+  };
+
+  close = async () => {
+    await this.resetState();
+  };
+}
+
+exports.CDPDataCollector = CDPDataCollector;

e2e/cypress/plugins/benchmark/formatting.js

@@ -0,0 +1,94 @@
+const { fromPairs } = require('lodash');
+
+const isLivePerformanceAppStats = (data) =>
+  data.some((st) => {
+    const stat = st?.[MeasurementName.DataRenderDelay];
+    return Array.isArray(stat) && Boolean(stat?.length);
+  });
+
+const formatAppStats = (allStats) => {
+  if (!isLivePerformanceAppStats(allStats)) {
+    return {};
+  }
+
+  const names = Object.keys(MeasurementName);
+
+  return fromPairs(
+    names.map((name) => {
+      const statsForMeasurement = allStats.map((s) => s[name]);
+      const res = {
+        total: {
+          count: [],
+          avg: [],
+        },
+        lastInterval: {
+          avg: [],
+          min: [],
+          max: [],
+          count: [],
+        },
+      };
+
+      statsForMeasurement.forEach((s) => {
+        const total = s.reduce(
+          (prev, next) => {
+            prev.count += next.count;
+            prev.avg += next.avg;
+            return prev;
+          },
+          { count: 0, avg: 0 }
+        );
+        res.total.count.push(Math.round(total.count));
+        res.total.avg.push(Math.round(total.avg / s.length));
+
+        const lastInterval = s[s.length - 1];
+
+        res.lastInterval.avg.push(Math.round(lastInterval?.avg));
+        res.lastInterval.min.push(Math.round(lastInterval?.min));
+        res.lastInterval.max.push(Math.round(lastInterval?.max));
+        res.lastInterval.count.push(Math.round(lastInterval?.count));
+      });
+
+      return [name, res];
+    })
+  );
+};
+
+const emptyFormattedCDPData = () => ({
+  minorGC: [],
+  majorGC: [],
+  droppedFrames: [],
+  fps: [],
+  tracingDataLossOccurred: false,
+  longTaskWarnings: [],
+});
+
+const isCDPData = (data) => data.every((d) => typeof d.eventCounts === 'object');
+
+const formatCDPData = (data) => {
+  if (!isCDPData(data)) {
+    return emptyFormattedCDPData();
+  }
+
+  return data.reduce((acc, next) => {
+    acc.majorGC.push(next.eventCounts.MajorGC ?? 0);
+    acc.minorGC.push(next.eventCounts.MinorGC ?? 0);
+    acc.fps.push(Math.round(next.fps) ?? 0);
+    acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss);
+    acc.droppedFrames.push(next.eventCounts.DroppedFrame ?? 0);
+    acc.longTaskWarnings.push(next.warnings.LongTask ?? 0);
+    return acc;
+  }, emptyFormattedCDPData());
+};
+
+const formatResults = (results) => {
+  return {
+    ...formatAppStats(results.map(({ appStats }) => appStats)),
+    ...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])),
+
+    __raw: results,
+  };
+};
+
+exports.formatResults = formatResults;
+exports.formatAppStats = formatAppStats;

e2e/cypress/plugins/benchmark/index.js

@@ -0,0 +1,90 @@
+const fs = require('fs');
+const { fromPairs } = require('lodash');
+
+const { CDPDataCollector } = require('./CDPDataCollector');
+const { formatResults } = require('./formatting');
+
+const remoteDebuggingPortOptionPrefix = '--remote-debugging-port=';
+
+const getOrAddRemoteDebuggingPort = (args) => {
+  const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix));
+
+  if (existing) {
+    return Number(existing.substring(remoteDebuggingPortOptionPrefix.length));
+  }
+
+  const port = 40000 + Math.round(Math.random() * 25000);
+  args.push(`${remoteDebuggingPortOptionPrefix}${port}`);
+  return port;
+};
+
+let collectors = [];
+let results = [];
+
+const startBenchmarking = async ({ testName }) => {
+  await Promise.all(collectors.map((coll) => coll.start({ id: testName })));
+
+  return true;
+};
+
+const stopBenchmarking = async ({ testName, appStats }) => {
+  const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })]));
+
+  results.push({
+    collectorsData: fromPairs(data),
+    appStats: appStats,
+  });
+
+  return true;
+};
+const afterRun = async () => {
+  await Promise.all(collectors.map((coll) => coll.close()));
+  collectors = [];
+  results = [];
+};
+
+const afterSpec = (resultsFolder) => async (spec) => {
+  fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2));
+
+  results = [];
+};
+
+const initialize = (on, config) => {
+  const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER'];
+
+  if (!fs.existsSync(resultsFolder)) {
+    fs.mkdirSync(resultsFolder, { recursive: true });
+    console.log(`Created folder for benchmark results ${resultsFolder}`);
+  }
+
+  on('before:browser:launch', async (browser, options) => {
+    if (browser.family !== 'chromium' || browser.name === 'electron') {
+      throw new Error('benchmarking plugin requires chrome');
+    }
+
+    const { args } = options;
+
+    const port = getOrAddRemoteDebuggingPort(args);
+    collectors.push(new CDPDataCollector({ port }));
+
+    args.push('--start-fullscreen');
+
+    console.log(
+      `initialized benchmarking plugin with ${collectors.length} collectors: ${collectors
+        .map((col) => col.getName())
+        .join(', ')}`
+    );
+
+    return options;
+  });
+
+  on('task', {
+    startBenchmarking,
+    stopBenchmarking,
+  });
+
+  on('after:run', afterRun);
+  on('after:spec', afterSpec(resultsFolder));
+};
+
+exports.initialize = initialize;

e2e/cypress/plugins/compareScreenshots.js

@@ -0,0 +1,49 @@
+'use strict';
+const BlinkDiff = require('blink-diff');
+const { resolve } = require('path');
+
+// @todo use npmjs.com/pixelmatch or an available cypress plugin
+const compareScreenshots = async ({ config, screenshotsFolder, specName }) => {
+  const name = config.name || config; // @todo use `??`
+  const threshold = config.threshold || 0.001; // @todo use `??`
+
+  const imageAPath = `${screenshotsFolder}/${specName}/${name}.png`;
+  const imageBPath = resolve(`${screenshotsFolder}/../expected/${specName}/${name}.png`);
+
+  const imageOutputPath = screenshotsFolder.endsWith('actual') ? imageAPath.replace('.png', '.diff.png') : undefined;
+
+  const { code } = await new Promise((resolve, reject) => {
+    new BlinkDiff({
+      imageAPath,
+      imageBPath,
+      imageOutputPath,
+      threshold,
+      thresholdType: BlinkDiff.THRESHOLD_PERCENT,
+    }).run((error, result) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    });
+  });
+
+  if (code <= 1) {
+    let msg = `\nThe screenshot [${imageAPath}] differs from [${imageBPath}]`;
+    msg += '\n';
+    msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
+    msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
+    msg += '\n    - Replace the expected image with the outcome and retry.';
+    msg += '\n';
+    throw new Error(msg);
+  } else {
+    // Must return a value
+    return true;
+  }
+};
+
+module.exports = compareScreenshots;

e2e/cypress/plugins/extendConfig.js

@@ -0,0 +1,79 @@
+'use strict';
+const {
+  promises: { readFile },
+} = require('fs');
+const { resolve } = require('path');
+
+// @todo use https://github.com/bahmutov/cypress-extends when possible
+module.exports = async (baseConfig) => {
+  // From CLI
+  const {
+    env: { CWD, UPDATE_SCREENSHOTS },
+  } = baseConfig;
+
+  if (CWD) {
+    // @todo: https://github.com/cypress-io/cypress/issues/6406
+    const jsonReporter = require.resolve('@mochajs/json-file-reporter');
+
+    // @todo `baseUrl: env.CYPRESS_BASEURL`
+    const projectConfig = {
+      fixturesFolder: `${CWD}/cypress/fixtures`,
+      integrationFolder: `${CWD}/cypress/integration`,
+      reporter: jsonReporter,
+      reporterOptions: {
+        output: `${CWD}/cypress/report.json`,
+      },
+      screenshotsFolder: `${CWD}/cypress/screenshots/${UPDATE_SCREENSHOTS ? 'expected' : 'actual'}`,
+      videosFolder: `${CWD}/cypress/videos`,
+    };
+
+    const customProjectConfig = await readFile(`${CWD}/cypress.json`, 'utf8')
+      .then(JSON.parse)
+      .then((config) => {
+        const pathKeys = [
+          'fileServerFolder',
+          'fixturesFolder',
+          'ignoreTestFiles',
+          'integrationFolder',
+          'pluginsFile',
+          'screenshotsFolder',
+          'supportFile',
+          'testFiles',
+          'videosFolder',
+        ];
+
+        return Object.fromEntries(
+          Object.entries(config).map(([key, value]) => {
+            if (pathKeys.includes(key)) {
+              return [key, resolve(CWD, value)];
+            } else {
+              return [key, value];
+            }
+          })
+        );
+      })
+      .catch((error) => {
+        if (error.code === 'ENOENT') {
+          // File is optional
+          return {};
+        } else {
+          // Unexpected error
+          throw error;
+        }
+      });
+
+    return {
+      ...baseConfig,
+      ...projectConfig,
+      ...customProjectConfig,
+      reporterOptions: {
+        ...baseConfig.reporterOptions,
+        ...projectConfig.reporterOptions,
+        ...customProjectConfig.reporterOptions,
+      },
+    };
+  } else {
+    // Temporary legacy support for Grafana core (using `yarn start`)
+    return baseConfig;
+  }
+};

e2e/cypress/plugins/index.js

@@ -0,0 +1,73 @@
+const fs = require('fs');
+const path = require('path');
+
+const benchmarkPlugin = require('./benchmark');
+const compareScreenshots = require('./compareScreenshots');
+const extendConfig = require('./extendConfig');
+const readProvisions = require('./readProvisions');
+const typescriptPreprocessor = require('./typescriptPreprocessor');
+
+module.exports = (on, config) => {
+  if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) {
+    benchmarkPlugin.initialize(on, config);
+  }
+
+  on('file:preprocessor', typescriptPreprocessor);
+  on('task', { compareScreenshots, readProvisions });
+  on('task', {
+    log({ message, optional }) {
+      optional ? console.log(message, optional) : console.log(message);
+      return null;
+    },
+  });
+  on('task', {
+    getJSONFilesFromDir: async ({ projectPath, relativePath }) => {
+      const directoryPath = path.join(projectPath, relativePath);
+      const jsonFiles = fs.readdirSync(directoryPath);
+      return jsonFiles
+        .filter((fileName) => /.json$/i.test(fileName))
+        .map((fileName) => {
+          const fileBuffer = fs.readFileSync(path.join(directoryPath, fileName));
+          return JSON.parse(fileBuffer);
+        });
+    },
+  });
+
+  // Make recordings higher resolution
+  // https://www.cypress.io/blog/2021/03/01/generate-high-resolution-videos-and-screenshots/
+  on('before:browser:launch', (browser = {}, launchOptions) => {
+    console.log('launching browser %s is headless? %s', browser.name, browser.isHeadless);
+
+    // the browser width and height we want to get
+    // our screenshots and videos will be of that resolution
+    const width = 1920;
+    const height = 1080;
+
+    console.log('setting the browser window size to %d x %d', width, height);
+
+    if (browser.name === 'chrome' && browser.isHeadless) {
+      launchOptions.args.push(`--window-size=${width},${height}`);
+
+      // force screen to be non-retina and just use our given resolution
+      launchOptions.args.push('--force-device-scale-factor=1');
+    }
+
+    if (browser.name === 'electron' && browser.isHeadless) {
+      // might not work on CI for some reason
+      launchOptions.preferences.width = width;
+      launchOptions.preferences.height = height;
+    }
+
+    if (browser.name === 'firefox' && browser.isHeadless) {
+      launchOptions.args.push(`--width=${width}`);
+      launchOptions.args.push(`--height=${height}`);
+    }
+
+    // IMPORTANT: return the updated browser launch options
+    return launchOptions;
+  });
+
+  // Always extend with this library's config and return for diffing
+  // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674
+  return extendConfig(config);
+};

e2e/cypress/plugins/readProvisions.js

@@ -0,0 +1,14 @@
+'use strict';
+const {
+  promises: { readFile },
+} = require('fs');
+const { resolve: resolvePath } = require('path');
+const { parse: parseYml } = require('yaml');
+
+const readProvision = (filePath) => readFile(filePath, 'utf8').then((contents) => parseYml(contents));
+
+const readProvisions = (filePaths) => Promise.all(filePaths.map(readProvision));
+
+// Paths are relative to <project-root>/provisioning
+module.exports = ({ CWD, filePaths }) =>
+  readProvisions(filePaths.map((filePath) => resolvePath(CWD, 'provisioning', filePath)));

e2e/cypress/plugins/typescriptPreprocessor.js

@@ -0,0 +1,42 @@
+const wp = require('@cypress/webpack-preprocessor');
+const { resolve } = require('path');
+
+const anyNodeModules = /node_modules/;
+const packageRoot = resolve(`${__dirname}/../../`);
+const packageModules = `${packageRoot}/node_modules`;
+
+const webpackOptions = {
+  module: {
+    rules: [
+      {
+        include: (modulePath) => {
+          if (!anyNodeModules.test(modulePath)) {
+            // Is a file within the project
+            return true;
+          } else {
+            // Is a file within this package
+            return modulePath.startsWith(packageRoot) && !modulePath.startsWith(packageModules);
+          }
+        },
+        test: /\.ts$/,
+        use: [
+          {
+            loader: 'ts-loader',
+            options: {
+              transpileOnly: true,
+            },
+          },
+        ],
+      },
+    ],
+  },
+  resolve: {
+    extensions: ['.ts', '.js'],
+  },
+};
+
+const options = {
+  webpackOptions,
+};
+
+module.exports = wp(options);

e2e/cypress/support/commands.js

@@ -0,0 +1,36 @@
+import 'cypress-file-upload';
+
+Cypress.Commands.add('compareScreenshots', (config) => {
+  cy.task('compareScreenshots', {
+    config,
+    screenshotsFolder: Cypress.config('screenshotsFolder'),
+    specName: Cypress.spec.name,
+  });
+});
+
+Cypress.Commands.add('logToConsole', (message, optional) => {
+  cy.task('log', { message: '(' + new Date().toISOString() + ') ' + message, optional });
+});
+
+Cypress.Commands.add('readProvisions', (filePaths) => {
+  cy.task('readProvisions', {
+    CWD: Cypress.env('CWD'),
+    filePaths,
+  });
+});
+
+Cypress.Commands.add('getJSONFilesFromDir', (dirPath) => {
+  return cy.task('getJSONFilesFromDir', {
+    // CWD is set for plugins in the cli but not for the main grafana repo: https://github.com/grafana/grafana/blob/main/packages/grafana-e2e/cli.js#L12
+    projectPath: Cypress.env('CWD') || Cypress.config().parentTestsFolder,
+    relativePath: dirPath,
+  });
+});
+
+Cypress.Commands.add('startBenchmarking', (testName) => {
+  return cy.task('startBenchmarking', { testName });
+});
+
+Cypress.Commands.add('stopBenchmarking', (testName, appStats) => {
+  return cy.task('stopBenchmarking', { testName, appStats });
+});

e2e/cypress/support/e2e.js

@@ -0,0 +1,45 @@
+require('./commands');
+
+Cypress.Screenshot.defaults({
+  screenshotOnRunFailure: false,
+});
+
+const COMMAND_DELAY = 1000;
+
+if (Cypress.env('SLOWMO')) {
+  const commandsToModify = ['clear', 'click', 'contains', 'reload', 'then', 'trigger', 'type', 'visit'];
+
+  commandsToModify.forEach((command) => {
+    // @ts-ignore -- https://github.com/cypress-io/cypress/issues/7807
+    Cypress.Commands.overwrite(command, (originalFn, ...args) => {
+      const origVal = originalFn(...args);
+
+      return new Promise((resolve) => {
+        setTimeout(() => resolve(origVal), COMMAND_DELAY);
+      });
+    });
+  });
+}
+
+// @todo remove when possible: https://github.com/cypress-io/cypress/issues/95
+Cypress.on('window:before:load', (win) => {
+  // @ts-ignore
+  delete win.fetch;
+});
+
+// See https://github.com/quasarframework/quasar/issues/2233 for details
+const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/;
+Cypress.on('uncaught:exception', (err) => {
+  /* returning false here prevents Cypress from failing the test */
+  if (resizeObserverLoopErrRe.test(err.message)) {
+    return false;
+  }
+  return true;
+});
+
+// uncomment below to prevent Cypress from failing tests when unhandled errors are thrown
+// Cypress.on('uncaught:exception', (err, runnable) => {
+//   // returning false here prevents Cypress from
+//   // failing the test
+//   return false;
+// });

e2e/cypress/support/index.d.ts

@@ -0,0 +1,18 @@
+/// <reference types="cypress" />
+
+interface CompareScreenshotsConfig {
+  name: string;
+  threshold?: number;
+}
+
+declare namespace Cypress {
+  interface Chainable {
+    compareScreenshots(config: CompareScreenshotsConfig | string): Chainable;
+    logToConsole(message: string, optional?: any): void;
+    readProvisions(filePaths: string[]): Chainable;
+    getJSONFilesFromDir(dirPath: string): Chainable;
+    startBenchmarking(testName: string): void;
+    stopBenchmarking(testName: string, appStats: Record<string, unknown>): void;
+    checkHealthRetryable(fn: Function, retryCount: number): Chainable;
+  }
+}

e2e/dashboards-suite/Repeating_a_panel_horizontally.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const PAGE_UNDER_TEST = 'WVpf2jp7z/repeating-a-panel-horizontally';
 
 describe('Repeating a panel horizontally', () => {
@@ -9,7 +9,7 @@ describe('Repeating a panel horizontally', () => {
   it('should be able to repeat a panel horizontally', () => {
     e2e.flows.openDashboard({ uid: PAGE_UNDER_TEST });
     let prevLeft = Number.NEGATIVE_INFINITY;
-    let prevTop = null;
+    let prevTop: number | null = null;
     const panelTitles = ['Panel Title 1', 'Panel Title 2', 'Panel Title 3'];
     panelTitles.forEach((title) => {
       e2e.components.Panels.Panel.title(title)
@@ -30,7 +30,7 @@ describe('Repeating a panel horizontally', () => {
   it('responds to changes to the variables', () => {
     e2e.flows.openDashboard({ uid: PAGE_UNDER_TEST });
     let prevLeft = Number.NEGATIVE_INFINITY;
-    let prevTop = null;
+    let prevTop: number | null = null;
     const panelTitles = ['Panel Title 1', 'Panel Title 2', 'Panel Title 3'];
     panelTitles.forEach((title) => {
       e2e.components.Panels.Panel.title(title).should('be.visible');
@@ -68,7 +68,7 @@ describe('Repeating a panel horizontally', () => {
     // Have to manually add the queryParams to the url because they have the same name
     e2e.flows.openDashboard({ uid: `${PAGE_UNDER_TEST}?var-horizontal=1&var-horizontal=3` });
     let prevLeft = Number.NEGATIVE_INFINITY;
-    let prevTop = null;
+    let prevTop: number | null = null;
     const panelsShown = ['Panel Title 1', 'Panel Title 3'];
     const panelsNotShown = ['Panel Title 2'];
     // Check correct panels are displayed

e2e/dashboards-suite/Repeating_a_panel_vertically.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const PAGE_UNDER_TEST = 'OY8Ghjt7k/repeating-a-panel-vertically';
 
 describe('Repeating a panel vertically', () => {
@@ -10,7 +10,7 @@ describe('Repeating a panel vertically', () => {
     e2e.flows.openDashboard({ uid: PAGE_UNDER_TEST });
 
     let prevTop = Number.NEGATIVE_INFINITY;
-    let prevLeft = null;
+    let prevLeft: number | null = null;
     const panelTitles = ['Panel Title 1', 'Panel Title 2', 'Panel Title 3'];
     panelTitles.forEach((title) => {
       e2e.components.Panels.Panel.title(title)
@@ -31,7 +31,7 @@ describe('Repeating a panel vertically', () => {
     e2e.flows.openDashboard({ uid: PAGE_UNDER_TEST });
 
     let prevTop = Number.NEGATIVE_INFINITY;
-    let prevLeft = null;
+    let prevLeft: number | null = null;
     const panelTitles = ['Panel Title 1', 'Panel Title 2', 'Panel Title 3'];
     panelTitles.forEach((title) => {
       e2e.components.Panels.Panel.title(title).should('be.visible');
@@ -69,7 +69,7 @@ describe('Repeating a panel vertically', () => {
     e2e.flows.openDashboard({ uid: `${PAGE_UNDER_TEST}?var-vertical=1&var-vertical=3` });
 
     let prevTop = Number.NEGATIVE_INFINITY;
-    let prevLeft = null;
+    let prevLeft: number | null = null;
     const panelsShown = ['Panel Title 1', 'Panel Title 3'];
     const panelsNotShown = ['Panel Title 2'];
     panelsShown.forEach((title) => {

e2e/dashboards-suite/Repeating_a_row_with_a_non_repeating_panel.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const PAGE_UNDER_TEST = 'k3PEoCpnk/repeating-a-row-with-a-non-repeating-panel-and-horizontal-repeating-panel';
 const DASHBOARD_NAME = 'Repeating a row with a non-repeating panel and horizontal repeating panel';
 

e2e/dashboards-suite/Repeating_an_empty_row.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const PAGE_UNDER_TEST = 'dtpl2Ctnk/repeating-an-empty-row';
 
 describe('Repeating empty rows', () => {

e2e/dashboards-suite/dashboard-browse-nested.spec.ts

@@ -1,6 +1,7 @@
-import { e2e } from '@grafana/e2e';
 import { selectors } from '@grafana/e2e-selectors';
 
+import { e2e } from '../utils';
+
 import { makeNewDashboardRequestBody } from './utils/makeDashboard';
 
 const NUM_ROOT_FOLDERS = 60;

e2e/dashboards-suite/dashboard-browse.spec.ts

@@ -1,6 +1,5 @@
-import { e2e } from '@grafana/e2e';
-
 import testDashboard from '../dashboards/TestDashboard.json';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Dashboard browse',

e2e/dashboards-suite/dashboard-public-create.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Create a public dashboard',

e2e/dashboards-suite/dashboard-public-templating.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Create a public dashboard with template variables shows a template variable warning',

e2e/dashboards-suite/dashboard-templating.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Dashboard templating',

e2e/dashboards-suite/dashboard-time-zone.spec.ts

@@ -9,7 +9,7 @@ import {
   toDate,
 } from 'date-fns';
 
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Dashboard time zone support',
@@ -36,7 +36,7 @@ e2e.scenario({
     const timesInUtc: Record<string, string> = {};
 
     for (const title of panelsToCheck) {
-      e2e.components.Panels.Panel.containerByTitle(title)
+      e2e.components.Panels.Panel.title(title)
         .should('be.visible')
         .within(() =>
           e2e.components.Panels.Visualization.Graph.xAxis
@@ -61,11 +61,11 @@ e2e.scenario({
     e2e.components.Select.option().should('be.visible').contains(toTimeZone).click();
 
     // click to go back to the dashboard.
-    e2e.components.BackButton.backArrow().click({ force: true });
+    e2e.pages.Dashboard.Settings.Actions.close().click();
     e2e.components.RefreshPicker.runButtonV2().should('be.visible').click();
 
     for (const title of panelsToCheck) {
-      e2e.components.Panels.Panel.containerByTitle(title)
+      e2e.components.Panels.Panel.title(title)
         .should('be.visible')
         .within(() =>
           e2e.components.Panels.Visualization.Graph.xAxis
@@ -76,9 +76,7 @@ e2e.scenario({
               const inUtc = timesInUtc[title];
               const inTz = element.text();
               const isCorrect = isTimeCorrect(inUtc, inTz, offset);
-              expect(isCorrect, `Expect the panel "${title}" to have the new timezone applied but isn't`).to.be.equal(
+              expect(isCorrect).to.be.equal(true);
-                true
-              );
             })
         );
     }

e2e/dashboards-suite/dashboard-timepicker.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Dashboard timepicker',

e2e/dashboards-suite/import-dashboard.spec.ts

@@ -1,6 +1,5 @@
-import { e2e } from '@grafana/e2e';
-
 import testDashboard from '../dashboards/TestDashboard.json';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Import Dashboards Test',

e2e/dashboards-suite/load-options-from-url.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = '-Y-tnEDWk/templating-nested-template-variables';
 

e2e/dashboards-suite/new-constant-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'kVi2Gex7z/test-variable-output';
 const DASHBOARD_NAME = 'Test variable output';

e2e/dashboards-suite/new-custom-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'kVi2Gex7z/test-variable-output';
 const DASHBOARD_NAME = 'Test variable output';

e2e/dashboards-suite/new-datasource-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'kVi2Gex7z/test-variable-output';
 const DASHBOARD_NAME = 'Test variable output';

e2e/dashboards-suite/new-interval-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'kVi2Gex7z/test-variable-output';
 const DASHBOARD_NAME = 'Test variable output';

e2e/dashboards-suite/new-query-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = '-Y-tnEDWk/templating-nested-template-variables';
 const DASHBOARD_NAME = 'Templating - Nested Template Variables';

e2e/dashboards-suite/new-text-box-variable.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'kVi2Gex7z/test-variable-output';
 const DASHBOARD_NAME = 'Test variable output';

e2e/dashboards-suite/set-options-from-ui.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = '-Y-tnEDWk/templating-nested-template-variables';
 

e2e/dashboards-suite/templating-dashboard-links-and-variables.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Templating',

e2e/dashboards-suite/textbox-variables.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PAGE_UNDER_TEST = 'AejrN1AMz';
 

e2e/datagrid-suite/datagrid-data-change.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const DASHBOARD_ID = 'c01bf42b-b783-4447-a304-8554cee1843b';
 const DATAGRID_SELECT_SERIES = 'Datagrid Select series';

e2e/datagrid-suite/datagrid-editing-features.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const DASHBOARD_ID = 'c01bf42b-b783-4447-a304-8554cee1843b';
 const DATAGRID_CANVAS = 'data-grid-canvas';

e2e/e2e-flow-suite/00-panel-menu-extension.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Panel menu ui extension flow',

e2e/geomap-suite/geomap-layer-types.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const DASHBOARD_ID = 'P2jR04WVk';
 

e2e/geomap-suite/geomap-map-controls.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const DASHBOARD_ID = 'P2jR04WVk';
 
 e2e.scenario({

e2e/geomap-suite/geomap-spatial-operations-transform.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const DASHBOARD_ID = 'P2jR04WVk';
 

e2e/panels-suite/panelEdit_base.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const PANEL_UNDER_TEST = 'Lines 500 data points';
 

e2e/panels-suite/panelEdit_queries.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const flakyTimeout = 10000;
 
@@ -39,12 +39,11 @@ e2e.scenario({
     e2e.components.QueryEditorRow.actionButton('Duplicate query').eq(0).should('be.visible').click();
 
     // We expect row with refId Band and A to exist and be visible
-    e2e.components.QueryEditorRows.rows().within((rows) => {
+    e2e.components.QueryEditorRows.rows().should('have.length', 2);
-      expect(rows.length).equals(2);
-    });
 
     // Change to CSV Metric Values scenario for A
     e2e.components.DataSource.TestData.QueryTab.scenarioSelectContainer()
+      .first()
       .should('be.visible')
       .within(() => {
         e2e().get('input[id*="test-data-scenario-select-"]').eq(0).should('be.visible').click();
@@ -95,7 +94,7 @@ const expectInspectorResultAndClose = (expectCallBack: (keys: JQuery<HTMLElement
 
   e2e.components.PanelInspector.Query.jsonObjectKeys({ timeout: flakyTimeout })
     .should('be.visible')
-    .within((keys) => expectCallBack(keys));
+    .should((keys) => expectCallBack(keys));
 
   e2e.components.Drawer.General.close().should('be.visible').click();
 };

e2e/panels-suite/panelEdit_transforms.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Panel edit tests - transformations',

e2e/run-suite

@@ -16,7 +16,7 @@ echo -e "Starting Cypress scenarios"
 
 args=("$@")
 
-CMD="start"
+CMD="cy:run"
 PARAMS=""
 CLEANUP=""
 
@@ -26,25 +26,24 @@ declare -A env=(
 )
 
 testFilesForSingleSuite="*.spec.ts"
-rootForEnterpriseSuite="extensions-suite"
+rootForEnterpriseSuite="./e2e/extensions-suite"
 
 declare -A cypressConfig=(
-  [integrationFolder]=../../e2e
+  [screenshotsFolder]=./e2e/"${args[0]}"/screenshots
-  [screenshotsFolder]=../../e2e/"${args[0]}"/screenshots
+  [fixturesFolder]=./e2e/cypress/fixtures
-  [videosFolder]=../../e2e/"${args[0]}"/videos
+  [videosFolder]=./e2e/"${args[0]}"/videos
-  [fileServerFolder]=./cypress
+  [downloadsFolder]=./e2e/cypress/downloads
-  [testFiles]=*-suite/*spec.ts
+  [fileServerFolder]=./e2e/cypress
+  [specPattern]=./e2e/*-suite/*spec.ts
   [defaultCommandTimeout]=30000
   [viewportWidth]=1920
   [viewportHeight]=1080
   [trashAssetsBeforeRuns]=false
   [videoUploadOnPasses]=false
-  [reporter]=../../e2e/log-reporter.js
+  [reporter]=./e2e/log-reporter.js
+  [baseUrl]=${BASE_URL:-"http://$HOST:$PORT"}
 )
 
-
-cd packages/grafana-e2e
-
 case "$1" in
     "debug")
         echo -e "Debug mode"
@@ -53,23 +52,22 @@ case "$1" in
         ;;
     "dev")
         echo "Dev mode"
-        CMD="open"
+        CMD="cy:open"
         ;;
     "benchmark")
         echo "Benchmark"
         PARAMS="--headed"
-        CMD="start-benchmark"
+        CMD="cy:benchmark"
         env[BENCHMARK_PLUGIN_ENABLED]=true
-        env[BENCHMARK_PLUGIN_RESULTS_FOLDER]=../../e2e/benchmarks/"${args[1]}"/results
+        env[BENCHMARK_PLUGIN_RESULTS_FOLDER]=./e2e/benchmarks/"${args[1]}"/results
         cypressConfig[video]=false
-        cypressConfig[integrationFolder]=../../e2e/benchmarks/"${args[1]}"
+        cypressConfig[screenshotsFolder]=./e2e/benchmarks/"${args[1]}"/screenshots
-        cypressConfig[screenshotsFolder]=../../e2e/benchmarks/"${args[1]}"/screenshots
+        cypressConfig[specPattern]=./e2e/benchmarks/"${args[1]}"/$testFilesForSingleSuite
-        cypressConfig[testFiles]=$testFilesForSingleSuite
         ;;
     "enterprise")
         echo "Enterprise"
-        CLEANUP="rm -rf ../../e2e/extensions-suite"
+        CLEANUP="rm -rf ./e2e/extensions-suite"
-        SETUP="cp -Lr ../../e2e/extensions ../../e2e/extensions-suite"
+        SETUP="cp -Lr ./e2e/extensions ./e2e/extensions-suite"
         enterpriseSuite=$(basename "${args[1]}")
         case "$2" in
             "debug")
@@ -80,18 +78,17 @@ case "$1" in
                 ;;
             "dev")
                 echo "Dev mode"
-                CMD="open"
+                CMD="cy:open"
                 enterpriseSuite=$(basename "${args[2]}")
                 ;;
         esac
-        cypressConfig[testFiles]=$rootForEnterpriseSuite/$enterpriseSuite/*-suite/*.spec.ts
+        cypressConfig[specPattern]=$rootForEnterpriseSuite/$enterpriseSuite/*-suite/*.spec.ts
         $CLEANUP && $SETUP
         ;;
     "")
         ;;
     *)
-        cypressConfig[integrationFolder]=../../e2e/"${args[0]}"
+        cypressConfig[specPattern]=./e2e/"${args[0]}"/$testFilesForSingleSuite
-        cypressConfig[testFiles]=$testFilesForSingleSuite
         cypressConfig[video]=${args[1]}
         ;;
 esac

e2e/shared/smokeTestScenario.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 export const smokeTestScenario = {
   describeName: 'Smoke tests',

e2e/shared/tsconfig.json

@@ -1,5 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "include": ["**/*.ts", "../../packages/grafana-e2e/cypress/support/index.d.ts"],
-  "resolveJsonModule": true
-}

e2e/smoke-tests-suite/1-smoketests.spec.ts

@@ -1,5 +1,4 @@
-import { e2e } from '@grafana/e2e';
-
 import { smokeTestScenario } from '../shared/smokeTestScenario';
+import { e2e } from '../utils';
 
 e2e.scenario(smokeTestScenario);

e2e/smoke-tests-suite/panels_smokescreen.spec.ts

@@ -1,6 +1,7 @@
-import { e2e } from '@grafana/e2e';
 import { GrafanaBootConfig } from '@grafana/runtime';
 
+import { e2e } from '../utils';
+
 e2e.scenario({
   describeName: 'Panels smokescreen',
   itName: 'Tests each panel type in the panel edit view to ensure no crash',

e2e/sql-suite/mysql.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 import datasetResponse from './datasets-response.json';
 import fieldsResponse from './fields-response.json';

e2e/tsconfig.json

@@ -4,5 +4,5 @@
     "resolveJsonModule": true
   },
   "extends": "@grafana/tsconfig/base.json",
-  "include": ["**/*.ts"]
+  "include": ["**/*.ts", "cypress/support/e2e.js", "cypress/support/index.d.ts"]
 }

e2e/utils/flows/addDashboard.ts

@@ -0,0 +1,303 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { e2e } from '../index';
+import { getDashboardUid } from '../support/url';
+
+import { DeleteDashboardConfig } from './deleteDashboard';
+import { selectOption } from './selectOption';
+import { setDashboardTimeRange, TimeRangeConfig } from './setDashboardTimeRange';
+
+export interface AddAnnotationConfig {
+  dataSource: string;
+  dataSourceForm?: () => void;
+  name: string;
+}
+
+export interface AddDashboardConfig {
+  annotations: AddAnnotationConfig[];
+  timeRange: TimeRangeConfig;
+  title: string;
+  variables: PartialAddVariableConfig[];
+}
+
+interface AddVariableDefault {
+  hide: string;
+  type: string;
+}
+
+interface AddVariableOptional {
+  constantValue?: string;
+  dataSource?: string;
+  label?: string;
+  query?: string;
+  regex?: string;
+  variableQueryForm?: (config: AddVariableConfig) => void;
+}
+
+interface AddVariableRequired {
+  name: string;
+}
+
+export type PartialAddVariableConfig = Partial<AddVariableDefault> & AddVariableOptional & AddVariableRequired;
+export type AddVariableConfig = AddVariableDefault & AddVariableOptional & AddVariableRequired;
+
+/**
+ * This flow is used to add a dashboard with whatever configuration specified.
+ * @param config Configuration object. Currently supports configuring dashboard time range, annotations, and variables (support dependant on type).
+ * @see{@link AddDashboardConfig}
+ *
+ * @example
+ * ```
+ * // Configuring a simple dashboard
+ * addDashboard({
+ *    timeRange: {
+ *      from: '2022-10-03 00:00:00',
+ *      to: '2022-10-03 23:59:59',
+ *      zone: 'Coordinated Universal Time',
+ *    },
+ *    title: 'Test Dashboard',
+ * })
+ * ```
+ *
+ * @example
+ * ```
+ * // Configuring a dashboard with annotations
+ * addDashboard({
+ *    title: 'Test Dashboard',
+ *    annotations: [
+ *      {
+ *        // This should match the datasource name
+ *        dataSource: 'azure-monitor',
+ *        name: 'Test Annotation',
+ *        dataSourceForm: () => {
+ *          // Insert steps to create annotation using datasource form
+ *        }
+ *      }
+ *    ]
+ * })
+ * ```
+ *
+ * @see{@link AddAnnotationConfig}
+ *
+ * @example
+ * ```
+ * // Configuring a dashboard with variables
+ * addDashboard({
+ *    title: 'Test Dashboard',
+ *    variables: [
+ *      {
+ *        name: 'test-query-variable',
+ *        label: 'Testing Query',
+ *        hide: '',
+ *        type: e2e.flows.VARIABLE_TYPE_QUERY,
+ *        dataSource: 'azure-monitor',
+ *        variableQueryForm: () => {
+ *          // Insert steps to create variable using datasource form
+ *        },
+ *      },
+ *      {
+ *        name: 'test-constant-variable',
+ *        label: 'Testing Constant',
+ *        type: e2e.flows.VARIABLE_TYPE_CONSTANT,
+ *        constantValue: 'constant',
+ *      }
+ *    ]
+ * })
+ * ```
+ *
+ * @see{@link AddVariableConfig}
+ *
+ * @see{@link https://github.com/grafana/grafana/blob/main/e2e/cloud-plugins-suite/azure-monitor.spec.ts Azure Monitor Tests for full examples}
+ */
+export const addDashboard = (config?: Partial<AddDashboardConfig>) => {
+  const fullConfig: AddDashboardConfig = {
+    annotations: [],
+    title: `e2e-${uuidv4()}`,
+    variables: [],
+    ...config,
+    timeRange: {
+      from: '2020-01-01 00:00:00',
+      to: '2020-01-01 06:00:00',
+      zone: 'Coordinated Universal Time',
+      ...config?.timeRange,
+    },
+  };
+
+  const { annotations, timeRange, title, variables } = fullConfig;
+
+  e2e().logToConsole('Adding dashboard with title:', title);
+
+  e2e.pages.AddDashboard.visit();
+
+  if (annotations.length > 0 || variables.length > 0) {
+    e2e.components.PageToolbar.item('Dashboard settings').click();
+    addAnnotations(annotations);
+
+    fullConfig.variables = addVariables(variables);
+
+    e2e.components.BackButton.backArrow().should('be.visible').click({ force: true });
+  }
+
+  setDashboardTimeRange(timeRange);
+
+  e2e.components.PageToolbar.item('Save dashboard').click();
+  e2e.pages.SaveDashboardAsModal.newName().clear().type(title, { force: true });
+  e2e.pages.SaveDashboardAsModal.save().click();
+  e2e.flows.assertSuccessNotification();
+  e2e.pages.AddDashboard.itemButton('Create new panel button').should('be.visible');
+
+  e2e().logToConsole('Added dashboard with title:', title);
+
+  return e2e()
+    .url()
+    .should('contain', '/d/')
+    .then((url: string) => {
+      const uid = getDashboardUid(url);
+
+      e2e.getScenarioContext().then(({ addedDashboards }: any) => {
+        e2e.setScenarioContext({
+          addedDashboards: [...addedDashboards, { title, uid } as DeleteDashboardConfig],
+        });
+      });
+
+      // @todo remove `wrap` when possible
+      return e2e().wrap(
+        {
+          config: fullConfig,
+          uid,
+        },
+        { log: false }
+      );
+    });
+};
+
+const addAnnotation = (config: AddAnnotationConfig, isFirst: boolean) => {
+  if (isFirst) {
+    if (e2e.pages.Dashboard.Settings.Annotations.List.addAnnotationCTAV2) {
+      e2e.pages.Dashboard.Settings.Annotations.List.addAnnotationCTAV2().click();
+    } else {
+      e2e.pages.Dashboard.Settings.Annotations.List.addAnnotationCTA().click();
+    }
+  } else {
+    cy.contains('New query').click();
+  }
+
+  const { dataSource, dataSourceForm, name } = config;
+
+  selectOption({
+    container: e2e.components.DataSourcePicker.container(),
+    optionText: dataSource,
+  });
+
+  e2e.pages.Dashboard.Settings.Annotations.Settings.name().clear().type(name);
+
+  if (dataSourceForm) {
+    dataSourceForm();
+  }
+};
+
+const addAnnotations = (configs: AddAnnotationConfig[]) => {
+  if (configs.length > 0) {
+    e2e.pages.Dashboard.Settings.General.sectionItems('Annotations').click();
+  }
+
+  return configs.forEach((config, i) => addAnnotation(config, i === 0));
+};
+
+export const VARIABLE_HIDE_LABEL = 'Label';
+export const VARIABLE_HIDE_NOTHING = '';
+export const VARIABLE_HIDE_VARIABLE = 'Variable';
+
+export const VARIABLE_TYPE_AD_HOC_FILTERS = 'Ad hoc filters';
+export const VARIABLE_TYPE_CONSTANT = 'Constant';
+export const VARIABLE_TYPE_DATASOURCE = 'Datasource';
+export const VARIABLE_TYPE_QUERY = 'Query';
+
+const addVariable = (config: PartialAddVariableConfig, isFirst: boolean): AddVariableConfig => {
+  const fullConfig = {
+    hide: VARIABLE_HIDE_NOTHING,
+    type: VARIABLE_TYPE_QUERY,
+    ...config,
+  };
+
+  if (isFirst) {
+    if (e2e.pages.Dashboard.Settings.Variables.List.addVariableCTAV2) {
+      e2e.pages.Dashboard.Settings.Variables.List.addVariableCTAV2().click();
+    } else {
+      e2e.pages.Dashboard.Settings.Variables.List.addVariableCTA().click();
+    }
+  } else {
+    e2e.pages.Dashboard.Settings.Variables.List.newButton().click();
+  }
+
+  const { constantValue, dataSource, label, name, query, regex, type, variableQueryForm } = fullConfig;
+
+  // This field is key to many reactive changes
+  if (type !== VARIABLE_TYPE_QUERY) {
+    e2e.pages.Dashboard.Settings.Variables.Edit.General.generalTypeSelectV2()
+      .should('be.visible')
+      .within(() => {
+        e2e.components.Select.singleValue().should('have.text', 'Query').parent().click();
+      });
+    e2e.pages.Dashboard.Settings.Variables.Edit.General.generalTypeSelectV2().find('input').type(`${type}{enter}`);
+  }
+
+  if (label) {
+    e2e.pages.Dashboard.Settings.Variables.Edit.General.generalLabelInputV2().type(label);
+  }
+
+  e2e.pages.Dashboard.Settings.Variables.Edit.General.generalNameInputV2().clear().type(name);
+
+  if (
+    dataSource &&
+    (type === VARIABLE_TYPE_AD_HOC_FILTERS || type === VARIABLE_TYPE_DATASOURCE || type === VARIABLE_TYPE_QUERY)
+  ) {
+    e2e.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsDataSourceSelect()
+      .should('be.visible')
+      .within(() => {
+        e2e.components.DataSourcePicker.inputV2().type(`${dataSource}{enter}`);
+      });
+  }
+
+  if (constantValue && type === VARIABLE_TYPE_CONSTANT) {
+    e2e.pages.Dashboard.Settings.Variables.Edit.ConstantVariable.constantOptionsQueryInputV2().type(constantValue);
+  }
+
+  if (type === VARIABLE_TYPE_QUERY) {
+    if (query) {
+      e2e.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput().type(query);
+    }
+
+    if (regex) {
+      e2e.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2().type(regex);
+    }
+
+    if (variableQueryForm) {
+      variableQueryForm(fullConfig);
+    }
+  }
+
+  // Avoid flakiness
+  e2e().focused().blur();
+
+  e2e.pages.Dashboard.Settings.Variables.Edit.General.previewOfValuesOption()
+    .should('exist')
+    .within((previewOfValues) => {
+      if (type === VARIABLE_TYPE_CONSTANT) {
+        expect(previewOfValues.text()).equals(constantValue);
+      }
+    });
+
+  e2e.pages.Dashboard.Settings.Variables.Edit.General.submitButton().click();
+  e2e.pages.Dashboard.Settings.Variables.Edit.General.applyButton().click();
+
+  return fullConfig;
+};
+
+const addVariables = (configs: PartialAddVariableConfig[]): AddVariableConfig[] => {
+  if (configs.length > 0) {
+    e2e.components.Tab.title('Variables').click();
+  }
+
+  return configs.map((config, i) => addVariable(config, i === 0));
+};

e2e/utils/flows/addDataSource.ts

@@ -0,0 +1,116 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { e2e } from '../index';
+
+import { DeleteDataSourceConfig } from './deleteDataSource';
+
+export interface AddDataSourceConfig {
+  basicAuth: boolean;
+  basicAuthPassword: string;
+  basicAuthUser: string;
+  expectedAlertMessage: string | RegExp;
+  form: () => void;
+  name: string;
+  skipTlsVerify: boolean;
+  type: string;
+  timeout?: number;
+  awaitHealth?: boolean;
+}
+
+// @todo this actually returns type `Cypress.Chainable<AddDaaSourceConfig>`
+export const addDataSource = (config?: Partial<AddDataSourceConfig>) => {
+  const fullConfig: AddDataSourceConfig = {
+    basicAuth: false,
+    basicAuthPassword: '',
+    basicAuthUser: '',
+    expectedAlertMessage: 'Data source is working',
+    form: () => {},
+    name: `e2e-${uuidv4()}`,
+    skipTlsVerify: false,
+    type: 'TestData',
+    ...config,
+  };
+
+  const {
+    basicAuth,
+    basicAuthPassword,
+    basicAuthUser,
+    expectedAlertMessage,
+    form,
+    name,
+    skipTlsVerify,
+    type,
+    timeout,
+    awaitHealth,
+  } = fullConfig;
+
+  if (awaitHealth) {
+    e2e()
+      .intercept(/health/)
+      .as('health');
+  }
+
+  e2e().logToConsole('Adding data source with name:', name);
+  e2e.pages.AddDataSource.visit();
+  e2e.pages.AddDataSource.dataSourcePluginsV2(type)
+    .scrollIntoView()
+    .should('be.visible') // prevents flakiness
+    .click();
+
+  e2e.pages.DataSource.name().clear();
+  e2e.pages.DataSource.name().type(name);
+
+  if (basicAuth) {
+    e2e().contains('label', 'Basic auth').scrollIntoView().click();
+    e2e()
+      .contains('.gf-form-group', 'Basic Auth Details')
+      .should('be.visible')
+      .scrollIntoView()
+      .within(() => {
+        if (basicAuthUser) {
+          e2e().get('[placeholder=user]').type(basicAuthUser);
+        }
+        if (basicAuthPassword) {
+          e2e().get('[placeholder=Password]').type(basicAuthPassword);
+        }
+      });
+  }
+
+  if (skipTlsVerify) {
+    e2e().contains('label', 'Skip TLS Verify').scrollIntoView().click();
+  }
+
+  form();
+
+  e2e.pages.DataSource.saveAndTest().click();
+
+  if (awaitHealth) {
+    e2e().wait('@health', { timeout: timeout ?? e2e.config().defaultCommandTimeout });
+  }
+
+  // use the timeout passed in if it exists, otherwise, continue to use the default
+  e2e.pages.DataSource.alert()
+    .should('exist')
+    .contains(expectedAlertMessage, {
+      timeout: timeout ?? e2e.config().defaultCommandTimeout,
+    });
+  e2e().logToConsole('Added data source with name:', name);
+
+  return e2e()
+    .url()
+    .then(() => {
+      e2e.getScenarioContext().then(({ addedDataSources }: any) => {
+        e2e.setScenarioContext({
+          addedDataSources: [...addedDataSources, { name } as DeleteDataSourceConfig],
+        });
+      });
+
+      // @todo remove `wrap` when possible
+      return e2e().wrap(
+        {
+          config: fullConfig,
+        },
+        { log: false }
+      );
+    });
+};

e2e/utils/flows/addPanel.ts

@@ -0,0 +1,15 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { getScenarioContext } from '../support/scenarioContext';
+
+import { configurePanel, PartialAddPanelConfig } from './configurePanel';
+
+export const addPanel = (config?: Partial<PartialAddPanelConfig>) =>
+  getScenarioContext().then(({ lastAddedDataSource }: any) =>
+    configurePanel({
+      dataSourceName: lastAddedDataSource,
+      panelTitle: `e2e-${uuidv4()}`,
+      ...config,
+      isEdit: false,
+    })
+  );

e2e/utils/flows/assertSuccessNotification.ts

@@ -0,0 +1,9 @@
+import { e2e } from '../index';
+
+export const assertSuccessNotification = () => {
+  if (e2e.components.Alert.alertV2) {
+    e2e.components.Alert.alertV2('success').should('exist');
+  } else {
+    e2e.components.Alert.alert('success').should('exist');
+  }
+};

e2e/utils/flows/configurePanel.ts

@@ -0,0 +1,192 @@
+import { e2e } from '..';
+import { getScenarioContext } from '../support/scenarioContext';
+
+import { setDashboardTimeRange } from './setDashboardTimeRange';
+import { TimeRangeConfig } from './setTimeRange';
+
+interface AddPanelOverrides {
+  dataSourceName: string;
+  queriesForm: (config: AddPanelConfig) => void;
+  panelTitle: string;
+}
+
+interface EditPanelOverrides {
+  queriesForm?: (config: EditPanelConfig) => void;
+  panelTitle: string;
+}
+
+interface ConfigurePanelDefault {
+  chartData: {
+    method: string;
+    route: string | RegExp;
+  };
+  dashboardUid: string;
+  matchScreenshot: boolean;
+  saveDashboard: boolean;
+  screenshotName: string;
+  visitDashboardAtStart: boolean; // @todo remove when possible
+}
+
+interface ConfigurePanelOptional {
+  dataSourceName?: string;
+  queriesForm?: (config: ConfigurePanelConfig) => void;
+  panelTitle?: string;
+  timeRange?: TimeRangeConfig;
+  visualizationName?: string;
+  timeout?: number;
+}
+
+interface ConfigurePanelRequired {
+  isEdit: boolean;
+}
+
+export type PartialConfigurePanelConfig = Partial<ConfigurePanelDefault> &
+  ConfigurePanelOptional &
+  ConfigurePanelRequired;
+
+export type ConfigurePanelConfig = ConfigurePanelDefault & ConfigurePanelOptional & ConfigurePanelRequired;
+
+export type PartialAddPanelConfig = PartialConfigurePanelConfig & AddPanelOverrides;
+export type AddPanelConfig = ConfigurePanelConfig & AddPanelOverrides;
+
+export type PartialEditPanelConfig = PartialConfigurePanelConfig & EditPanelOverrides;
+export type EditPanelConfig = ConfigurePanelConfig & EditPanelOverrides;
+
+// @todo this actually returns type `Cypress.Chainable<AddPanelConfig | EditPanelConfig | ConfigurePanelConfig>`
+export const configurePanel = (config: PartialAddPanelConfig | PartialEditPanelConfig | PartialConfigurePanelConfig) =>
+  getScenarioContext().then(({ lastAddedDashboardUid }: any) => {
+    const fullConfig: AddPanelConfig | EditPanelConfig | ConfigurePanelConfig = {
+      chartData: {
+        method: 'POST',
+        route: '/api/ds/query',
+      },
+      dashboardUid: lastAddedDashboardUid,
+      matchScreenshot: false,
+      saveDashboard: true,
+      screenshotName: 'panel-visualization',
+      visitDashboardAtStart: true,
+      ...config,
+    };
+
+    const {
+      chartData,
+      dashboardUid,
+      dataSourceName,
+      isEdit,
+      matchScreenshot,
+      panelTitle,
+      queriesForm,
+      screenshotName,
+      timeRange,
+      visitDashboardAtStart,
+      visualizationName,
+      timeout,
+    } = fullConfig;
+
+    if (visitDashboardAtStart) {
+      e2e.flows.openDashboard({ uid: dashboardUid });
+    }
+
+    if (isEdit) {
+      e2e.components.Panels.Panel.title(panelTitle).click();
+      e2e.components.Panels.Panel.headerItems('Edit').click();
+    } else {
+      try {
+        e2e.components.PageToolbar.itemButton('Add button').should('be.visible');
+        e2e.components.PageToolbar.itemButton('Add button').click();
+      } catch (e) {
+        // Depending on the screen size, the "Add" button might be hidden
+        e2e.components.PageToolbar.item('Show more items').click();
+        e2e.components.PageToolbar.item('Add button').last().click();
+      }
+      e2e.pages.AddDashboard.itemButton('Add new visualization menu item').should('be.visible');
+      e2e.pages.AddDashboard.itemButton('Add new visualization menu item').click();
+    }
+
+    if (timeRange) {
+      setDashboardTimeRange(timeRange);
+    }
+
+    // @todo alias '/**/*.js*' as '@pluginModule' when possible: https://github.com/cypress-io/cypress/issues/1296
+
+    e2e().intercept(chartData.method, chartData.route).as('chartData');
+
+    if (dataSourceName) {
+      e2e.components.DataSourcePicker.container().click().type(`${dataSourceName}{downArrow}{enter}`);
+    }
+
+    // @todo instead wait for '@pluginModule' if not already loaded
+    e2e().wait(2000);
+
+    // `panelTitle` is needed to edit the panel, and unlikely to have its value changed at that point
+    const changeTitle = panelTitle && !isEdit;
+
+    if (changeTitle || visualizationName) {
+      if (changeTitle && panelTitle) {
+        e2e.components.PanelEditor.OptionsPane.fieldLabel('Panel options Title').type(`{selectall}${panelTitle}`);
+      }
+
+      if (visualizationName) {
+        e2e.components.PluginVisualization.item(visualizationName).scrollIntoView().click();
+
+        // @todo wait for '@pluginModule' if not a core visualization and not already loaded
+        e2e().wait(2000);
+      }
+    } else {
+      // Consistently closed
+      closeOptions();
+    }
+
+    if (queriesForm) {
+      queriesForm(fullConfig);
+
+      // Wait for a possible complex visualization to render (or something related, as this isn't necessary on the dashboard page)
+      // Can't assert that its HTML changed because a new query could produce the same results
+      e2e().wait(1000);
+    }
+
+    // @todo enable when plugins have this implemented
+    //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click();
+    //e2e().wait('@chartData');
+    //e2e.components.Panels.Panel.containerByTitle(panelTitle).find('.panel-content').contains('No data');
+    //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click();
+    //e2e().wait('@chartData');
+
+    // Avoid annotations flakiness
+    e2e.components.RefreshPicker.runButtonV2().first().click({ force: true });
+
+    // Wait for RxJS
+    e2e().wait(timeout ?? e2e.config().defaultCommandTimeout);
+
+    if (matchScreenshot) {
+      let visualization;
+
+      visualization = e2e.components.Panels.Panel.containerByTitle(panelTitle).find('.panel-content');
+
+      visualization.scrollIntoView().screenshot(screenshotName);
+      e2e().compareScreenshots(screenshotName);
+    }
+
+    // @todo remove `wrap` when possible
+    return e2e().wrap({ config: fullConfig }, { log: false });
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+const closeOptions = () => e2e.components.PanelEditor.toggleVizOptions().click();
+
+export const VISUALIZATION_ALERT_LIST = 'Alert list';
+export const VISUALIZATION_BAR_GAUGE = 'Bar gauge';
+export const VISUALIZATION_CLOCK = 'Clock';
+export const VISUALIZATION_DASHBOARD_LIST = 'Dashboard list';
+export const VISUALIZATION_GAUGE = 'Gauge';
+export const VISUALIZATION_GRAPH = 'Graph';
+export const VISUALIZATION_HEAT_MAP = 'Heatmap';
+export const VISUALIZATION_LOGS = 'Logs';
+export const VISUALIZATION_NEWS = 'News';
+export const VISUALIZATION_PIE_CHART = 'Pie Chart';
+export const VISUALIZATION_PLUGIN_LIST = 'Plugin list';
+export const VISUALIZATION_POLYSTAT = 'Polystat';
+export const VISUALIZATION_STAT = 'Stat';
+export const VISUALIZATION_TABLE = 'Table';
+export const VISUALIZATION_TEXT = 'Text';
+export const VISUALIZATION_WORLD_MAP = 'Worldmap Panel';

e2e/utils/flows/deleteDashboard.ts

@@ -0,0 +1,51 @@
+import { e2e } from '../index';
+import { fromBaseUrl } from '../support/url';
+
+export interface DeleteDashboardConfig {
+  quick?: boolean;
+  title: string;
+  uid: string;
+}
+
+export const deleteDashboard = ({ quick = false, title, uid }: DeleteDashboardConfig) => {
+  e2e().logToConsole('Deleting dashboard with uid:', uid);
+
+  if (quick) {
+    quickDelete(uid);
+  } else {
+    uiDelete(uid, title);
+  }
+
+  e2e().logToConsole('Deleted dashboard with uid:', uid);
+
+  e2e.getScenarioContext().then(({ addedDashboards }: any) => {
+    e2e.setScenarioContext({
+      addedDashboards: addedDashboards.filter((dashboard: DeleteDashboardConfig) => {
+        return dashboard.title !== title && dashboard.uid !== uid;
+      }),
+    });
+  });
+};
+
+const quickDelete = (uid: string) => {
+  e2e().request('DELETE', fromBaseUrl(`/api/dashboards/uid/${uid}`));
+};
+
+const uiDelete = (uid: string, title: string) => {
+  e2e.pages.Dashboard.visit(uid);
+  e2e.components.PageToolbar.item('Dashboard settings').click();
+  e2e.pages.Dashboard.Settings.General.deleteDashBoard().click();
+  e2e.pages.ConfirmModal.delete().click();
+  e2e.flows.assertSuccessNotification();
+
+  e2e.pages.Dashboards.visit();
+
+  // @todo replace `e2e.pages.Dashboards.dashboards` with this when argument is empty
+  if (e2e.components.Search.dashboardItems) {
+    e2e.components.Search.dashboardItems().each((item) => e2e().wrap(item).should('not.contain', title));
+  } else {
+    e2e()
+      .get('[aria-label^="Dashboard search item "]')
+      .each((item) => e2e().wrap(item).should('not.contain', title));
+  }
+};

e2e/utils/flows/deleteDataSource.ts

@@ -0,0 +1,46 @@
+import { e2e } from '../index';
+import { fromBaseUrl } from '../support/url';
+
+export interface DeleteDataSourceConfig {
+  id: string;
+  name: string;
+  quick?: boolean;
+}
+
+export const deleteDataSource = ({ id, name, quick = false }: DeleteDataSourceConfig) => {
+  e2e().logToConsole('Deleting data source with name:', name);
+
+  if (quick) {
+    quickDelete(name);
+  } else {
+    uiDelete(name);
+  }
+
+  e2e().logToConsole('Deleted data source with name:', name);
+
+  e2e.getScenarioContext().then(({ addedDataSources }: any) => {
+    e2e.setScenarioContext({
+      addedDataSources: addedDataSources.filter((dataSource: DeleteDataSourceConfig) => {
+        return dataSource.id !== id && dataSource.name !== name;
+      }),
+    });
+  });
+};
+
+const quickDelete = (name: string) => {
+  e2e().request('DELETE', fromBaseUrl(`/api/datasources/name/${name}`));
+};
+
+const uiDelete = (name: string) => {
+  e2e.pages.DataSources.visit();
+  e2e.pages.DataSources.dataSources(name).click();
+  e2e.pages.DataSource.delete().click();
+  e2e.pages.ConfirmModal.delete().click();
+
+  e2e.pages.DataSources.visit();
+
+  // @todo replace `e2e.pages.DataSources.dataSources` with this when argument is empty
+  e2e()
+    .get('[aria-label^="Data source list item "]')
+    .each((item) => e2e().wrap(item).should('not.contain', name));
+};

e2e/utils/flows/editPanel.ts

@@ -0,0 +1,7 @@
+import { configurePanel, PartialEditPanelConfig } from './configurePanel';
+
+export const editPanel = (config: Partial<PartialEditPanelConfig>) =>
+  configurePanel({
+    ...config,
+    isEdit: true,
+  });

e2e/utils/flows/importDashboard.ts

@@ -0,0 +1,70 @@
+import { e2e } from '../index';
+import { fromBaseUrl, getDashboardUid } from '../support/url';
+
+import { DeleteDashboardConfig } from '.';
+
+type Panel = {
+  title: string;
+  [key: string]: unknown;
+};
+
+export type Dashboard = { title: string; panels: Panel[]; uid: string; [key: string]: unknown };
+
+/**
+ * Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading
+ * @param dashboardToImport a sample dashboard
+ * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skip panel validation
+ */
+export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number, skipPanelValidation?: boolean) => {
+  e2e().visit(fromBaseUrl('/dashboard/import'));
+
+  // Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
+  e2e.components.DashboardImportPage.textarea().should('be.visible');
+  e2e.components.DashboardImportPage.textarea().click();
+  e2e.components.DashboardImportPage.textarea().invoke('val', JSON.stringify(dashboardToImport));
+  e2e.components.DashboardImportPage.submit().should('be.visible').click();
+  e2e.components.ImportDashboardForm.name().should('be.visible').click().clear().type(dashboardToImport.title);
+  e2e.components.ImportDashboardForm.submit().should('be.visible').click();
+
+  // wait for dashboard to load
+  e2e().wait(queryTimeout || 6000);
+
+  // save the newly imported dashboard to context so it'll get properly deleted later
+  e2e()
+    .url()
+    .should('contain', '/d/')
+    .then((url: string) => {
+      const uid = getDashboardUid(url);
+
+      e2e.getScenarioContext().then(({ addedDashboards }: { addedDashboards: DeleteDashboardConfig[] }) => {
+        e2e.setScenarioContext({
+          addedDashboards: [...addedDashboards, { title: dashboardToImport.title, uid }],
+        });
+      });
+
+      expect(dashboardToImport.uid).to.equal(uid);
+    });
+
+  if (!skipPanelValidation) {
+    dashboardToImport.panels.forEach((panel) => {
+      // Look at the json data
+      e2e.components.Panels.Panel.menu(panel.title).click({ force: true }); // force click because menu is hidden and show on hover
+      e2e.components.Panels.Panel.menuItems('Inspect').should('be.visible').click();
+      e2e.components.Tab.title('JSON').should('be.visible').click();
+      e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
+      e2e.components.Select.option().should('be.visible').contains('Panel data').click();
+
+      // ensures that panel has loaded without knowingly hitting an error
+      // note: this does not prove that data came back as we expected it,
+      // it could get `state: Done` for no data for example
+      // but it ensures we didn't hit a 401 or 500 or something like that
+      e2e.components.CodeEditor.container()
+        .should('be.visible')
+        .contains(/"state": "(Done|Streaming)"/);
+
+      // need to close panel
+      e2e.components.Drawer.General.close().click();
+    });
+  }
+};

e2e/utils/flows/importDashboards.ts

@@ -0,0 +1,21 @@
+import { e2e } from '../index';
+
+import { importDashboard, Dashboard } from './importDashboard';
+
+/**
+ * Smoke test several dashboard json files from a test directory
+ * and validate that all the panels in each import finish loading their queries
+ * @param dirPath the relative path to a directory which contains json files representing dashboards,
+ * for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards`
+ * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skips panel validation
+ */
+export const importDashboards = async (dirPath: string, queryTimeout?: number, skipPanelValidation?: boolean) => {
+  e2e()
+    .getJSONFilesFromDir(dirPath)
+    .then((jsonFiles: Dashboard[]) => {
+      jsonFiles.forEach((file) => {
+        importDashboard(file, queryTimeout || 6000, skipPanelValidation);
+      });
+    });
+};

e2e/utils/flows/index.ts

@@ -0,0 +1,36 @@
+export * from './addDashboard';
+export * from './addDataSource';
+export * from './addPanel';
+export * from './assertSuccessNotification';
+export * from './deleteDashboard';
+export * from './deleteDataSource';
+export * from './editPanel';
+export * from './login';
+export * from './openDashboard';
+export * from './openPanelMenuItem';
+export * from './revertAllChanges';
+export * from './saveDashboard';
+export * from './selectOption';
+export * from './setTimeRange';
+export * from './importDashboard';
+export * from './importDashboards';
+export * from './userPreferences';
+
+export {
+  VISUALIZATION_ALERT_LIST,
+  VISUALIZATION_BAR_GAUGE,
+  VISUALIZATION_CLOCK,
+  VISUALIZATION_DASHBOARD_LIST,
+  VISUALIZATION_GAUGE,
+  VISUALIZATION_GRAPH,
+  VISUALIZATION_HEAT_MAP,
+  VISUALIZATION_LOGS,
+  VISUALIZATION_NEWS,
+  VISUALIZATION_PIE_CHART,
+  VISUALIZATION_PLUGIN_LIST,
+  VISUALIZATION_POLYSTAT,
+  VISUALIZATION_STAT,
+  VISUALIZATION_TABLE,
+  VISUALIZATION_TEXT,
+  VISUALIZATION_WORLD_MAP,
+} from './configurePanel';

e2e/utils/flows/login.ts

@@ -0,0 +1,42 @@
+import { e2e } from '../index';
+import { fromBaseUrl } from '../support/url';
+
+const DEFAULT_USERNAME = 'admin';
+const DEFAULT_PASSWORD = 'admin';
+
+const loginApi = (username: string, password: string) => {
+  cy.request({
+    method: 'POST',
+    url: fromBaseUrl('/login'),
+    body: {
+      user: username,
+      password,
+    },
+  });
+};
+
+const loginUi = (username: string, password: string) => {
+  e2e().logToConsole('Logging in with username:', username);
+  e2e.pages.Login.visit();
+  e2e.pages.Login.username()
+    .should('be.visible') // prevents flakiness
+    .type(username);
+  e2e.pages.Login.password().type(password);
+  e2e.pages.Login.submit().click();
+
+  // Local tests will have insecure credentials
+  if (password === DEFAULT_PASSWORD) {
+    e2e.pages.Login.skip().should('be.visible').click();
+  }
+
+  e2e().get('.login-page').should('not.exist');
+};
+
+export const login = (username = DEFAULT_USERNAME, password = DEFAULT_PASSWORD, loginViaApi = true) => {
+  if (loginViaApi) {
+    loginApi(username, password);
+  } else {
+    loginUi(username, password);
+  }
+  e2e().logToConsole('Logged in with username:', username);
+};

e2e/utils/flows/openDashboard.ts

@@ -0,0 +1,36 @@
+import { e2e } from '../index';
+import { getScenarioContext } from '../support/scenarioContext';
+
+import { setDashboardTimeRange, TimeRangeConfig } from './setDashboardTimeRange';
+
+interface OpenDashboardDefault {
+  uid: string;
+}
+
+interface OpenDashboardOptional {
+  timeRange?: TimeRangeConfig;
+  queryParams?: object;
+}
+
+export type PartialOpenDashboardConfig = Partial<OpenDashboardDefault> & OpenDashboardOptional;
+export type OpenDashboardConfig = OpenDashboardDefault & OpenDashboardOptional;
+
+// @todo this actually returns type `Cypress.Chainable<OpenDashboardConfig>`
+export const openDashboard = (config?: PartialOpenDashboardConfig) =>
+  getScenarioContext().then(({ lastAddedDashboardUid }: any) => {
+    const fullConfig: OpenDashboardConfig = {
+      uid: lastAddedDashboardUid,
+      ...config,
+    };
+
+    const { timeRange, uid, queryParams } = fullConfig;
+
+    e2e.pages.Dashboard.visit(uid, queryParams);
+
+    if (timeRange) {
+      setDashboardTimeRange(timeRange);
+    }
+
+    // @todo remove `wrap` when possible
+    return e2e().wrap({ config: fullConfig }, { log: false });
+  });

e2e/utils/flows/openPanelMenuItem.ts

@@ -0,0 +1,57 @@
+import { e2e } from '../index';
+
+export enum PanelMenuItems {
+  Edit = 'Edit',
+  Inspect = 'Inspect',
+  More = 'More...',
+  Extensions = 'Extensions',
+}
+
+export const openPanelMenuItem = (menu: PanelMenuItems, panelTitle = 'Panel Title') => {
+  // we changed the way we open the panel menu in react panels with the new panel header
+  detectPanelType(panelTitle, (isAngularPanel) => {
+    if (isAngularPanel) {
+      e2e.components.Panels.Panel.title(panelTitle).should('be.visible').click();
+      e2e.components.Panels.Panel.headerItems(menu).should('be.visible').click();
+    } else {
+      e2e.components.Panels.Panel.menu(panelTitle).click({ force: true }); // force click because menu is hidden and show on hover
+      e2e.components.Panels.Panel.menuItems(menu).should('be.visible').click();
+    }
+  });
+};
+
+export const openPanelMenuExtension = (extensionTitle: string, panelTitle = 'Panel Title') => {
+  const menuItem = PanelMenuItems.Extensions;
+  // we changed the way we open the panel menu in react panels with the new panel header
+  detectPanelType(panelTitle, (isAngularPanel) => {
+    if (isAngularPanel) {
+      e2e.components.Panels.Panel.title(panelTitle).should('be.visible').click();
+      e2e.components.Panels.Panel.headerItems(menuItem)
+        .should('be.visible')
+        .parent()
+        .parent()
+        .invoke('addClass', 'open');
+      e2e.components.Panels.Panel.headerItems(extensionTitle).should('be.visible').click();
+    } else {
+      e2e.components.Panels.Panel.menu(panelTitle).click({ force: true }); // force click because menu is hidden and show on hover
+      e2e.components.Panels.Panel.menuItems(menuItem).trigger('mouseover', { force: true });
+      e2e.components.Panels.Panel.menuItems(extensionTitle).click({ force: true });
+    }
+  });
+};
+
+function detectPanelType(panelTitle: string, detected: (isAngularPanel: boolean) => void) {
+  e2e.components.Panels.Panel.title(panelTitle).then((el) => {
+    const isAngularPanel = el.find('plugin-component.ng-scope').length > 0;
+
+    if (isAngularPanel) {
+      Cypress.log({
+        name: 'detectPanelType',
+        displayName: 'detector',
+        message: 'Angular panel detected, will use legacy selectors.',
+      });
+    }
+
+    detected(isAngularPanel);
+  });
+}

e2e/utils/flows/revertAllChanges.ts

@@ -0,0 +1,12 @@
+import { e2e } from '../index';
+
+export const revertAllChanges = () => {
+  e2e.getScenarioContext().then(({ addedDashboards, addedDataSources, hasChangedUserPreferences }) => {
+    addedDashboards.forEach((dashboard: any) => e2e.flows.deleteDashboard({ ...dashboard, quick: true }));
+    addedDataSources.forEach((dataSource: any) => e2e.flows.deleteDataSource({ ...dataSource, quick: true }));
+
+    if (hasChangedUserPreferences) {
+      e2e.flows.setDefaultUserPreferences();
+    }
+  });
+};

e2e/utils/flows/saveDashboard.ts

@@ -0,0 +1,9 @@
+import { e2e } from '../index';
+
+export const saveDashboard = () => {
+  e2e.components.PageToolbar.item('Save dashboard').click();
+
+  e2e.pages.SaveDashboardModal.save().click();
+
+  e2e.flows.assertSuccessNotification();
+};

e2e/utils/flows/selectOption.ts

@@ -0,0 +1,43 @@
+import { e2e } from '../index';
+
+export interface SelectOptionConfig {
+  clickToOpen?: boolean;
+  container: any;
+  forceClickOption?: boolean;
+  optionText: string | RegExp;
+}
+
+// @todo this actually returns type `Cypress.Chainable`
+export const selectOption = (config: SelectOptionConfig): any => {
+  const fullConfig: SelectOptionConfig = {
+    clickToOpen: true,
+    forceClickOption: false,
+    ...config,
+  };
+
+  const { clickToOpen, container, forceClickOption, optionText } = fullConfig;
+
+  container.within(() => {
+    if (clickToOpen) {
+      e2e()
+        .get('[class$="-input-suffix"]', { timeout: 1000 })
+        .then((element) => {
+          expect(Cypress.dom.isAttached(element)).to.eq(true);
+          e2e().get('[class$="-input-suffix"]', { timeout: 1000 }).click({ force: true });
+        });
+    }
+  });
+
+  return e2e.components.Select.option()
+    .filter((_, { textContent }) => {
+      if (textContent === null) {
+        return false;
+      } else if (typeof optionText === 'string') {
+        return textContent.includes(optionText);
+      } else {
+        return optionText.test(textContent);
+      }
+    })
+    .scrollIntoView()
+    .click({ force: forceClickOption });
+};

e2e/utils/flows/setDashboardTimeRange.ts

@@ -0,0 +1,5 @@
+import { setTimeRange, TimeRangeConfig } from './setTimeRange';
+
+export type { TimeRangeConfig };
+
+export const setDashboardTimeRange = (config: TimeRangeConfig) => setTimeRange(config);

e2e/utils/flows/setTimeRange.ts

@@ -0,0 +1,40 @@
+import { e2e } from '../index';
+
+import { selectOption } from './selectOption';
+
+export interface TimeRangeConfig {
+  from: string;
+  to: string;
+  zone?: string;
+}
+
+export const setTimeRange = ({ from, to, zone }: TimeRangeConfig) => {
+  e2e.components.TimePicker.openButton().click();
+
+  if (zone) {
+    e2e().contains('button', 'Change time settings').click();
+    e2e().log('setting time zone to ' + zone);
+
+    if (e2e.components.TimeZonePicker.containerV2) {
+      selectOption({
+        clickToOpen: true,
+        container: e2e.components.TimeZonePicker.containerV2(),
+        optionText: zone,
+      });
+    } else {
+      selectOption({
+        clickToOpen: true,
+        container: e2e.components.TimeZonePicker.container(),
+        optionText: zone,
+      });
+    }
+  }
+
+  // For smaller screens
+  e2e.components.TimePicker.absoluteTimeRangeTitle().click();
+
+  e2e.components.TimePicker.fromField().clear().type(from);
+  e2e.components.TimePicker.toField().clear().type(to);
+
+  e2e.components.TimePicker.applyTimeRange().click();
+};

e2e/utils/flows/userPreferences.ts

@@ -0,0 +1,25 @@
+import { Preferences as UserPreferencesDTO } from '@grafana/schema/src/raw/preferences/x/preferences_types.gen';
+
+import { e2e } from '..';
+import { fromBaseUrl } from '../support/url';
+
+const defaultUserPreferences = {
+  timezone: '', // "Default" option
+} as const; // TODO: when we update typescript >4.9 change to `as const satisfies UserPreferencesDTO`
+
+// Only accept preferences we have defaults for as arguments. To allow a new preference to be set, add a default for it
+type UserPreferences = Pick<UserPreferencesDTO, keyof typeof defaultUserPreferences>;
+
+export function setUserPreferences(prefs: UserPreferences) {
+  e2e.setScenarioContext({ hasChangedUserPreferences: prefs !== defaultUserPreferences });
+
+  return cy.request({
+    method: 'PUT',
+    url: fromBaseUrl('/api/user/preferences'),
+    body: prefs,
+  });
+}
+
+export function setDefaultUserPreferences() {
+  return setUserPreferences(defaultUserPreferences);
+}

e2e/utils/index.ts

@@ -0,0 +1,26 @@
+import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors';
+
+import * as flows from './flows';
+import { e2eFactory } from './support';
+import { benchmark } from './support/benchmark';
+import { e2eScenario, ScenarioArguments } from './support/scenario';
+import { getScenarioContext, setScenarioContext } from './support/scenarioContext';
+import * as typings from './typings';
+
+const e2eObject = {
+  env: (args: string) => Cypress.env(args),
+  config: () => Cypress.config(),
+  blobToBase64String: (blob: Blob) => Cypress.Blob.blobToBase64String(blob),
+  imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url),
+  scenario: (args: ScenarioArguments) => e2eScenario(args),
+  benchmark,
+  pages: e2eFactory({ selectors: selectors.pages }),
+  typings,
+  components: e2eFactory({ selectors: selectors.components }),
+  flows,
+  getScenarioContext,
+  setScenarioContext,
+  getSelectors: <T extends Selectors>(selectors: E2ESelectors<T>) => e2eFactory({ selectors }),
+};
+
+export const e2e: (() => Cypress.cy) & typeof e2eObject = Object.assign(() => cy, e2eObject);

e2e/utils/support/benchmark.ts

@@ -0,0 +1,79 @@
+import { e2e } from '../';
+
+export interface BenchmarkArguments {
+  name: string;
+  dashboard: {
+    folder: string;
+    delayAfterOpening: number;
+    skipPanelValidation: boolean;
+  };
+  repeat: number;
+  duration: number;
+  appStats?: {
+    startCollecting?: (window: Window) => void;
+    collect: (window: Window) => Record<string, unknown>;
+  };
+  skipScenario?: boolean;
+}
+
+export const benchmark = ({
+  name,
+  skipScenario = false,
+  repeat,
+  duration,
+  appStats,
+  dashboard,
+}: BenchmarkArguments) => {
+  if (skipScenario) {
+    describe(name, () => {
+      it.skip(name, () => {});
+    });
+  } else {
+    describe(name, () => {
+      before(() => {
+        cy.session('login', () => e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD'), true), {
+          cacheAcrossSpecs: true,
+        });
+      });
+
+      beforeEach(() => {
+        e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation);
+      });
+
+      afterEach(() => e2e.flows.revertAllChanges());
+
+      Array(repeat)
+        .fill(0)
+        .map((_, i) => {
+          const testName = `${name}-${i}`;
+          return it(testName, () => {
+            e2e.flows.openDashboard();
+
+            e2e().wait(dashboard.delayAfterOpening);
+
+            if (appStats) {
+              const startCollecting = appStats.startCollecting;
+              if (startCollecting) {
+                e2e()
+                  .window()
+                  .then((win) => startCollecting(win));
+              }
+
+              e2e().startBenchmarking(testName);
+              e2e().wait(duration);
+
+              e2e()
+                .window()
+                .then((win) => {
+                  e2e().stopBenchmarking(testName, appStats.collect(win));
+                });
+            } else {
+              e2e().startBenchmarking(testName);
+              e2e().wait(duration);
+              e2e().stopBenchmarking(testName, {});
+            }
+          });
+        });
+    });
+  }
+};

e2e/utils/support/index.ts

@@ -0,0 +1,4 @@
+export * from './localStorage';
+export * from './scenarioContext';
+export * from './selector';
+export * from './types';

e2e/utils/support/localStorage.ts

@@ -0,0 +1,23 @@
+import { e2e } from '../index';
+
+// @todo this actually returns type `Cypress.Chainable`
+const get = (key: string): any =>
+  e2e()
+    .wrap({ getLocalStorage: () => localStorage.getItem(key) }, { log: false })
+    .invoke('getLocalStorage');
+
+// @todo this actually returns type `Cypress.Chainable`
+export const getLocalStorage = (key: string): any =>
+  get(key).then((value: any) => {
+    if (value === null) {
+      return value;
+    } else {
+      return JSON.parse(value);
+    }
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+export const requireLocalStorage = (key: string): any =>
+  get(key) // `getLocalStorage()` would turn 'null' into `null`
+    .should('not.equal', null)
+    .then((value: any) => JSON.parse(value as string));

e2e/utils/support/scenario.ts

@@ -0,0 +1,53 @@
+import { e2e } from '../';
+
+export interface ScenarioArguments {
+  describeName: string;
+  itName: string;
+  scenario: Function;
+  skipScenario?: boolean;
+  addScenarioDataSource?: boolean;
+  addScenarioDashBoard?: boolean;
+  loginViaApi?: boolean;
+}
+
+export const e2eScenario = ({
+  describeName,
+  itName,
+  scenario,
+  skipScenario = false,
+  addScenarioDataSource = false,
+  addScenarioDashBoard = false,
+  loginViaApi = true,
+}: ScenarioArguments) => {
+  describe(describeName, () => {
+    if (skipScenario) {
+      it.skip(itName, () => scenario());
+    } else {
+      before(() => {
+        cy.session(
+          'login',
+          () => {
+            e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD'), loginViaApi);
+          },
+          {
+            cacheAcrossSpecs: true,
+          }
+        );
+        e2e.flows.setDefaultUserPreferences();
+      });
+
+      beforeEach(() => {
+        if (addScenarioDataSource) {
+          e2e.flows.addDataSource();
+        }
+        if (addScenarioDashBoard) {
+          e2e.flows.addDashboard();
+        }
+      });
+
+      afterEach(() => e2e.flows.revertAllChanges());
+
+      it(itName, () => scenario());
+    }
+  });
+};

e2e/utils/support/scenarioContext.ts

@@ -0,0 +1,61 @@
+import { DeleteDashboardConfig } from '../flows/deleteDashboard';
+import { DeleteDataSourceConfig } from '../flows/deleteDataSource';
+import { e2e } from '../index';
+
+export interface ScenarioContext {
+  addedDashboards: DeleteDashboardConfig[];
+  addedDataSources: DeleteDataSourceConfig[];
+  lastAddedDashboard: string; // @todo rename to `lastAddedDashboardTitle`
+  lastAddedDashboardUid: string;
+  lastAddedDataSource: string; // @todo rename to `lastAddedDataSourceName`
+  lastAddedDataSourceId: string;
+  hasChangedUserPreferences: boolean;
+  [key: string]: any;
+}
+
+const scenarioContext: ScenarioContext = {
+  addedDashboards: [],
+  addedDataSources: [],
+  hasChangedUserPreferences: false,
+  get lastAddedDashboard() {
+    return lastProperty(this.addedDashboards, 'title');
+  },
+  get lastAddedDashboardUid() {
+    return lastProperty(this.addedDashboards, 'uid');
+  },
+  get lastAddedDataSource() {
+    return lastProperty(this.addedDataSources, 'name');
+  },
+  get lastAddedDataSourceId() {
+    return lastProperty(this.addedDataSources, 'id');
+  },
+};
+
+const lastProperty = <T extends DeleteDashboardConfig | DeleteDataSourceConfig, K extends keyof T>(
+  items: T[],
+  key: K
+) => items[items.length - 1]?.[key] ?? '';
+
+export const getScenarioContext = (): Cypress.Chainable<ScenarioContext> =>
+  e2e()
+    .wrap(
+      {
+        getScenarioContext: (): ScenarioContext => ({ ...scenarioContext }),
+      },
+      { log: false }
+    )
+    .invoke({ log: false }, 'getScenarioContext');
+
+export const setScenarioContext = (newContext: Partial<ScenarioContext>): Cypress.Chainable<ScenarioContext> =>
+  e2e()
+    .wrap(
+      {
+        setScenarioContext: () => {
+          Object.entries(newContext).forEach(([key, value]) => {
+            scenarioContext[key] = value;
+          });
+        },
+      },
+      { log: false }
+    )
+    .invoke({ log: false }, 'setScenarioContext');

e2e/utils/support/selector.ts

@@ -0,0 +1,11 @@
+export interface SelectorApi {
+  fromAriaLabel: (selector: string) => string;
+  fromDataTestId: (selector: string) => string;
+  fromSelector: (selector: string) => string;
+}
+
+export const Selector: SelectorApi = {
+  fromAriaLabel: (selector: string) => `[aria-label="${selector}"]`,
+  fromDataTestId: (selector: string) => `[data-testid="${selector}"]`,
+  fromSelector: (selector: string) => selector,
+};

e2e/utils/support/types.ts

@@ -0,0 +1,138 @@
+import { CssSelector, FunctionSelector, Selectors, StringSelector, UrlSelector } from '@grafana/e2e-selectors';
+
+import { e2e } from '../index';
+
+import { Selector } from './selector';
+import { fromBaseUrl } from './url';
+
+export type VisitFunction = (args?: string, queryParams?: object) => Cypress.Chainable<Window>;
+export type E2EVisit = { visit: VisitFunction };
+export type E2EFunction = ((text?: string, options?: CypressOptions) => Cypress.Chainable<JQuery<HTMLElement>>) &
+  E2EFunctionWithOnlyOptions;
+export type E2EFunctionWithOnlyOptions = (options?: CypressOptions) => Cypress.Chainable<JQuery<HTMLElement>>;
+
+export type TypeSelectors<S> = S extends StringSelector
+  ? E2EFunctionWithOnlyOptions
+  : S extends FunctionSelector
+  ? E2EFunction
+  : S extends CssSelector
+  ? E2EFunction
+  : S extends UrlSelector
+  ? E2EVisit & Omit<E2EFunctions<S>, 'url'>
+  : S extends Record<any, any>
+  ? E2EFunctions<S>
+  : S;
+
+export type E2EFunctions<S extends Selectors> = {
+  [P in keyof S]: TypeSelectors<S[P]>;
+};
+
+export type E2EObjects<S extends Selectors> = E2EFunctions<S>;
+
+export type E2EFactoryArgs<S extends Selectors> = { selectors: S };
+
+export type CypressOptions = Partial<Cypress.Loggable & Cypress.Timeoutable & Cypress.Withinable & Cypress.Shadow>;
+
+const processSelectors = <S extends Selectors>(e2eObjects: E2EFunctions<S>, selectors: S): E2EFunctions<S> => {
+  const logOutput = (data: any) => e2e().logToConsole('Retrieving Selector:', data);
+  const keys = Object.keys(selectors);
+  for (let index = 0; index < keys.length; index++) {
+    const key = keys[index];
+    const value = selectors[key];
+
+    if (key === 'url') {
+      // @ts-ignore
+      e2eObjects['visit'] = (args?: string, queryParams?: object) => {
+        let parsedUrl = '';
+        if (typeof value === 'string') {
+          parsedUrl = fromBaseUrl(value);
+        }
+
+        if (typeof value === 'function' && args) {
+          parsedUrl = fromBaseUrl(value(args));
+        }
+
+        e2e().logToConsole('Visiting', parsedUrl);
+        if (queryParams) {
+          return e2e().visit({ url: parsedUrl, qs: queryParams });
+        } else {
+          return e2e().visit(parsedUrl);
+        }
+      };
+
+      continue;
+    }
+
+    if (typeof value === 'string') {
+      // @ts-ignore
+      e2eObjects[key] = (options?: CypressOptions) => {
+        logOutput(value);
+        const selector = value.startsWith('data-testid')
+          ? Selector.fromDataTestId(value)
+          : Selector.fromAriaLabel(value);
+
+        return e2e().get(selector, options);
+      };
+
+      continue;
+    }
+
+    if (typeof value === 'function') {
+      // @ts-ignore
+      e2eObjects[key] = function (textOrOptions?: string | CypressOptions, options?: CypressOptions) {
+        // the input can only be ()
+        if (arguments.length === 0) {
+          const selector = value(undefined as unknown as string);
+
+          logOutput(selector);
+          return e2e().get(selector);
+        }
+
+        // the input can be (text) or (options)
+        if (arguments.length === 1) {
+          if (typeof textOrOptions === 'string') {
+            const selectorText = value(textOrOptions);
+            const selector = selectorText.startsWith('data-testid')
+              ? Selector.fromDataTestId(selectorText)
+              : Selector.fromAriaLabel(selectorText);
+
+            logOutput(selector);
+            return e2e().get(selector);
+          }
+          const selector = value(undefined as unknown as string);
+
+          logOutput(selector);
+          return e2e().get(selector, textOrOptions);
+        }
+
+        // the input can only be (text, options)
+        if (arguments.length === 2 && typeof textOrOptions === 'string') {
+          const text = textOrOptions;
+          const selectorText = value(text);
+          const selector = text.startsWith('data-testid')
+            ? Selector.fromDataTestId(selectorText)
+            : Selector.fromAriaLabel(selectorText);
+
+          logOutput(selector);
+          return e2e().get(selector, options);
+        }
+      };
+
+      continue;
+    }
+
+    if (typeof value === 'object') {
+      // @ts-ignore
+      e2eObjects[key] = processSelectors({}, value);
+    }
+  }
+
+  return e2eObjects;
+};
+
+export const e2eFactory = <S extends Selectors>({ selectors }: E2EFactoryArgs<S>): E2EObjects<S> => {
+  const e2eObjects: E2EFunctions<S> = {} as E2EFunctions<S>;
+  processSelectors(e2eObjects, selectors);
+
+  return { ...e2eObjects };
+};

e2e/utils/support/url.ts

@@ -0,0 +1,14 @@
+import { e2e } from '../index';
+
+const getBaseUrl = () => e2e.env('BASE_URL') || e2e.config().baseUrl || 'http://localhost:3000';
+
+export const fromBaseUrl = (url = '') => new URL(url, getBaseUrl()).href;
+
+export const getDashboardUid = (url: string): string => {
+  const matches = new URL(url).pathname.match(/\/d\/([^/]+)/);
+  if (!matches) {
+    throw new Error(`Couldn't parse uid from ${url}`);
+  } else {
+    return matches[1];
+  }
+};

e2e/utils/typings/index.ts

@@ -0,0 +1 @@
+export { undo } from './undo';

e2e/utils/typings/undo.ts

@@ -0,0 +1,19 @@
+// https://nodejs.org/api/os.html#os_os_platform
+enum Platform {
+  osx = 'darwin',
+  windows = 'win32',
+  linux = 'linux',
+  aix = 'aix',
+  freebsd = 'freebsd',
+  openbsd = 'openbsd',
+  sunos = 'sunos',
+}
+
+export const undo = () => {
+  switch (Cypress.platform) {
+    case Platform.osx:
+      return '{cmd}z';
+    default:
+      return '{ctrl}z';
+  }
+};

e2e/various-suite/bar-gauge.spec.ts

@@ -1,6 +1,7 @@
-import { e2e } from '@grafana/e2e';
 import { selectors } from '@grafana/e2e-selectors';
 
+import { e2e } from '../utils';
+
 e2e.scenario({
   describeName: 'Bar Gauge Panel',
   itName: 'Bar Gauge rendering e2e tests',

e2e/various-suite/exemplars.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 const dataSourceName = 'PromExemplar';
 const addDataSource = () => {

e2e/various-suite/explore.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Explore',

e2e/various-suite/filter-annotations.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 const DASHBOARD_ID = 'ed155665';
 
 e2e.scenario({

e2e/various-suite/gauge.spec.ts

@@ -1,4 +1,4 @@
-import { e2e } from '@grafana/e2e';
+import { e2e } from '../utils';
 
 e2e.scenario({
   describeName: 'Gauge Panel',