grafana/.github/workflows/pr-e2e-tests.yml

name: End-to-end tests

on:
  pull_request:
  push:
    branches:
      - main
      - release-*.*.*

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

permissions: {}

jobs:
  detect-changes:
    name: Detect whether code changed
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      changed: ${{ steps.detect-changes.outputs.e2e }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: true # required to get more history in the changed-files action
          fetch-depth: 2
      - name: Detect changes
        id: detect-changes
        uses: ./.github/actions/change-detection
        with:
          self: .github/workflows/pr-e2e-tests.yml

  build-grafana:
    needs: detect-changes
    if: needs.detect-changes.outputs.changed == 'true'
    name: Build & Package Grafana
    runs-on: ubuntu-latest-16-cores
    permissions:
      contents: read
    outputs:
      artifact: ${{ steps.artifact.outputs.artifact }}
    steps:
      - uses: actions/checkout@v4
        with:
          path: ./grafana
          persist-credentials: false
      - uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go -C grafana run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}/grafana" > out.txt
      - run: mv "$(cat out.txt)" grafana.tar.gz
      - run: echo "artifact=grafana-e2e-${{github.run_number}}" >> "$GITHUB_OUTPUT"
        id: artifact
      - uses: actions/upload-artifact@v4
        id: upload
        with:
          retention-days: 1
          name: ${{ steps.artifact.outputs.artifact }}
          path: grafana.tar.gz

  build-e2e-runner:
    needs: detect-changes
    if: needs.detect-changes.outputs.changed == 'true'
    name: Build E2E test runner
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      artifact: ${{ steps.artifact.outputs.artifact }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache: ${{ !github.event.pull_request.head.repo.fork }}
      - name: Build E2E test runner
        id: artifact
        run: |
          set -euo pipefail
          # We want a static binary, so we need to set CGO_ENABLED=0
          CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/
          echo "artifact=e2e-runner-${{github.run_number}}" >> "$GITHUB_OUTPUT"          
      - uses: actions/upload-artifact@v4
        id: upload
        with:
          retention-days: 1
          name: ${{ steps.artifact.outputs.artifact }}
          path: e2e-runner

  run-e2e-tests:
    needs:
      - build-grafana
      - build-e2e-runner
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: various-suite
            path: e2e/various-suite
          - suite: dashboards-suite
            path: e2e/dashboards-suite
          - suite: smoke-tests-suite
            path: e2e/smoke-tests-suite
          - suite: panels-suite
            path: e2e/panels-suite
          - suite: various-suite (old arch)
            path: e2e/old-arch/various-suite
            flags: --flags="--env DISABLE_SCENES=true"
          - suite: dashboards-suite (old arch)
            path: e2e/old-arch/dashboards-suite
            flags: --flags="--env DISABLE_SCENES=true"
          - suite: smoke-tests-suite (old arch)
            path: e2e/old-arch/smoke-tests-suite
            flags: --flags="--env DISABLE_SCENES=true"
          - suite: panels-suite (old arch)
            path: e2e/old-arch/panels-suite
            flags: --flags="--env DISABLE_SCENES=true"
    name: ${{ matrix.suite }}
    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-grafana.outputs.artifact }}
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-e2e-runner.outputs.artifact }}
      - name: chmod +x
        run: chmod +x ./e2e-runner
      - name: Run E2E tests
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/e2e --package=grafana.tar.gz
            --suite=${{ matrix.path }}
            ${{ matrix.flags }}
      - name: Set suite name
        id: set-suite-name
        if: success() || failure()
        env:
          SUITE: ${{ matrix.path }}
        run: |
          set -euo pipefail
          echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"          
      - uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: ${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }}
          path: videos
          retention-days: 1

  run-a11y-test:
    needs:
      - build-grafana
    name: A11y test
    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-grafana.outputs.artifact }}
      - name: Run PR a11y test
        if: github.event_name == 'pull_request'
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/a11y --package=grafana.tar.gz
      - name: Run non-PR a11y test
        if: github.event_name != 'pull_request'
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/a11y --package=grafana.tar.gz --no-threshold-fail

  # This is the job that is actually required by rulesets.
  # We want to only require one job instead of all the individual tests.
  # Future work also allows us to start skipping some tests based on changed files.
  required-e2e-tests:
    needs:
      - run-e2e-tests
      # a11y test is not listed on purpose: it is not an important E2E test.
      # It is also totally fine to fail right now.
    # always() is the best function here.
    # success() || failure() will skip this function if any need is also skipped.
    # That means conditional test suites will fail the entire requirement check.
    if: always()

    name: All E2E tests complete
    runs-on: ubuntu-latest
    steps:
      - name: Check test suites
        env:
          NEEDS: ${{ toJson(needs) }}
        run: |
          FAILURES="$(echo "$NEEDS" | jq 'with_entries(select(.value.result == "failure")) | map_values(.result)')"
          echo "$FAILURES"
          if [ "$(echo "$FAILURES" | jq '. | length')" != "0" ]; then
            exit 1
          fi
          echo "All OK!"