Actions: Shard test suite (#105166)

.github/CODEOWNERS

@@ -66,6 +66,7 @@
 /build.go @grafana/grafana-backend-services-squad
 /scripts/modowners/ @grafana/grafana-backend-services-squad
 /scripts/go-workspace @grafana/grafana-app-platform-squad
+/scripts/ci/backend-tests @grafana/grafana-operator-experience-squad
 /hack/ @grafana/grafana-app-platform-squad
 
 /pkg/apis/provisioning @grafana/grafana-git-ui-sync-team

.github/workflows/backend-unit-tests.yml

@@ -24,7 +24,15 @@ jobs:
     # Run this workflow only for PRs from forks; if it gets merged into `main` or `release-*`, 
     # the `pr-backend-unit-tests-enterprise` workflow will run instead
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true
-    name: Grafana
+    strategy:
+      matrix:
+        shard: [
+          1/8, 2/8, 3/8, 4/8,
+          5/8, 6/8, 7/8, 8/8,
+        ]
+      fail-fast: false
+
+    name: Grafana (${{ matrix.shard }})
     runs-on: ubuntu-latest-8-cores
     continue-on-error: true
     permissions:
@@ -42,12 +50,24 @@ jobs:
       - name: Generate Go code
         run: make gen-go
       - name: Run unit tests
-        run: make test-go-unit
+        env:
+          SHARD: ${{ matrix.shard }}
+        run: |
+          readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/shard.sh -N"$SHARD")"
+          go test -short -timeout=30m "${PACKAGES[@]}"
 
   grafana-enterprise:
     # Run this workflow for non-PR events (like pushes to `main` or `release-*`) OR for internal PRs (PRs not from forks)
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
-    name: Grafana Enterprise
+    strategy:
+      matrix:
+        shard: [
+          1/8, 2/8, 3/8, 4/8,
+          5/8, 6/8, 7/8, 8/8,
+        ]
+      fail-fast: false
+
+    name: Grafana Enterprise (${{ matrix.shard }})
     runs-on: ubuntu-latest-8-cores
     permissions:
       contents: read
@@ -68,4 +88,8 @@ jobs:
       - name: Generate Go code
         run: make gen-go
       - name: Run unit tests
-        run: make test-go-unit
+        env:
+          SHARD: ${{ matrix.shard }}
+        run: |
+          readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/shard.sh -N"$SHARD")"
+          go test -short -timeout=30m "${PACKAGES[@]}"

.github/workflows/pr-test-integration.yml

@@ -6,6 +6,10 @@ on:
       - main
       - release-*.*.*
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -15,7 +19,15 @@ permissions: {}
 
 jobs:
   sqlite:
-    name: Sqlite
+    strategy:
+      matrix:
+        shard: [
+          1/8, 2/8, 3/8, 4/8,
+          5/8, 6/8, 7/8, 8/8,
+        ]
+      fail-fast: false
+
+    name: Sqlite (${{ matrix.shard }})
     runs-on: ubuntu-latest-8-cores
     permissions:
       contents: read
@@ -29,14 +41,24 @@ jobs:
         with:
           go-version-file: go.mod
           cache: true
-      - run: |
+      - name: Generate Go code
-          go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)"
+        run: make gen-go
-          IFS=' ' read -ra packages <<< "$go_packages"
+      - name: Run tests
-
+        env:
-          make gen-go
+          SHARD: ${{ matrix.shard }}
-          go test -tags=sqlite -timeout=5m -run '^TestIntegration' "${packages[@]}"
+        run: |
+          readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)"
+          go test -tags=sqlite -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}"
   mysql:
-    name: MySQL
+    strategy:
+      matrix:
+        shard: [
+          1/8, 2/8, 3/8, 4/8,
+          5/8, 6/8, 7/8, 8/8,
+        ]
+      fail-fast: false
+
+    name: MySQL (${{ matrix.shard }})
     runs-on: ubuntu-latest-8-cores
     permissions:
       contents: read
@@ -64,19 +86,33 @@ jobs:
         with:
           go-version-file: go.mod
           cache: true
-      - run: |
+      - name: Setup MySQL devenv
-          go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)"
+        run: mysql -h 127.0.0.1 -P 3306 -u root -prootpass < devenv/docker/blocks/mysql_tests/setup.sql
-          IFS=' ' read -ra packages <<< "$go_packages"
+      - name: Generate Go code
-
+        run: make gen-go
-          sudo apt-get update -yq && sudo apt-get install mariadb-client
+      - name: Run tests
-          mariadb -h 127.0.0.1 -P 3306 -u root -prootpass --disable-ssl-verify-server-cert < devenv/docker/blocks/mysql_tests/setup.sql
+        env:
-          make gen-go
+          SHARD: ${{ matrix.shard }}
-          go test -tags=mysql -p=1 -timeout=5m -run '^TestIntegration' "${packages[@]}"
+        run: |
+          readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)"
+          go test -p=1 -tags=mysql -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}"
   postgres:
-    name: Postgres
+    strategy:
+      matrix:
+        shard: [
+          1/8, 2/8, 3/8, 4/8,
+          5/8, 6/8, 7/8, 8/8,
+        ]
+      fail-fast: false
+
+    name: Postgres (${{ matrix.shard }})
     runs-on: ubuntu-latest-8-cores
     permissions:
       contents: read
+    env:
+      GRAFANA_TEST_DB: postgres
+      PGPASSWORD: grafanatest
+      POSTGRES_HOST: 127.0.0.1
     services:
       postgres:
         image: postgres:12.3-alpine
@@ -96,15 +132,13 @@ jobs:
         with:
           go-version-file: go.mod
           cache: true
-      - env:
+      - name: Setup Postgres devenv
-          GRAFANA_TEST_DB: postgres
+        run: psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
-          PGPASSWORD: grafanatest
+      - name: Generate Go code
-          POSTGRES_HOST: 127.0.0.1
+        run: make gen-go
+      - name: Run tests
+        env:
+          SHARD: ${{ matrix.shard }}
         run: |
-          go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)"
+          readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)"
-          IFS=' ' read -ra packages <<< "$go_packages"
+          go test -p=1 -tags=postgres -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}"
-
-          sudo apt-get update -yq && sudo apt-get install postgresql-client
-          psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
-          make gen-go
-          go test -p=1 -tags=postgres -timeout=5m -run '^TestIntegration' "${packages[@]}"

Makefile

@@ -18,15 +18,16 @@ GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev)
 GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS))
 GO_BUILD_FLAGS += $(GO_RACE_FLAG)
 GO_TEST_FLAGS += $(if $(GO_BUILD_TAGS),-tags=$(GO_BUILD_TAGS))
-GO_TEST_OUTPUT := $(shell [ -n "$(GO_TEST_OUTPUT)" ] && echo '-json | tee $(GO_TEST_OUTPUT) | tparse -all')
 GIT_BASE = remotes/origin/main
 
 # GNU xargs has flag -r, and BSD xargs (e.g. MacOS) has that behaviour by default
 XARGSR = $(shell xargs --version 2>&1 | grep -q GNU && echo xargs -r || echo xargs)
 
-targets := $(shell echo '$(sources)' | tr "," " ")
+# Test sharding to replicate CI behaviour locally.
+SHARD ?= 1
+SHARDS ?= 1
 
-GO_INTEGRATION_TESTS := $(shell find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)
+targets := $(shell echo '$(sources)' | tr "," " ")
 
 .PHONY: all
 all: deps build
@@ -267,8 +268,9 @@ test-go: test-go-unit test-go-integration
 
 .PHONY: test-go-unit
 test-go-unit: ## Run unit tests for backend with flags.
-	@echo "backend unit tests"
+	@echo "backend unit tests ($(SHARD)/$(SHARDS))"
-	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -v -short -timeout=30m $(GO_TEST_FILES) $(GO_TEST_OUTPUT)
+	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -v -short -timeout=30m \
+		$(shell ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
 
 .PHONY: test-go-unit-pretty
 test-go-unit-pretty: check-tparse
@@ -281,7 +283,8 @@ test-go-unit-pretty: check-tparse
 .PHONY: test-go-integration
 test-go-integration: ## Run integration tests for backend with flags.
 	@echo "test backend integration tests"
-	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=$(GO_INTEGRATION_COVER_PROFILE)  -timeout=5m $(GO_INTEGRATION_TESTS) $(GO_TEST_OUTPUT)
+	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=$(GO_INTEGRATION_COVER_PROFILE) -timeout=5m \
+		$(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
 
 .PHONY: test-go-integration-alertmanager
 test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block).
@@ -303,32 +306,29 @@ test-go-integration-postgres: devenv-postgres ## Run integration tests for postg
 	@echo "test backend integration postgres tests"
 	$(GO) clean -testcache
 	GRAFANA_TEST_DB=postgres \
-	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS)
+	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m \
+		$(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
 
 .PHONY: test-go-integration-mysql
 test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backend with flags.
 	@echo "test backend integration mysql tests"
 	GRAFANA_TEST_DB=mysql \
-	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS)
+	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m \
+		$(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
 
 .PHONY: test-go-integration-redis
 test-go-integration-redis: ## Run integration tests for redis cache.
 	@echo "test backend integration redis tests"
 	$(GO) clean -testcache
-	REDIS_URL=localhost:6379 $(GO) test $(GO_TEST_FLAGS) -run IntegrationRedis -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS)
+	REDIS_URL=localhost:6379 $(GO) test $(GO_TEST_FLAGS) -run IntegrationRedis -covermode=atomic -timeout=2m \
+		$(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
 
 .PHONY: test-go-integration-memcached
 test-go-integration-memcached: ## Run integration tests for memcached cache.
 	@echo "test backend integration memcached tests"
 	$(GO) clean -testcache
-	MEMCACHED_HOSTS=localhost:11211 $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -run IntegrationMemcached -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS)
+	MEMCACHED_HOSTS=localhost:11211 $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -run IntegrationMemcached -covermode=atomic -timeout=2m \
-
+		$(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s)
-.PHONY: test-go-integration-spanner
-test-go-integration-spanner: ## Run integration tests for Spanner backend with flags. Uses spanner-emulator on localhost:9010 and localhost:9020.
-	@if [ "${WIRE_TAGS}" != "enterprise" ]; then echo "Spanner integration test require enterprise setup"; exit 1; fi
-	@echo "test backend integration spanner tests"
-	GRAFANA_TEST_DB=spanner \
-	$(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -v -run "^TestIntegration" -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS)
 
 .PHONY: test-js
 test-js: ## Run tests for frontend.

scripts/ci/backend-tests/pkgs-with-tests-named.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+    {
+        echo "pkgs-with-tests-named.sh: Find packages with tests in them, filtered by the test names."
+        echo "usage: $0 [-h] [-d <directory>] -b <beginning_with> [-s]"
+        echo
+        echo "  -h: Show this help message."
+        echo "  -b: Tests beginning with this name will be included."
+        echo "      Can only be used once. If not specified, all directories will be included."
+        echo "  -d: The directory to find packages with tests in."
+        echo "      Can be a path or a /... style pattern."
+        echo "      Can be repeated to specify multiple directories."
+        echo "      Default: ./..."
+        echo "  -s: Split final package list with spaces rather than newlines."
+    } >&2
+}
+
+beginningWith=""
+dirs=()
+s=0
+while getopts ":hb:c:d:s" opt; do
+    case $opt in
+        h)
+            usage
+            exit 0
+            ;;
+        b)
+            beginningWith="$OPTARG"
+            ;;
+        d)
+            dirs+=("$OPTARG")
+            ;;
+        s)
+            s=1
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+if [[ ${#dirs[@]} -eq 0 ]]; then
+    dirs+=("./...")
+fi
+if [ -z "$beginningWith" ]; then
+    for pkg in "${dirs[@]}"; do
+        if [ $s -eq 1 ]; then
+            printf "%s " "$pkg"
+        else
+            printf "%s\n" "$pkg"
+        fi
+    done
+    exit 0
+fi
+
+readarray -t PACKAGES <<< "$(go list -f '{{.Dir}}' -e "${dirs[@]}")"
+
+for i in "${!PACKAGES[@]}"; do
+    readarray -t PKG_FILES <<< "$(find "${PACKAGES[$i]}" -type f -name '*_test.go')"
+    if [ ${#PKG_FILES[@]} -eq 0 ] || [ ${#PKG_FILES[@]} -eq 1 ] && [ -z "${PKG_FILES[0]}" ]; then
+        unset "PACKAGES[$i]"
+        continue
+    fi
+    if ! grep -q "^func $beginningWith" "${PKG_FILES[@]}"; then
+        unset "PACKAGES[$i]"
+    fi
+done
+
+for pkg in "${PACKAGES[@]}"; do
+    if [ $s -eq 1 ]; then
+        printf "%s " "$pkg"
+    else
+        printf "%s\n" "$pkg"
+    fi
+done

scripts/ci/backend-tests/shard.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+    {
+        echo "shard.sh: Shard tests for parallel execution in CI."
+        echo "usage: $0 [-h] -n <shard> -m <total_shards> [-d <directory>] [-s]"
+        echo
+        echo "  -h: Show this help message."
+        echo "  -n: The shard number (1-indexed)."
+        echo "  -m: The total number of shards. Must be equal to or greater than -n."
+        echo "  -N: The shard in shard notation (n/m), corresponding to -n and -m."
+        echo "  -d: The directory to find packages with tests in."
+        echo "      Can be a path or a /... style pattern."
+        echo "      Can be repeated to specify multiple directories."
+        echo "      Can be - to read from stdin."
+        echo "      Default: ./..."
+        echo "  -s: Split final package list with spaces rather than newlines."
+    } >&2
+}
+
+is_int() {
+    # we can't just return the result of the regex match shellcheck is unhappy...
+    if [[ "$1" =~ ^[0-9]+$ ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+n=0
+m=0
+dirs=()
+s=0
+while getopts ":hn:m:d:sN:" opt; do
+    case $opt in
+        h)
+            usage
+            exit 0
+            ;;
+        n)
+            if ! is_int "$OPTARG"; then
+                echo "Error: -n must be an integer." >&2
+                usage
+                exit 1
+            fi
+            n=$OPTARG
+            ;;
+        m)
+            if ! is_int "$OPTARG"; then
+                echo "Error: -m must be an integer." >&2
+                usage
+                exit 1
+            fi
+            m=$OPTARG
+            ;;
+        N)
+            if [[ "$OPTARG" =~ ^([0-9]+)/([0-9]+)$ ]]; then
+                n="${BASH_REMATCH[1]}"
+                m="${BASH_REMATCH[2]}"
+            else
+                echo "Error: -N must be in the form n/m." >&2
+                usage
+                exit 1
+            fi
+            ;;
+        d)
+            dirs+=("$OPTARG")
+            ;;
+        s)
+            s=1
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            exit 1
+            ;;
+        :)
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+if [[ $n -eq 0 || $m -eq 0 ]]; then
+    echo "Error: -n and -m are required." >&2
+    usage
+    exit 1
+fi
+if [[ $n -lt 1 || $m -lt 1 ]]; then
+    echo "Error: -n and -m must be greater than 0." >&2
+    usage
+    exit 1
+fi
+if [[ $n -gt $m ]]; then
+    echo "Error: -n must be less than or equal to -m." >&2
+    usage
+    exit 1
+fi
+if [[ ${#dirs[@]} -eq 0 ]]; then
+    dirs+=("./...")
+fi
+# If dirs is just ("-"), read from stdin instead.
+if [[ ${#dirs[@]} -eq 1 && "${dirs[0]}" == "-" ]]; then
+    dirs=()
+    while IFS= read -r line; do
+        dirs+=("$line")
+    done
+fi
+if [[ $n -eq 1 && $m -eq 1 ]]; then
+    # If there is only one shard, just return all packages.
+    for pkg in "${dirs[@]}"; do
+        if [ $s -eq 1 ]; then
+            printf "%s " "$pkg"
+        else
+            printf "%s\n" "$pkg"
+        fi
+    done
+    exit 0
+fi
+
+readarray -t PACKAGES <<< "$(go list -f '{{.Dir}}' -e "${dirs[@]}")"
+if [[ ${#PACKAGES[@]} -eq 0 ]]; then
+    echo "No packages found in directories: ${dirs[*]}" >&2
+    exit 1
+fi
+
+for i in "${!PACKAGES[@]}"; do
+    if [ -z "$(find "${PACKAGES[i]}" -maxdepth 1 -type f -name '*_test.go' -printf '.' -quit)" ]; then
+        # There are no test files in this package.
+        unset 'PACKAGES[i]'
+    fi
+done
+
+for i in "${!PACKAGES[@]}"; do
+    if (( (i % m) + 1 != n )); then
+        unset 'PACKAGES[i]'
+    fi
+done
+
+for pkg in "${PACKAGES[@]}"; do
+    if [ $s -eq 1 ]; then
+        printf "%s " "$pkg"
+    else
+        printf "%s\n" "$pkg"
+    fi
+done