From 08c55b60ab07eb9d36fb6e9c2316ed7cff271f0f Mon Sep 17 00:00:00 2001 From: Mariell Hoversholm Date: Tue, 13 May 2025 13:08:19 +0200 Subject: [PATCH] Actions: Shard test suite (#105166) --- .github/CODEOWNERS | 1 + .github/workflows/backend-unit-tests.yml | 32 +++- .github/workflows/pr-test-integration.yml | 90 +++++++---- Makefile | 34 ++-- .../ci/backend-tests/pkgs-with-tests-named.sh | 79 ++++++++++ scripts/ci/backend-tests/shard.sh | 149 ++++++++++++++++++ 6 files changed, 336 insertions(+), 49 deletions(-) create mode 100755 scripts/ci/backend-tests/pkgs-with-tests-named.sh create mode 100755 scripts/ci/backend-tests/shard.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 607a91d45cd..f4484f2ee89 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -66,6 +66,7 @@ /build.go @grafana/grafana-backend-services-squad /scripts/modowners/ @grafana/grafana-backend-services-squad /scripts/go-workspace @grafana/grafana-app-platform-squad +/scripts/ci/backend-tests @grafana/grafana-operator-experience-squad /hack/ @grafana/grafana-app-platform-squad /pkg/apis/provisioning @grafana/grafana-git-ui-sync-team diff --git a/.github/workflows/backend-unit-tests.yml b/.github/workflows/backend-unit-tests.yml index a1357d2e87b..caf230e1f42 100644 --- a/.github/workflows/backend-unit-tests.yml +++ b/.github/workflows/backend-unit-tests.yml @@ -24,7 +24,15 @@ jobs: # Run this workflow only for PRs from forks; if it gets merged into `main` or `release-*`, # the `pr-backend-unit-tests-enterprise` workflow will run instead if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true - name: Grafana + strategy: + matrix: + shard: [ + 1/8, 2/8, 3/8, 4/8, + 5/8, 6/8, 7/8, 8/8, + ] + fail-fast: false + + name: Grafana (${{ matrix.shard }}) runs-on: ubuntu-latest-8-cores continue-on-error: true permissions: @@ -42,12 +50,24 @@ jobs: - name: Generate Go code run: make gen-go - name: Run unit tests - run: make test-go-unit + env: + SHARD: ${{ matrix.shard }} + run: | + readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/shard.sh -N"$SHARD")" + go test -short -timeout=30m "${PACKAGES[@]}" grafana-enterprise: # Run this workflow for non-PR events (like pushes to `main` or `release-*`) OR for internal PRs (PRs not from forks) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false - name: Grafana Enterprise + strategy: + matrix: + shard: [ + 1/8, 2/8, 3/8, 4/8, + 5/8, 6/8, 7/8, 8/8, + ] + fail-fast: false + + name: Grafana Enterprise (${{ matrix.shard }}) runs-on: ubuntu-latest-8-cores permissions: contents: read @@ -68,4 +88,8 @@ jobs: - name: Generate Go code run: make gen-go - name: Run unit tests - run: make test-go-unit + env: + SHARD: ${{ matrix.shard }} + run: | + readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/shard.sh -N"$SHARD")" + go test -short -timeout=30m "${PACKAGES[@]}" diff --git a/.github/workflows/pr-test-integration.yml b/.github/workflows/pr-test-integration.yml index c79cc569299..53138fd9cc7 100644 --- a/.github/workflows/pr-test-integration.yml +++ b/.github/workflows/pr-test-integration.yml @@ -6,6 +6,10 @@ on: - main - release-*.*.* pull_request: + types: + - opened + - synchronize + - reopened concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,7 +19,15 @@ permissions: {} jobs: sqlite: - name: Sqlite + strategy: + matrix: + shard: [ + 1/8, 2/8, 3/8, 4/8, + 5/8, 6/8, 7/8, 8/8, + ] + fail-fast: false + + name: Sqlite (${{ matrix.shard }}) runs-on: ubuntu-latest-8-cores permissions: contents: read @@ -29,14 +41,24 @@ jobs: with: go-version-file: go.mod cache: true - - run: | - go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)" - IFS=' ' read -ra packages <<< "$go_packages" - - make gen-go - go test -tags=sqlite -timeout=5m -run '^TestIntegration' "${packages[@]}" + - name: Generate Go code + run: make gen-go + - name: Run tests + env: + SHARD: ${{ matrix.shard }} + run: | + readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)" + go test -tags=sqlite -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}" mysql: - name: MySQL + strategy: + matrix: + shard: [ + 1/8, 2/8, 3/8, 4/8, + 5/8, 6/8, 7/8, 8/8, + ] + fail-fast: false + + name: MySQL (${{ matrix.shard }}) runs-on: ubuntu-latest-8-cores permissions: contents: read @@ -64,19 +86,33 @@ jobs: with: go-version-file: go.mod cache: true - - run: | - go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)" - IFS=' ' read -ra packages <<< "$go_packages" - - sudo apt-get update -yq && sudo apt-get install mariadb-client - mariadb -h 127.0.0.1 -P 3306 -u root -prootpass --disable-ssl-verify-server-cert < devenv/docker/blocks/mysql_tests/setup.sql - make gen-go - go test -tags=mysql -p=1 -timeout=5m -run '^TestIntegration' "${packages[@]}" + - name: Setup MySQL devenv + run: mysql -h 127.0.0.1 -P 3306 -u root -prootpass < devenv/docker/blocks/mysql_tests/setup.sql + - name: Generate Go code + run: make gen-go + - name: Run tests + env: + SHARD: ${{ matrix.shard }} + run: | + readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)" + go test -p=1 -tags=mysql -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}" postgres: - name: Postgres + strategy: + matrix: + shard: [ + 1/8, 2/8, 3/8, 4/8, + 5/8, 6/8, 7/8, 8/8, + ] + fail-fast: false + + name: Postgres (${{ matrix.shard }}) runs-on: ubuntu-latest-8-cores permissions: contents: read + env: + GRAFANA_TEST_DB: postgres + PGPASSWORD: grafanatest + POSTGRES_HOST: 127.0.0.1 services: postgres: image: postgres:12.3-alpine @@ -96,15 +132,13 @@ jobs: with: go-version-file: go.mod cache: true - - env: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: 127.0.0.1 + - name: Setup Postgres devenv + run: psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql + - name: Generate Go code + run: make gen-go + - name: Run tests + env: + SHARD: ${{ matrix.shard }} run: | - go_packages="$(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)" - IFS=' ' read -ra packages <<< "$go_packages" - - sudo apt-get update -yq && sudo apt-get install postgresql-client - psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - make gen-go - go test -p=1 -tags=postgres -timeout=5m -run '^TestIntegration' "${packages[@]}" + readarray -t PACKAGES <<< "$(./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -N"$SHARD" -d-)" + go test -p=1 -tags=postgres -timeout=5m -run '^TestIntegration' "${PACKAGES[@]}" diff --git a/Makefile b/Makefile index 92a7d239f3e..d8cfafb4f7b 100644 --- a/Makefile +++ b/Makefile @@ -18,15 +18,16 @@ GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev) GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS)) GO_BUILD_FLAGS += $(GO_RACE_FLAG) GO_TEST_FLAGS += $(if $(GO_BUILD_TAGS),-tags=$(GO_BUILD_TAGS)) -GO_TEST_OUTPUT := $(shell [ -n "$(GO_TEST_OUTPUT)" ] && echo '-json | tee $(GO_TEST_OUTPUT) | tparse -all') GIT_BASE = remotes/origin/main # GNU xargs has flag -r, and BSD xargs (e.g. MacOS) has that behaviour by default XARGSR = $(shell xargs --version 2>&1 | grep -q GNU && echo xargs -r || echo xargs) -targets := $(shell echo '$(sources)' | tr "," " ") +# Test sharding to replicate CI behaviour locally. +SHARD ?= 1 +SHARDS ?= 1 -GO_INTEGRATION_TESTS := $(shell find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) +targets := $(shell echo '$(sources)' | tr "," " ") .PHONY: all all: deps build @@ -267,8 +268,9 @@ test-go: test-go-unit test-go-integration .PHONY: test-go-unit test-go-unit: ## Run unit tests for backend with flags. - @echo "backend unit tests" - $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -v -short -timeout=30m $(GO_TEST_FILES) $(GO_TEST_OUTPUT) + @echo "backend unit tests ($(SHARD)/$(SHARDS))" + $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -v -short -timeout=30m \ + $(shell ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-go-unit-pretty test-go-unit-pretty: check-tparse @@ -281,7 +283,8 @@ test-go-unit-pretty: check-tparse .PHONY: test-go-integration test-go-integration: ## Run integration tests for backend with flags. @echo "test backend integration tests" - $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=$(GO_INTEGRATION_COVER_PROFILE) -timeout=5m $(GO_INTEGRATION_TESTS) $(GO_TEST_OUTPUT) + $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=$(GO_INTEGRATION_COVER_PROFILE) -timeout=5m \ + $(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-go-integration-alertmanager test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block). @@ -303,32 +306,29 @@ test-go-integration-postgres: devenv-postgres ## Run integration tests for postg @echo "test backend integration postgres tests" $(GO) clean -testcache GRAFANA_TEST_DB=postgres \ - $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS) + $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m \ + $(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-go-integration-mysql test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backend with flags. @echo "test backend integration mysql tests" GRAFANA_TEST_DB=mysql \ - $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS) + $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m \ + $(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-go-integration-redis test-go-integration-redis: ## Run integration tests for redis cache. @echo "test backend integration redis tests" $(GO) clean -testcache - REDIS_URL=localhost:6379 $(GO) test $(GO_TEST_FLAGS) -run IntegrationRedis -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS) + REDIS_URL=localhost:6379 $(GO) test $(GO_TEST_FLAGS) -run IntegrationRedis -covermode=atomic -timeout=2m \ + $(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-go-integration-memcached test-go-integration-memcached: ## Run integration tests for memcached cache. @echo "test backend integration memcached tests" $(GO) clean -testcache - MEMCACHED_HOSTS=localhost:11211 $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -run IntegrationMemcached -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS) - -.PHONY: test-go-integration-spanner -test-go-integration-spanner: ## Run integration tests for Spanner backend with flags. Uses spanner-emulator on localhost:9010 and localhost:9020. - @if [ "${WIRE_TAGS}" != "enterprise" ]; then echo "Spanner integration test require enterprise setup"; exit 1; fi - @echo "test backend integration spanner tests" - GRAFANA_TEST_DB=spanner \ - $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -p=1 -count=1 -v -run "^TestIntegration" -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS) + MEMCACHED_HOSTS=localhost:11211 $(GO) test $(GO_RACE_FLAG) $(GO_TEST_FLAGS) -run IntegrationMemcached -covermode=atomic -timeout=2m \ + $(shell ./scripts/ci/backend-tests/pkgs-with-tests-named.sh -b TestIntegration | ./scripts/ci/backend-tests/shard.sh -n$(SHARD) -m$(SHARDS) -s) .PHONY: test-js test-js: ## Run tests for frontend. diff --git a/scripts/ci/backend-tests/pkgs-with-tests-named.sh b/scripts/ci/backend-tests/pkgs-with-tests-named.sh new file mode 100755 index 00000000000..ea8cf286880 --- /dev/null +++ b/scripts/ci/backend-tests/pkgs-with-tests-named.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + { + echo "pkgs-with-tests-named.sh: Find packages with tests in them, filtered by the test names." + echo "usage: $0 [-h] [-d ] -b [-s]" + echo + echo " -h: Show this help message." + echo " -b: Tests beginning with this name will be included." + echo " Can only be used once. If not specified, all directories will be included." + echo " -d: The directory to find packages with tests in." + echo " Can be a path or a /... style pattern." + echo " Can be repeated to specify multiple directories." + echo " Default: ./..." + echo " -s: Split final package list with spaces rather than newlines." + } >&2 +} + +beginningWith="" +dirs=() +s=0 +while getopts ":hb:c:d:s" opt; do + case $opt in + h) + usage + exit 0 + ;; + b) + beginningWith="$OPTARG" + ;; + d) + dirs+=("$OPTARG") + ;; + s) + s=1 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [[ ${#dirs[@]} -eq 0 ]]; then + dirs+=("./...") +fi +if [ -z "$beginningWith" ]; then + for pkg in "${dirs[@]}"; do + if [ $s -eq 1 ]; then + printf "%s " "$pkg" + else + printf "%s\n" "$pkg" + fi + done + exit 0 +fi + +readarray -t PACKAGES <<< "$(go list -f '{{.Dir}}' -e "${dirs[@]}")" + +for i in "${!PACKAGES[@]}"; do + readarray -t PKG_FILES <<< "$(find "${PACKAGES[$i]}" -type f -name '*_test.go')" + if [ ${#PKG_FILES[@]} -eq 0 ] || [ ${#PKG_FILES[@]} -eq 1 ] && [ -z "${PKG_FILES[0]}" ]; then + unset "PACKAGES[$i]" + continue + fi + if ! grep -q "^func $beginningWith" "${PKG_FILES[@]}"; then + unset "PACKAGES[$i]" + fi +done + +for pkg in "${PACKAGES[@]}"; do + if [ $s -eq 1 ]; then + printf "%s " "$pkg" + else + printf "%s\n" "$pkg" + fi +done diff --git a/scripts/ci/backend-tests/shard.sh b/scripts/ci/backend-tests/shard.sh new file mode 100755 index 00000000000..6e2702c610a --- /dev/null +++ b/scripts/ci/backend-tests/shard.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + { + echo "shard.sh: Shard tests for parallel execution in CI." + echo "usage: $0 [-h] -n -m [-d ] [-s]" + echo + echo " -h: Show this help message." + echo " -n: The shard number (1-indexed)." + echo " -m: The total number of shards. Must be equal to or greater than -n." + echo " -N: The shard in shard notation (n/m), corresponding to -n and -m." + echo " -d: The directory to find packages with tests in." + echo " Can be a path or a /... style pattern." + echo " Can be repeated to specify multiple directories." + echo " Can be - to read from stdin." + echo " Default: ./..." + echo " -s: Split final package list with spaces rather than newlines." + } >&2 +} + +is_int() { + # we can't just return the result of the regex match shellcheck is unhappy... + if [[ "$1" =~ ^[0-9]+$ ]]; then + return 0 + else + return 1 + fi +} + +n=0 +m=0 +dirs=() +s=0 +while getopts ":hn:m:d:sN:" opt; do + case $opt in + h) + usage + exit 0 + ;; + n) + if ! is_int "$OPTARG"; then + echo "Error: -n must be an integer." >&2 + usage + exit 1 + fi + n=$OPTARG + ;; + m) + if ! is_int "$OPTARG"; then + echo "Error: -m must be an integer." >&2 + usage + exit 1 + fi + m=$OPTARG + ;; + N) + if [[ "$OPTARG" =~ ^([0-9]+)/([0-9]+)$ ]]; then + n="${BASH_REMATCH[1]}" + m="${BASH_REMATCH[2]}" + else + echo "Error: -N must be in the form n/m." >&2 + usage + exit 1 + fi + ;; + d) + dirs+=("$OPTARG") + ;; + s) + s=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + usage + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [[ $n -eq 0 || $m -eq 0 ]]; then + echo "Error: -n and -m are required." >&2 + usage + exit 1 +fi +if [[ $n -lt 1 || $m -lt 1 ]]; then + echo "Error: -n and -m must be greater than 0." >&2 + usage + exit 1 +fi +if [[ $n -gt $m ]]; then + echo "Error: -n must be less than or equal to -m." >&2 + usage + exit 1 +fi +if [[ ${#dirs[@]} -eq 0 ]]; then + dirs+=("./...") +fi +# If dirs is just ("-"), read from stdin instead. +if [[ ${#dirs[@]} -eq 1 && "${dirs[0]}" == "-" ]]; then + dirs=() + while IFS= read -r line; do + dirs+=("$line") + done +fi +if [[ $n -eq 1 && $m -eq 1 ]]; then + # If there is only one shard, just return all packages. + for pkg in "${dirs[@]}"; do + if [ $s -eq 1 ]; then + printf "%s " "$pkg" + else + printf "%s\n" "$pkg" + fi + done + exit 0 +fi + +readarray -t PACKAGES <<< "$(go list -f '{{.Dir}}' -e "${dirs[@]}")" +if [[ ${#PACKAGES[@]} -eq 0 ]]; then + echo "No packages found in directories: ${dirs[*]}" >&2 + exit 1 +fi + +for i in "${!PACKAGES[@]}"; do + if [ -z "$(find "${PACKAGES[i]}" -maxdepth 1 -type f -name '*_test.go' -printf '.' -quit)" ]; then + # There are no test files in this package. + unset 'PACKAGES[i]' + fi +done + +for i in "${!PACKAGES[@]}"; do + if (( (i % m) + 1 != n )); then + unset 'PACKAGES[i]' + fi +done + +for pkg in "${PACKAGES[@]}"; do + if [ $s -eq 1 ]; then + printf "%s " "$pkg" + else + printf "%s\n" "$pkg" + fi +done