Replace remaining calls to testing.Short where possible. (#110765)

* Replace remaining calls to testing.Short where possible.
* Update style guide.
* Revert change in TestAlertmanager_ExtraDedupStage, as it doesn't work.
* Make TestAlertRulePostExport into integration test.

contribute/backend/style-guide.md

@@ -60,13 +60,12 @@ You only need to define `TestMain` in one `_test.go` file within each package.
 
 We run unit and integration tests separately, to help keep our CI pipeline running smoothly and provide a better developer experience.
 
-To properly mark a test as being an integration test, you must format your test function definition as follows, with the function name starting with `TestIntegration` and the check for `testing.Short()`:
+To properly mark a test as being an integration test, you must format your test function definition as follows, with the function name starting with `TestIntegration` and the check for running in Short mode by using `testutil.SkipIntegrationTestInShortMode(t)` function:
 
 ```go
 func TestIntegrationFoo(t *testing.T) {
-    if testing.Short() {
+    testutil.SkipIntegrationTestInShortMode(t)
-        t.Skip("skipping integration test")
+
-    }
     // function body
 }
 ```

pkg/cmd/grafana-cli/commands/install_command_test.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/tests/testinfra"
-	"github.com/stretchr/testify/mock"
+	"github.com/grafana/grafana/pkg/util/testutil"
-	"github.com/stretchr/testify/require"
 )
 
 func TestValidateInput(t *testing.T) {
@@ -81,6 +83,43 @@ func TestValidateInput(t *testing.T) {
 	}
 }
 
+func TestIntegrationPluginRepoConfig(t *testing.T) {
+	testutil.SkipIntegrationTestInShortMode(t)
+
+	t.Run("Should use config parameter if it is set", func(t *testing.T) {
+		grafDir, cfgPath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			GrafanaComAPIURL:      "https://grafana-dev.com",
+			GrafanaComSSOAPIToken: "token3",
+		})
+
+		c, err := commandstest.NewCliContext(map[string]string{
+			"config":   cfgPath,
+			"homepath": grafDir,
+		})
+		require.NoError(t, err)
+		repoURL := c.PluginRepoURL()
+		require.Equal(t, "https://grafana-dev.com/plugins", repoURL)
+
+		token := c.GcomToken()
+		require.Equal(t, "token3", token)
+	})
+
+	t.Run("Should use config overrides parameter if it is set alongside config parameter", func(t *testing.T) {
+		// GrafanaComApiUrl is set to the default path https://grafana.com/api
+		grafDir, cfgPath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{})
+
+		// overriding the GrafanaComApiUrl to https://grafana-dev.com
+		c, err := commandstest.NewCliContext(map[string]string{
+			"config":          cfgPath,
+			"homepath":        grafDir,
+			"configOverrides": "cfg:grafana_com.api_url=https://grafana-dev.com",
+		})
+		require.NoError(t, err)
+		repoURL := c.PluginRepoURL()
+		require.Equal(t, "https://grafana-dev.com/plugins", repoURL)
+	})
+}
+
 func TestValidatePluginRepoConfig(t *testing.T) {
 	t.Run("Should use provided repo parameter for installation", func(t *testing.T) {
 		c, err := commandstest.NewCliContext(map[string]string{
@@ -100,44 +139,4 @@ func TestValidatePluginRepoConfig(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, "https://example.com", c.PluginRepoURL())
 	})
-
-	t.Run("Should use config parameter if it is set", func(t *testing.T) {
-		if testing.Short() {
-			t.Skip("skipping integration test")
-		}
-		grafDir, cfgPath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			GrafanaComAPIURL:      "https://grafana-dev.com",
-			GrafanaComSSOAPIToken: "token3",
-		})
-
-		c, err := commandstest.NewCliContext(map[string]string{
-			"config":   cfgPath,
-			"homepath": grafDir,
-		})
-		require.NoError(t, err)
-		repoURL := c.PluginRepoURL()
-		require.Equal(t, "https://grafana-dev.com/plugins", repoURL)
-
-		token := c.GcomToken()
-		require.Equal(t, "token3", token)
-	})
-
-	t.Run("Should use config overrides parameter if it is set alongside config parameter", func(t *testing.T) {
-		if testing.Short() {
-			t.Skip("skipping integration test")
-		}
-
-		// GrafanaComApiUrl is set to the default path https://grafana.com/api
-		grafDir, cfgPath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{})
-
-		// overriding the GrafanaComApiUrl to https://grafana-dev.com
-		c, err := commandstest.NewCliContext(map[string]string{
-			"config":          cfgPath,
-			"homepath":        grafDir,
-			"configOverrides": "cfg:grafana_com.api_url=https://grafana-dev.com",
-		})
-		require.NoError(t, err)
-		repoURL := c.PluginRepoURL()
-		require.Equal(t, "https://grafana-dev.com/plugins", repoURL)
-	})
 }

pkg/registry/apis/iam/resourcepermission/sql_test.go

@@ -206,10 +206,8 @@ func TestIntegration_ResourcePermSqlBackend_getResourcePermission(t *testing.T)
 	}
 }
 
-func TestResourcePermSqlBackend_deleteResourcePermission(t *testing.T) {
+func TestIntegrationResourcePermSqlBackend_deleteResourcePermission(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test in short mode")
-	}
 
 	backend := setupBackend(t)
 	sql, err := backend.dbProvider(context.Background())

pkg/registry/apis/iam/resourcepermission/storage_backend_test.go

@@ -330,10 +330,6 @@ func TestWriteEvent_Add(t *testing.T) {
 }
 
 func TestWriteEvent_Delete(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
 	backend := setupBackend(t)
 	sql, err := backend.dbProvider(context.Background())
 	require.NoError(t, err)

pkg/services/accesscontrol/resourcepermissions/store_test.go

@@ -753,10 +753,6 @@ func retrievePermissionsHelper(store *store, t *testing.T) []orgPermission {
 }
 
 func TestStore_StoreActionSet(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test")
-	}
-
 	type actionSetTest struct {
 		desc     string
 		resource string

pkg/services/annotations/annotationsimpl/loki/historian_store_test.go

@@ -10,11 +10,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/grafana/grafana/pkg/services/ngalert/lokiclient"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/maps"
 
+	"github.com/grafana/grafana/pkg/services/ngalert/lokiclient"
+
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"

pkg/services/ngalert/store/provisioning_store_test.go

@@ -156,10 +156,8 @@ func TestIntegrationProvisioningStore(t *testing.T) {
 	}
 }
 
-func TestSetProvenance_DeadlockScenarios(t *testing.T) {
+func TestIntegrationSetProvenance_DeadlockScenarios(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test")
-	}
 
 	ng, dbStore := tests.SetupTestEnv(t, testAlertingIntervalSeconds)
 	dbStore.FeatureToggles = featuremgmt.WithFeatures(featuremgmt.FlagAlertingProvenanceLockWrites)

pkg/tests/alertmanager/alertmanager_test.go

@@ -8,6 +8,7 @@ import (
 )
 
 func TestAlertmanager_ExtraDedupStage(t *testing.T) {
+	// TODO: rename test and call testutil.SkipIntegrationTestInShortMode(t)
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}

pkg/tests/api/alerting/api_ruler_test.go

@@ -711,7 +711,7 @@ func TestIntegrationAlertRuleNestedPermissions(t *testing.T) {
 	})
 }
 
-func TestAlertRulePostExport(t *testing.T) {
+func TestIntegrationAlertRulePostExport(t *testing.T) {
 	testinfra.SQLiteIntegrationTest(t)
 
 	// Setup Grafana and its Database

pkg/tests/api/folders/api_folders_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/grafana-openapi-client-go/client/folders"
 	"github.com/grafana/grafana-openapi-client-go/models"
+
 	"github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
@@ -17,6 +18,8 @@ import (
 	"github.com/grafana/grafana/pkg/tests/testinfra"
 	"github.com/grafana/grafana/pkg/tests/testsuite"
 	"github.com/grafana/grafana/pkg/util/retryer"
+	"github.com/grafana/grafana/pkg/util/testutil"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,10 +28,9 @@ func TestMain(m *testing.M) {
 	testsuite.Run(m)
 }
 
-func TestGetFolders(t *testing.T) {
+func TestIntegrationGetFolders(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test")
+
-	}
 	// Setup Grafana and its Database
 	dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
 		DisableLegacyAlerting: true,

pkg/tests/apis/iam/service_account_integration_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tests/apis"
 	"github.com/grafana/grafana/pkg/tests/testinfra"
+	"github.com/grafana/grafana/pkg/util/testutil"
+
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -27,9 +29,7 @@ var gvrServiceAccounts = schema.GroupVersionResource{
 }
 
 func TestIntegrationServiceAccounts(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test")
-	}
 
 	// TODO: Figure out why rest.Mode4 is failing
 	modes := []rest.DualWriterMode{rest.Mode0, rest.Mode1, rest.Mode2, rest.Mode3}

pkg/tests/apis/plugins/discovery_test.go

@@ -4,12 +4,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/util/testutil"
 )
 
-func TestPluginsIntegrationDiscovery(t *testing.T) {
+func TestIntegrationPluginsIntegrationDiscovery(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test")
-	}
 
 	t.Run("discovery", func(t *testing.T) {
 		helper := setupHelper(t)

pkg/tests/apis/provisioning/exportjob_test.go

@@ -16,10 +16,8 @@ import (
 	"github.com/grafana/grafana/pkg/util/testutil"
 )
 
-func TestProvisioning_ExportUnifiedToRepository(t *testing.T) {
+func TestIntegrationProvisioning_ExportUnifiedToRepository(t *testing.T) {
-	if testing.Short() {
+	testutil.SkipIntegrationTestInShortMode(t)
-		t.Skip("skipping integration test")
-	}
 
 	helper := runGrafana(t)
 	ctx := context.Background()

pkg/tests/testinfra/testinfra.go

@@ -12,14 +12,16 @@ import (
 	"testing"
 	"time"
 
-	"github.com/grafana/grafana/pkg/configprovider"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/ini.v1"
 
+	"github.com/grafana/grafana/pkg/configprovider"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
+	"github.com/grafana/grafana/pkg/util/testutil"
+
 	"github.com/grafana/dskit/kv"
 
 	"github.com/grafana/grafana/pkg/api"
@@ -570,8 +572,9 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) {
 
 func SQLiteIntegrationTest(t *testing.T) {
 	t.Helper()
+	testutil.SkipIntegrationTestInShortMode(t)
 
-	if testing.Short() || !db.IsTestDbSQLite() {
+	if !db.IsTestDbSQLite() {
 		t.Skip("skipping integration test")
 	}
 }

pkg/tsdb/influxdb/fsql/fsql_test.go

@@ -55,7 +55,7 @@ func (suite *FSQLTestSuite) AfterTest(suiteName, testName string) {
 	suite.server.Shutdown()
 }
 
-func TestFSQLTestSuite(t *testing.T) {
+func TestIntegrationFSQLTestSuite(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}

pkg/tsdb/mysql/mysql_snapshot_test.go

@@ -36,11 +36,8 @@ func TestIntegrationMySQLSnapshots(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
-	shouldRunTest := func() bool {
-		if testing.Short() {
-			return false
-		}
 
+	shouldRunTest := func() bool {
 		testDbName, present := os.LookupEnv("GRAFANA_TEST_DB")
 
 		if present && testDbName == "mysql" {