Chore: Increase number of backend test retries in `grabpl` to 5 in release pipelines (#34493)

* Increase number of backend test retries to 5

* Exclude release-branch pipelines

* Fixes according to reviewer's comments

* Refactor

* Remove unused arguments

* Remove magic number

.drone.yml

@@ -755,8 +755,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition oss
-  - ./bin/grabpl integration-tests --edition oss
+  - ./bin/grabpl test-backend --edition oss --tries 5
+  - ./bin/grabpl integration-tests --edition oss --tries 5
   depends_on:
   - initialize
 
@@ -1130,8 +1130,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition enterprise
-  - ./bin/grabpl integration-tests --edition enterprise
+  - ./bin/grabpl test-backend --edition enterprise --tries 5
+  - ./bin/grabpl integration-tests --edition enterprise --tries 5
   depends_on:
   - initialize
 
@@ -1196,8 +1196,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition enterprise2
-  - ./bin/grabpl integration-tests --edition enterprise2
+  - ./bin/grabpl test-backend --edition enterprise2 --tries 5
+  - ./bin/grabpl integration-tests --edition enterprise2 --tries 5
   depends_on:
   - initialize
 
@@ -1710,8 +1710,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition oss
-  - ./bin/grabpl integration-tests --edition oss
+  - ./bin/grabpl test-backend --edition oss --tries 5
+  - ./bin/grabpl integration-tests --edition oss --tries 5
   depends_on:
   - initialize
 
@@ -2074,8 +2074,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition enterprise
-  - ./bin/grabpl integration-tests --edition enterprise
+  - ./bin/grabpl test-backend --edition enterprise --tries 5
+  - ./bin/grabpl integration-tests --edition enterprise --tries 5
   depends_on:
   - initialize
 
@@ -2140,8 +2140,8 @@ steps:
   image: grafana/build-container:1.4.1
   commands:
   - "[ $(grep FocusConvey -R pkg | wc -l) -eq \"0\" ] || exit 1"
-  - ./bin/grabpl test-backend --edition enterprise2
-  - ./bin/grabpl integration-tests --edition enterprise2
+  - ./bin/grabpl test-backend --edition enterprise2 --tries 5
+  - ./bin/grabpl integration-tests --edition enterprise2 --tries 5
   depends_on:
   - initialize
 

scripts/lib.star

@@ -434,7 +434,12 @@ def build_plugins_step(edition, sign=False):
         ],
     }
 
-def test_backend_step(edition):
+def test_backend_step(edition, tries=None):
+    test_backend_cmd = './bin/grabpl test-backend --edition {}'.format(edition)
+    integration_tests_cmd = './bin/grabpl integration-tests --edition {}'.format(edition)
+    if tries:
+        test_backend_cmd += ' --tries {}'.format(tries)
+        integration_tests_cmd += ' --tries {}'.format(tries)
     return {
         'name': 'test-backend' + enterprise2_sfx(edition),
         'image': build_image,
@@ -445,9 +450,9 @@ def test_backend_step(edition):
             # First make sure that there are no tests with FocusConvey
             '[ $(grep FocusConvey -R pkg | wc -l) -eq "0" ] || exit 1',
             # Then execute non-integration tests in parallel, since it should be safe
-            './bin/grabpl test-backend --edition {}'.format(edition),
+            test_backend_cmd,
             # Then execute integration tests in serial
-            './bin/grabpl integration-tests --edition {}'.format(edition),
+            integration_tests_cmd,
         ],
     }
 

scripts/release.star

@@ -67,11 +67,14 @@ def get_steps(edition, ver_mode):
     should_publish = ver_mode in ('release', 'test-release',)
     should_upload = should_publish or ver_mode in ('release-branch',)
     include_enterprise2 = edition == 'enterprise'
+    tries = None
+    if should_publish:
+        tries = 5
 
     steps = [
         codespell_step(),
         shellcheck_step(),
-        test_backend_step(edition=edition),
+        test_backend_step(edition=edition, tries=tries),
         lint_backend_step(edition=edition),
         test_frontend_step(),
         build_backend_step(edition=edition, ver_mode=ver_mode),
@@ -84,7 +87,7 @@ def get_steps(edition, ver_mode):
     if include_enterprise2:
         edition2 = 'enterprise2'
         steps.extend([
-            test_backend_step(edition=edition2),
+            test_backend_step(edition=edition2, tries=tries),
             lint_backend_step(edition=edition2),
             build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
         ])