Splitup x-pack testing in more build steps (#101987)

We want to keep pull request and intake build time as short as possible.

While testing this I noticed though that all those checkPartX runs are basically building core modules e.g. like :server by themself not leveraging the gradle cache ideally as all 4 checkPart jobs running in parallel likely run :server:compileJava in parallel. Not sure this is a big deal. I remember within the gradle team we had similar issues and addressed this by
quick dedicated "populating build cache" pipeline steps. Not something we need to look into now, but at one point investigating this approach might be valuable for certain scenarios.

.buildkite/pipelines/intake.template.yml

@@ -32,6 +32,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/intake.yml

@@ -33,6 +33,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/pull-request/part-4-fips.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "Team:Security"
+steps:
+  - label: part-4-fips
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-windows.yml

@@ -0,0 +1,14 @@
+config:
+  allow-labels: "test-windows"
+steps:
+  - label: part-4-windows
+    command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-windows-2022
+      machineType: custom-32-98304
+      diskType: pd-ssd
+      diskSizeGb: 350
+    env:
+      GRADLE_TASK: checkPart4

.buildkite/pipelines/pull-request/part-4.yml

@@ -0,0 +1,11 @@
+config:
+  skip-target-branches: "7.17"
+steps:
+  - label: part-4
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

build.gradle

@@ -161,8 +161,10 @@ tasks.register("verifyVersions") {
     String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key
     String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
     if (versionMapping != expectedMapping) {
-      throw new GradleException("Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
-        "'${expectedMapping}'. Update .backportrc.json.")
+      throw new GradleException(
+        "Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
+          "'${expectedMapping}'. Update .backportrc.json."
+      )
     }
   }
 }
@@ -248,6 +250,8 @@ allprojects {
   plugins.withId('lifecycle-base') {
     if (project.path.startsWith(":x-pack:")) {
       if (project.path.contains("security") || project.path.contains(":ml")) {
+        tasks.register('checkPart4') { dependsOn 'check' }
+      } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") ||  project.path.contains("smoke-test")) {
         tasks.register('checkPart3') { dependsOn 'check' }
       } else {
         tasks.register('checkPart2') { dependsOn 'check' }