diff --git a/Jenkinsfile b/Jenkinsfile
index bc63364fc6a..07db6e4b648 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,19 +26,26 @@ def setupGradle() {
 }
 
 def doValidation() {
-  sh '''
+  sh """
     ./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
         spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \
         --profile --no-daemon --continue -PxmlSpotBugsReport=true
-  '''
+  """
 }
 
-def doTest(target = "unitTest integrationTest") {
-  sh """
-    ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
-        --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
-        -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5
-  """
+def isChangeRequest(env) {
+  env.CHANGE_ID != null && !env.CHANGE_ID.isEmpty()
+}
+
+def retryFlagsString(env) {
+    if (isChangeRequest(env)) " -PmaxTestRetries=1 -PmaxTestRetryFailures=5"
+    else ""
+}
+
+def doTest(env, target = "unitTest integrationTest") {
+  sh """./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
+      --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
+      -PignoreFailures=true -PmaxParallelForks=2""" + retryFlagsString(env)
   junit '**/build/test-results/**/TEST-*.xml'
 }
 
@@ -95,10 +102,16 @@ def tryStreamsArchetype() {
 
 pipeline {
   agent none
+  
+  options {
+    disableConcurrentBuilds()
+  }
+  
   stages {
     stage('Build') {
       parallel {
-        stage('JDK 8') {
+
+        stage('JDK 8 and Scala 2.12') {
           agent { label 'ubuntu' }
           tools {
             jdk 'jdk_1.8_latest'
@@ -114,12 +127,12 @@ pipeline {
           steps {
             setupGradle()
             doValidation()
-            doTest()
+            doTest(env)
             tryStreamsArchetype()
           }
         }
 
-        stage('JDK 11') {
+        stage('JDK 11 and Scala 2.13') {
           agent { label 'ubuntu' }
           tools {
             jdk 'jdk_11_latest'
@@ -134,12 +147,12 @@ pipeline {
           steps {
             setupGradle()
             doValidation()
-            doTest()
+            doTest(env)
             echo 'Skipping Kafka Streams archetype test for Java 11'
           }
         }
-       
-        stage('JDK 15') {
+
+        stage('JDK 15 and Scala 2.13') {
           agent { label 'ubuntu' }
           tools {
             jdk 'jdk_15_latest'
@@ -154,7 +167,7 @@ pipeline {
           steps {
             setupGradle()
             doValidation()
-            doTest()
+            doTest(env)
             echo 'Skipping Kafka Streams archetype test for Java 15'
           }
         }
@@ -172,11 +185,102 @@ pipeline {
             setupGradle()
             doValidation()
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-              doTest('unitTest')
+              doTest(env, 'unitTest')
             }
             echo 'Skipping Kafka Streams archetype test for ARM build'
           }
         }
+        
+        // To avoid excessive Jenkins resource usage, we only run the stages
+        // above at the PR stage. The ones below are executed after changes
+        // are pushed to trunk and/or release branches. We achieve this via
+        // the `when` clause.
+        
+        stage('JDK 8 and Scala 2.13') {
+          when {
+            not { changeRequest() }
+            beforeAgent true
+          }
+          agent { label 'ubuntu' }
+          tools {
+            jdk 'jdk_1.8_latest'
+            maven 'maven_3_latest'
+          }
+          options {
+            timeout(time: 8, unit: 'HOURS') 
+            timestamps()
+          }
+          environment {
+            SCALA_VERSION=2.13
+          }
+          steps {
+            setupGradle()
+            doValidation()
+            doTest(env)
+            tryStreamsArchetype()
+          }
+        }
+
+        stage('JDK 11 and Scala 2.12') {
+          when {
+            not { changeRequest() }
+            beforeAgent true
+          }
+          agent { label 'ubuntu' }
+          tools {
+            jdk 'jdk_11_latest'
+          }
+          options {
+            timeout(time: 8, unit: 'HOURS') 
+            timestamps()
+          }
+          environment {
+            SCALA_VERSION=2.12
+          }
+          steps {
+            setupGradle()
+            doValidation()
+            doTest(env)
+            echo 'Skipping Kafka Streams archetype test for Java 11'
+          }
+        }
+
+        stage('JDK 15 and Scala 2.12') {
+          when {
+            not { changeRequest() }
+            beforeAgent true
+          }
+          agent { label 'ubuntu' }
+          tools {
+            jdk 'jdk_15_latest'
+          }
+          options {
+            timeout(time: 8, unit: 'HOURS') 
+            timestamps()
+          }
+          environment {
+            SCALA_VERSION=2.12
+          }
+          steps {
+            setupGradle()
+            doValidation()
+            doTest(env)
+            echo 'Skipping Kafka Streams archetype test for Java 15'
+          }
+        }
+      }
+    }
+  }
+  
+  post {
+    always {
+      script {
+        if (!isChangeRequest(env)) {
+          step([$class: 'Mailer',
+               notifyEveryUnstableBuild: true,
+               recipients: "dev@kafka.apache.org",
+               sendToIndividuals: false])
+        }
       }
     }
   }
diff --git a/release.py b/release.py
index 5f2cede4976..751e375f7a9 100755
--- a/release.py
+++ b/release.py
@@ -731,7 +731,7 @@ https://kafka.apache.org/%(docs_version)s/documentation.html
 https://kafka.apache.org/%(docs_version)s/protocol.html
 
 * Successful Jenkins builds for the %(dev_branch)s branch:
-Unit/integration tests: https://builds.apache.org/job/kafka-%(dev_branch)s-jdk8/<BUILD NUMBER>/
+Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s/<BUILD NUMBER>/
 System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s/<BUILD_NUMBER>/
 
 /**************************************