MINOR: Reduce scala compilation time by 15% via scalac backend parallelism (#11739)

Introduce `maxScalacThreads` and set the default to the lowest of `8`
and the number of processors available to the JVM. The number `8` was
picked empirically, the sweet spot is between 6 and 10.

On my desktop, `./gradlew clean core:compileScala core:compileTestScala`
improved from around 60s to 51s ( with this change.

While at it, we improve the build output to include more useful
information at the start: build id, max parallel forks, max scala
threads and max test retries.

Reviewers: Manikumar Reddy <manikumar.reddy@gmail.com>, Sean Li

README.md

@@ -230,13 +230,15 @@ The following options should be set with a `-P` switch, for example `./gradlew -
 
 * `commitId`: sets the build commit ID as .git/HEAD might not be correct if there are local commits added for build purposes.
 * `mavenUrl`: sets the URL of the maven deployment repository (`file://path/to/repo` can be used to point to a local repository).
-* `maxParallelForks`: limits the maximum number of processes for each task.
+* `maxParallelForks`: maximum number of test processes to start in parallel. Defaults to the number of processors available to the JVM.
+* `maxScalacThreads`: maximum number of worker threads for the scalac backend. Defaults to the lowest of `8` and the number of processors
+available to the JVM. The value must be between 1 and 16 (inclusive). 
 * `ignoreFailures`: ignore test failures from junit
 * `showStandardStreams`: shows standard out and standard error of the test JVM(s) on the console.
 * `skipSigning`: skips signing of artifacts.
 * `testLoggingEvents`: unit test events to be logged, separated by comma. For example `./gradlew -PtestLoggingEvents=started,passed,skipped,failed test`.
 * `xmlSpotBugsReport`: enable XML reports for spotBugs. This also disables HTML reports as only one can be enabled at a time.
-* `maxTestRetries`: the maximum number of retries for a failing test case.
+* `maxTestRetries`: maximum number of retries for a failing test case.
 * `maxTestRetryFailures`: maximum number of test failures before retrying is disabled for subsequent tests.
 * `enableTestCoverage`: enables test coverage plugins and tasks, including bytecode enhancement of classes required to track said
 coverage. Note that this introduces some overhead when running tests and hence why it's disabled by default (the overhead

build.gradle

@@ -119,7 +119,9 @@ ext {
       "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
     )
 
-  userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null
+  maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors()
+  maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacParallelism.toInteger() :
+      Math.min(Runtime.runtime.availableProcessors(), 8)
   userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures : false
 
   userMaxTestRetries = project.hasProperty('maxTestRetries') ? maxTestRetries.toInteger() : 0
@@ -146,7 +148,26 @@ ext {
 
   generatedDocsDir = new File("${project.rootDir}/docs/generated")
 
-  commitId = project.hasProperty('commitId') ? commitId : null
+  commitId = determineCommitId()
+}
+
+def determineCommitId() {
+  def takeFromHash = 16
+  if (project.hasProperty('commitId2')) {
+    commitId2.take(takeFromHash)
+  } else if (file("$rootDir/.git/HEAD").exists()) {
+    def headRef = file("$rootDir/.git/HEAD").text
+    if (headRef.contains('ref: ')) {
+      headRef = headRef.replaceAll('ref: ', '').trim()
+      if (file("$rootDir/.git/$headRef").exists()) {
+        file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
+      }
+    } else {
+      headRef.trim().take(takeFromHash)
+    }
+  } else {
+    "unknown"
+  }
 }
 
 apply from: file('wrapper.gradle')
@@ -186,7 +207,8 @@ if (file('.git').exists()) {
 } else {
   rat.enabled = false
 }
-println("Starting build with version $version using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Starting build with version $version (commit id ${commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Build properties: maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
 
 subprojects {
 
@@ -391,7 +413,7 @@ subprojects {
   }
 
   test {
-    maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+    maxParallelForks = maxTestForks
     ignoreFailures = userIgnoreFailures
 
     maxHeapSize = defaultMaxHeapSize
@@ -416,7 +438,7 @@ subprojects {
   }
 
   task integrationTest(type: Test, dependsOn: compileJava) {
-    maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+    maxParallelForks = maxTestForks
     ignoreFailures = userIgnoreFailures
 
     maxHeapSize = defaultMaxHeapSize
@@ -449,7 +471,7 @@ subprojects {
   }
 
   task unitTest(type: Test, dependsOn: compileJava) {
-    maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+    maxParallelForks = maxTestForks
     ignoreFailures = userIgnoreFailures
 
     maxHeapSize = defaultMaxHeapSize
@@ -561,6 +583,7 @@ subprojects {
   }
 
   tasks.withType(ScalaCompile) {
+
     scalaCompileOptions.additionalParameters = [
       "-deprecation",
       "-unchecked",
@@ -570,6 +593,7 @@ subprojects {
       "-language:postfixOps",
       "-language:implicitConversions",
       "-language:existentials",
+      "-Ybackend-parallelism", maxScalacThreads.toString(),
       "-Xlint:constant",
       "-Xlint:delayedinit-select",
       "-Xlint:doc-detached",
@@ -689,25 +713,6 @@ subprojects {
     task reportCoverage(dependsOn: [coverageGen])
   }
 
-  task determineCommitId {
-    def takeFromHash = 16
-    if (commitId) {
-      commitId = commitId.take(takeFromHash)
-    } else if (file("$rootDir/.git/HEAD").exists()) {
-      def headRef = file("$rootDir/.git/HEAD").text
-      if (headRef.contains('ref: ')) {
-        headRef = headRef.replaceAll('ref: ', '').trim()
-        if (file("$rootDir/.git/$headRef").exists()) {
-          commitId = file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
-        }
-      } else {
-        commitId = headRef.trim().take(takeFromHash)
-      }
-    } else {
-      commitId = "unknown"
-    }
-  }
-
 }
 
 gradle.taskGraph.whenReady { taskGraph ->
@@ -1214,7 +1219,7 @@ project(':clients') {
     testImplementation libs.jacksonJaxrsJsonProvider
   }
 
-  task createVersionFile(dependsOn: determineCommitId) {
+  task createVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }
@@ -1329,7 +1334,7 @@ project(':raft') {
     testRuntimeOnly libs.slf4jlog4j
   }
 
-  task createVersionFile(dependsOn: determineCommitId) {
+  task createVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }
@@ -1408,7 +1413,7 @@ project(':server-common') {
     testRuntimeOnly libs.slf4jlog4j
   }
 
-  task createVersionFile(dependsOn: determineCommitId) {
+  task createVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }
@@ -1464,7 +1469,7 @@ project(':storage:api') {
     testRuntimeOnly libs.slf4jlog4j
   }
 
-  task createVersionFile(dependsOn: determineCommitId) {
+  task createVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }
@@ -1530,7 +1535,7 @@ project(':storage') {
     testRuntimeOnly libs.slf4jlog4j
   }
 
-  task createVersionFile(dependsOn: determineCommitId) {
+  task createVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }
@@ -1806,7 +1811,7 @@ project(':streams') {
     duplicatesStrategy 'exclude'
   }
 
-  task createStreamsVersionFile(dependsOn: determineCommitId) {
+  task createStreamsVersionFile() {
     ext.receiptFile = file("$buildDir/kafka/$buildStreamsVersionFileName")
     outputs.file receiptFile
     outputs.upToDateWhen { false }