Support JMH benchmarks from Jenkins Test Harness (#4135)

* Support running JMH benchmarks from JTH 2.51

* Fix NPE caused by ExtensionFinderTest
* A test for running a sample benchmark
* Bump up JTH to 2.54
* git ignore the report produced by benchmarks

* Fix BindException in BootFailureTest

* Fix NPE during `tearDown()` in CustomPluginManagerTest

JenkinsRule >= 2.50 does not ignore errors during Jenkins shutdown

test-pom/pom.xml

@@ -64,7 +64,7 @@ THE SOFTWARE.
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>jenkins-test-harness</artifactId>
-      <version>2.49</version>
+      <version>2.54</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>

test/.gitignore

@@ -1,2 +1,3 @@
 !*.zip
 hs_err_pid*.log
+jmh-report.json

test/src/test/java/benchmarks/BenchmarkTest.java

@@ -0,0 +1,47 @@
+package benchmarks;
+
+import org.junit.Test;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+
+public class BenchmarkTest {
+    /**
+     * Runs a sample benchmark to make sure that benchmarks execute successfully and generate a report.
+     * <p>
+     * To run your benchmarks, create a benchmark runner similar to this class and use
+     * {@link jenkins.benchmark.jmh.BenchmarkFinder} to automatically find classes for benchmark which are annotated
+     * with {@link jenkins.benchmark.jmh.JmhBenchmark}.
+     * @throws Exception when the benchmark fails to run or throws an exception.
+     * @see <a href="https://jenkins.io/blog/2019/06/21/performance-testing-jenkins/">Blog post on writing benchmarks</a>
+     */
+    @Test
+    public void runSampleBenchmark() throws Exception {
+        // run the minimum possible number of iterations
+        ChainedOptionsBuilder options = new OptionsBuilder()
+                                            .mode(Mode.AverageTime)
+                                            .forks(1)
+                                            .result("jmh-report.json")
+                                            .resultFormat(ResultFormatType.JSON)
+                                            .operationsPerInvocation(1)
+                                            .threads(1)
+                                            .warmupForks(0)
+                                            .warmupIterations(0)
+                                            .measurementBatchSize(1)
+                                            .measurementIterations(1)
+                                            .timeUnit(TimeUnit.MICROSECONDS)
+                                            .shouldFailOnError(true)
+                                            // just run the SampleBenchmark, not other benchmarks
+                                            .include(SampleBenchmark.class.getName() + ".*");
+        new Runner(options.build()).run();
+        assertTrue(Files.exists(Paths.get("jmh-report.json")));
+    }
+}

test/src/test/java/benchmarks/SampleBenchmark.java

@@ -0,0 +1,20 @@
+package benchmarks;
+
+import jenkins.benchmark.jmh.JmhBenchmark;
+import jenkins.benchmark.jmh.JmhBenchmarkState;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * A sample benchmark.
+ */
+@JmhBenchmark
+public class SampleBenchmark {
+    public static class MyState extends JmhBenchmarkState {
+    }
+
+    @Benchmark
+    public void benchmark(MyState state, Blackhole blackhole) {
+        blackhole.consume(state.getJenkins().getSystemMessage());
+    }
+}

test/src/test/java/hudson/CustomPluginManagerTest.java

@@ -67,8 +67,12 @@ public class CustomPluginManagerTest {
             }
 
             @Override
-            public void tearDown(JenkinsRule jenkinsRule, WithCustomLocalPluginManager recipe) throws Exception {
-                System.setProperty(LocalPluginManager.CUSTOM_PLUGIN_MANAGER, oldValue);
+            public void tearDown(JenkinsRule jenkinsRule, WithCustomLocalPluginManager recipe) {
+                if (oldValue != null) {
+                    System.setProperty(LocalPluginManager.CUSTOM_PLUGIN_MANAGER, oldValue);
+                } else {
+                    System.clearProperty(LocalPluginManager.CUSTOM_PLUGIN_MANAGER);
+                }
             }
         }
     }

test/src/test/java/hudson/ExtensionFinderTest.java

@@ -92,7 +92,9 @@ public class ExtensionFinderTest extends HudsonTestCase {
     @Extension
     public static class ModuleImpl extends AbstractModule {
         protected void configure() {
-            if (TestEnvironment.get().testCase instanceof ExtensionFinderTest) {
+            TestEnvironment environment = TestEnvironment.get();
+            // JMH benchmarks do not initialize TestEnvironment, so check for null
+            if (environment != null && environment.testCase instanceof ExtensionFinderTest) {
                 bind(String.class).annotatedWith(LionKing.class).toInstance("lion king");
             }
         }

test/src/test/java/hudson/util/BootFailureTest.java

@@ -48,6 +48,7 @@ public class BootFailureTest {
 
         @Override
         public Hudson newHudson() throws Exception {
+            localPort = 0;
             ServletContext ws = createWebServer();
             wa = new WebAppMain() {
                 @Override