KAFKA-3989; Initial support for adding a JMH benchmarking module

Author: bbejeck <bbejeck@gmail.com>

Reviewers: Ewen Cheslack-Postava <ewen@confluent.io>, Ismael Juma <ismael@juma.me.uk>

Closes #1712 from bbejeck/KAFKA-3989_create_jmh_benchmarking_module

build.gradle

@@ -27,6 +27,7 @@ buildscript {
     classpath "org.ajoberstar:grgit:1.7.0"
     classpath 'com.github.ben-manes:gradle-versions-plugin:0.13.0'
     classpath 'org.scoverage:gradle-scoverage:2.1.0'
+    classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4'
   }
 }
 
@@ -822,6 +823,48 @@ project(':streams:examples') {
   }
 }
 
+project(':jmh-benchmarks') {
+
+  apply plugin: 'com.github.johnrengelman.shadow'
+
+  shadowJar {
+    baseName = 'kafka-jmh-benchmarks-all'
+    classifier = null
+    version = null
+  }
+
+  dependencies {
+      compile project(':clients')
+      compile project(':streams')
+      compile 'org.openjdk.jmh:jmh-core:1.17.5'
+      compile 'org.openjdk.jmh:jmh-generator-annprocess:1.17.5'
+      compile 'org.openjdk.jmh:jmh-core-benchmarks:1.17.5'
+  }
+
+  jar {
+    manifest {
+      attributes "Main-Class": "org.openjdk.jmh.Main"
+    }
+  }
+
+
+  task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) {
+
+    main="-jar"
+
+    doFirst {
+      if (System.getProperty("jmhArgs")) {
+          args System.getProperty("jmhArgs").split(',')
+      }
+      args = [shadowJar.archivePath, *args]
+    }
+  }
+
+  javadoc {
+     enabled = false
+  }
+}
+
 project(':log4j-appender') {
   archivesBaseName = "kafka-log4j-appender"
 

checkstyle/import-control.xml

@@ -181,6 +181,15 @@
     </subpackage>
   </subpackage>
 
+  <subpackage name="jmh">
+    <allow pkg="org.openjdk.jmh.annotations" />
+    <allow pkg="org.openjdk.jmh.runner" />
+    <allow pkg="org.openjdk.jmh.runner.options" />
+    <allow pkg="org.apache.kafka.common" />
+    <allow pkg="org.apache.kafka.streams" />
+    <allow pkg="org.github.jamm" />
+  </subpackage>
+
   <subpackage name="log4jappender">
     <allow pkg="org.apache.log4j" />
     <allow pkg="org.apache.kafka.clients" />

jmh-benchmarks/README.md

@@ -0,0 +1,61 @@
+###JMH-Benchmark module
+
+This module contains benchmarks written using [JMH](http://openjdk.java.net/projects/code-tools/jmh/) from OpenJDK.
+Writing correct micro-benchmarks is Java (or another JVM language) is difficult and there are many non-obvious pitfalls (many
+due to compiler optimizations). JMH is a framework for running and analyzing benchmarks (micro or macro) written in Java (or
+another JVM language).
+
+For help in writing correct JMH tests, the best place to start is the [sample code](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/) provided
+by the JMH project.
+
+Typically, JMH is expected to run as a separate project in Maven. The jmh-benchmarks module uses
+the [gradle shadow jar](https://github.com/johnrengelman/shadow) plugin to emulate this behavior, by creating the required
+uber-jar file containing the benchmarking code and required JMH classes.  
+
+JMH is highly configurable and users are encouraged to look through the samples for suggestions
+on what options are available. A good tutorial for using JMH can be found [here](http://tutorials.jenkov.com/java-performance/jmh.html#return-value-from-benchmark-method)
+
+###Gradle Tasks / Running benchmarks in gradle
+
+If no benchmark mode is specified, the default is used which is throughput. It is assumed that users run
+the gradle tasks with './gradlew' from the root of the Kafka project.
+
+*  jmh-benchmarks:shadowJar - creates the uber jar required to run the benchmarks.
+
+*  jmh-benchmarks:jmh - runs the `clean` and `shadowJar` tasks followed by all the benchmarks.
+ 
+### Using the jmh script
+If you want to set specific JMH flags or only run a certain test(s) passing arguments via
+gradle tasks is cumbersome.  Instead you can use the `jhm.sh` script.  NOTE: It is assumed users run
+the jmh.sh script from the jmh-benchmarks module.
+
+* Run a specific test setting fork-mode (number iterations) to 2 :`./jmh.sh -f 2 LRUCacheBenchmark`
+
+* By default all JMH output goes to stdout.  To run a benchmark and capture the results in a file:
+`./jmh.sh -f 2 -o benchmarkResults.txt LRUCacheBenchmark`
+NOTE: For now this script needs to be run from the jmh-benchmarks directory.
+ 
+### Running JMH outside of gradle
+The JMH benchmarks can be run outside of gradle as you would with any executable jar file:
+`java -jar <kafka-repo-dir>/jmh-benchmarks/build/libs/kafka-jmh-benchmarks-all.jar -f2 LRUCacheBenchmark`
+
+### JMH Options
+Some common JMH options are:
+```text
+ 
+   -e <regexp+>                Benchmarks to exclude from the run. 
+ 
+   -f <int>                    How many times to fork a single benchmark. Use 0 to 
+                               disable forking altogether. Warning: disabling 
+                               forking may have detrimental impact on benchmark 
+                               and infrastructure reliability, you might want 
+                               to use different warmup mode instead. 
+ 
+   -o <filename>               Redirect human-readable output to a given file. 
+ 
+  
+ 
+   -v <mode>                   Verbosity mode. Available modes are: [SILENT, NORMAL, 
+                               EXTRA] 
+```
+To view all options run jmh with the -h flag. 

jmh-benchmarks/jmh.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+base_dir=$(dirname $0)
+jmh_project_name="jmh-benchmarks"
+
+if [ ${base_dir} == "." ]; then
+     gradlew_dir=".."
+elif [ ${base_dir} == ${jmh_project_name} ]; then
+     gradlew_dir="."
+else
+    echo "JMH Benchmarks need to be run from the root of the kafka repository or the 'jmh-benchmarks' directory"
+    exit
+fi
+
+gradleCmd="${gradlew_dir}/gradlew"
+libDir="${base_dir}/build/libs"
+
+echo "running gradlew :jmh-benchmarks:clean :jmh-benchmarks:shadowJar in quiet mode"
+
+$gradleCmd  -q :jmh-benchmarks:clean :jmh-benchmarks:shadowJar
+
+echo "gradle build done"
+
+echo "running JMH with args [$@]"
+
+java -jar ${libDir}/kafka-jmh-benchmarks-all.jar "$@"
+
+echo "JMH benchmarks done"

jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.jmh.cache;
+
+import org.apache.kafka.common.cache.LRUCache;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+/**
+ * This is a simple example of a JMH benchmark.
+ *
+ * The sample code provided by the JMH project is a great place to start learning how to write correct benchmarks:
+ * http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/
+ */
+@State(Scope.Thread)
+public class LRUCacheBenchmark {
+
+    private LRUCache<String, String> lruCache;
+
+    private final String key = "the_key_to_use";
+    private final String value = "the quick brown fox jumped over the lazy dog the olympics are about to start";
+    int counter;
+
+
+    @Setup(Level.Trial)
+    public void setUpCaches() {
+        lruCache = new LRUCache<>(100);
+    }
+
+    @Benchmark
+    public String testCachePerformance() {
+        counter++;
+        lruCache.put(key + counter, value + counter);
+        return lruCache.get(key + counter);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(LRUCacheBenchmark.class.getSimpleName())
+                .forks(2)
+                .build();
+
+        new Runner(opt).run();
+    }
+
+}

settings.gradle

@@ -14,4 +14,4 @@
 // limitations under the License.
 
 include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:examples', 'log4j-appender',
-        'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file'
+        'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file', 'jmh-benchmarks'