From 79f85039d7be1b4266f06715a487f3635558ded6 Mon Sep 17 00:00:00 2001 From: bbejeck Date: Mon, 6 Mar 2017 10:47:36 +0000 Subject: [PATCH] KAFKA-3989; Initial support for adding a JMH benchmarking module Author: bbejeck Reviewers: Ewen Cheslack-Postava , Ismael Juma Closes #1712 from bbejeck/KAFKA-3989_create_jmh_benchmarking_module --- build.gradle | 43 ++++++++++++ checkstyle/import-control.xml | 9 +++ jmh-benchmarks/README.md | 61 +++++++++++++++++ jmh-benchmarks/jmh.sh | 42 ++++++++++++ .../kafka/jmh/cache/LRUCacheBenchmark.java | 68 +++++++++++++++++++ settings.gradle | 2 +- 6 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 jmh-benchmarks/README.md create mode 100755 jmh-benchmarks/jmh.sh create mode 100644 jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java diff --git a/build.gradle b/build.gradle index caac99d1e25..57beebeed21 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ buildscript { classpath "org.ajoberstar:grgit:1.7.0" classpath 'com.github.ben-manes:gradle-versions-plugin:0.13.0' classpath 'org.scoverage:gradle-scoverage:2.1.0' + classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4' } } @@ -822,6 +823,48 @@ project(':streams:examples') { } } +project(':jmh-benchmarks') { + + apply plugin: 'com.github.johnrengelman.shadow' + + shadowJar { + baseName = 'kafka-jmh-benchmarks-all' + classifier = null + version = null + } + + dependencies { + compile project(':clients') + compile project(':streams') + compile 'org.openjdk.jmh:jmh-core:1.17.5' + compile 'org.openjdk.jmh:jmh-generator-annprocess:1.17.5' + compile 'org.openjdk.jmh:jmh-core-benchmarks:1.17.5' + } + + jar { + manifest { + attributes "Main-Class": "org.openjdk.jmh.Main" + } + } + + + task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) { + + main="-jar" + + doFirst { + if (System.getProperty("jmhArgs")) { + args System.getProperty("jmhArgs").split(',') + } + args = [shadowJar.archivePath, *args] + } + } + + javadoc { + enabled = false + } +} + project(':log4j-appender') { archivesBaseName = "kafka-log4j-appender" diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 6c72e63c1e2..fa98593b0ff 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -181,6 +181,15 @@ + + + + + + + + + diff --git a/jmh-benchmarks/README.md b/jmh-benchmarks/README.md new file mode 100644 index 00000000000..53807ea06ff --- /dev/null +++ b/jmh-benchmarks/README.md @@ -0,0 +1,61 @@ +###JMH-Benchmark module + +This module contains benchmarks written using [JMH](http://openjdk.java.net/projects/code-tools/jmh/) from OpenJDK. +Writing correct micro-benchmarks is Java (or another JVM language) is difficult and there are many non-obvious pitfalls (many +due to compiler optimizations). JMH is a framework for running and analyzing benchmarks (micro or macro) written in Java (or +another JVM language). + +For help in writing correct JMH tests, the best place to start is the [sample code](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/) provided +by the JMH project. + +Typically, JMH is expected to run as a separate project in Maven. The jmh-benchmarks module uses +the [gradle shadow jar](https://github.com/johnrengelman/shadow) plugin to emulate this behavior, by creating the required +uber-jar file containing the benchmarking code and required JMH classes. + +JMH is highly configurable and users are encouraged to look through the samples for suggestions +on what options are available. A good tutorial for using JMH can be found [here](http://tutorials.jenkov.com/java-performance/jmh.html#return-value-from-benchmark-method) + +###Gradle Tasks / Running benchmarks in gradle + +If no benchmark mode is specified, the default is used which is throughput. It is assumed that users run +the gradle tasks with './gradlew' from the root of the Kafka project. + +* jmh-benchmarks:shadowJar - creates the uber jar required to run the benchmarks. + +* jmh-benchmarks:jmh - runs the `clean` and `shadowJar` tasks followed by all the benchmarks. + +### Using the jmh script +If you want to set specific JMH flags or only run a certain test(s) passing arguments via +gradle tasks is cumbersome. Instead you can use the `jhm.sh` script. NOTE: It is assumed users run +the jmh.sh script from the jmh-benchmarks module. + +* Run a specific test setting fork-mode (number iterations) to 2 :`./jmh.sh -f 2 LRUCacheBenchmark` + +* By default all JMH output goes to stdout. To run a benchmark and capture the results in a file: +`./jmh.sh -f 2 -o benchmarkResults.txt LRUCacheBenchmark` +NOTE: For now this script needs to be run from the jmh-benchmarks directory. + +### Running JMH outside of gradle +The JMH benchmarks can be run outside of gradle as you would with any executable jar file: +`java -jar /jmh-benchmarks/build/libs/kafka-jmh-benchmarks-all.jar -f2 LRUCacheBenchmark` + +### JMH Options +Some common JMH options are: +```text + + -e Benchmarks to exclude from the run. + + -f How many times to fork a single benchmark. Use 0 to + disable forking altogether. Warning: disabling + forking may have detrimental impact on benchmark + and infrastructure reliability, you might want + to use different warmup mode instead. + + -o Redirect human-readable output to a given file. + + + + -v Verbosity mode. Available modes are: [SILENT, NORMAL, + EXTRA] +``` +To view all options run jmh with the -h flag. diff --git a/jmh-benchmarks/jmh.sh b/jmh-benchmarks/jmh.sh new file mode 100755 index 00000000000..7f3927aa843 --- /dev/null +++ b/jmh-benchmarks/jmh.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +base_dir=$(dirname $0) +jmh_project_name="jmh-benchmarks" + +if [ ${base_dir} == "." ]; then + gradlew_dir=".." +elif [ ${base_dir} == ${jmh_project_name} ]; then + gradlew_dir="." +else + echo "JMH Benchmarks need to be run from the root of the kafka repository or the 'jmh-benchmarks' directory" + exit +fi + +gradleCmd="${gradlew_dir}/gradlew" +libDir="${base_dir}/build/libs" + +echo "running gradlew :jmh-benchmarks:clean :jmh-benchmarks:shadowJar in quiet mode" + +$gradleCmd -q :jmh-benchmarks:clean :jmh-benchmarks:shadowJar + +echo "gradle build done" + +echo "running JMH with args [$@]" + +java -jar ${libDir}/kafka-jmh-benchmarks-all.jar "$@" + +echo "JMH benchmarks done" diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java new file mode 100644 index 00000000000..ecf73f9ab59 --- /dev/null +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/cache/LRUCacheBenchmark.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.jmh.cache; + +import org.apache.kafka.common.cache.LRUCache; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * This is a simple example of a JMH benchmark. + * + * The sample code provided by the JMH project is a great place to start learning how to write correct benchmarks: + * http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/ + */ +@State(Scope.Thread) +public class LRUCacheBenchmark { + + private LRUCache lruCache; + + private final String key = "the_key_to_use"; + private final String value = "the quick brown fox jumped over the lazy dog the olympics are about to start"; + int counter; + + + @Setup(Level.Trial) + public void setUpCaches() { + lruCache = new LRUCache<>(100); + } + + @Benchmark + public String testCachePerformance() { + counter++; + lruCache.put(key + counter, value + counter); + return lruCache.get(key + counter); + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(LRUCacheBenchmark.class.getSimpleName()) + .forks(2) + .build(); + + new Runner(opt).run(); + } + +} diff --git a/settings.gradle b/settings.gradle index 29d38950a8a..f0fdf07128c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,4 +14,4 @@ // limitations under the License. include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:examples', 'log4j-appender', - 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file' + 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file', 'jmh-benchmarks'