KAFKA-18773 Migrate the log4j1 config to log4j 2 for native image and README (#18872)

- update reflection-config.json and resource-config.json to include log4j2 and jackson
- remove unused jackson scala library
- fix the incorrect path of log4j2.yaml
- adopt workaround (--standalone) to make this PR work and it will be fixed by KAFKA-18737)

Reviewers: TengYao Chi <kitingiao@gmail.com>, Chia-Ping Tsai <chia7712@gmail.com>

build.gradle

@@ -1072,7 +1072,6 @@ project(':core') {
     implementation libs.argparse4j
     implementation libs.commonsValidator
     implementation libs.jacksonDatabind
-    implementation libs.jacksonModuleScala
     implementation libs.jacksonDataformatCsv
     implementation libs.jacksonJDK8Datatypes
     implementation libs.jacksonDatabindYaml

core/src/main/scala/kafka/docker/KafkaDockerWrapper.scala

@@ -110,7 +110,7 @@ object KafkaDockerWrapper extends Logging {
       case Some(str) => str
       case None => throw new RuntimeException("CLUSTER_ID environment variable is not set.")
     }
-    Array("format", "--cluster-id=" + clusterId, "-c", s"${configsPath.toString}/server.properties")
+    Array("format", "--cluster-id=" + clusterId, "-c", s"${configsPath.toString}/server.properties", "--standalone")
   }
 
   private def prepareConfigs(defaultConfigsPath: Path, mountedConfigsPath: Path, finalConfigsPath: Path): Unit = {
@@ -250,10 +250,6 @@ object KafkaDockerWrapper extends Logging {
     }
     val config = yaml.getConfiguration
 
-    if (config == null && loggerFromEnv.isEmpty && rootOption.isEmpty) {
-      return
-    }
-
     if (config == null) {
       generateDefaultLog4jConfig(loggerFromEnv, rootOption, filepath, mapper)
     } else {

core/src/test/scala/unit/kafka/docker/KafkaDockerWrapperTest.scala

@@ -133,7 +133,7 @@ class KafkaDockerWrapperTest {
     val configsPath = Paths.get("/path/to/configs")
     val envVars = Map("CLUSTER_ID" -> "MYwKGPhXQZidgd0qMv8Mkw")
 
-    val expected = Array("format", "--cluster-id=MYwKGPhXQZidgd0qMv8Mkw", "-c", "/path/to/configs/server.properties")
+    val expected = Array("format", "--cluster-id=MYwKGPhXQZidgd0qMv8Mkw", "-c", "/path/to/configs/server.properties", "--standalone")
     val actual = KafkaDockerWrapper.formatStorageCmd(configsPath, envVars)
 
     assertArrayEquals(expected.toArray[Object], actual.toArray[Object])

docker/examples/README.md

@@ -48,11 +48,11 @@ It is also possible to use the input file to have a common set of configurations
         
 - To provide configs to log4j property files, following points should be considered:
   - log4j properties provided via environment variables will be appended to the default properties file (log4j properties files bundled with Kafka).
-  - `KAFKA_LOG4J_ROOT_LOGLEVEL` can be provided to set the value of `log4j.rootLogger` in log4j.properties and `tools-log4j.properties`.
+  - `KAFKA_LOG4J_ROOT_LOGLEVEL` can be provided to set the value of `log4j.rootLogger` in `log4j2.yaml` and `tools-log4j2.yaml`.
-  - log4j loggers can be added to log4j.properties by setting them in `KAFKA_LOG4J_LOGGERS` environment variable in a single comma separated string.
+  - log4j loggers can be added to `log4j2.yaml` by setting them in `KAFKA_LOG4J_LOGGERS` environment variable in a single comma separated string.
       - Example:
           - Assuming that `KAFKA_LOG4J_LOGGERS='property1=value1,property2=value2'` environment variable is provided to Docker container.
-          - `log4j.logger.property1=value1` and `log4j.logger.property2=value2` will be added to the `log4j.properties` file inside Docker container.
+          - `log4j.logger.property1=value1` and `log4j.logger.property2=value2` will be added to the `log4j2.yaml` file inside Docker container.
 
 Running in SSL mode
 -------------------

docker/native/README.md

@@ -15,6 +15,11 @@ These metadata configs are provided in the `native-image-configs` directory.
 GraalVM provide an option to create these configs automatically by running the application normally with the [native-image agent](https://www.graalvm.org/latest/reference-manual/native-image/metadata/AutomaticMetadataCollection/) attached.
 The configs in the `native-image-configs` directory are generated by running existing Apache Kafka System Tests(using GraalVM JIT and attaching the native-image agent to the process) as they are quite exhaustive.
 
+To update reachability metadata, users can set following environment variable and run related jar files. New metadata configs will be generated in the `native-image-configs` directory.
+```bash
+export JAVA_TOOL_OPTIONS="-agentlib:native-image-agent=config-merge-dir=/path/to/kafka/docker/native/native-image-configs"
+```
+
 ## Limitations of Native Apache Kafka executable
 - Dynamic Features: For any new dynamic feature added or modified in Apache Kafka, the corresponding metadata configurations must be added or updated in the `native-image-configs`. As of now these configs are static.
 - Runtime Jars: Native Kafka Executable does not support capabilities that require a new runtime jar provided by the user, as the jar's information is not available during build time. In such cases, a new native Kafka binary must be built, including that jar in the classpath during the build.

docker/native/launch

@@ -41,12 +41,11 @@ fi
 result=$(/opt/kafka/kafka.Kafka setup \
       --default-configs-dir /etc/kafka/docker \
       --mounted-configs-dir /mnt/shared/config \
-      --final-configs-dir /opt/kafka/config \
+      --final-configs-dir /opt/kafka/config 2>&1) || \
-      -Dlog4j.configuration=file:/opt/kafka/config/tools-log4j.properties 2>&1) || \
       echo $result | grep -i "already formatted" || \
       { echo $result && (exit 1) }
 
 echo "WARNING: THIS IS AN EXPERIMENTAL DOCKER IMAGE RECOMMENDED FOR LOCAL TESTING AND DEVELOPMENT PURPOSES."
 
-KAFKA_LOG4J_CMD_OPTS="-Dkafka.logs.dir=/opt/kafka/logs/ -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"
+KAFKA_LOG4J_CMD_OPTS="-Dkafka.logs.dir=/opt/kafka/logs/ -Dlog4j2.configurationFile=file:/opt/kafka/config/log4j2.yaml"
 exec /opt/kafka/kafka.Kafka start --config /opt/kafka/config/server.properties $KAFKA_LOG4J_CMD_OPTS $KAFKA_JMX_OPTS ${KAFKA_OPTS-}

docker/native/native-image-configs/jni-config.json

@@ -24,12 +24,32 @@
 {
   "name":"java.lang.OutOfMemoryError"
 },
+{
+  "name":"java.lang.String",
+  "methods":[{"name":"lastIndexOf","parameterTypes":["int"] }, {"name":"substring","parameterTypes":["int"] }]
+},
+{
+  "name":"java.lang.System",
+  "methods":[{"name":"getProperty","parameterTypes":["java.lang.String"] }, {"name":"setProperty","parameterTypes":["java.lang.String","java.lang.String"] }]
+},
+{
+  "name":"java.lang.VersionProps",
+  "methods":[{"name":"print","parameterTypes":["boolean"] }]
+},
 {
   "name":"java.util.Arrays",
   "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }]
 },
+{
+  "name":"kafka.Kafka",
+  "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }]
+},
+{
+  "name":"kafka.tools.StorageTool",
+  "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }]
+},
 {
   "name":"sun.management.VMManagementImpl",
   "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}]
 }
-]
+]

docker/native/native-image-configs/predefined-classes-config.json

@@ -5,3 +5,4 @@
     ]
   }
 ]
+

docker/native/native-image-configs/proxy-config.json

@@ -2,4 +2,4 @@
   {
     "interfaces":["sun.misc.SignalHandler"]
   }
-]
+]

docker/native/native-image-configs/resource-config.json

@@ -1,11 +1,17 @@
 {
   "resources":{
   "includes":[{
+    "pattern":"\\QMETA-INF/log4j-provider.properties\\E"
+  }, {
     "pattern":"\\QMETA-INF/maven/org.xerial.snappy/snappy-java/pom.properties\\E"
   }, {
     "pattern":"\\QMETA-INF/native/libnetty_transport_native_epoll.so\\E"
   }, {
     "pattern":"\\QMETA-INF/native/libnetty_transport_native_epoll_x86_64.so\\E"
+  }, {
+    "pattern":"\\QMETA-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat\\E"
+  }, {
+    "pattern":"\\QMETA-INF/services/com.fasterxml.jackson.databind.Module\\E"
   }, {
     "pattern":"\\QMETA-INF/services/java.lang.System$LoggerFinder\\E"
   }, {
@@ -18,40 +24,110 @@
     "pattern":"\\QMETA-INF/services/java.rmi.server.RMIClassLoaderSpi\\E"
   }, {
     "pattern":"\\QMETA-INF/services/java.time.zone.ZoneRulesProvider\\E"
+  }, {
+    "pattern":"\\QMETA-INF/services/java.util.spi.ResourceBundleControlProvider\\E"
   }, {
     "pattern":"\\QMETA-INF/services/javax.management.remote.JMXConnectorServerProvider\\E"
   }, {
     "pattern":"\\QMETA-INF/services/javax.security.auth.spi.LoginModule\\E"
   }, {
-    "pattern":"\\Qkafka/kafka-version.properties\\E"
+    "pattern":"\\QMETA-INF/services/org.apache.logging.log4j.core.util.ContextDataProvider\\E"
   }, {
-    "pattern":"\\Qlinux/amd64/libzstd-jni-1.5.6-6.so\\E"
+    "pattern":"\\QMETA-INF/services/org.apache.logging.log4j.core.util.WatchEventService\\E"
+  }, {
+    "pattern":"\\QMETA-INF/services/org.apache.logging.log4j.spi.Provider\\E"
+  }, {
+    "pattern":"\\QMETA-INF/services/org.apache.logging.log4j.util.PropertySource\\E"
+  }, {
+    "pattern":"\\Qcom/fasterxml/jackson/module/scala/build.properties\\E"
+  }, {
+    "pattern":"\\Qjava/lang/Iterable.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/AbstractCollection.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/AbstractList.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/ArrayList.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/Collection.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/LinkedHashMap.tasty\\E"
+  }, {
+    "pattern":"\\Qjava/util/List.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Appenders.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Configuration.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Log4jConfiguration.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Logger.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Loggers.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Properties.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/docker/Root.tasty\\E"
+  }, {
+    "pattern":"\\Qkafka/kafka-version.properties\\E"
   }, {
     "pattern":"\\Qlinux/aarch64/libzstd-jni-1.5.6-6.so\\E"
   }, {
-    "pattern":"\\Qnet/jpountz/util/linux/amd64/liblz4-java.so\\E"
+    "pattern":"\\Qlinux/amd64/libzstd-jni-1.5.6-6.so\\E"
+  }, {
+    "pattern":"\\Qlog4j2.StatusLogger.properties\\E"
+  }, {
+    "pattern":"\\Qlog4j2.component.properties\\E"
+  }, {
+    "pattern":"\\Qlog4j2.system.properties\\E"
   }, {
     "pattern":"\\Qnet/jpountz/util/linux/aarch64/liblz4-java.so\\E"
+  }, {
+    "pattern":"\\Qnet/jpountz/util/linux/amd64/liblz4-java.so\\E"
   }, {
     "pattern":"\\Qorg-xerial-snappy.properties\\E"
   }, {
     "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"
   }, {
     "pattern":"\\Qorg/xerial/snappy/VERSION\\E"
+  }, {
+    "pattern":"\\Qorg/xerial/snappy/native/Linux/aarch64/libsnappyjava.so\\E"
   }, {
     "pattern":"\\Qorg/xerial/snappy/native/Linux/x86_64/libsnappyjava.so\\E"
   }, {
-    "pattern":"\\Qorg/xerial/snappy/native/Linux/aarch64/libsnappyjava.so\\E"
+    "pattern":"\\Qscala/collection/convert/JavaCollectionWrappers$SeqWrapper.class\\E"
+  }, {
+    "pattern":"\\Qscala/collection/convert/JavaCollectionWrappers.tasty\\E"
+  }, {
+    "pattern":"\\Qscala/collection/convert/JavaCollectionWrappers/SeqWrapper.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/lang/Iterable.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/AbstractCollection.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/AbstractList.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/ArrayList.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/Collection.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/LinkedHashMap.tasty\\E"
+  }, {
+    "pattern":"java.base:\\Qjava/util/List.tasty\\E"
   }, {
     "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt67b/nfkc.nrm\\E"
   }, {
     "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt67b/uprops.icu\\E"
+  }, {
+    "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfc.nrm\\E"
   }, {
     "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfkc.nrm\\E"
   }, {
     "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/uprops.icu\\E"
   }, {
     "pattern":"java.base:\\Qsun/net/idn/uidna.spp\\E"
+  }, {
+    "pattern":"java.base:\\Qsun/text/resources/LineBreakIteratorData\\E"
   }, {
     "pattern":"jdk.jfr:\\Qjdk/jfr/internal/query/view.ini\\E"
   }]},
@@ -62,4 +138,4 @@
     "name":"sun.security.util.Resources",
     "classNames":["sun.security.util.Resources"]
   }]
-}
+}

docker/native/native-image-configs/serialization-config.json

@@ -125,4 +125,4 @@
   ],
   "proxies":[
   ]
-}
+}

gradle/dependencies.gradle

@@ -150,7 +150,6 @@ libs += [
   jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson",
   jacksonDatabindYaml: "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$versions.jackson",
   jacksonDataformatCsv: "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson",
-  jacksonModuleScala: "com.fasterxml.jackson.module:jackson-module-scala_$versions.baseScala:$versions.jackson",
   jacksonJDK8Datatypes: "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$versions.jackson",
   jacksonBlackbird: "com.fasterxml.jackson.module:jackson-module-blackbird:$versions.jackson",
   jacksonJakartarsJsonProvider: "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:$versions.jackson",