From b37b89c6686f5270b37d39cf3144bccb4f5bceb2 Mon Sep 17 00:00:00 2001 From: TengYao Chi Date: Sat, 14 Dec 2024 01:14:31 +0800 Subject: [PATCH] KAFKA-9366 Upgrade log4j to log4j2 (#17373) This pull request replaces Log4j with Log4j2 across the entire project, including dependencies, configurations, and code. The notable changes are listed below: 1. Introduce Log4j2 Instead of Log4j 2. Change Configuration File Format from Properties to YAML 3. Adds warnings to notify users if they are still using Log4j properties, encouraging them to transition to Log4j2 configurations Co-authored-by: Lee Dongjin Reviewers: Luke Chen , Mickael Maison , Chia-Ping Tsai --- LICENSE-binary | 8 +- README.md | 6 +- bin/connect-distributed.sh | 8 +- bin/connect-mirror-maker.sh | 8 +- bin/connect-standalone.sh | 8 +- bin/kafka-server-start.sh | 8 +- bin/windows/connect-distributed.bat | 9 +- bin/windows/connect-standalone.bat | 9 +- bin/windows/kafka-server-start.bat | 9 +- build.gradle | 139 ++++++--- checkstyle/import-control-core.xml | 19 +- checkstyle/import-control.xml | 14 +- .../clients/consumer/KafkaConsumerTest.java | 2 +- .../CoordinatorRequestManagerTest.java | 2 +- .../clients/producer/KafkaProducerTest.java | 2 +- .../common/utils/LogCaptureAppender.java | 126 +++++--- clients/src/test/resources/log4j2.yaml | 38 +++ config/connect-log4j.properties | 39 --- config/connect-log4j2.yaml | 44 +++ config/log4j.properties | 93 ------ config/log4j2.yaml | 158 ++++++++++ .../file/src/test/resources/log4j.properties | 28 -- connect/file/src/test/resources/log4j2.yaml | 35 +++ .../mirror/MirrorSourceConnectorTest.java | 2 +- .../src/test/resources/log4j.properties | 33 -- connect/mirror/src/test/resources/log4j2.yaml | 41 +++ .../kafka/connect/runtime/AbstractHerder.java | 2 +- .../apache/kafka/connect/runtime/Loggers.java | 101 ++++--- .../rest/resources/LoggingResource.java | 2 +- .../kafka/connect/runtime/LoggersTest.java | 133 ++++---- .../SourceTaskOffsetCommitterTest.java | 2 +- .../connect/runtime/WorkerSourceTaskTest.java | 2 +- .../runtime/rest/ConnectRestServerTest.java | 1 - .../src/test/resources/log4j.properties | 37 --- .../runtime/src/test/resources/log4j2.yaml | 48 +++ .../scala/kafka/utils/Log4jController.scala | 129 ++++---- .../test/java/kafka/admin/AclCommandTest.java | 2 +- core/src/test/resources/log4j2.yaml | 38 +++ .../api/PlaintextAdminIntegrationTest.scala | 47 +-- .../test/scala/other/kafka.log4j.properties | 22 -- .../UncleanLeaderElectionTest.scala | 15 +- .../unit/kafka/network/SocketServerTest.scala | 9 +- gradle/dependencies.gradle | 17 +- .../src/test/resources/log4j2.yaml | 26 +- metadata/src/test/resources/log4j2.yaml | 36 +++ raft/bin/test-kraft-server-start.sh | 1 + raft/config/kraft-log4j2.yaml | 39 +++ raft/src/test/resources/log4j2.yaml | 38 +++ .../src/test/resources/log4j.properties | 21 -- .../src/test/resources/log4j2.yaml | 28 +- .../src/test/resources/log4j2.yaml | 25 +- storage/src/test/resources/log4j.properties | 28 -- storage/src/test/resources/log4j2.yaml | 57 ++++ .../src/test/resources/log4j.properties | 36 --- .../src/test/resources/log4j2.yaml | 65 ++++ .../src/main/resources/log4j.properties | 19 -- .../src/main/resources/log4j2.yaml | 26 +- .../kafka/streams/StreamsConfigTest.java | 2 +- .../internals/InternalTopicManagerTest.java | 2 +- .../internals/PartitionGroupTest.java | 2 +- .../internals/RecordCollectorTest.java | 5 +- .../internals/StoreChangelogReaderTest.java | 2 +- .../processor/internals/TaskManagerTest.java | 2 +- streams/src/test/resources/log4j.properties | 36 --- streams/src/test/resources/log4j2.yaml | 65 ++++ .../src/test/resources/log4j.properties | 34 --- .../src/test/resources/log4j2.yaml | 40 +++ .../src/test/resources/log4j.properties | 21 -- .../test-utils/src/test/resources/log4j2.yaml | 35 +++ test-common/src/main/resources/log4j2.yaml | 35 +++ tests/kafkatest/services/connect.py | 15 +- tests/kafkatest/services/console_consumer.py | 14 +- tests/kafkatest/services/kafka/kafka.py | 8 +- .../services/kafka/templates/log4j2.yaml | 283 ++++++++++++++++++ tests/kafkatest/services/kafka/util.py | 17 ++ .../performance/consumer_performance.py | 9 +- .../performance/end_to_end_latency.py | 10 +- .../performance/producer_performance.py | 9 +- .../performance/templates/tools_log4j2.yaml | 24 +- tests/kafkatest/services/streams.py | 39 ++- .../services/templates/connect_log4j2.yaml | 35 +++ .../services/templates/tools_log4j2.yaml | 39 +++ .../services/transactional_message_copier.py | 10 +- .../services/trogdor/templates/log4j2.yaml | 42 +++ tests/kafkatest/services/trogdor/trogdor.py | 18 +- .../kafkatest/services/verifiable_consumer.py | 8 +- .../kafkatest/services/verifiable_producer.py | 9 +- .../streams/streams_relational_smoke_test.py | 14 +- .../streams/templates/log4j2_template.yaml | 30 +- .../tools/other/ReplicationQuotasTestRig.java | 4 +- tools/src/test/resources/log4j.properties | 22 -- tools/src/test/resources/log4j2.yaml | 39 +++ trogdor/src/test/resources/log4j.properties | 22 -- trogdor/src/test/resources/log4j2.yaml | 39 +++ 94 files changed, 2041 insertions(+), 939 deletions(-) create mode 100644 clients/src/test/resources/log4j2.yaml delete mode 100644 config/connect-log4j.properties create mode 100644 config/connect-log4j2.yaml delete mode 100644 config/log4j.properties create mode 100644 config/log4j2.yaml delete mode 100644 connect/file/src/test/resources/log4j.properties create mode 100644 connect/file/src/test/resources/log4j2.yaml delete mode 100644 connect/mirror/src/test/resources/log4j.properties create mode 100644 connect/mirror/src/test/resources/log4j2.yaml delete mode 100644 connect/runtime/src/test/resources/log4j.properties create mode 100644 connect/runtime/src/test/resources/log4j2.yaml create mode 100644 core/src/test/resources/log4j2.yaml delete mode 100644 core/src/test/scala/other/kafka.log4j.properties rename shell/src/test/resources/log4j.properties => group-coordinator/src/test/resources/log4j2.yaml (57%) create mode 100644 metadata/src/test/resources/log4j2.yaml create mode 100644 raft/config/kraft-log4j2.yaml create mode 100644 raft/src/test/resources/log4j2.yaml delete mode 100644 server-common/src/test/resources/log4j.properties rename group-coordinator/src/test/resources/log4j.properties => server-common/src/test/resources/log4j2.yaml (57%) rename metadata/src/test/resources/log4j.properties => shell/src/test/resources/log4j2.yaml (61%) delete mode 100644 storage/src/test/resources/log4j.properties create mode 100644 storage/src/test/resources/log4j2.yaml delete mode 100644 streams/integration-tests/src/test/resources/log4j.properties create mode 100644 streams/integration-tests/src/test/resources/log4j2.yaml delete mode 100644 streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j.properties rename raft/src/test/resources/log4j.properties => streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j2.yaml (60%) delete mode 100644 streams/src/test/resources/log4j.properties create mode 100644 streams/src/test/resources/log4j2.yaml delete mode 100644 streams/streams-scala/src/test/resources/log4j.properties create mode 100644 streams/streams-scala/src/test/resources/log4j2.yaml delete mode 100644 streams/test-utils/src/test/resources/log4j.properties create mode 100644 streams/test-utils/src/test/resources/log4j2.yaml create mode 100644 test-common/src/main/resources/log4j2.yaml create mode 100644 tests/kafkatest/services/kafka/templates/log4j2.yaml rename core/src/test/resources/log4j.properties => tests/kafkatest/services/performance/templates/tools_log4j2.yaml (60%) create mode 100644 tests/kafkatest/services/templates/connect_log4j2.yaml create mode 100644 tests/kafkatest/services/templates/tools_log4j2.yaml create mode 100644 tests/kafkatest/services/trogdor/templates/log4j2.yaml rename clients/src/test/resources/log4j.properties => tests/kafkatest/tests/streams/templates/log4j2_template.yaml (58%) delete mode 100644 tools/src/test/resources/log4j.properties create mode 100644 tools/src/test/resources/log4j2.yaml delete mode 100644 trogdor/src/test/resources/log4j.properties create mode 100644 trogdor/src/test/resources/log4j2.yaml diff --git a/LICENSE-binary b/LICENSE-binary index 08092f6def9..8e5f7c14342 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -220,6 +220,7 @@ jackson-annotations-2.16.2 jackson-core-2.16.2 jackson-databind-2.16.2 jackson-dataformat-csv-2.16.2 +jackson-dataformat-yaml-2.16.2 jackson-datatype-jdk8-2.16.2 jackson-jaxrs-base-2.16.2 jackson-jaxrs-json-provider-2.16.2 @@ -239,6 +240,11 @@ jetty-servlets-9.4.56.v20240826 jetty-util-9.4.56.v20240826 jetty-util-ajax-9.4.56.v20240826 jose4j-0.9.4 +log4j-api-2.24.1 +log4j-core-2.24.1 +log4j-core-test-2.24.1 +log4j-slf4j-impl-2.24.1 +log4j-1.2-api-2.24.1 lz4-java-1.8.0 maven-artifact-3.9.6 metrics-core-4.1.12.1 @@ -254,7 +260,6 @@ netty-transport-native-epoll-4.1.115.Final netty-transport-native-unix-common-4.1.115.Final opentelemetry-proto-1.0.0-alpha plexus-utils-3.5.1 -reload4j-1.2.25 rocksdbjni-7.9.2 scala-library-2.13.15 scala-logging_2.13-3.9.5 @@ -312,7 +317,6 @@ argparse4j-0.7.0, see: licenses/argparse-MIT classgraph-4.8.173, see: licenses/classgraph-MIT jopt-simple-5.0.4, see: licenses/jopt-simple-MIT slf4j-api-1.7.36, see: licenses/slf4j-MIT -slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT pcollections-4.0.1, see: licenses/pcollections-MIT --------------------------------------- diff --git a/README.md b/README.md index b8275004aac..03069291a5e 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,10 @@ Follow instructions in https://kafka.apache.org/quickstart ./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate ### Running a particular unit/integration test with log4j output ### -By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j.properties` file in the module's `src/test/resources` directory. +By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j2.yml` file in the module's `src/test/resources` directory. -For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j.properties#L21) in `clients/src/test/resources/log4j.properties` -to `log4j.logger.org.apache.kafka=INFO` and then run: +For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j2.yml#L35) in `clients/src/test/resources/log4j2.yml` +to `level: INFO` and then run: ./gradlew cleanTest clients:test --tests NetworkClientTest diff --git a/bin/connect-distributed.sh b/bin/connect-distributed.sh index b8088ad9234..a9d185493ef 100755 --- a/bin/connect-distributed.sh +++ b/bin/connect-distributed.sh @@ -22,8 +22,12 @@ fi base_dir=$(dirname $0) -if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then - export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties" +if [ -z "$KAFKA_LOG4J_OPTS" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml" +elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2 + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2 + echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2 fi if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then diff --git a/bin/connect-mirror-maker.sh b/bin/connect-mirror-maker.sh index 8e2b2e162da..90305754772 100755 --- a/bin/connect-mirror-maker.sh +++ b/bin/connect-mirror-maker.sh @@ -22,8 +22,12 @@ fi base_dir=$(dirname $0) -if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then - export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties" +if [ -z "$KAFKA_LOG4J_OPTS" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml" +elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2 + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2 + echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2 fi if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then diff --git a/bin/connect-standalone.sh b/bin/connect-standalone.sh index bef78d658fd..92e8dc9c8ee 100755 --- a/bin/connect-standalone.sh +++ b/bin/connect-standalone.sh @@ -22,8 +22,12 @@ fi base_dir=$(dirname $0) -if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then - export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties" +if [ -z "$KAFKA_LOG4J_OPTS" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml" +elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2 + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2 + echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2 fi if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then diff --git a/bin/kafka-server-start.sh b/bin/kafka-server-start.sh index 5a53126172d..6539746160f 100755 --- a/bin/kafka-server-start.sh +++ b/bin/kafka-server-start.sh @@ -21,8 +21,12 @@ then fi base_dir=$(dirname $0) -if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then - export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" +if [ -z "$KAFKA_LOG4J_OPTS" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/log4j2.yaml" +elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2 + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2 + echo You can also use the \$KAFKA_HOME/config/log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2 fi if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then diff --git a/bin/windows/connect-distributed.bat b/bin/windows/connect-distributed.bat index 0535085bde5..43c338fc26a 100644 --- a/bin/windows/connect-distributed.bat +++ b/bin/windows/connect-distributed.bat @@ -27,7 +27,14 @@ popd rem Log4j settings IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( - set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties + set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml +) ELSE ( + echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul + IF %ERRORLEVEL% == 0 ( + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2 + echo You can also use the %BASE_DIR%/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2 + ) ) "%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectDistributed %* diff --git a/bin/windows/connect-standalone.bat b/bin/windows/connect-standalone.bat index 12ebb21dc9a..bac8bbd1291 100644 --- a/bin/windows/connect-standalone.bat +++ b/bin/windows/connect-standalone.bat @@ -27,7 +27,14 @@ popd rem Log4j settings IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( - set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties + set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml +) ELSE ( + echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul + IF %ERRORLEVEL% == 0 ( + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. + echo You can also use the %BASE_DIR%/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. + ) ) "%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectStandalone %* diff --git a/bin/windows/kafka-server-start.bat b/bin/windows/kafka-server-start.bat index 8624eda9ff0..ff29321d000 100644 --- a/bin/windows/kafka-server-start.bat +++ b/bin/windows/kafka-server-start.bat @@ -21,7 +21,14 @@ IF [%1] EQU [] ( SetLocal IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( - set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties + set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%~dp0../../config/log4j2.yaml +) ELSE ( + echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul + IF %ERRORLEVEL% == 0 ( + echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. + echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. + echo You can also use the %~dp0../../config/log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. + ) ) IF ["%KAFKA_HEAP_OPTS%"] EQU [""] ( rem detect OS architecture diff --git a/build.gradle b/build.gradle index 51a6658f1be..78bf91d799f 100644 --- a/build.gradle +++ b/build.gradle @@ -138,7 +138,7 @@ ext { } runtimeTestLibs = [ - libs.slf4jReload4j, + libs.slf4jLog4j2, libs.junitPlatformLanucher, project(":test-common:test-common-runtime") ] @@ -178,12 +178,14 @@ allprojects { libs.scalaLibrary, libs.scalaReflect, libs.jacksonAnnotations, + libs.jacksonDatabindYaml, // be explicit about the Netty dependency version instead of relying on the version set by // ZooKeeper (potentially older and containing CVEs) libs.nettyHandler, libs.nettyTransportNativeEpoll, - // be explicit about the reload4j version instead of relying on the transitive versions - libs.reload4j + libs.log4j2Api, + libs.log4j2Core, + libs.log4j1Bridge2Api ) } } @@ -963,13 +965,15 @@ project(':server') { implementation libs.slf4jApi - compileOnly libs.reload4j + compileOnly libs.log4j2Api + compileOnly libs.log4j2Core + compileOnly libs.log4j1Bridge2Api testImplementation project(':clients').sourceSets.test.output testImplementation libs.mockitoCore testImplementation libs.junitJupiter - testImplementation libs.slf4jReload4j + testImplementation libs.slf4jLog4j2 testRuntimeOnly runtimeTestLibs } @@ -1028,7 +1032,7 @@ project(':share') { testImplementation libs.junitJupiter testImplementation libs.mockitoCore - testImplementation libs.slf4jReload4j + testImplementation libs.slf4jLog4j2 testRuntimeOnly runtimeTestLibs } @@ -1099,15 +1103,17 @@ project(':core') { implementation libs.dropwizardMetrics exclude module: 'slf4j-log4j12' exclude module: 'log4j' - // Both Kafka and Zookeeper use slf4j. ZooKeeper moved from log4j to logback in v3.8.0, but Kafka relies on reload4j. + // Both Kafka and Zookeeper use slf4j. ZooKeeper moved from log4j to logback in v3.8.0. // We are removing Zookeeper's dependency on logback so we have a singular logging backend. exclude module: 'logback-classic' exclude module: 'logback-core' } // ZooKeeperMain depends on commons-cli but declares the dependency as `provided` implementation libs.commonsCli - - compileOnly libs.reload4j + implementation libs.log4j2Core + implementation libs.log4j2Api + implementation libs.log4j1Bridge2Api + implementation libs.jacksonDatabindYaml testImplementation project(':clients').sourceSets.test.output testImplementation project(':group-coordinator').sourceSets.test.output @@ -1137,7 +1143,7 @@ project(':core') { testImplementation libs.apachedsMavibotPartition testImplementation libs.apachedsJdbmPartition testImplementation libs.junitJupiter - testImplementation libs.slf4jReload4j + testImplementation libs.slf4jLog4j2 testImplementation libs.caffeine testRuntimeOnly runtimeTestLibs @@ -1169,9 +1175,6 @@ project(':core') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.compileClasspath) { - include('reload4j*jar') - } from (configurations.runtimeClasspath) { exclude('kafka-clients*') } @@ -1384,11 +1387,14 @@ project(':metadata') { implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.metrics - compileOnly libs.reload4j + compileOnly libs.log4j2Api + compileOnly libs.log4j2Core + compileOnly libs.log4j1Bridge2Api + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.jqwik testImplementation libs.mockitoCore - testImplementation libs.slf4jReload4j + testImplementation libs.slf4jLog4j2 testImplementation project(':clients').sourceSets.test.output testImplementation project(':raft').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output @@ -1513,6 +1519,7 @@ project(':group-coordinator') { testImplementation project(':clients').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':coordinator-common').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore @@ -1575,9 +1582,10 @@ project(':test-common') { implementation project(':storage') implementation project(':server-common') implementation libs.slf4jApi + implementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore - + testRuntimeOnly runtimeTestLibs } @@ -1883,12 +1891,18 @@ project(':clients') { testImplementation libs.bcpkix testImplementation libs.jacksonJakartarsJsonProvider + testImplementation libs.jacksonDatabindYaml testImplementation libs.jose4j testImplementation libs.junitJupiter - testImplementation libs.reload4j + testImplementation libs.log4j2Api + testImplementation libs.log4j2Core + testImplementation libs.log4j1Bridge2Api + testImplementation libs.spotbugs testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter // supports MockitoExtension + testCompileOnly libs.bndlib + testRuntimeOnly libs.jacksonDatabind testRuntimeOnly libs.jacksonJDK8Datatypes testRuntimeOnly runtimeTestLibs @@ -2046,11 +2060,13 @@ project(':raft') { implementation project(':clients') implementation libs.slf4jApi implementation libs.jacksonDatabind + implementation libs.jacksonDatabindYaml testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.jqwik @@ -2146,6 +2162,7 @@ project(':server-common') { testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore @@ -2279,6 +2296,7 @@ project(':storage') { testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation libs.hamcrest + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.bcpkix @@ -2441,13 +2459,18 @@ project(':tools') { implementation libs.jacksonDataformatCsv implementation libs.jacksonJDK8Datatypes implementation libs.slf4jApi - implementation libs.slf4jReload4j + implementation libs.slf4jLog4j2 + implementation libs.log4j2Api + implementation libs.log4j2Core + implementation libs.log4j1Bridge2Api implementation libs.joptSimple implementation libs.re2j implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation implementation libs.jacksonJakartarsJsonProvider + compileOnly libs.spotbugs + testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation project(':server') @@ -2473,7 +2496,6 @@ project(':tools') { testImplementation(libs.jfreechart) { exclude group: 'junit', module: 'junit' } - testImplementation libs.reload4j testImplementation libs.apachedsCoreApi testImplementation libs.apachedsInterceptorKerberos testImplementation libs.apachedsProtocolShared @@ -2513,7 +2535,9 @@ project(':trogdor') { implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api implementation libs.jacksonJakartarsJsonProvider implementation libs.jerseyContainerServlet @@ -2534,13 +2558,16 @@ project(':trogdor') { implementation project(':group-coordinator:group-coordinator-api') testImplementation project(':clients') - testImplementation libs.junitJupiter testImplementation project(':clients').sourceSets.test.output + testImplementation project(':group-coordinator') + testImplementation libs.junitJupiter testImplementation libs.mockitoCore - testImplementation project(':group-coordinator') - testRuntimeOnly runtimeTestLibs + testRuntimeOnly libs.log4j2Api + testRuntimeOnly libs.log4j2Core + testRuntimeOnly libs.log4j1Bridge2Api + testRuntimeOnly libs.junitPlatformLanucher } javadoc { @@ -2585,6 +2612,7 @@ project(':shell') { testImplementation project(':core') testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testRuntimeOnly runtimeTestLibs @@ -2629,9 +2657,13 @@ project(':streams') { // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle testCompileOnly project(':streams:test-utils') + testCompileOnly libs.bndlib testImplementation project(':clients').sourceSets.test.output - testImplementation libs.reload4j + testImplementation libs.log4j2Api + testImplementation libs.log4j2Core + testImplementation libs.log4j1Bridge2Api + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.bcpkix testImplementation libs.hamcrest @@ -2774,6 +2806,7 @@ project(':streams:streams-scala') { testImplementation project(':clients').sourceSets.test.output testImplementation project(':streams:test-utils') + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testRuntimeOnly runtimeTestLibs @@ -2830,10 +2863,13 @@ project(':streams:integration-tests') { testImplementation project(':transaction-coordinator') testImplementation libs.bcpkix testImplementation libs.hamcrest + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.junitPlatformSuiteEngine // supports suite test testImplementation libs.mockitoCore - testImplementation libs.reload4j + testImplementation libs.log4j2Api + testImplementation libs.log4j2Core + testImplementation libs.log4j1Bridge2Api testImplementation libs.slf4jApi testImplementation project(':streams:test-utils') @@ -2873,6 +2909,7 @@ project(':streams:test-utils') { implementation libs.slf4jApi testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.hamcrest @@ -2904,7 +2941,7 @@ project(':streams:examples') { implementation(project(':connect:json')) implementation project(':streams') - implementation libs.slf4jReload4j + implementation libs.slf4jLog4j2 testImplementation project(':streams:test-utils') testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest @@ -3299,7 +3336,7 @@ project(':jmh-benchmarks') { implementation libs.jacksonDatabind implementation libs.metrics implementation libs.mockitoCore - implementation libs.slf4jReload4j + implementation libs.slf4jLog4j2 implementation libs.scalaLibrary } @@ -3343,7 +3380,9 @@ project(':connect:api') { dependencies { api project(':clients') implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api implementation libs.jakartaRsApi testImplementation libs.junitJupiter @@ -3379,7 +3418,9 @@ project(':connect:transforms') { api project(':connect:api') implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api testImplementation libs.junitJupiter @@ -3419,7 +3460,9 @@ project(':connect:json') { api libs.jacksonBlackbird implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api testImplementation libs.junitJupiter @@ -3464,8 +3507,10 @@ project(':connect:runtime') { api project(':connect:transforms') implementation libs.slf4jApi - implementation libs.reload4j - implementation libs.slf4jReload4j + implementation libs.slf4jLog4j2 + implementation libs.log4j2Api + implementation libs.log4j2Core + implementation libs.log4j1Bridge2Api implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation implementation libs.jacksonAnnotations implementation libs.jacksonJakartarsJsonProvider @@ -3489,6 +3534,9 @@ project(':connect:runtime') { implementation libs.mavenArtifact implementation libs.swaggerAnnotations + compileOnly libs.bndlib + compileOnly libs.spotbugs + // We use this library to generate OpenAPI docs for the REST API, but we don't want or need it at compile // or run time. So, we add it to a separate configuration, which we use later on during docs generation swagger libs.swaggerJaxrs2 @@ -3507,11 +3555,14 @@ project(':connect:runtime') { testImplementation project(':server-common').sourceSets.test.output testImplementation project(':test-common:test-common-api') + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter testImplementation libs.httpclient + testCompileOnly libs.bndlib + testRuntimeOnly libs.bcpkix testRuntimeOnly runtimeTestLibs } @@ -3606,10 +3657,14 @@ project(':connect:file') { dependencies { implementation project(':connect:api') implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api + testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output testImplementation project(':connect:runtime') testImplementation project(':connect:runtime').sourceSets.test.output @@ -3646,7 +3701,9 @@ project(':connect:basic-auth-extension') { dependencies { implementation project(':connect:api') implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api implementation libs.jakartaRsApi implementation libs.jaxAnnotationApi @@ -3691,7 +3748,9 @@ project(':connect:mirror') { implementation libs.argparse4j implementation libs.jacksonAnnotations implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api implementation libs.jacksonAnnotations implementation libs.jacksonJakartarsJsonProvider implementation libs.jerseyContainerServlet @@ -3713,7 +3772,11 @@ project(':connect:mirror') { implementation libs.swaggerAnnotations testImplementation libs.junitJupiter - testImplementation libs.reload4j + testImplementation libs.log4j2Api + testImplementation libs.log4j2Core + testImplementation libs.log4j1Bridge2Api + testImplementation libs.bndlib + testImplementation libs.jacksonDatabindYaml testImplementation libs.mockitoCore testImplementation project(':clients').sourceSets.test.output testImplementation project(':connect:runtime').sourceSets.test.output @@ -3781,7 +3844,9 @@ project(':connect:mirror-client') { dependencies { implementation project(':clients') implementation libs.slf4jApi - runtimeOnly libs.reload4j + runtimeOnly libs.log4j2Api + runtimeOnly libs.log4j2Core + runtimeOnly libs.log4j1Bridge2Api testImplementation libs.junitJupiter testImplementation project(':clients').sourceSets.test.output diff --git a/checkstyle/import-control-core.xml b/checkstyle/import-control-core.xml index a8dc78160e3..3cfd0ce663c 100644 --- a/checkstyle/import-control-core.xml +++ b/checkstyle/import-control-core.xml @@ -114,7 +114,14 @@ - + + + + + + + + @@ -136,7 +143,7 @@ - + @@ -146,6 +153,14 @@ + + + + + + + + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 76aa4a2b175..0b9e7dd717b 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -201,7 +201,7 @@ - + @@ -225,7 +225,7 @@ - + @@ -308,6 +308,10 @@ + + + + @@ -391,7 +395,7 @@ - + @@ -553,6 +557,7 @@ + @@ -568,7 +573,7 @@ - + @@ -580,6 +585,7 @@ + diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index 36d15c0fe94..33ca2844305 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -104,7 +104,7 @@ import org.apache.kafka.test.MockConsumerInterceptor; import org.apache.kafka.test.MockMetricsReporter; import org.apache.kafka.test.TestUtils; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java index 4f59db3d863..7e805dc3cd3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java @@ -32,7 +32,7 @@ import org.apache.kafka.common.utils.LogCaptureAppender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java index d72e576a78d..08d6d994abc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java @@ -94,7 +94,7 @@ import org.apache.kafka.test.MockProducerInterceptor; import org.apache.kafka.test.MockSerializer; import org.apache.kafka.test.TestUtils; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; diff --git a/clients/src/test/java/org/apache/kafka/common/utils/LogCaptureAppender.java b/clients/src/test/java/org/apache/kafka/common/utils/LogCaptureAppender.java index 1194b9a5de2..2df74c8681c 100644 --- a/clients/src/test/java/org/apache/kafka/common/utils/LogCaptureAppender.java +++ b/clients/src/test/java/org/apache/kafka/common/utils/LogCaptureAppender.java @@ -16,31 +16,37 @@ */ package org.apache.kafka.common.utils; -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.LoggingEvent; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.config.Property; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseable { - private final List events = new LinkedList<>(); +public class LogCaptureAppender extends AbstractAppender implements AutoCloseable { + private final List events = new LinkedList<>(); private final List logLevelChanges = new LinkedList<>(); + private final List loggers = new ArrayList<>(); public static class LogLevelChange { + private final Level originalLevel; + private final Class clazz; public LogLevelChange(final Level originalLevel, final Class clazz) { this.originalLevel = originalLevel; this.clazz = clazz; } - - private final Level originalLevel; - - private final Class clazz; - } @SuppressWarnings("OptionalUsedAsFieldOrParameterType") @@ -74,31 +80,53 @@ public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseabl } } + public LogCaptureAppender(String name) { + super(name, null, null, true, Property.EMPTY_ARRAY); + } + public static LogCaptureAppender createAndRegister() { - final LogCaptureAppender logCaptureAppender = new LogCaptureAppender(); - Logger.getRootLogger().addAppender(logCaptureAppender); + final LogCaptureAppender logCaptureAppender = new LogCaptureAppender("LogCaptureAppender"); + Logger logger = LogManager.getRootLogger(); + logCaptureAppender.addToLogger(logger); return logCaptureAppender; } public static LogCaptureAppender createAndRegister(final Class clazz) { - final LogCaptureAppender logCaptureAppender = new LogCaptureAppender(); - Logger.getLogger(clazz).addAppender(logCaptureAppender); + final LogCaptureAppender logCaptureAppender = new LogCaptureAppender("LogCaptureAppender"); + Logger logger = LogManager.getLogger(clazz); + logCaptureAppender.addToLogger(logger); return logCaptureAppender; } - public void setClassLogger(final Class clazz, Level level) { - logLevelChanges.add(new LogLevelChange(Logger.getLogger(clazz).getLevel(), clazz)); - Logger.getLogger(clazz).setLevel(level); + public void addToLogger(Logger logger) { + org.apache.logging.log4j.core.Logger coreLogger = (org.apache.logging.log4j.core.Logger) logger; + this.start(); + coreLogger.addAppender(this); + loggers.add(coreLogger); } - public static void unregister(final LogCaptureAppender logCaptureAppender) { - Logger.getRootLogger().removeAppender(logCaptureAppender); + public void setClassLogger(final Class clazz, Level level) { + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + String loggerName = clazz.getName(); + LoggerConfig loggerConfig = config.getLoggerConfig(loggerName); + + Level originalLevel = loggerConfig.getLevel(); + logLevelChanges.add(new LogLevelChange(originalLevel, clazz)); + + if (!loggerConfig.getName().equals(loggerName)) { + LoggerConfig newLoggerConfig = new LoggerConfig(loggerName, level, true); + config.addLogger(loggerName, newLoggerConfig); + } else { + loggerConfig.setLevel(level); + } + ctx.updateLoggers(); } @Override - protected void append(final LoggingEvent event) { + public void append(final LogEvent event) { synchronized (events) { - events.add(event); + events.add(event.toImmutable()); } } @@ -112,8 +140,8 @@ public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseabl public List getMessages() { final LinkedList result = new LinkedList<>(); synchronized (events) { - for (final LoggingEvent event : events) { - result.add(event.getRenderedMessage()); + for (final LogEvent event : events) { + result.add(event.getMessage().getFormattedMessage()); } } return result; @@ -122,25 +150,26 @@ public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseabl public List getEvents() { final LinkedList result = new LinkedList<>(); synchronized (events) { - for (final LoggingEvent event : events) { - final String[] throwableStrRep = event.getThrowableStrRep(); + for (final LogEvent event : events) { + final Throwable throwable = event.getThrown(); final Optional throwableString; final Optional throwableClassName; - if (throwableStrRep == null) { + if (throwable == null) { throwableString = Optional.empty(); throwableClassName = Optional.empty(); } else { - final StringBuilder throwableStringBuilder = new StringBuilder(); - - for (final String s : throwableStrRep) { - throwableStringBuilder.append(s); - } - - throwableString = Optional.of(throwableStringBuilder.toString()); - throwableClassName = Optional.of(event.getThrowableInformation().getThrowable().getClass().getName()); + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter); + throwable.printStackTrace(printWriter); + throwableString = Optional.of(stringWriter.toString()); + throwableClassName = Optional.of(throwable.getClass().getName()); } - result.add(new Event(event.getLevel().toString(), event.getRenderedMessage(), throwableString, throwableClassName)); + result.add(new Event( + event.getLevel().toString(), + event.getMessage().getFormattedMessage(), + throwableString, + throwableClassName)); } } return result; @@ -148,15 +177,30 @@ public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseabl @Override public void close() { + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + for (final LogLevelChange logLevelChange : logLevelChanges) { - Logger.getLogger(logLevelChange.clazz).setLevel(logLevelChange.originalLevel); + String loggerName = logLevelChange.clazz.getName(); + LoggerConfig loggerConfig = config.getLoggerConfig(loggerName); + if (!loggerConfig.getName().equals(loggerName)) { + LoggerConfig newLoggerConfig = new LoggerConfig(loggerName, logLevelChange.originalLevel, true); + config.addLogger(loggerName, newLoggerConfig); + } else { + loggerConfig.setLevel(logLevelChange.originalLevel); + } } logLevelChanges.clear(); - unregister(this); + ctx.updateLoggers(); + + unregister(); } - @Override - public boolean requiresLayout() { - return false; + public void unregister() { + for (org.apache.logging.log4j.core.Logger logger : loggers) { + logger.removeAppender(this); + } + loggers.clear(); + this.stop(); } } diff --git a/clients/src/test/resources/log4j2.yaml b/clients/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..bfe8b3835a0 --- /dev/null +++ b/clients/src/test/resources/log4j2.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: OFF + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: ERROR + # We are testing for a particular INFO log message in CommonNameLoggingTrustManagerFactoryWrapper + - name: org.apache.kafka.common.security.ssl.CommonNameLoggingTrustManagerFactoryWrapper + level: INFO diff --git a/config/connect-log4j.properties b/config/connect-log4j.properties deleted file mode 100644 index 979cb3869f9..00000000000 --- a/config/connect-log4j.properties +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -log4j.rootLogger=INFO, stdout, connectAppender - -# Send the logs to the console. -# -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout - -# Send the logs to a file, rolling the file at midnight local time. For example, the `File` option specifies the -# location of the log files (e.g. ${kafka.logs.dir}/connect.log), and at midnight local time the file is closed -# and copied in the same directory but with a filename that ends in the `DatePattern` option. -# -log4j.appender.connectAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.connectAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log -log4j.appender.connectAppender.layout=org.apache.log4j.PatternLayout - -# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information -# in the log messages, where appropriate. This makes it easier to identify those log messages that apply to a -# specific connector. -# -connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n - -log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} -log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern} diff --git a/config/connect-log4j2.yaml b/config/connect-log4j2.yaml new file mode 100644 index 00000000000..89a9a967365 --- /dev/null +++ b/config/connect-log4j2.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "kafka.logs.dir" + value: "." + - name: "logPattern" + value: "[%d] %p %X{connector.context}%m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + RollingFile: + - name: ConnectAppender + fileName: "${sys:kafka.logs.dir}/connect.log" + filePattern: "${sys:kafka.logs.dir}/connect-%d{yyyy-MM-dd-HH}.log" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + - ref: ConnectAppender diff --git a/config/log4j.properties b/config/log4j.properties deleted file mode 100644 index bcf2b9daa4e..00000000000 --- a/config/log4j.properties +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Unspecified loggers and loggers with additivity=true output to server.log and stdout -# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise -log4j.rootLogger=INFO, stdout, kafkaAppender - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log -log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log -log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log -log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log -log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log -log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log -log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) -log4j.logger.kafka=INFO -log4j.logger.org.apache.kafka=INFO - -# Change to DEBUG or TRACE to enable request logging -log4j.logger.kafka.request.logger=WARN, requestAppender -log4j.additivity.kafka.request.logger=false - -# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output -# related to the handling of requests -#log4j.logger.kafka.network.Processor=TRACE, requestAppender -#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender -#log4j.additivity.kafka.server.KafkaApis=false -log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender -log4j.additivity.kafka.network.RequestChannel$=false - -# Change the line below to adjust KRaft mode controller logging -log4j.logger.org.apache.kafka.controller=INFO, controllerAppender -log4j.additivity.org.apache.kafka.controller=false - -# Change the line below to adjust ZK mode controller logging -log4j.logger.kafka.controller=TRACE, controllerAppender -log4j.additivity.kafka.controller=false - -log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender -log4j.additivity.kafka.log.LogCleaner=false - -log4j.logger.state.change.logger=INFO, stateChangeAppender -log4j.additivity.state.change.logger=false - -# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses -log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender -log4j.additivity.kafka.authorizer.logger=false - diff --git a/config/log4j2.yaml b/config/log4j2.yaml new file mode 100644 index 00000000000..2b000d407e2 --- /dev/null +++ b/config/log4j2.yaml @@ -0,0 +1,158 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unspecified loggers and loggers with additivity=true output to server.log and stdout +# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise +Configuration: + Properties: + Property: + # Fallback if the system property is not set + - name: "kafka.logs.dir" + value: "." + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + # Appenders configuration + # See: https://logging.apache.org/log4j/2.x/manual/appenders.html + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + RollingFile: + - name: KafkaAppender + fileName: "${sys:kafka.logs.dir}/server.log" + filePattern: "${sys:kafka.logs.dir}/server.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # State Change appender + - name: StateChangeAppender + fileName: "${sys:kafka.logs.dir}/state-change.log" + filePattern: "${sys:kafka.logs.dir}/stage-change.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Request appender + - name: RequestAppender + fileName: "${sys:kafka.logs.dir}/kafka-request.log" + filePattern: "${sys:kafka.logs.dir}/kafka-request.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Cleaner appender + - name: CleanerAppender + fileName: "${sys:kafka.logs.dir}/log-cleaner.log" + filePattern: "${sys:kafka.logs.dir}/log-cleaner.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Controller appender + - name: ControllerAppender + fileName: "${sys:kafka.logs.dir}/controller.log" + filePattern: "${sys:kafka.logs.dir}/controller.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + # Authorizer appender + - name: AuthorizerAppender + fileName: "${sys:kafka.logs.dir}/kafka-authorizer.log" + filePattern: "${sys:kafka.logs.dir}/kafka-authorizer.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + modulate: true + interval: 1 + + # Loggers configuration + # See: https://logging.apache.org/log4j/2.x/manual/configuration.html#configuring-loggers + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + - ref: KafkaAppender + Logger: + # Kafka logger + - name: kafka + level: INFO + # Kafka org.apache logger + - name: org.apache.kafka + level: INFO + # Kafka request logger + - name: kafka.request.logger + level: WARN + additivity: false + AppenderRef: + ref: RequestAppender + # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE + # for additional output related to the handling of requests +# - name: kafka.network.Processor +# level: TRACE +# additivity: false +# AppenderRef: +# ref: RequestAppender +# - name: kafka.server.KafkaApis +# level: TRACE +# additivity: false +# AppenderRef: +# ref: RequestAppender + # Kafka network RequestChannel$ logger + - name: kafka.network.RequestChannel$ + level: WARN + additivity: false + AppenderRef: + ref: RequestAppender + # KRaft mode controller logger + - name: org.apache.kafka.controller + level: INFO + additivity: false + AppenderRef: + ref: ControllerAppender + # ZK mode controller logger + - name: kafka.controller + level: TRACE + additivity: false + AppenderRef: + ref: ControllerAppender + # LogCleaner logger + - name: kafka.log.LogCleaner + level: INFO + additivity: false + AppenderRef: + ref: CleanerAppender + # State change logger + - name: state.change.logger + level: INFO + additivity: false + AppenderRef: + ref: StateChangeAppender + # Authorizer logger + - name: kafka.authorizer.logger + level: INFO + additivity: false + AppenderRef: + ref: AuthorizerAppender \ No newline at end of file diff --git a/connect/file/src/test/resources/log4j.properties b/connect/file/src/test/resources/log4j.properties deleted file mode 100644 index 548e8c33cfb..00000000000 --- a/connect/file/src/test/resources/log4j.properties +++ /dev/null @@ -1,28 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -# -# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information -# in the log message, where appropriate. This makes it easier to identify those log messages that apply to a -# specific connector. Simply add this parameter to the log layout configuration below to include the contextual information. -# -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n - -log4j.logger.kafka=WARN diff --git a/connect/file/src/test/resources/log4j2.yaml b/connect/file/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..1e9f550fa6d --- /dev/null +++ b/connect/file/src/test/resources/log4j2.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %X{connector.context}%m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: WARN diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java index 588baf8c090..21bcc7cbad5 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java @@ -40,7 +40,7 @@ import org.apache.kafka.connect.connector.ConnectorContext; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.source.ExactlyOnceSupport; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/connect/mirror/src/test/resources/log4j.properties b/connect/mirror/src/test/resources/log4j.properties deleted file mode 100644 index c4ca6a2388f..00000000000 --- a/connect/mirror/src/test/resources/log4j.properties +++ /dev/null @@ -1,33 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -# -# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information -# in the log message, where appropriate. This makes it easier to identify those log messages that apply to a -# specific connector. Simply add this parameter to the log layout configuration below to include the contextual information. -# -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n -# -# The following line includes no MDC context parameters: -#log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n (%t) - -log4j.logger.kafka=WARN -log4j.logger.state.change.logger=OFF -log4j.logger.org.apache.kafka.connect=DEBUG diff --git a/connect/mirror/src/test/resources/log4j2.yaml b/connect/mirror/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..b63606d0ba5 --- /dev/null +++ b/connect/mirror/src/test/resources/log4j2.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %X{connector.context}%m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: WARN + + - name: state.change.logger + level: "OFF" + + - name: org.apache.kafka.connect + level: DEBUG diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java index 43772802070..c4f53ad4137 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java @@ -65,7 +65,7 @@ import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.Stage; import org.apache.kafka.connect.util.TemporaryStage; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.apache.maven.artifact.versioning.InvalidVersionSpecificationException; import org.apache.maven.artifact.versioning.VersionRange; import org.slf4j.Logger; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Loggers.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Loggers.java index 9e59b13d34a..0c16d0d6f01 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Loggers.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Loggers.java @@ -19,19 +19,23 @@ package org.apache.kafka.connect.runtime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.runtime.rest.entities.LoggerLevel; -import org.apache.log4j.Level; -import org.apache.log4j.LogManager; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Manages logging levels on a single worker. Supports dynamic adjustment and querying @@ -46,6 +50,10 @@ public class Loggers { /** * Log4j uses "root" (case-insensitive) as name of the root logger. + * Note: In log4j, the root logger's name was "root" and Kafka also followed that name for dynamic logging control feature. + * + * The root logger's name is changed in log4j2 to empty string (see: [[LogManager.ROOT_LOGGER_NAME]]) but for backward- + * compatibility. Kafka keeps its original root logger name. It is why here is a dedicated definition for the root logger name. */ private static final String ROOT_LOGGER_NAME = "root"; @@ -66,18 +74,17 @@ public class Loggers { public synchronized LoggerLevel level(String logger) { Objects.requireNonNull(logger, "Logger may not be null"); - org.apache.log4j.Logger foundLogger = null; + org.apache.logging.log4j.Logger foundLogger = null; if (ROOT_LOGGER_NAME.equalsIgnoreCase(logger)) { foundLogger = rootLogger(); } else { - Enumeration en = currentLoggers(); + List currentLoggers = currentLoggers(); // search within existing loggers for the given name. // using LogManger.getLogger() will create a logger if it doesn't exist // (potential leak since these don't get cleaned up). - while (en.hasMoreElements()) { - org.apache.log4j.Logger l = en.nextElement(); - if (logger.equals(l.getName())) { - foundLogger = l; + for (org.apache.logging.log4j.Logger currentLogger : currentLoggers) { + if (logger.equals(currentLogger.getName())) { + foundLogger = currentLogger; break; } } @@ -98,14 +105,12 @@ public class Loggers { public synchronized Map allLevels() { Map result = new TreeMap<>(); - Enumeration enumeration = currentLoggers(); - Collections.list(enumeration) - .stream() - .filter(logger -> logger.getLevel() != null) + currentLoggers().stream() + .filter(logger -> !logger.getLevel().equals(Level.OFF)) .forEach(logger -> result.put(logger.getName(), loggerLevel(logger))); - org.apache.log4j.Logger root = rootLogger(); - if (root.getLevel() != null) { + org.apache.logging.log4j.Logger root = rootLogger(); + if (!root.getLevel().equals(Level.OFF)) { result.put(ROOT_LOGGER_NAME, loggerLevel(root)); } @@ -124,10 +129,10 @@ public class Loggers { Objects.requireNonNull(level, "Level may not be null"); log.info("Setting level of namespace {} and children to {}", namespace, level); - List childLoggers = loggers(namespace); + List childLoggers = loggers(namespace); List result = new ArrayList<>(); - for (org.apache.log4j.Logger logger: childLoggers) { + for (org.apache.logging.log4j.Logger logger: childLoggers) { setLevel(logger, level); result.add(logger.getName()); } @@ -143,25 +148,24 @@ public class Loggers { * @return all loggers that fall under the given namespace; never null, and will always contain * at least one logger (the ancestor logger for the namespace) */ - private synchronized List loggers(String namespace) { + private synchronized List loggers(String namespace) { Objects.requireNonNull(namespace, "Logging namespace may not be null"); if (ROOT_LOGGER_NAME.equalsIgnoreCase(namespace)) { - List result = Collections.list(currentLoggers()); + List result = currentLoggers(); result.add(rootLogger()); return result; } - List result = new ArrayList<>(); - org.apache.log4j.Logger ancestorLogger = lookupLogger(namespace); - Enumeration en = currentLoggers(); + List result = new ArrayList<>(); + org.apache.logging.log4j.Logger ancestorLogger = lookupLogger(namespace); + List currentLoggers = currentLoggers(); boolean present = false; - while (en.hasMoreElements()) { - org.apache.log4j.Logger current = en.nextElement(); - if (current.getName().startsWith(namespace)) { - result.add(current); + for (org.apache.logging.log4j.Logger currentLogger : currentLoggers) { + if (currentLogger.getName().startsWith(namespace)) { + result.add(currentLogger); } - if (namespace.equals(current.getName())) { + if (namespace.equals(currentLogger.getName())) { present = true; } } @@ -174,43 +178,46 @@ public class Loggers { } // visible for testing - org.apache.log4j.Logger lookupLogger(String logger) { + org.apache.logging.log4j.Logger lookupLogger(String logger) { return LogManager.getLogger(logger); } - @SuppressWarnings("unchecked") - // visible for testing - Enumeration currentLoggers() { - return LogManager.getCurrentLoggers(); + List currentLoggers() { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + Collection loggerConfigs = context.getConfiguration().getLoggers().values(); + return loggerConfigs.stream() + .map(LoggerConfig::getName) + .distinct() + .map(LogManager::getLogger) + .collect(Collectors.toCollection(ArrayList::new)); } // visible for testing - org.apache.log4j.Logger rootLogger() { + org.apache.logging.log4j.Logger rootLogger() { return LogManager.getRootLogger(); } - private void setLevel(org.apache.log4j.Logger logger, Level level) { - Level currentLevel = logger.getLevel(); - if (currentLevel == null) - currentLevel = logger.getEffectiveLevel(); + private void setLevel(org.apache.logging.log4j.Logger logger, Level level) { + String loggerName = logger.getName(); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + LoggerConfig loggerConfig = context.getConfiguration().getLoggerConfig(loggerName); + Level currentLevel = loggerConfig.getLevel(); if (level.equals(currentLevel)) { - log.debug("Skipping update for logger {} since its level is already {}", logger.getName(), level); + log.debug("Skipping update for logger {} since its level is already {}", loggerName, level); return; } - log.debug("Setting level of logger {} (excluding children) to {}", logger.getName(), level); - logger.setLevel(level); - lastModifiedTimes.put(logger.getName(), time.milliseconds()); + log.debug("Setting level of logger {} (excluding children) to {}", loggerName, level); + Configurator.setLevel(loggerName, level); + lastModifiedTimes.put(loggerName, time.milliseconds()); } - private LoggerLevel loggerLevel(org.apache.log4j.Logger logger) { - Level level = logger.getLevel(); - if (level == null) - level = logger.getEffectiveLevel(); - + private LoggerLevel loggerLevel(org.apache.logging.log4j.Logger logger) { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + LoggerConfig loggerConfig = context.getConfiguration().getLoggerConfig(logger.getName()); + Level level = loggerConfig.getLevel(); Long lastModified = lastModifiedTimes.get(logger.getName()); return new LoggerLevel(Objects.toString(level), lastModified); } - } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java index 85be83061eb..dbbfb46375d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/LoggingResource.java @@ -21,7 +21,7 @@ import org.apache.kafka.connect.runtime.Herder; import org.apache.kafka.connect.runtime.rest.entities.LoggerLevel; import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.slf4j.LoggerFactory; import java.util.List; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/LoggersTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/LoggersTest.java index 3dbe688a076..23e5f753cde 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/LoggersTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/LoggersTest.java @@ -20,9 +20,13 @@ import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.runtime.rest.entities.LoggerLevel; -import org.apache.log4j.Hierarchy; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -30,13 +34,12 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Vector; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -59,12 +62,15 @@ public class LoggersTest { @Test public void testGetLoggersIgnoresNullLevels() { - Logger root = logger("root"); + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Logger root = loggerContext.getRootLogger(); + Configurator.setLevel(root, Level.OFF); - Logger a = logger("a"); - a.setLevel(null); - Logger b = logger("b"); - b.setLevel(Level.INFO); + Logger a = loggerContext.getLogger("a"); + Configurator.setLevel(a, null); + + Logger b = loggerContext.getLogger("b"); + Configurator.setLevel(b, Level.INFO); Loggers loggers = new TestLoggers(root, a, b); @@ -78,14 +84,15 @@ public class LoggersTest { @Test public void testGetLoggerFallsBackToEffectiveLogLevel() { - Logger root = logger("root"); - root.setLevel(Level.ERROR); + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Logger root = loggerContext.getRootLogger(); + Configurator.setLevel(root, Level.ERROR); - Hierarchy hierarchy = new Hierarchy(root); - Logger a = hierarchy.getLogger("a"); - a.setLevel(null); - Logger b = hierarchy.getLogger("b"); - b.setLevel(Level.INFO); + Logger a = loggerContext.getLogger("a"); + Configurator.setLevel(a, null); + + Logger b = loggerContext.getLogger("b"); + Configurator.setLevel(b, Level.INFO); Loggers loggers = new TestLoggers(root, a, b); @@ -96,14 +103,15 @@ public class LoggersTest { @Test public void testGetUnknownLogger() { - Logger root = logger("root"); - root.setLevel(Level.ERROR); + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Logger root = loggerContext.getRootLogger(); + Configurator.setLevel(root, Level.ERROR); - Hierarchy hierarchy = new Hierarchy(root); - Logger a = hierarchy.getLogger("a"); - a.setLevel(null); - Logger b = hierarchy.getLogger("b"); - b.setLevel(Level.INFO); + Logger a = loggerContext.getLogger("a"); + Configurator.setLevel(a, null); + + Logger b = loggerContext.getLogger("b"); + Configurator.setLevel(b, Level.INFO); Loggers loggers = new TestLoggers(root, a, b); @@ -113,17 +121,18 @@ public class LoggersTest { @Test public void testSetLevel() { - Logger root = logger("root"); - root.setLevel(Level.ERROR); + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Logger root = loggerContext.getRootLogger(); + Configurator.setLevel(root, Level.ERROR); - Logger x = logger("a.b.c.p.X"); - Logger y = logger("a.b.c.p.Y"); - Logger z = logger("a.b.c.p.Z"); - Logger w = logger("a.b.c.s.W"); - x.setLevel(Level.INFO); - y.setLevel(Level.INFO); - z.setLevel(Level.INFO); - w.setLevel(Level.INFO); + Logger x = loggerContext.getLogger("a.b.c.p.X"); + Logger y = loggerContext.getLogger("a.b.c.p.Y"); + Logger z = loggerContext.getLogger("a.b.c.p.Z"); + Logger w = loggerContext.getLogger("a.b.c.s.W"); + Configurator.setLevel(x, Level.INFO); + Configurator.setLevel(y, Level.INFO); + Configurator.setLevel(z, Level.INFO); + Configurator.setLevel(w, Level.INFO); // We don't explicitly register a logger for a.b.c.p, so it won't appear in the list of current loggers; // one should be created by the Loggers instance when we set the level @@ -166,25 +175,37 @@ public class LoggersTest { @Test public void testSetRootLevel() { - Logger root = logger("root"); - root.setLevel(Level.ERROR); + // In this test case, we focus on setting the level for the root logger. + // Ideally, we want to start with a "clean" configuration to conduct this test case. + // By programmatically creating a new configuration at the beginning, we can ensure + // that this test case is not affected by existing Log4j configurations. + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + String rootLoggerName = "root"; + LoggerConfig rootConfig = new LoggerConfig(rootLoggerName, Level.ERROR, false); + config.addLogger(rootLoggerName, rootConfig); + loggerContext.updateLoggers(); - Logger p = logger("a.b.c.p"); - Logger x = logger("a.b.c.p.X"); - Logger y = logger("a.b.c.p.Y"); - Logger z = logger("a.b.c.p.Z"); - Logger w = logger("a.b.c.s.W"); - x.setLevel(Level.INFO); - y.setLevel(Level.INFO); - z.setLevel(Level.INFO); - w.setLevel(Level.INFO); + Logger root = LogManager.getLogger(rootLoggerName); + Configurator.setLevel(root, Level.ERROR); + + Logger p = loggerContext.getLogger("a.b.c.p"); + Logger x = loggerContext.getLogger("a.b.c.p.X"); + Logger y = loggerContext.getLogger("a.b.c.p.Y"); + Logger z = loggerContext.getLogger("a.b.c.p.Z"); + Logger w = loggerContext.getLogger("a.b.c.s.W"); + Configurator.setLevel(p, Level.INFO); + Configurator.setLevel(x, Level.INFO); + Configurator.setLevel(y, Level.INFO); + Configurator.setLevel(z, Level.INFO); + Configurator.setLevel(w, Level.INFO); Loggers loggers = new TestLoggers(root, x, y, z, w); - List modified = loggers.setLevel("root", Level.DEBUG); - assertEquals(Arrays.asList("a.b.c.p.X", "a.b.c.p.Y", "a.b.c.p.Z", "a.b.c.s.W", "root"), modified); + List modified = loggers.setLevel(rootLoggerName, Level.DEBUG); + assertEquals(Arrays.asList("a.b.c.p.X", "a.b.c.p.Y", "a.b.c.p.Z", "a.b.c.s.W", rootLoggerName), modified); - assertNull(p.getLevel()); + assertEquals(p.getLevel(), Level.INFO); assertEquals(root.getLevel(), Level.DEBUG); @@ -194,7 +215,7 @@ public class LoggersTest { assertEquals(z.getLevel(), Level.DEBUG); Map expectedLevels = new HashMap<>(); - expectedLevels.put("root", new LoggerLevel(Level.DEBUG.toString(), INITIAL_TIME)); + expectedLevels.put(rootLoggerName, new LoggerLevel(Level.DEBUG.toString(), INITIAL_TIME)); expectedLevels.put("a.b.c.p.X", new LoggerLevel(Level.DEBUG.toString(), INITIAL_TIME)); expectedLevels.put("a.b.c.p.Y", new LoggerLevel(Level.DEBUG.toString(), INITIAL_TIME)); expectedLevels.put("a.b.c.p.Z", new LoggerLevel(Level.DEBUG.toString(), INITIAL_TIME)); @@ -206,7 +227,8 @@ public class LoggersTest { @Test public void testSetLevelNullArguments() { - Logger root = logger("root"); + LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + Logger root = loggerContext.getRootLogger(); Loggers loggers = new TestLoggers(root); assertThrows(NullPointerException.class, () -> loggers.setLevel(null, Level.INFO)); assertThrows(NullPointerException.class, () -> loggers.setLevel("root", null)); @@ -229,12 +251,12 @@ public class LoggersTest { @Override Logger lookupLogger(String logger) { - return currentLoggers.computeIfAbsent(logger, l -> new Logger(logger) { }); + return currentLoggers.computeIfAbsent(logger, LogManager::getLogger); } @Override - Enumeration currentLoggers() { - return new Vector<>(currentLoggers.values()).elements(); + List currentLoggers() { + return new ArrayList<>(currentLoggers.values()); } @Override @@ -242,9 +264,4 @@ public class LoggersTest { return rootLogger; } } - - private Logger logger(String name) { - return new Logger(name) { }; - } - } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SourceTaskOffsetCommitterTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SourceTaskOffsetCommitterTest.java index bd70ed357c6..106659d0f8f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SourceTaskOffsetCommitterTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SourceTaskOffsetCommitterTest.java @@ -22,7 +22,7 @@ import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; import org.apache.kafka.connect.util.ConnectorTaskId; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java index 77d56a207d7..3ddbf164494 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java @@ -54,7 +54,7 @@ import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.TopicAdmin; import org.apache.kafka.connect.util.TopicCreationGroup; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/ConnectRestServerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/ConnectRestServerTest.java index 58ca25ce0f0..13727c5b438 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/ConnectRestServerTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/ConnectRestServerTest.java @@ -353,7 +353,6 @@ public class ConnectRestServerTest { server.stop(); Collection logMessages = restServerAppender.getMessages(); - LogCaptureAppender.unregister(restServerAppender); restServerAppender.close(); String expectedlogContent = "\"GET / HTTP/1.1\" " + response.getStatusLine().getStatusCode(); assertTrue(logMessages.stream().anyMatch(logMessage -> logMessage.contains(expectedlogContent))); diff --git a/connect/runtime/src/test/resources/log4j.properties b/connect/runtime/src/test/resources/log4j.properties deleted file mode 100644 index de7180c282a..00000000000 --- a/connect/runtime/src/test/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -# -# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information -# in the log message, where appropriate. This makes it easier to identify those log messages that apply to a -# specific connector. Simply add this parameter to the log layout configuration below to include the contextual information. -# -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n -# -# The following line includes no MDC context parameters: -#log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n (%t) - -log4j.logger.kafka=WARN -log4j.logger.state.change.logger=OFF -log4j.logger.org.apache.kafka.connect=DEBUG - -# Troubleshooting KAFKA-17493. -log4j.logger.org.apache.kafka.consumer=DEBUG -log4j.logger.org.apache.kafka.coordinator.group=DEBUG \ No newline at end of file diff --git a/connect/runtime/src/test/resources/log4j2.yaml b/connect/runtime/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..45faa635378 --- /dev/null +++ b/connect/runtime/src/test/resources/log4j2.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %X{connector.context}%m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: WARN + + - name: state.change.logger + level: "OFF" + + - name: org.apache.kafka.connect + level: DEBUG + + # Troubleshooting KAFKA-17493. + - name: org.apache.kafka.consumer + level: DEBUG + + - name: org.apache.kafka.coordinator.group + level: DEBUG diff --git a/core/src/main/scala/kafka/utils/Log4jController.scala b/core/src/main/scala/kafka/utils/Log4jController.scala index 0d54c74e075..4bc022dadfe 100755 --- a/core/src/main/scala/kafka/utils/Log4jController.scala +++ b/core/src/main/scala/kafka/utils/Log4jController.scala @@ -17,83 +17,90 @@ package kafka.utils +import org.apache.kafka.common.utils.Utils +import org.apache.logging.log4j.core.LoggerContext +import org.apache.logging.log4j.core.config.Configurator +import org.apache.logging.log4j.{Level, LogManager} + import java.util import java.util.Locale - -import org.apache.kafka.common.utils.Utils -import org.apache.log4j.{Level, LogManager, Logger} - -import scala.collection.mutable import scala.jdk.CollectionConverters._ object Log4jController { + + /** + * Note: In log4j, the root logger's name was "root" and Kafka also followed that name for dynamic logging control feature. + * + * The root logger's name is changed in log4j2 to empty string (see: [[LogManager.ROOT_LOGGER_NAME]]) but for backward- + * compatibility. Kafka keeps its original root logger name. It is why here is a dedicated definition for the root logger name. + */ val ROOT_LOGGER = "root" - private def resolveLevel(logger: Logger): String = { - var name = logger.getName - var level = logger.getLevel - while (level == null) { - val index = name.lastIndexOf(".") - if (index > 0) { - name = name.substring(0, index) - val ancestor = existingLogger(name) - if (ancestor != null) { - level = ancestor.getLevel - } - } else { - level = existingLogger(ROOT_LOGGER).getLevel - } - } - level.toString + /** + * Returns a map of the log4j loggers and their assigned log level. + * If a logger does not have a log level assigned, we return the log level of the first ancestor with a level configured. + */ + def loggers: Map[String, String] = { + val logContext = LogManager.getContext(false).asInstanceOf[LoggerContext] + val rootLoggerLevel = logContext.getRootLogger.getLevel.toString + + // Loggers defined in the configuration + val configured = logContext.getConfiguration.getLoggers.asScala + .values + .filterNot(_.getName.equals(LogManager.ROOT_LOGGER_NAME)) + .map { logger => + logger.getName -> logger.getLevel.toString + }.toMap + + // Loggers actually running + val actual = logContext.getLoggers.asScala + .filterNot(_.getName.equals(LogManager.ROOT_LOGGER_NAME)) + .map { logger => + logger.getName -> logger.getLevel.toString + }.toMap + + (configured ++ actual) + (ROOT_LOGGER -> rootLoggerLevel) } /** - * Returns a map of the log4j loggers and their assigned log level. - * If a logger does not have a log level assigned, we return the root logger's log level - */ - def loggers: mutable.Map[String, String] = { - val logs = new mutable.HashMap[String, String]() - val rootLoggerLvl = existingLogger(ROOT_LOGGER).getLevel.toString - logs.put(ROOT_LOGGER, rootLoggerLvl) - - val loggers = LogManager.getCurrentLoggers - while (loggers.hasMoreElements) { - val logger = loggers.nextElement().asInstanceOf[Logger] - if (logger != null) { - logs.put(logger.getName, resolveLevel(logger)) - } - } - logs - } - - /** - * Sets the log level of a particular logger - */ + * Sets the log level of a particular logger. If the given logLevel is not an available log4j level + * (i.e., one of OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL) it falls back to DEBUG. + * + * @see [[Level.toLevel]] + */ def logLevel(loggerName: String, logLevel: String): Boolean = { - val log = existingLogger(loggerName) - if (!Utils.isBlank(loggerName) && !Utils.isBlank(logLevel) && log != null) { - log.setLevel(Level.toLevel(logLevel.toUpperCase(Locale.ROOT))) + if (Utils.isBlank(loggerName) || Utils.isBlank(logLevel)) + return false + + val level = Level.toLevel(logLevel.toUpperCase(Locale.ROOT)) + + if (loggerName == ROOT_LOGGER) { + Configurator.setAllLevels(LogManager.ROOT_LOGGER_NAME, level) true + } else { + if (loggerExists(loggerName) && level != null) { + Configurator.setAllLevels(loggerName, level) + true + } + else false } - else false } def unsetLogLevel(loggerName: String): Boolean = { - val log = existingLogger(loggerName) - if (!Utils.isBlank(loggerName) && log != null) { - log.setLevel(null) + if (loggerName == ROOT_LOGGER) { + Configurator.setAllLevels(LogManager.ROOT_LOGGER_NAME, null) true + } else { + if (loggerExists(loggerName)) { + Configurator.setAllLevels(loggerName, null) + true + } + else false } - else false } - def loggerExists(loggerName: String): Boolean = existingLogger(loggerName) != null - - private def existingLogger(loggerName: String) = - if (loggerName == ROOT_LOGGER) - LogManager.getRootLogger - else LogManager.exists(loggerName) + def loggerExists(loggerName: String): Boolean = loggers.contains(loggerName) } /** @@ -113,15 +120,7 @@ class Log4jController extends Log4jControllerMBean { def getLogLevel(loggerName: String): String = { - val log = Log4jController.existingLogger(loggerName) - if (log != null) { - val level = log.getLevel - if (level != null) - log.getLevel.toString - else - Log4jController.resolveLevel(log) - } - else "No such logger." + Log4jController.loggers.getOrElse(loggerName, "No such logger.") } def setLogLevel(loggerName: String, level: String): Boolean = Log4jController.logLevel(loggerName, level) diff --git a/core/src/test/java/kafka/admin/AclCommandTest.java b/core/src/test/java/kafka/admin/AclCommandTest.java index 4f9bfff0f9d..e71c348c272 100644 --- a/core/src/test/java/kafka/admin/AclCommandTest.java +++ b/core/src/test/java/kafka/admin/AclCommandTest.java @@ -40,7 +40,7 @@ import org.apache.kafka.common.utils.SecurityUtils; import org.apache.kafka.metadata.authorizer.StandardAuthorizer; import org.apache.kafka.test.TestUtils; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/core/src/test/resources/log4j2.yaml b/core/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..016a542689b --- /dev/null +++ b/core/src/test/resources/log4j2.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: OFF + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: WARN + + - name: org.apache.kafka + level: WARN diff --git a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala index cff801b2b50..890aff6da1e 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala @@ -57,9 +57,9 @@ import org.apache.kafka.security.authorizer.AclEntry import org.apache.kafka.server.config.{QuotaConfig, ServerConfigs, ServerLogConfigs, ZkConfigs} import org.apache.kafka.storage.internals.log.{CleanerConfig, LogConfig, LogFileUtils} import org.apache.kafka.test.TestUtils.{DEFAULT_MAX_WAIT_MS, assertFutureThrows} -import org.apache.log4j.PropertyConfigurator +import org.apache.logging.log4j.core.config.Configurator import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo, Timeout} +import org.junit.jupiter.api.{BeforeEach, TestInfo, Timeout} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.{MethodSource, ValueSource} import org.slf4j.LoggerFactory @@ -89,18 +89,11 @@ class PlaintextAdminIntegrationTest extends BaseAdminIntegrationTest { @BeforeEach override def setUp(testInfo: TestInfo): Unit = { super.setUp(testInfo) + Configurator.reconfigure(); brokerLoggerConfigResource = new ConfigResource( ConfigResource.Type.BROKER_LOGGER, brokers.head.config.brokerId.toString) } - @AfterEach - override def tearDown(): Unit = { - // Due to the fact that log4j is not re-initialized across tests, changing a logger's log level persists - // across test classes. We need to clean up the changes done after testing. - resetLogging() - super.tearDown() - } - @ParameterizedTest @Timeout(30) @ValueSource(strings = Array("kraft")) @@ -3766,6 +3759,27 @@ class PlaintextAdminIntegrationTest extends BaseAdminIntegrationTest { assertEquals(newAncestorLogLevel, newAncestorLoggerConfig.get("kafka.server.ControllerServer").value()) } + @ParameterizedTest + @ValueSource(strings = Array("kraft")) + def testIncrementalAlterConfigsForLog4jLogLevelsCanSetToRootLogger(quorum: String): Unit = { + client = createAdminClient + val initialLoggerConfig = describeBrokerLoggers() + val initialRootLogLevel = initialLoggerConfig.get(Log4jController.ROOT_LOGGER).value() + val newRootLogLevel = LogLevelConfig.DEBUG_LOG_LEVEL + + val alterRootLoggerEntry = Seq( + new AlterConfigOp(new ConfigEntry(Log4jController.ROOT_LOGGER, newRootLogLevel), AlterConfigOp.OpType.SET) + ).asJavaCollection + + alterBrokerLoggers(alterRootLoggerEntry, validateOnly = true) + val validatedRootLoggerConfig = describeBrokerLoggers() + assertEquals(initialRootLogLevel, validatedRootLoggerConfig.get(Log4jController.ROOT_LOGGER).value()) + + alterBrokerLoggers(alterRootLoggerEntry) + val changedRootLoggerConfig = describeBrokerLoggers() + assertEquals(newRootLogLevel, changedRootLoggerConfig.get(Log4jController.ROOT_LOGGER).value()) + } + @ParameterizedTest @ValueSource(strings = Array("kraft")) def testIncrementalAlterConfigsForLog4jLogLevelsCannotResetRootLogger(quorum: String): Unit = { @@ -4108,17 +4122,4 @@ object PlaintextAdminIntegrationTest { assertEquals(LogConfig.DEFAULT_COMPRESSION_TYPE, configs.get(brokerResource).get(ServerConfigs.COMPRESSION_TYPE_CONFIG).value) } - - /** - * Resets the logging configuration after the test. - */ - def resetLogging(): Unit = { - org.apache.log4j.LogManager.resetConfiguration() - val stream = this.getClass.getResourceAsStream("/log4j.properties") - try { - PropertyConfigurator.configure(stream) - } finally { - stream.close() - } - } } diff --git a/core/src/test/scala/other/kafka.log4j.properties b/core/src/test/scala/other/kafka.log4j.properties deleted file mode 100644 index 1a53fd5d286..00000000000 --- a/core/src/test/scala/other/kafka.log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, KAFKA - -log4j.appender.KAFKA=kafka.log4j.KafkaAppender - -log4j.appender.KAFKA.Port=9092 -log4j.appender.KAFKA.Host=localhost -log4j.appender.KAFKA.Topic=test-logger -log4j.appender.KAFKA.Serializer=kafka.AppenderStringSerializer diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala index b12e60980fc..3d78bf27aff 100755 --- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala +++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala @@ -34,13 +34,14 @@ import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, AlterConfigsResult, ConfigEntry} import org.apache.kafka.server.config.ReplicationConfigs import org.apache.kafka.server.metrics.KafkaYammerMetrics -import org.apache.log4j.{Level, Logger} +import org.apache.logging.log4j.{Level, LogManager} import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import com.yammer.metrics.core.Meter import org.apache.kafka.metadata.LeaderConstants +import org.apache.logging.log4j.core.config.Configurator class UncleanLeaderElectionTest extends QuorumTestHarness { val brokerId1 = 0 @@ -63,8 +64,8 @@ class UncleanLeaderElectionTest extends QuorumTestHarness { val partitionId = 0 val topicPartition = new TopicPartition(topic, partitionId) - val kafkaApisLogger = Logger.getLogger(classOf[kafka.server.KafkaApis]) - val networkProcessorLogger = Logger.getLogger(classOf[kafka.network.Processor]) + val kafkaApisLogger = LogManager.getLogger(classOf[kafka.server.KafkaApis]) + val networkProcessorLogger = LogManager.getLogger(classOf[kafka.network.Processor]) @BeforeEach override def setUp(testInfo: TestInfo): Unit = { @@ -80,8 +81,8 @@ class UncleanLeaderElectionTest extends QuorumTestHarness { } // temporarily set loggers to a higher level so that tests run quietly - kafkaApisLogger.setLevel(Level.FATAL) - networkProcessorLogger.setLevel(Level.FATAL) + Configurator.setLevel(kafkaApisLogger.getName, Level.FATAL) + Configurator.setLevel(networkProcessorLogger.getName, Level.FATAL) } @AfterEach @@ -90,8 +91,8 @@ class UncleanLeaderElectionTest extends QuorumTestHarness { brokers.foreach(broker => CoreUtils.delete(broker.config.logDirs)) // restore log levels - kafkaApisLogger.setLevel(Level.ERROR) - networkProcessorLogger.setLevel(Level.ERROR) + Configurator.setLevel(kafkaApisLogger.getName, Level.ERROR) + Configurator.setLevel(networkProcessorLogger.getName, Level.ERROR) admin.close() diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index ca4156eacd2..362796381c2 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -45,7 +45,8 @@ import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.network.ConnectionDisconnectListener import org.apache.kafka.server.quota.{ThrottleCallback, ThrottledChannel} import org.apache.kafka.test.{TestSslUtils, TestUtils => JTestUtils} -import org.apache.log4j.Level +import org.apache.logging.log4j.{Level, LogManager} +import org.apache.logging.log4j.core.config.Configurator import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api._ @@ -88,7 +89,7 @@ class SocketServerTest { var server: SocketServer = _ val sockets = new ArrayBuffer[Socket] - private val kafkaLogger = org.apache.log4j.LogManager.getLogger("kafka") + private val kafkaLogger = LogManager.getLogger("kafka") private var logLevelToRestore: Level = _ def endpoint: EndPoint = { KafkaConfig.fromProps(props, doLog = false).dataPlaneListeners.head @@ -102,7 +103,7 @@ class SocketServerTest { server.enableRequestProcessing(Map.empty).get(1, TimeUnit.MINUTES) // Run the tests with TRACE logging to exercise request logging path logLevelToRestore = kafkaLogger.getLevel - kafkaLogger.setLevel(Level.TRACE) + Configurator.setLevel(kafkaLogger.getName, Level.TRACE) assertTrue(server.controlPlaneRequestChannelOpt.isEmpty) } @@ -112,7 +113,7 @@ class SocketServerTest { shutdownServerAndMetrics(server) sockets.foreach(_.close()) sockets.clear() - kafkaLogger.setLevel(logLevelToRestore) + Configurator.setLevel(kafkaLogger.getName, logLevelToRestore) TestUtils.clearYammerMetrics() } diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 1dd01d5e0eb..37677869afb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -54,10 +54,11 @@ versions += [ apacheds: "2.0.0-M24", argparse4j: "0.7.0", bcpkix: "1.78.1", - // Version >=3.1.2 includes an improvement to prevent hash DOS attacks, - // but currently, tests are failing in >=3.1.2. Therefore, we are temporarily using version 3.1.1. + // Version >=3.1.2 includes an improvement to prevent hash DOS attacks, + // but currently, tests are failing in >=3.1.2. Therefore, we are temporarily using version 3.1.1. // The failing tests should be fixed under KAFKA-18089, allowing us to upgrade to >=3.1.2. caffeine: "3.1.1", + bndlib: "7.0.0", checkstyle: project.hasProperty('checkstyleVersion') ? checkstyleVersion : "10.20.2", commonsCli: "1.4", commonsIo: "2.14.0", // ZooKeeper dependency. Do not use, this is going away. @@ -106,6 +107,7 @@ versions += [ kafka_37: "3.7.1", kafka_38: "3.8.1", kafka_39: "3.9.0", + log4j2: "2.24.1", // When updating lz4 make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid lz4: "1.8.0", mavenArtifact: "3.9.6", @@ -115,7 +117,6 @@ versions += [ opentelemetryProto: "1.0.0-alpha", protobuf: "3.25.5", // a dependency of opentelemetryProto pcollections: "4.0.1", - reload4j: "1.2.25", re2j: "1.7", rocksDB: "7.9.2", // When updating the scalafmt version please also update the version field in checkstyle/.scalafmt.conf. scalafmt now @@ -148,6 +149,7 @@ libs += [ apachedsJdbmPartition: "org.apache.directory.server:apacheds-jdbm-partition:$versions.apacheds", argparse4j: "net.sourceforge.argparse4j:argparse4j:$versions.argparse4j", bcpkix: "org.bouncycastle:bcpkix-jdk18on:$versions.bcpkix", + bndlib:"biz.aQute.bnd:biz.aQute.bndlib:$versions.bndlib", caffeine: "com.github.ben-manes.caffeine:caffeine:$versions.caffeine", classgraph: "io.github.classgraph:classgraph:$versions.classgraph", commonsCli: "commons-cli:commons-cli:$versions.commonsCli", @@ -155,6 +157,7 @@ libs += [ commonsValidator: "commons-validator:commons-validator:$versions.commonsValidator", jacksonAnnotations: "com.fasterxml.jackson.core:jackson-annotations:$versions.jackson", jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson", + jacksonDatabindYaml: "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$versions.jackson", jacksonDataformatCsv: "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson", jacksonModuleScala: "com.fasterxml.jackson.module:jackson-module-scala_$versions.baseScala:$versions.jackson", jacksonJDK8Datatypes: "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$versions.jackson", @@ -204,6 +207,10 @@ libs += [ kafkaStreams_37: "org.apache.kafka:kafka-streams:$versions.kafka_37", kafkaStreams_38: "org.apache.kafka:kafka-streams:$versions.kafka_38", kafkaStreams_39: "org.apache.kafka:kafka-streams:$versions.kafka_39", + log4j1Bridge2Api: "org.apache.logging.log4j:log4j-1.2-api:$versions.log4j2", + log4j2Api: "org.apache.logging.log4j:log4j-api:$versions.log4j2", + log4j2Core: "org.apache.logging.log4j:log4j-core:$versions.log4j2", + log4j2CoreTest: "org.apache.logging.log4j:log4j-core-test:$versions.log4j2", lz4: "org.lz4:lz4-java:$versions.lz4", metrics: "com.yammer.metrics:metrics-core:$versions.metrics", dropwizardMetrics: "io.dropwizard.metrics:metrics-core:$versions.dropwizardMetrics", @@ -214,15 +221,15 @@ libs += [ pcollections: "org.pcollections:pcollections:$versions.pcollections", opentelemetryProto: "io.opentelemetry.proto:opentelemetry-proto:$versions.opentelemetryProto", protobuf: "com.google.protobuf:protobuf-java:$versions.protobuf", - reload4j: "ch.qos.reload4j:reload4j:$versions.reload4j", re2j: "com.google.re2j:re2j:$versions.re2j", rocksDBJni: "org.rocksdb:rocksdbjni:$versions.rocksDB", scalaLibrary: "org.scala-lang:scala-library:$versions.scala", scalaLogging: "com.typesafe.scala-logging:scala-logging_$versions.baseScala:$versions.scalaLogging", scalaReflect: "org.scala-lang:scala-reflect:$versions.scala", slf4jApi: "org.slf4j:slf4j-api:$versions.slf4j", - slf4jReload4j: "org.slf4j:slf4j-reload4j:$versions.slf4j", + slf4jLog4j2: "org.apache.logging.log4j:log4j-slf4j-impl:$versions.log4j2", snappy: "org.xerial.snappy:snappy-java:$versions.snappy", + spotbugs: "com.github.spotbugs:spotbugs-annotations:$versions.spotbugs", swaggerAnnotations: "io.swagger.core.v3:swagger-annotations:$swaggerVersion", swaggerJaxrs2: "io.swagger.core.v3:swagger-jaxrs2:$swaggerVersion", zookeeper: "org.apache.zookeeper:zookeeper:$versions.zookeeper", diff --git a/shell/src/test/resources/log4j.properties b/group-coordinator/src/test/resources/log4j2.yaml similarity index 57% rename from shell/src/test/resources/log4j.properties rename to group-coordinator/src/test/resources/log4j2.yaml index a72a9693de2..59b02951909 100644 --- a/shell/src/test/resources/log4j.properties +++ b/group-coordinator/src/test/resources/log4j2.yaml @@ -1,6 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at @@ -12,8 +9,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=DEBUG, stdout +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: DEBUG + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: DEBUG diff --git a/metadata/src/test/resources/log4j2.yaml b/metadata/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..fd94a4974e2 --- /dev/null +++ b/metadata/src/test/resources/log4j2.yaml @@ -0,0 +1,36 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: DEBUG + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: DEBUG diff --git a/raft/bin/test-kraft-server-start.sh b/raft/bin/test-kraft-server-start.sh index 701bc1864a4..ad7d755752f 100755 --- a/raft/bin/test-kraft-server-start.sh +++ b/raft/bin/test-kraft-server-start.sh @@ -17,6 +17,7 @@ base_dir=$(dirname $0) if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + echo "DEPRECATED: using log4j 1.x configuration. To use log4j 2.x configuration, run with: 'export KAFKA_LOG4J_OPTS=\"-Dlog4j.configurationFile=file:$base_dir/../config/kraft-log4j2.yml\"'" export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/kraft-log4j.properties" fi diff --git a/raft/config/kraft-log4j2.yaml b/raft/config/kraft-log4j2.yaml new file mode 100644 index 00000000000..3bfd01ca5cf --- /dev/null +++ b/raft/config/kraft-log4j2.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + Appenders: + Console: + name: STDERR + target: SYSTEM_ERR + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDERR + Logger: + - name: org.apache.kafka.raft + level: INFO + + - name: org.apache.kafka.snapshot + level: INFO diff --git a/raft/src/test/resources/log4j2.yaml b/raft/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..50d9e781b8e --- /dev/null +++ b/raft/src/test/resources/log4j2.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: OFF + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka.raft + level: ERROR + + - name: org.apache.kafka.snapshot + level: ERROR diff --git a/server-common/src/test/resources/log4j.properties b/server-common/src/test/resources/log4j.properties deleted file mode 100644 index be36f90299a..00000000000 --- a/server-common/src/test/resources/log4j.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.org.apache.kafka=INFO diff --git a/group-coordinator/src/test/resources/log4j.properties b/server-common/src/test/resources/log4j2.yaml similarity index 57% rename from group-coordinator/src/test/resources/log4j.properties rename to server-common/src/test/resources/log4j2.yaml index 9c8357947d9..be546a18b55 100644 --- a/group-coordinator/src/test/resources/log4j.properties +++ b/server-common/src/test/resources/log4j2.yaml @@ -1,9 +1,9 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -12,10 +12,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=DEBUG, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" -log4j.logger.org.apache.kafka=DEBUG + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: INFO diff --git a/metadata/src/test/resources/log4j.properties b/shell/src/test/resources/log4j2.yaml similarity index 61% rename from metadata/src/test/resources/log4j.properties rename to shell/src/test/resources/log4j2.yaml index 9c8357947d9..c229cbce316 100644 --- a/metadata/src/test/resources/log4j.properties +++ b/shell/src/test/resources/log4j2.yaml @@ -1,9 +1,9 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -12,10 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=DEBUG, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" -log4j.logger.org.apache.kafka=DEBUG + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: DEBUG + AppenderRef: + - ref: STDOUT diff --git a/storage/src/test/resources/log4j.properties b/storage/src/test/resources/log4j.properties deleted file mode 100644 index 7ee388a407f..00000000000 --- a/storage/src/test/resources/log4j.properties +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=OFF, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.appender.fileAppender=org.apache.log4j.RollingFileAppender -log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.fileAppender.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n -log4j.appender.fileAppender.File=storage.log - -log4j.logger.org.apache.kafka.server.log.remote.storage=INFO -log4j.logger.org.apache.kafka.server.log.remote.metadata.storage=INFO -log4j.logger.kafka.log.remote=INFO diff --git a/storage/src/test/resources/log4j2.yaml b/storage/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..e2050ad723d --- /dev/null +++ b/storage/src/test/resources/log4j2.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + - name: "fileLogPattern" + value: "%d [%t] %-5p %c %x - %m%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + RollingFile: + - name: FileAppender + fileName: storage.log + filePattern: "storage-%d{yyyy-MM-dd}.log" + PatternLayout: + pattern: "${fileLogPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + + Loggers: + Root: + level: OFF + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka.server.log.remote.storage + level: INFO + AppenderRef: + - ref: FileAppender + + - name: org.apache.kafka.server.log.remote.metadata.storage + level: INFO + AppenderRef: + - ref: FileAppender + + - name: kafka.log.remote + level: INFO + AppenderRef: + - ref: FileAppender diff --git a/streams/integration-tests/src/test/resources/log4j.properties b/streams/integration-tests/src/test/resources/log4j.properties deleted file mode 100644 index 104b46df20f..00000000000 --- a/streams/integration-tests/src/test/resources/log4j.properties +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.kafka=ERROR -log4j.logger.state.change.logger=ERROR -log4j.logger.org.apache.kafka=ERROR -log4j.logger.org.apache.kafka.clients=ERROR - -# These are the only logs we will likely ever find anything useful in to debug Streams test failures -log4j.logger.org.apache.kafka.clients.consumer=INFO -log4j.logger.org.apache.kafka.clients.producer=INFO -log4j.logger.org.apache.kafka.streams=INFO - -# printing out the configs takes up a huge amount of the allotted characters, -# and provides little value as we can always figure out the test configs without the logs -log4j.logger.org.apache.kafka.clients.producer.ProducerConfig=ERROR -log4j.logger.org.apache.kafka.clients.consumer.ConsumerConfig=ERROR -log4j.logger.org.apache.kafka.clients.admin.AdminClientConfig=ERROR -log4j.logger.org.apache.kafka.streams.StreamsConfig=ERROR diff --git a/streams/integration-tests/src/test/resources/log4j2.yaml b/streams/integration-tests/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..0942036a33c --- /dev/null +++ b/streams/integration-tests/src/test/resources/log4j2.yaml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: ERROR + + - name: state.change.logger + level: ERROR + + - name: org.apache.kafka + level: ERROR + + - name: org.apache.kafka.clients + level: ERROR + + - name: org.apache.kafka.clients.consumer + level: INFO + + - name: org.apache.kafka.clients.producer + level: INFO + + - name: org.apache.kafka.streams + level: INFO + + - name: org.apache.kafka.clients.producer.ProducerConfig + level: ERROR + + - name: org.apache.kafka.clients.consumer.ConsumerConfig + level: ERROR + + - name: org.apache.kafka.clients.admin.AdminClientConfig + level: ERROR + + - name: org.apache.kafka.streams.StreamsConfig + level: ERROR diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j.properties b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j.properties deleted file mode 100644 index b620f1bb390..00000000000 --- a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j.properties +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n \ No newline at end of file diff --git a/raft/src/test/resources/log4j.properties b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j2.yaml similarity index 60% rename from raft/src/test/resources/log4j.properties rename to streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j2.yaml index 6d90f6dd348..0c112dd06d6 100644 --- a/raft/src/test/resources/log4j.properties +++ b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/resources/log4j2.yaml @@ -1,9 +1,9 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -12,11 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=OFF, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n +Configuration: + Properties: + Property: + - name: "logPattern" + value: "%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n" -log4j.logger.org.apache.kafka.raft=ERROR -log4j.logger.org.apache.kafka.snapshot=ERROR + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java index 4467e252b92..5895d3a632a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java @@ -44,7 +44,7 @@ import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor; import org.apache.kafka.streams.state.BuiltInDslStoreSuppliers; import org.apache.kafka.streams.utils.TestUtils.RecordingProcessorWrapper; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java index b0152da0be4..14470db2efa 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicManagerTest.java @@ -53,7 +53,7 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java index 95a5210ae34..e29af81095b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java @@ -40,7 +40,7 @@ import org.apache.kafka.test.InternalMockProcessorContext; import org.apache.kafka.test.MockSourceNode; import org.apache.kafka.test.MockTimestampExtractor; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java index f7ec5784890..66a581fb8f4 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java @@ -59,7 +59,8 @@ import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; import org.apache.kafka.test.InternalMockProcessorContext; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.filter.ThresholdFilter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1283,7 +1284,7 @@ public class RecordCollectorTest { try (final LogCaptureAppender logCaptureAppender = LogCaptureAppender.createAndRegister(RecordCollectorImpl.class)) { - logCaptureAppender.setThreshold(Level.INFO); + logCaptureAppender.addFilter(ThresholdFilter.createFilter(Level.INFO, null, null)); collector.send(topic, "3", "0", null, null, stringSerializer, stringSerializer, sinkNodeName, context, streamPartitioner); collector.flush(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java index 7d3167bf32e..320494c5348 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java @@ -45,7 +45,7 @@ import org.apache.kafka.test.MockStandbyUpdateListener; import org.apache.kafka.test.MockStateRestoreListener; import org.apache.kafka.test.StreamsTestUtils; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java index b55c9a12fcf..6d812e0119e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java @@ -55,7 +55,7 @@ import org.apache.kafka.streams.processor.internals.tasks.DefaultTaskManager; import org.apache.kafka.streams.processor.internals.testutil.DummyStreamsConfig; import org.apache.kafka.streams.state.internals.OffsetCheckpoint; -import org.apache.log4j.Level; +import org.apache.logging.log4j.Level; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/streams/src/test/resources/log4j.properties b/streams/src/test/resources/log4j.properties deleted file mode 100644 index 104b46df20f..00000000000 --- a/streams/src/test/resources/log4j.properties +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.kafka=ERROR -log4j.logger.state.change.logger=ERROR -log4j.logger.org.apache.kafka=ERROR -log4j.logger.org.apache.kafka.clients=ERROR - -# These are the only logs we will likely ever find anything useful in to debug Streams test failures -log4j.logger.org.apache.kafka.clients.consumer=INFO -log4j.logger.org.apache.kafka.clients.producer=INFO -log4j.logger.org.apache.kafka.streams=INFO - -# printing out the configs takes up a huge amount of the allotted characters, -# and provides little value as we can always figure out the test configs without the logs -log4j.logger.org.apache.kafka.clients.producer.ProducerConfig=ERROR -log4j.logger.org.apache.kafka.clients.consumer.ConsumerConfig=ERROR -log4j.logger.org.apache.kafka.clients.admin.AdminClientConfig=ERROR -log4j.logger.org.apache.kafka.streams.StreamsConfig=ERROR diff --git a/streams/src/test/resources/log4j2.yaml b/streams/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..0942036a33c --- /dev/null +++ b/streams/src/test/resources/log4j2.yaml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: kafka + level: ERROR + + - name: state.change.logger + level: ERROR + + - name: org.apache.kafka + level: ERROR + + - name: org.apache.kafka.clients + level: ERROR + + - name: org.apache.kafka.clients.consumer + level: INFO + + - name: org.apache.kafka.clients.producer + level: INFO + + - name: org.apache.kafka.streams + level: INFO + + - name: org.apache.kafka.clients.producer.ProducerConfig + level: ERROR + + - name: org.apache.kafka.clients.consumer.ConsumerConfig + level: ERROR + + - name: org.apache.kafka.clients.admin.AdminClientConfig + level: ERROR + + - name: org.apache.kafka.streams.StreamsConfig + level: ERROR diff --git a/streams/streams-scala/src/test/resources/log4j.properties b/streams/streams-scala/src/test/resources/log4j.properties deleted file mode 100644 index 93ffc165654..00000000000 --- a/streams/streams-scala/src/test/resources/log4j.properties +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2018 Lightbend Inc. -# Copyright (C) 2017-2018 Alexis Seigneurin. -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=INFO, R - -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender - -log4j.appender.R=org.apache.log4j.RollingFileAppender -log4j.appender.R.File=logs/kafka-streams-scala.log - -log4j.appender.R.MaxFileSize=100KB -# Keep one backup file -log4j.appender.R.MaxBackupIndex=1 - -# A1 uses PatternLayout. -log4j.appender.R.layout=org.apache.log4j.PatternLayout -log4j.appender.R.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/streams/streams-scala/src/test/resources/log4j2.yaml b/streams/streams-scala/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..6e1d18834be --- /dev/null +++ b/streams/streams-scala/src/test/resources/log4j2.yaml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "%-4r [%t] %-5p %c %x - %m%n" + + Appenders: + Console: + name: A1 + RollingFile: + - name: R + fileName: logs/kafka-streams-scala.log + filePattern: "streams-scala-%d{yyyy-MM-dd}.log" + PatternLayout: + pattern: "${logPattern}" + SizeBasedTriggeringPolicy: + size: "100KB" + DefaultRolloverStrategy: + max: 1 + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: R diff --git a/streams/test-utils/src/test/resources/log4j.properties b/streams/test-utils/src/test/resources/log4j.properties deleted file mode 100644 index be36f90299a..00000000000 --- a/streams/test-utils/src/test/resources/log4j.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.org.apache.kafka=INFO diff --git a/streams/test-utils/src/test/resources/log4j2.yaml b/streams/test-utils/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..be546a18b55 --- /dev/null +++ b/streams/test-utils/src/test/resources/log4j2.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: INFO diff --git a/test-common/src/main/resources/log4j2.yaml b/test-common/src/main/resources/log4j2.yaml new file mode 100644 index 00000000000..be546a18b55 --- /dev/null +++ b/test-common/src/main/resources/log4j2.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + Logger: + - name: org.apache.kafka + level: INFO diff --git a/tests/kafkatest/services/connect.py b/tests/kafkatest/services/connect.py index c84a3ec43c3..e09eba30b3e 100644 --- a/tests/kafkatest/services/connect.py +++ b/tests/kafkatest/services/connect.py @@ -25,7 +25,7 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_connect class ConnectServiceBase(KafkaPathResolverMixin, Service): @@ -38,7 +38,6 @@ class ConnectServiceBase(KafkaPathResolverMixin, Service): LOG_FILE = os.path.join(PERSISTENT_ROOT, "connect.log") STDOUT_FILE = os.path.join(PERSISTENT_ROOT, "connect.stdout") STDERR_FILE = os.path.join(PERSISTENT_ROOT, "connect.stderr") - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "connect-log4j.properties") PID_FILE = os.path.join(PERSISTENT_ROOT, "connect.pid") EXTERNAL_CONFIGS_FILE = os.path.join(PERSISTENT_ROOT, "connect-external-configs.properties") CONNECT_REST_PORT = 8083 @@ -340,7 +339,8 @@ class ConnectStandaloneService(ConnectServiceBase): return self.nodes[0] def start_cmd(self, node, connector_configs): - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE + cmd = "( export KAFKA_LOG4J_OPTS=\"%s%s\"; " % \ + (get_log4j_config_param(node), os.path.join(self.PERSISTENT_ROOT, get_log4j_config_for_connect(node))) heap_kafka_opts = "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=%s" % \ self.logs["connect_heap_dump_file"]["path"] other_kafka_opts = self.security_config.kafka_opts.strip('\"') @@ -364,7 +364,8 @@ class ConnectStandaloneService(ConnectServiceBase): if self.external_config_template_func: node.account.create_file(self.EXTERNAL_CONFIGS_FILE, self.external_config_template_func(node)) node.account.create_file(self.CONFIG_FILE, self.config_template_func(node)) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE)) + node.account.create_file(os.path.join(self.PERSISTENT_ROOT, get_log4j_config_for_connect(node)), + self.render(get_log4j_config_for_connect(node), log_file=self.LOG_FILE)) remote_connector_configs = [] for idx, template in enumerate(self.connector_config_templates): target_file = os.path.join(self.PERSISTENT_ROOT, "connect-connector-" + str(idx) + ".properties") @@ -400,7 +401,8 @@ class ConnectDistributedService(ConnectServiceBase): # connector_configs argument is intentionally ignored in distributed service. def start_cmd(self, node, connector_configs): - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE + cmd = ("( export KAFKA_LOG4J_OPTS=\"%s%s\"; " % + (get_log4j_config_param(node), os.path.join(self.PERSISTENT_ROOT, get_log4j_config_for_connect(node)))) heap_kafka_opts = "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=%s" % \ self.logs["connect_heap_dump_file"]["path"] other_kafka_opts = self.security_config.kafka_opts.strip('\"') @@ -421,7 +423,8 @@ class ConnectDistributedService(ConnectServiceBase): if self.external_config_template_func: node.account.create_file(self.EXTERNAL_CONFIGS_FILE, self.external_config_template_func(node)) node.account.create_file(self.CONFIG_FILE, self.config_template_func(node)) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE)) + node.account.create_file(os.path.join(self.PERSISTENT_ROOT, get_log4j_config_for_connect(node)), + self.render(get_log4j_config_for_connect(node), log_file=self.LOG_FILE)) if self.connector_config_templates: raise DucktapeError("Config files are not valid in distributed mode, submit connectors via the REST API") diff --git a/tests/kafkatest/services/console_consumer.py b/tests/kafkatest/services/console_consumer.py index 3e65efc3483..9755faa1969 100644 --- a/tests/kafkatest/services/console_consumer.py +++ b/tests/kafkatest/services/console_consumer.py @@ -21,8 +21,8 @@ from ducktape.utils.util import wait_until from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from kafkatest.services.monitor.jmx import JmxMixin, JmxTool -from kafkatest.version import DEV_BRANCH, LATEST_3_7 -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.version import DEV_BRANCH, LATEST_3_7, get_version, LATEST_4_0 +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_tools """ The console consumer is a tool that reads data from Kafka and outputs it to standard output. @@ -36,7 +36,6 @@ class ConsoleConsumer(KafkaPathResolverMixin, JmxMixin, BackgroundThreadService) STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console_consumer.stderr") LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") LOG_FILE = os.path.join(LOG_DIR, "console_consumer.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "console_consumer.properties") JMX_TOOL_LOG = os.path.join(PERSISTENT_ROOT, "jmx_tool.log") JMX_TOOL_ERROR_LOG = os.path.join(PERSISTENT_ROOT, "jmx_tool.err.log") @@ -146,7 +145,8 @@ class ConsoleConsumer(KafkaPathResolverMixin, JmxMixin, BackgroundThreadService) args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['stderr'] = ConsoleConsumer.STDERR_CAPTURE args['log_dir'] = ConsoleConsumer.LOG_DIR - args['log4j_config'] = ConsoleConsumer.LOG4J_CONFIG + args['log4j_param'] = get_log4j_config_param(node) + args['log4j_config'] = get_log4j_config_for_tools(node) args['config_file'] = ConsoleConsumer.CONFIG_FILE args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['jmx_port'] = self.jmx_port @@ -160,7 +160,7 @@ class ConsoleConsumer(KafkaPathResolverMixin, JmxMixin, BackgroundThreadService) cmd = fix_opts_for_new_jvm(node) cmd += "export JMX_PORT=%(jmx_port)s; " \ "export LOG_DIR=%(log_dir)s; " \ - "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j_config)s\"; " \ + "export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j_config)s\"; " \ "export KAFKA_OPTS=%(kafka_opts)s; " \ "%(console_consumer)s " \ "--topic %(topic)s " \ @@ -226,8 +226,8 @@ class ConsoleConsumer(KafkaPathResolverMixin, JmxMixin, BackgroundThreadService) node.account.create_file(ConsoleConsumer.CONFIG_FILE, prop_file) # Create and upload log properties - log_config = self.render('tools_log4j.properties', log_file=ConsoleConsumer.LOG_FILE) - node.account.create_file(ConsoleConsumer.LOG4J_CONFIG, log_config) + log_config = self.render(get_log4j_config_for_tools(node), log_file=ConsoleConsumer.LOG_FILE) + node.account.create_file(get_log4j_config_for_tools(node), log_config) # Run and capture output cmd = self.start_cmd(node) diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py index acfa5c7f6c2..b713aacb04e 100644 --- a/tests/kafkatest/services/kafka/kafka.py +++ b/tests/kafkatest/services/kafka/kafka.py @@ -33,7 +33,7 @@ from kafkatest.services.security.listener_security_config import ListenerSecurit from kafkatest.services.security.security_config import SecurityConfig from kafkatest.version import DEV_BRANCH from kafkatest.version import KafkaVersion -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config class KafkaListener: @@ -145,7 +145,6 @@ class KafkaService(KafkaPathResolverMixin, JmxMixin, Service): """ PERSISTENT_ROOT = "/mnt/kafka" STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "server-start-stdout-stderr.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "kafka-log4j.properties") # Logs such as controller.log, server.log, etc all go here OPERATIONAL_LOG_DIR = os.path.join(PERSISTENT_ROOT, "kafka-operational-logs") OPERATIONAL_LOG_INFO_DIR = os.path.join(OPERATIONAL_LOG_DIR, "info") @@ -805,7 +804,7 @@ class KafkaService(KafkaPathResolverMixin, JmxMixin, Service): kafka_mode = self.context.globals.get("kafka_mode", "") cmd = f"export KAFKA_MODE={kafka_mode}; " cmd += "export JMX_PORT=%d; " % self.jmx_port - cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG + cmd += "export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), os.path.join(self.PERSISTENT_ROOT, get_log4j_config(node))) heap_kafka_opts = "-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=%s" % \ self.logs["kafka_heap_dump_file"]["path"] security_kafka_opts = self.security_config.kafka_opts.strip('\"') @@ -874,7 +873,8 @@ class KafkaService(KafkaPathResolverMixin, JmxMixin, Service): self.logger.info("kafka.properties:") self.logger.info(prop_file) node.account.create_file(KafkaService.CONFIG_FILE, prop_file) - node.account.create_file(self.LOG4J_CONFIG, self.render('log4j.properties', log_dir=KafkaService.OPERATIONAL_LOG_DIR)) + node.account.create_file(os.path.join(self.PERSISTENT_ROOT, get_log4j_config(node)), + self.render(get_log4j_config(node), log_dir=KafkaService.OPERATIONAL_LOG_DIR)) if self.quorum_info.using_kraft: # format log directories if necessary diff --git a/tests/kafkatest/services/kafka/templates/log4j2.yaml b/tests/kafkatest/services/kafka/templates/log4j2.yaml new file mode 100644 index 00000000000..22e3f118f68 --- /dev/null +++ b/tests/kafkatest/services/kafka/templates/log4j2.yaml @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Configuration: + Properties: + Property: + - name: "log_dir" + value: {{ log_dir }} + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + RollingFile: + - name: KafkaInfoAppender + fileName: "${log_dir}/info/server.log" + filePattern: "${log_dir}/info/server.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: StateChangeInfoAppender + fileName: "${log_dir}/info/state-change.log" + filePattern: "${log_dir}/info/state-change.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: RequestInfoAppender + fileName: "${log_dir}/info/kafka-request.log" + filePattern: "${log_dir}/info/kafka-request.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: CleanerInfoAppender + fileName: "${log_dir}/info/log-cleaner.log" + filePattern: "${log_dir}/info/log-cleaner.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: ControllerInfoAppender + fileName: "${log_dir}/info/controller.log" + filePattern: "${log_dir}/info/controller.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: AuthorizerInfoAppender + fileName: "${log_dir}/info/kafka-authorizer.log" + filePattern: "${log_dir}/info/kafka-authorizer.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: INFO + onMatch: ACCEPT + + - name: KafkaDebugAppender + fileName: "${log_dir}/debug/server.log" + filePattern: "${log_dir}/debug/server.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + - name: StateChangeDebugAppender + fileName: "${log_dir}/debug/state-change.log" + filePattern: "${log_dir}/debug/state-change.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + - name: RequestDebugAppender + fileName: "${log_dir}/debug/kafka-request.log" + filePattern: "${log_dir}/debug/kafka-request.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + - name: CleanerDebugAppender + fileName: "${log_dir}/debug/log-cleaner.log" + filePattern: "${log_dir}/debug/log-cleaner.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + - name: ControllerDebugAppender + fileName: "${log_dir}/debug/controller.log" + filePattern: "${log_dir}/debug/controller.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + - name: AuthorizerDebugAppender + fileName: "${log_dir}/debug/kafka-authorizer.log" + filePattern: "${log_dir}/debug/kafka-authorizer.log.%d{yyyy-MM-dd-HH}" + PatternLayout: + pattern: "${logPattern}" + TimeBasedTriggeringPolicy: + interval: 1 + Filters: + ThresholdFilter: + level: DEBUG + onMatch: ACCEPT + + Loggers: + Root: + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: STDOUT + + Logger: + - name: kafka.producer.async.DefaultEventHandler + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender + + - name: kafka.client.ClientUtils + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender + + - name: kafka.perf + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender + + - name: kafka.perf.ProducerPerformance$ProducerThread + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender + + - name: kafka + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender + + - name: kafka.network.RequestChannel$ + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: RequestInfoAppender + - ref: RequestDebugAppender + + - name: kafka.network.Processor + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: RequestInfoAppender + - ref: RequestDebugAppender + + - name: kafka.server.KafkaApis + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: RequestInfoAppender + - ref: RequestDebugAppender + + - name: kafka.request.logger + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: RequestInfoAppender + - ref: RequestDebugAppender + + - name: org.apache.kafka.raft + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: ControllerInfoAppender + - ref: ControllerDebugAppender + + - name: org.apache.kafka.controller + level: {{ log_level|default("DEBUG") }} + AppenderRef: + - ref: ControllerInfoAppender + - ref: ControllerDebugAppender + + - name: kafka.controller + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: ControllerInfoAppender + - ref: ControllerDebugAppender + + - name: kafka.log.LogCleaner + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: CleanerInfoAppender + - ref: CleanerDebugAppender + + - name: state.change.logger + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: StateChangeInfoAppender + - ref: StateChangeDebugAppender + + - name: kafka.authorizer.logger + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: AuthorizerInfoAppender + - ref: AuthorizerDebugAppender + + - name: org.apache.kafka.coordinator.group + level: {{ log_level|default("DEBUG") }} + additivity: false + AppenderRef: + - ref: KafkaInfoAppender + - ref: KafkaDebugAppender diff --git a/tests/kafkatest/services/kafka/util.py b/tests/kafkatest/services/kafka/util.py index 0965fd9d4e4..a2e22ac32b7 100644 --- a/tests/kafkatest/services/kafka/util.py +++ b/tests/kafkatest/services/kafka/util.py @@ -16,6 +16,7 @@ from collections import namedtuple from kafkatest.utils.remote_account import java_version +from kafkatest.version import LATEST_4_0, get_version TopicPartition = namedtuple('TopicPartition', ['topic', 'partition']) @@ -30,4 +31,20 @@ def fix_opts_for_new_jvm(node): return "" +def get_log4j_config_param(node): + return '-Dlog4j2.configurationFile=file:' if get_version(node) >= LATEST_4_0 else '-Dlog4j.configuration=file:' +def get_log4j_config(node): + return 'log4j2.yaml' if get_version(node) >= LATEST_4_0 else 'log4j.properties' + +def get_log4j_config_for_connect(node): + return 'connect_log4j2.yaml' if get_version(node) >= LATEST_4_0 else 'connect_log4j.properties' + +def get_log4j_config_for_tools(node): + return 'tools_log4j2.yaml' if get_version(node) >= LATEST_4_0 else 'tools_log4j.properties' + +def get_log4j_config_for_trogdor_coordinator(node): + return 'trogdor-coordinator-log4j2.yaml' if get_version(node) >= LATEST_4_0 else 'trogdor-coordinator-log4j.properties' + +def get_log4j_config_for_trogdor_agent(node): + return 'trogdor-agent-log4j2.yaml' if get_version(node) >= LATEST_4_0 else 'trogdor-agent-log4j.properties' diff --git a/tests/kafkatest/services/performance/consumer_performance.py b/tests/kafkatest/services/performance/consumer_performance.py index eea91cbfd90..28086e82818 100644 --- a/tests/kafkatest/services/performance/consumer_performance.py +++ b/tests/kafkatest/services/performance/consumer_performance.py @@ -16,7 +16,7 @@ import os -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_tools from kafkatest.services.performance import PerformanceService from kafkatest.version import V_2_5_0, DEV_BRANCH @@ -49,7 +49,6 @@ class ConsumerPerformanceService(PerformanceService): STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "consumer_performance.stdout") STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "consumer_performance.stderr") LOG_FILE = os.path.join(LOG_DIR, "consumer_performance.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "consumer.properties") logs = { @@ -111,7 +110,7 @@ class ConsumerPerformanceService(PerformanceService): cmd = fix_opts_for_new_jvm(node) cmd += "export LOG_DIR=%s;" % ConsumerPerformanceService.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts - cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\";" % ConsumerPerformanceService.LOG4J_CONFIG + cmd += " export KAFKA_LOG4J_OPTS=\"%s%s\";" % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += " %s" % self.path.script("kafka-consumer-perf-test.sh", node) for key, value in self.args(node.version).items(): cmd += " --%s %s" % (key, value) @@ -128,8 +127,8 @@ class ConsumerPerformanceService(PerformanceService): def _worker(self, idx, node): node.account.ssh("mkdir -p %s" % ConsumerPerformanceService.PERSISTENT_ROOT, allow_fail=False) - log_config = self.render('tools_log4j.properties', log_file=ConsumerPerformanceService.LOG_FILE) - node.account.create_file(ConsumerPerformanceService.LOG4J_CONFIG, log_config) + log_config = self.render(get_log4j_config_for_tools(node), log_file=ConsumerPerformanceService.LOG_FILE) + node.account.create_file(get_log4j_config_for_tools(node), log_config) node.account.create_file(ConsumerPerformanceService.CONFIG_FILE, str(self.security_config)) self.security_config.setup_node(node) diff --git a/tests/kafkatest/services/performance/end_to_end_latency.py b/tests/kafkatest/services/performance/end_to_end_latency.py index e7e0100e511..15915557705 100644 --- a/tests/kafkatest/services/performance/end_to_end_latency.py +++ b/tests/kafkatest/services/performance/end_to_end_latency.py @@ -15,9 +15,8 @@ import os -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_tools from kafkatest.services.performance import PerformanceService -from kafkatest.services.security.security_config import SecurityConfig from kafkatest.version import get_version, V_3_4_0, DEV_BRANCH @@ -31,7 +30,6 @@ class EndToEndLatencyService(PerformanceService): STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "end_to_end_latency.stdout") STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "end_to_end_latency.stderr") LOG_FILE = os.path.join(LOG_DIR, "end_to_end_latency.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "client.properties") logs = { @@ -76,7 +74,7 @@ class EndToEndLatencyService(PerformanceService): }) cmd = fix_opts_for_new_jvm(node) - cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % EndToEndLatencyService.LOG4J_CONFIG + cmd += "export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += "KAFKA_OPTS=%(kafka_opts)s %(kafka_run_class)s %(java_class_name)s " % args cmd += "%(bootstrap_servers)s %(topic)s %(num_records)d %(acks)d %(message_bytes)d %(config_file)s" % args @@ -88,9 +86,9 @@ class EndToEndLatencyService(PerformanceService): def _worker(self, idx, node): node.account.ssh("mkdir -p %s" % EndToEndLatencyService.PERSISTENT_ROOT, allow_fail=False) - log_config = self.render('tools_log4j.properties', log_file=EndToEndLatencyService.LOG_FILE) + log_config = self.render(get_log4j_config_for_tools(node), log_file=EndToEndLatencyService.LOG_FILE) - node.account.create_file(EndToEndLatencyService.LOG4J_CONFIG, log_config) + node.account.create_file(get_log4j_config_for_tools(node), log_config) client_config = str(self.security_config) client_config += "compression_type=%(compression_type)s" % self.args node.account.create_file(EndToEndLatencyService.CONFIG_FILE, client_config) diff --git a/tests/kafkatest/services/performance/producer_performance.py b/tests/kafkatest/services/performance/producer_performance.py index acb0aec8650..acfe4790d73 100644 --- a/tests/kafkatest/services/performance/producer_performance.py +++ b/tests/kafkatest/services/performance/producer_performance.py @@ -19,7 +19,7 @@ from ducktape.utils.util import wait_until from ducktape.cluster.remoteaccount import RemoteCommandError from kafkatest.directory_layout.kafka_path import TOOLS_JAR_NAME, TOOLS_DEPENDANT_TEST_LIBS_JAR_NAME -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_tools from kafkatest.services.monitor.http import HttpMetricsCollector from kafkatest.services.performance import PerformanceService from kafkatest.services.security.security_config import SecurityConfig @@ -33,7 +33,6 @@ class ProducerPerformanceService(HttpMetricsCollector, PerformanceService): STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "producer_performance.stderr") LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") LOG_FILE = os.path.join(LOG_DIR, "producer_performance.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") def __init__(self, context, num_nodes, kafka, topic, num_records, record_size, throughput, version=DEV_BRANCH, settings=None, intermediate_stats=False, client_id="producer-performance"): @@ -90,7 +89,7 @@ class ProducerPerformanceService(HttpMetricsCollector, PerformanceService): cmd += "for file in %s; do CLASSPATH=$CLASSPATH:$file; done; " % jar cmd += "export CLASSPATH; " - cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % ProducerPerformanceService.LOG4J_CONFIG + cmd += " export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += "KAFKA_OPTS=%(kafka_opts)s KAFKA_HEAP_OPTS=\"-XX:+HeapDumpOnOutOfMemoryError\" %(kafka_run_class)s org.apache.kafka.tools.ProducerPerformance " \ "--topic %(topic)s --num-records %(num_records)d --record-size %(record_size)d --throughput %(throughput)d --producer-props bootstrap.servers=%(bootstrap_servers)s client.id=%(client_id)s %(metrics_props)s" % args @@ -119,8 +118,8 @@ class ProducerPerformanceService(HttpMetricsCollector, PerformanceService): node.account.ssh("mkdir -p %s" % ProducerPerformanceService.PERSISTENT_ROOT, allow_fail=False) # Create and upload log properties - log_config = self.render('tools_log4j.properties', log_file=ProducerPerformanceService.LOG_FILE) - node.account.create_file(ProducerPerformanceService.LOG4J_CONFIG, log_config) + log_config = self.render(get_log4j_config_for_tools(node), log_file=ProducerPerformanceService.LOG_FILE) + node.account.create_file(get_log4j_config_for_tools(node), log_config) cmd = self.start_cmd(node) self.logger.debug("Producer performance %d command: %s", idx, cmd) diff --git a/core/src/test/resources/log4j.properties b/tests/kafkatest/services/performance/templates/tools_log4j2.yaml similarity index 60% rename from core/src/test/resources/log4j.properties rename to tests/kafkatest/services/performance/templates/tools_log4j2.yaml index 833d63e1e8e..5c5e1099f94 100644 --- a/core/src/test/resources/log4j.properties +++ b/tests/kafkatest/services/performance/templates/tools_log4j2.yaml @@ -1,9 +1,9 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -12,11 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=OFF, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n +Configuration: + Appenders: + File: + name: FILE + fileName: {{ log_file }} + append: true + immediateFlush: true + PatternLayout: + pattern: "[%d] %p %m (%c)%n" -log4j.logger.kafka=WARN -log4j.logger.org.apache.kafka=WARN + Loggers: + Root: + level: {{ log_level|default("INFO") }} + AppenderRef: + - ref: FILE diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index 3848fea686d..df8a0b39230 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -22,6 +22,7 @@ from ducktape.utils.util import wait_until from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from kafkatest.services.kafka import KafkaConfig from kafkatest.services.monitor.jmx import JmxMixin +from .kafka.util import get_log4j_config_param, get_log4j_config_for_tools STATE_DIR = "state.dir" @@ -37,7 +38,6 @@ class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service): STDERR_FILE = os.path.join(PERSISTENT_ROOT, "streams.stderr") JMX_LOG_FILE = os.path.join(PERSISTENT_ROOT, "jmx_tool.log") JMX_ERR_FILE = os.path.join(PERSISTENT_ROOT, "jmx_tool.err.log") - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") PID_FILE = os.path.join(PERSISTENT_ROOT, "streams.pid") CLEAN_NODE_ENABLED = True @@ -285,10 +285,11 @@ class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \ " %(config_file)s %(user_test_args1)s %(user_test_args2)s %(user_test_args3)s" \ " %(user_test_args4)s & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -305,7 +306,7 @@ class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service): node.account.mkdirs(self.PERSISTENT_ROOT) prop_file = self.prop_file() node.account.create_file(self.CONFIG_FILE, prop_file) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('tools_log4j.properties', log_file=self.LOG_FILE)) + node.account.create_file(get_log4j_config_for_tools(node), self.render(get_log4j_config_for_tools(node), log_file=self.LOG_FILE)) self.logger.info("Starting StreamsTest process on " + str(node.account)) with node.account.monitor_log(self.STDOUT_FILE) as monitor: @@ -363,11 +364,12 @@ class StreamsSmokeTestBaseService(StreamsTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['version'] = self.KAFKA_STREAMS_VERSION args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\";" \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\";" \ " INCLUDE_TEST_JARS=true UPGRADE_KAFKA_STREAMS_TEST_VERSION=%(version)s" \ " %(kafka_run_class)s %(streams_class_name)s" \ " %(config_file)s %(user_test_args1)s" \ @@ -419,11 +421,12 @@ class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['disable_auto_terminate'] = self.DISABLE_AUTO_TERMINATE args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \ " %(config_file)s %(user_test_args1)s %(disable_auto_terminate)s" \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -496,10 +499,11 @@ class StreamsBrokerDownResilienceService(StreamsTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \ " %(config_file)s %(user_test_args1)s %(user_test_args2)s %(user_test_args3)s" \ " %(user_test_args4)s & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -535,12 +539,13 @@ class StreamsResetter(StreamsTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['application.id'] = self.applicationId args['input.topics'] = self.topic args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "(export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "(export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\";" \ "%(kafka_run_class)s %(streams_class_name)s " \ "--bootstrap-server %(bootstrap.servers)s " \ "--force " \ @@ -630,11 +635,12 @@ class StreamsUpgradeTestJobRunnerService(StreamsTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['version'] = self.KAFKA_STREAMS_VERSION args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true UPGRADE_KAFKA_STREAMS_TEST_VERSION=%(version)s " \ " %(kafka_run_class)s %(streams_class_name)s %(config_file)s " \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -730,11 +736,12 @@ class CooperativeRebalanceUpgradeService(StreamsTestBaseService): args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE - args['log4j'] = self.LOG4J_CONFIG_FILE + args['log4j_param'] = get_log4j_config_param(node) + args['log4j'] = get_log4j_config_for_tools(node) args['version'] = self.KAFKA_STREAMS_VERSION args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) - cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + cmd = "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true UPGRADE_KAFKA_STREAMS_TEST_VERSION=%(version)s " \ " %(kafka_run_class)s %(streams_class_name)s %(config_file)s " \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args diff --git a/tests/kafkatest/services/templates/connect_log4j2.yaml b/tests/kafkatest/services/templates/connect_log4j2.yaml new file mode 100644 index 00000000000..71f9f0f39bd --- /dev/null +++ b/tests/kafkatest/services/templates/connect_log4j2.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + Appenders: + File: + - name: FILE + fileName: {{ log_file }} + append: true + immediateFlush: true + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: {{ log_level|default("INFO") }} + AppenderRef: + - ref: FILE diff --git a/tests/kafkatest/services/templates/tools_log4j2.yaml b/tests/kafkatest/services/templates/tools_log4j2.yaml new file mode 100644 index 00000000000..2f41025d485 --- /dev/null +++ b/tests/kafkatest/services/templates/tools_log4j2.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Appenders: + File: + name: FILE + fileName: {{ log_file }} + append: true + immediateFlush: true + PatternLayout: + pattern: "[%d] %p %m (%c)%n" + + Loggers: + Root: + level: {{ log_level|default("INFO") }} + AppenderRef: + - ref: FILE + + {% if loggers is defined %} + Logger: + # Add additional loggers dynamically if defined + {% for logger, log_level in loggers.items() %} + - name: {{ logger }} + level: {{ log_level }} + {% endfor %} + {% endif %} \ No newline at end of file diff --git a/tests/kafkatest/services/transactional_message_copier.py b/tests/kafkatest/services/transactional_message_copier.py index 564a23fdcc3..d1f918cd8e1 100644 --- a/tests/kafkatest/services/transactional_message_copier.py +++ b/tests/kafkatest/services/transactional_message_copier.py @@ -22,6 +22,9 @@ from ducktape.services.background_thread import BackgroundThreadService from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from ducktape.cluster.remoteaccount import RemoteCommandError +from kafkatest.services.kafka.util import get_log4j_config_param, get_log4j_config_for_tools + + class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService): """This service wraps org.apache.kafka.tools.TransactionalMessageCopier for use in system testing. @@ -31,7 +34,6 @@ class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "transactional_message_copier.stderr") LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") LOG_FILE = os.path.join(LOG_DIR, "transactional_message_copier.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") logs = { "transactional_message_copier_stdout": { @@ -75,9 +77,9 @@ class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService node.account.ssh("mkdir -p %s" % TransactionalMessageCopier.PERSISTENT_ROOT, allow_fail=False) # Create and upload log properties - log_config = self.render('tools_log4j.properties', + log_config = self.render(get_log4j_config_for_tools(node), log_file=TransactionalMessageCopier.LOG_FILE) - node.account.create_file(TransactionalMessageCopier.LOG4J_CONFIG, log_config) + node.account.create_file(get_log4j_config_for_tools(node).LOG4J_CONFIG, log_config) # Configure security self.security_config = self.kafka.security_config.client_config(node=node) self.security_config.setup_node(node) @@ -114,7 +116,7 @@ class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService def start_cmd(self, node, idx): cmd = "export LOG_DIR=%s;" % TransactionalMessageCopier.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts - cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % TransactionalMessageCopier.LOG4J_CONFIG + cmd += " export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += self.path.script("kafka-run-class.sh", node) + " org.apache.kafka.tools." + "TransactionalMessageCopier" cmd += " --broker-list %s" % self.kafka.bootstrap_servers(self.security_config.security_protocol) cmd += " --transactional-id %s" % self.transactional_id diff --git a/tests/kafkatest/services/trogdor/templates/log4j2.yaml b/tests/kafkatest/services/trogdor/templates/log4j2.yaml new file mode 100644 index 00000000000..42c1aa281e7 --- /dev/null +++ b/tests/kafkatest/services/trogdor/templates/log4j2.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c)%n" + + Appenders: + File: + - name: MyFileLogger + fileName: {{ log_path }} + PatternLayout: + pattern: "${logPattern}" + Loggers: + Root: + level: DEBUG + AppenderRef: + - ref: MyFileLogger + + Logger: + - name: kafka + level: DEBUG + + - name: org.apache.kafka + level: DEBUG + + - name: org.eclipse + level: INFO diff --git a/tests/kafkatest/services/trogdor/trogdor.py b/tests/kafkatest/services/trogdor/trogdor.py index 3b941fe9059..618c68d7851 100644 --- a/tests/kafkatest/services/trogdor/trogdor.py +++ b/tests/kafkatest/services/trogdor/trogdor.py @@ -22,6 +22,8 @@ from requests.packages.urllib3 import Retry from ducktape.services.service import Service from ducktape.utils.util import wait_until from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin +from kafkatest.services.kafka.util import get_log4j_config_param, get_log4j_config, \ + get_log4j_config_for_trogdor_coordinator, get_log4j_config_for_trogdor_agent class TrogdorService(KafkaPathResolverMixin, Service): @@ -48,8 +50,6 @@ class TrogdorService(KafkaPathResolverMixin, Service): AGENT_STDOUT_STDERR = os.path.join(PERSISTENT_ROOT, "trogdor-agent-stdout-stderr.log") COORDINATOR_LOG = os.path.join(PERSISTENT_ROOT, "trogdor-coordinator.log") AGENT_LOG = os.path.join(PERSISTENT_ROOT, "trogdor-agent.log") - COORDINATOR_LOG4J_PROPERTIES = os.path.join(PERSISTENT_ROOT, "trogdor-coordinator-log4j.properties") - AGENT_LOG4J_PROPERTIES = os.path.join(PERSISTENT_ROOT, "trogdor-agent-log4j.properties") CONFIG_PATH = os.path.join(PERSISTENT_ROOT, "trogdor.conf") DEFAULT_AGENT_PORT=8888 DEFAULT_COORDINATOR_PORT=8889 @@ -141,26 +141,26 @@ class TrogdorService(KafkaPathResolverMixin, Service): self._start_agent_node(node) def _start_coordinator_node(self, node): - node.account.create_file(TrogdorService.COORDINATOR_LOG4J_PROPERTIES, - self.render('log4j.properties', + node.account.create_file(get_log4j_config_for_trogdor_coordinator(node), + self.render(get_log4j_config(node), log_path=TrogdorService.COORDINATOR_LOG)) self._start_trogdor_daemon("coordinator", TrogdorService.COORDINATOR_STDOUT_STDERR, - TrogdorService.COORDINATOR_LOG4J_PROPERTIES, + get_log4j_config_for_trogdor_coordinator(node), TrogdorService.COORDINATOR_LOG, node) self.logger.info("Started trogdor coordinator on %s." % node.name) def _start_agent_node(self, node): - node.account.create_file(TrogdorService.AGENT_LOG4J_PROPERTIES, - self.render('log4j.properties', + node.account.create_file(get_log4j_config_for_trogdor_agent(node), + self.render(get_log4j_config(node), log_path=TrogdorService.AGENT_LOG)) self._start_trogdor_daemon("agent", TrogdorService.AGENT_STDOUT_STDERR, - TrogdorService.AGENT_LOG4J_PROPERTIES, + get_log4j_config_for_trogdor_agent(node), TrogdorService.AGENT_LOG, node) self.logger.info("Started trogdor agent on %s." % node.name) def _start_trogdor_daemon(self, daemon_name, stdout_stderr_capture_path, log4j_properties_path, log_path, node): - cmd = "export KAFKA_LOG4J_OPTS='-Dlog4j.configuration=file:%s'; " % log4j_properties_path + cmd = "export KAFKA_LOG4J_OPTS='%s%s'; " % (get_log4j_config_param(node), log4j_properties_path) cmd += "%s %s --%s.config %s --node-name %s 1>> %s 2>> %s &" % \ (self.path.script("trogdor.sh", node), daemon_name, diff --git a/tests/kafkatest/services/verifiable_consumer.py b/tests/kafkatest/services/verifiable_consumer.py index 4b93a785bfd..8264566f1c2 100644 --- a/tests/kafkatest/services/verifiable_consumer.py +++ b/tests/kafkatest/services/verifiable_consumer.py @@ -20,6 +20,7 @@ from ducktape.services.background_thread import BackgroundThreadService from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from kafkatest.services.kafka import TopicPartition, consumer_group +from kafkatest.services.kafka.util import get_log4j_config_param, get_log4j_config_for_tools from kafkatest.services.verifiable_client import VerifiableClientMixin from kafkatest.version import DEV_BRANCH, V_2_3_0, V_2_3_1, V_3_7_0, V_4_0_0 @@ -215,7 +216,6 @@ class VerifiableConsumer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "verifiable_consumer.stderr") LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") LOG_FILE = os.path.join(LOG_DIR, "verifiable_consumer.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "verifiable_consumer.properties") logs = { @@ -296,8 +296,8 @@ class VerifiableConsumer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou node.account.ssh("mkdir -p %s" % VerifiableConsumer.PERSISTENT_ROOT, allow_fail=False) # Create and upload log properties - log_config = self.render('tools_log4j.properties', log_file=VerifiableConsumer.LOG_FILE) - node.account.create_file(VerifiableConsumer.LOG4J_CONFIG, log_config) + log_config = self.render(get_log4j_config_for_tools(node), log_file=VerifiableConsumer.LOG_FILE) + node.account.create_file(get_log4j_config_for_tools(node), log_config) # Create and upload config file self.security_config = self.kafka.security_config.client_config(self.prop_file, node, @@ -380,7 +380,7 @@ class VerifiableConsumer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou cmd = "" cmd += "export LOG_DIR=%s;" % VerifiableConsumer.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts - cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % VerifiableConsumer.LOG4J_CONFIG + cmd += " export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += self.impl.exec_cmd(node) if self.on_record_consumed: cmd += " --verbose" diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py index ea6292d5772..6f473d8bb13 100644 --- a/tests/kafkatest/services/verifiable_producer.py +++ b/tests/kafkatest/services/verifiable_producer.py @@ -24,7 +24,7 @@ from kafkatest.services.kafka import TopicPartition from kafkatest.services.verifiable_client import VerifiableClientMixin from kafkatest.utils import is_int, is_int_with_prefix from kafkatest.version import get_version, V_2_5_0, DEV_BRANCH -from kafkatest.services.kafka.util import fix_opts_for_new_jvm +from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config_for_tools class VerifiableProducer(KafkaPathResolverMixin, VerifiableClientMixin, BackgroundThreadService): @@ -41,7 +41,6 @@ class VerifiableProducer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "verifiable_producer.stderr") LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") LOG_FILE = os.path.join(LOG_DIR, "verifiable_producer.log") - LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "verifiable_producer.properties") logs = { @@ -127,8 +126,8 @@ class VerifiableProducer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou node.account.ssh("mkdir -p %s" % VerifiableProducer.PERSISTENT_ROOT, allow_fail=False) # Create and upload log properties - log_config = self.render('tools_log4j.properties', log_file=VerifiableProducer.LOG_FILE) - node.account.create_file(VerifiableProducer.LOG4J_CONFIG, log_config) + log_config = self.render(get_log4j_config_for_tools(node), log_file=VerifiableProducer.LOG_FILE) + node.account.create_file(get_log4j_config_for_tools(node), log_config) # Configure security self.security_config = self.kafka.security_config.client_config(node=node, @@ -222,7 +221,7 @@ class VerifiableProducer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += fix_opts_for_new_jvm(node) - cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % VerifiableProducer.LOG4J_CONFIG + cmd += " export KAFKA_LOG4J_OPTS=\"%s%s\"; " % (get_log4j_config_param(node), get_log4j_config_for_tools(node)) cmd += self.impl.exec_cmd(node) version = get_version(node) if version >= V_2_5_0: diff --git a/tests/kafkatest/tests/streams/streams_relational_smoke_test.py b/tests/kafkatest/tests/streams/streams_relational_smoke_test.py index 55737a185c2..a879d95c418 100644 --- a/tests/kafkatest/tests/streams/streams_relational_smoke_test.py +++ b/tests/kafkatest/tests/streams/streams_relational_smoke_test.py @@ -18,8 +18,10 @@ from ducktape.mark import matrix from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until from kafkatest.services.kafka import quorum +from kafkatest.services.kafka.util import get_log4j_config_param, get_log4j_config_for_tools from kafkatest.services.streams import StreamsTestBaseService from kafkatest.tests.kafka_test import KafkaTest +from kafkatest.version import LATEST_4_0 class StreamsRelationalSmokeTestService(StreamsTestBaseService): @@ -33,14 +35,15 @@ class StreamsRelationalSmokeTestService(StreamsTestBaseService): self.mode = mode self.nodeId = nodeId self.processing_guarantee = processing_guarantee - self.log4j_template = 'log4j_template.properties' + self.log4j_template = "log4j2_template.yaml" if (self.node.version >= LATEST_4_0) else "log4j_template.properties" def start_cmd(self, node): - return "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + return "( export KAFKA_LOG4J_OPTS=\"%(log4j_param)s%(log4j)s\"; " \ "INCLUDE_TEST_JARS=true %(kafka_run_class)s org.apache.kafka.streams.tests.RelationalSmokeTest " \ " %(mode)s %(kafka)s %(nodeId)s %(processing_guarantee)s %(state_dir)s" \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % { - "log4j": self.LOG4J_CONFIG_FILE, + "log4j_param": get_log4j_config_param(node), + "log4j": get_log4j_config_for_tools(node), "kafka_run_class": self.path.script("kafka-run-class.sh", node), "mode": self.mode, "kafka": self.kafka.bootstrap_servers(), @@ -54,8 +57,9 @@ class StreamsRelationalSmokeTestService(StreamsTestBaseService): def start_node(self, node): node.account.mkdirs(self.PERSISTENT_ROOT) - node.account.create_file(self.LOG4J_CONFIG_FILE, - self.render("log4j_template.properties", log_file=self.LOG_FILE)) + node.account.create_file(get_log4j_config_for_tools(node), + self.render("log4j2_template.yaml" if node.version >= LATEST_4_0 else "log4j_template.properties", + log_file=self.LOG_FILE)) self.logger.info("Starting process on " + str(node.account)) node.account.ssh(self.start_cmd(node)) diff --git a/clients/src/test/resources/log4j.properties b/tests/kafkatest/tests/streams/templates/log4j2_template.yaml similarity index 58% rename from clients/src/test/resources/log4j.properties rename to tests/kafkatest/tests/streams/templates/log4j2_template.yaml index 0992580eca1..f94e7d437a9 100644 --- a/clients/src/test/resources/log4j.properties +++ b/tests/kafkatest/tests/streams/templates/log4j2_template.yaml @@ -12,12 +12,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=OFF, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n +# Define the root logger with appender file +Configuration: + Appenders: + File: + name: FILE + fileName: {{ log_file }} + append: true + immediateFlush: true + PatternLayout: + pattern: "[%d] %p %m (%c)%n" -log4j.logger.org.apache.kafka=ERROR -# We are testing for a particular INFO log message in CommonNameLoggingTrustManagerFactoryWrapper -log4j.logger.org.apache.kafka.common.security.ssl.CommonNameLoggingTrustManagerFactoryWrapper=INFO + Loggers: + Root: + level: {{ log_level|default("INFO") }} + AppenderRef: + - ref: FILE + + Logger: + {% if loggers is defined %} + {% for logger, log_level in loggers.items() %} + - name: {{ logger }} + level: {{ log_level }} + {% endfor %} + {% endif %} \ No newline at end of file diff --git a/tools/src/test/java/org/apache/kafka/tools/other/ReplicationQuotasTestRig.java b/tools/src/test/java/org/apache/kafka/tools/other/ReplicationQuotasTestRig.java index a6b2f13e3c4..0dedf567c49 100644 --- a/tools/src/test/java/org/apache/kafka/tools/other/ReplicationQuotasTestRig.java +++ b/tools/src/test/java/org/apache/kafka/tools/other/ReplicationQuotasTestRig.java @@ -41,7 +41,7 @@ import org.apache.kafka.metadata.PartitionRegistration; import org.apache.kafka.server.quota.QuotaType; import org.apache.kafka.tools.reassign.ReassignPartitionsCommand; -import org.apache.log4j.PropertyConfigurator; +import org.apache.logging.log4j.core.config.Configurator; import org.jfree.chart.ChartFactory; import org.jfree.chart.ChartFrame; import org.jfree.chart.JFreeChart; @@ -95,7 +95,7 @@ public class ReplicationQuotasTestRig { private static final String DIR; static { - PropertyConfigurator.configure("core/src/test/resources/log4j.properties"); + Configurator.reconfigure(); new File("Experiments").mkdir(); DIR = "Experiments/Run" + Long.valueOf(System.currentTimeMillis()).toString().substring(8); diff --git a/tools/src/test/resources/log4j.properties b/tools/src/test/resources/log4j.properties deleted file mode 100644 index 3aca07dc530..00000000000 --- a/tools/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.org.apache.kafka=INFO -log4j.logger.org.eclipse.jetty=INFO diff --git a/tools/src/test/resources/log4j2.yaml b/tools/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..aef7e561628 --- /dev/null +++ b/tools/src/test/resources/log4j2.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: INFO + AppenderRef: + - ref: STDOUT + + Logger: + - name: org.apache.kafka + level: ERROR + + - name: org.eclipse.jetty + level: ERROR diff --git a/trogdor/src/test/resources/log4j.properties b/trogdor/src/test/resources/log4j.properties deleted file mode 100644 index 5291604d49a..00000000000 --- a/trogdor/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=TRACE, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -log4j.logger.org.apache.kafka=INFO -log4j.logger.org.eclipse.jetty=INFO diff --git a/trogdor/src/test/resources/log4j2.yaml b/trogdor/src/test/resources/log4j2.yaml new file mode 100644 index 00000000000..4c3355e307e --- /dev/null +++ b/trogdor/src/test/resources/log4j2.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Configuration: + Properties: + Property: + - name: "logPattern" + value: "[%d] %p %m (%c:%L)%n" + + Appenders: + Console: + name: STDOUT + PatternLayout: + pattern: "${logPattern}" + + Loggers: + Root: + level: TRACE + AppenderRef: + - ref: STDOUT + + Logger: + - name: org.apache.kafka + level: ERROR + + - name: org.eclipse.jetty + level: ERROR