KAFKA-14661: Upgrade Zookeeper to 3.8.1 (#13260)

Reviewers: Divij Vaidya <diviv@amazon.com>, Mickael Maison <mickael.maison@gmail.com>

LICENSE-binary

@@ -261,8 +261,8 @@ scala-reflect-2.13.11
 scala-java8-compat_2.13-1.0.2
 snappy-java-1.1.10.3
 swagger-annotations-2.2.8
-zookeeper-3.6.4
+zookeeper-3.8.2
-zookeeper-jute-3.6.4
+zookeeper-jute-3.8.2
 
 ===============================================================================
 This product bundles various third-party components under other open source

build.gradle

@@ -893,6 +893,10 @@ project(':core') {
       implementation libs.dropwizardMetrics
       exclude module: 'slf4j-log4j12'
       exclude module: 'log4j'
+      // Both Kafka and Zookeeper use slf4j. ZooKeeper moved from log4j to logback in v3.8.0, but Kafka relies on reload4j.
+      // We are removing Zookeeper's dependency on logback so we have a singular logging backend.
+      exclude module: 'logback-classic'
+      exclude module: 'logback-core'
     }
     // ZooKeeperMain depends on commons-cli but declares the dependency as `provided`
     implementation libs.commonsCli

docs/upgrade.html

@@ -25,6 +25,18 @@
     <ul>
         <li>Apache Kafka now supports having both an IPv4 and an IPv6 listener on the same port. This change only applies to
             non advertised listeners (advertised listeners already have this feature)</li>
+        <li>The Apache Zookeeper dependency has been upgraded to 3.8.1 due to 3.6 reaching end-of-life. To bring both your
+            Kafka and Zookeeper clusters to the latest versions:
+            <ul>
+                <li><b>>=2.4</b> Kafka clusters can be updated directly.
+                    Zookeeper clusters which are running binaries bundled with Kafka versions 2.4 or above can be updated directly.</li>
+                <li><b><2.4</b> Kafka clusters first need to be updated to a version greater than 2.4 and smaller than 3.6.
+                Zookeeper clusters which are running binaries bundled with Kafka versions below 2.4 need to be updated to any
+                binaries bundled with Kafka versions greater than 2.4 and smaller than 3.6. You can then follow the first bullet-point.</li>
+            </ul>
+            For more detailed information please refer to the Compatibility, Deprecation, and Migration Plan section in
+            <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-902%3A+Upgrade+Zookeeper+to+3.8.1">KIP-902</a>.
+        </li>
     </ul>
 
 <h4><a id="upgrade_3_5_0" href="#upgrade_3_5_0">Upgrading to 3.5.0 from any version 0.8.x through 3.4.x</a></h4>

gradle/dependencies.gradle

@@ -148,7 +148,7 @@ versions += [
   swaggerAnnotations: "2.2.8",
   swaggerJaxrs2: "2.2.8",
   zinc: "1.9.2",
-  zookeeper: "3.6.4",
+  zookeeper: "3.8.2",
   zstd: "1.5.5-1"
 ]