KAFKA-8634: Update ZooKeeper to 3.5.5 (#6802)

ZooKeeper 3.5.5 is the first stable release in the 3.5.x series. The key new feature
in is TLS support, but there are a few more noteworthy features:

* Dynamic reconfiguration
* Local sessions
* New node types: Container, TTL
* Ability to remove watchers
* Multi-threaded commit processor
* Upgraded to Netty 4.1

See the release notes for more detail:
https://zookeeper.apache.org/doc/r3.5.5/releasenotes.html

In addition to the version bump, we:

* Add `commons-cli` dependency as it's required by `ZooKeeperMain`, but specified as
`provided` in their pom.
* Remove unnecessary `ZooKeeperMainWrapper`, the bug it worked around was fixed
upstream a long time ago.
* Ignore non zero exit in one system test invocation of `ZooKeeperMain`.
`ZooKeeperMainWrapper` always returned `0` and `ZooKeeperService.query` relies
on that for correct behavior.

Reviewers: Jason Gustafson <jason@confluent.io>

build.gradle

@@ -679,6 +679,8 @@ project(':core') {
       exclude module: 'log4j'
       exclude module: 'netty'
     }
+    // ZooKeeperMain depends on commons-cli but declares the dependency as `provided`
+    compile libs.commonsCli
 
     compileOnly libs.log4j
 

core/src/main/scala/kafka/tools/ZooKeeperMainWrapper.scala

@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package kafka.tools
-
-import kafka.utils.Exit
-import org.apache.zookeeper.ZooKeeperMain
-
-class ZooKeeperMainWrapper(args: Array[String]) extends ZooKeeperMain(args) {
-  def runCmd(): Unit = {
-    processCmd(this.cl)
-    Exit.exit(0)
-  }
-}
-
-/**
- * ZooKeeper 3.4.6 broke being able to pass commands on command line.
- * See ZOOKEEPER-1897.  This class is a hack to restore this facility.
- */
-object ZooKeeperMainWrapper {
-
-  def main(args: Array[String]): Unit = {
-    val main: ZooKeeperMainWrapper = new ZooKeeperMainWrapper(args)
-    main.runCmd()
-  }
-}

gradle/dependencies.gradle

@@ -65,6 +65,7 @@ versions += [
   argparse4j: "0.7.0",
   bcpkix: "1.61",
   checkstyle: "8.20",
+  commonsCli: "1.4",
   gradle: "5.4.1",
   gradleVersionsPlugin: "0.21.0",
   grgit: "3.1.1",
@@ -112,7 +113,7 @@ versions += [
   spotbugs: "3.1.12",
   spotbugsPlugin: "1.6.9",
   spotlessPlugin: "3.23.0",
-  zookeeper: "3.4.14",
+  zookeeper: "3.5.5",
   zstd: "1.4.0-1"
 ]
 
@@ -129,6 +130,7 @@ libs += [
   apachedsJdbmPartition: "org.apache.directory.server:apacheds-jdbm-partition:$versions.apacheds",
   argparse4j: "net.sourceforge.argparse4j:argparse4j:$versions.argparse4j",
   bcpkix: "org.bouncycastle:bcpkix-jdk15on:$versions.bcpkix",
+  commonsCli: "commons-cli:commons-cli:$versions.commonsCli",
   easymock: "org.easymock:easymock:$versions.easymock",
   jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson",
   jacksonDataformatCsv: "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson",

tests/kafkatest/services/zookeeper.py

@@ -159,7 +159,7 @@ class ZookeeperService(KafkaPathResolverMixin, Service):
 
         node = self.nodes[0]
         result = None
-        for line in node.account.ssh_capture(cmd):
+        for line in node.account.ssh_capture(cmd, allow_fail=True):
             # loop through all lines in the output, but only hold on to the first match
             if result is None:
                 match = re.match("^({.+})$", line)
@@ -188,4 +188,4 @@ class ZookeeperService(KafkaPathResolverMixin, Service):
 
     def java_cli_class_name(self):
         """ The class name of the Zookeeper tool within Kafka. """
-        return "kafka.tools.ZooKeeperMainWrapper"
+        return "org.apache.zookeeper.ZooKeeperMain"