KAFKA-18074: Add kafka client compatibility matrix (#18091)

Add new section for Kafka 4.0 compatibility metrics for user, so user
can check server and client in this section.

Reviewers: Divij Vaidya <diviv@amazon.com>, Matthias J. Sax <matthias@confluent.io>

docs/compatibility-summary.html

@@ -0,0 +1,27 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<script><!--#include virtual="js/templateData.js" --></script>
+
+<script id="compatibility-summary-template" type="text/x-handlebars-template">
+<p>
+    With the release of Kafka 4.0, significant changes have been introduced that impact compatibility across various
+    components. To assist users in planning upgrades and ensuring seamless interoperability, a comprehensive
+    <a href="/{{version}}/documentation/compatibility.html">compatibility matrix</a> has been prepared.
+</p>
+</script>
+<div class="p-compatibility-summary"></div>

docs/documentation.html

@@ -78,7 +78,9 @@
       <!--#include virtual="upgrade.html" -->
       <h3 class="anchor-heading"><a id="zk2kraft-summary" class="anchor-link"></a><a href="#zk2kraft-summary">1.6 KRaft vs ZooKeeper</a></h3>
       <!--#include virtual="zk2kraft-summary.html" -->
-      <h3 class="anchor-heading"><a id="docker" class="anchor-link"></a><a href="#docker">1.7 Docker</a></h3>
+      <h3 class="anchor-heading"><a id="compatibility-summary" class="anchor-link"></a><a href="#compatibility-summary">1.7 Compatibility</a></h3>
+      <!--#include virtual="compatibility-summary.html" -->
+      <h3 class="anchor-heading"><a id="docker" class="anchor-link"></a><a href="#docker">1.8 Docker</a></h3>
       <!--#include virtual="docker.html" -->
 
     <h2 class="anchor-heading"><a id="api" class="anchor-link"></a><a href="#api">2. APIs</a></h2>

docs/documentation/compatibility.html

@@ -0,0 +1,201 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--#include virtual="../../includes/_header.htm" -->
+<!--#include virtual="../../includes/_top.htm" -->
+<div class="content">
+    <div class="p-compatibility">
+        <h1>Compatibility</h1>
+        With the release of Kafka 4.0, significant changes have been introduced that impact compatibility across various
+        components. To assist users in planning upgrades and ensuring seamless interoperability, a comprehensive
+        compatibility matrix has been prepared.
+        <h2 class="anchor-heading">JDK Compatibility Across Kafka Versions</h2>
+        <table>
+            <tr>
+                <th>Module</th>
+                <th>Kafka Version</th>
+                <th>Java 11</th>
+                <th>Java 17</th>
+                <th>Java 23</th>
+            </tr>
+            <tr>
+                <td>Clients</td>
+                <td>4.0.0</td>
+                <td>✅</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>Streams</td>
+                <td>4.0.0</td>
+                <td>✅</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>Connect</td>
+                <td>4.0.0</td>
+                <td>❌</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>Server</td>
+                <td>4.0.0</td>
+                <td>❌</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+        </table>
+        <p><strong>Note: Java 8 is removed in Kafka 4.0 and is no longer supported.</strong></p>
+
+        <h2 class="anchor-heading">Server Compatibility</h2>
+        <table>
+            <tr>
+                <th>KRaft Cluster Version</th>
+                <th>Compatibility 4.0 Server (dynamic voter)</th>
+                <th>Compatibility 4.0 Server (static voter)</th>
+            </tr>
+            <tr>
+                <td>before 3.2.x</td>
+                <td>❌</td>
+                <td>❌</td>
+            </tr>
+            <tr>
+                <td>3.3.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.4.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.5.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.6.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.7.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.8.x</td>
+                <td>❌</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>3.9.x</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+            <tr>
+                <td>4.0.x</td>
+                <td>✅</td>
+                <td>✅</td>
+            </tr>
+        </table>
+        <p><strong>Note: Can’t upgrade server from static voter to dynamic
+            voter, see <a href="https://issues.apache.org/jira/browse/KAFKA-16538">KAFKA-16538</a>.</strong></p>
+
+        <h4>Client/Broker Forward Compatibility</h4>
+        <table>
+            <tr>
+                <th>Kafka Version</th>
+                <th>Module</th>
+                <th>Compatibility with Kafka 4.0</th>
+                <th>Key Differences/Limitations</th>
+            </tr>
+            <tr>
+                <td rowspan="3">0.x, 1.x, 2.0</td>
+                <td>Client</td>
+                <td>❌ Not Compatible</td>
+                <td>Pre-0.10.x protocols are fully removed in Kafka 4.0 (<a
+                        href="https://cwiki.apache.org/confluence/x/K5sODg">KIP-896</a>).
+                </td>
+            </tr>
+            <tr>
+                <td>Streams</td>
+                <td>❌ Not Compatible</td>
+                <td>Pre-0.10.x protocols are fully removed in Kafka 4.0 (<a
+                        href="https://cwiki.apache.org/confluence/x/K5sODg">KIP-896</a>).
+                </td>
+            </tr>
+            <tr>
+                <td>Connect</td>
+                <td>❌ Not Compatible</td>
+                <td>Pre-0.10.x protocols are fully removed in Kafka 4.0 (<a
+                        href="https://cwiki.apache.org/confluence/x/K5sODg">KIP-896</a>).
+                </td>
+            </tr>
+            <tr>
+                <td rowspan="3">2.1 ~ 2.8</td>
+                <td>Client</td>
+                <td>⚠️ Partially Compatible</td>
+                <td>
+                    More details in the <a href="/40/documentation.html#upgrade_400_notable_consumer">Consumer</a>, 
+                    <a href="/40/documentation.html#upgrade_400_notable_producer">Producer</a>, and
+                    <a href="/40/documentation.html#upgrade_400_notable_admin_client">Admin Client</a> section.
+                </td>
+            </tr>
+            <tr>
+                <td>Streams</td>
+                <td>⚠️ Limited Compatibility</td>
+                <td>
+                    More details in the <a href="/40/documentation.html#upgrade_400_notable_kafka_streams">Kafka Streams</a> section.
+                </td>
+            </tr>
+            <tr>
+                <td>Connect</td>
+                <td>⚠️ Limited Compatibility</td>
+                <td>
+                    More details in the <a href="/40/documentation.html#upgrade_400_notable_connect">Connect</a> section.
+                </td>
+            </tr>
+            <tr>
+                <td rowspan="3">3.x</td>
+                <td>Client</td>
+                <td>✅ Fully Compatible</td>
+                <td></td>
+            </tr>
+            <tr>
+                <td>Streams</td>
+                <td>✅ Fully Compatible</td>
+                <td></td>
+            </tr>
+            <tr>
+                <td>Connect</td>
+                <td>✅ Fully Compatible</td>
+                <td></td>
+            </tr>
+        </table>
+        <p>
+            Note: Starting with Kafka 4.0, the <code>--zookeeper</code> option in AdminClient commands has been removed. Users must
+            use the <code>--bootstrap-server</code> option to interact with the Kafka cluster. This change aligns with the transition
+            to KRaft mode.
+        </p>
+
+    </div>
+    <!--#include virtual="../../includes/_footer.htm" -->
+</div>

docs/toc.html

@@ -28,7 +28,8 @@
                 <li><a href="#ecosystem">1.4 Ecosystem</a>
                 <li><a href="#upgrade">1.5 Upgrading</a>
                 <li><a href="#zk2kraft-summary">1.6 KRaft vs ZooKeeper</a>
-                <li><a href="#docker">1.7 Docker</a>
+                <li><a href="#compatibility-summary">1.7 Compatibility</a>
+                <li><a href="#docker">1.8 Docker</a>
             </ul>
         
         <li><a href="#api">2. APIs</a>

docs/upgrade.html

@@ -101,7 +101,7 @@
         </li>
         <li>A number of deprecated classes, methods, configurations and tools have been removed.
             <ul>
-                <li><b>Common</b>
+                <li><a id="upgrade_400_notable_common" href="#upgrade_400_notable_common"><b>Common</b></a>
                     <ul>
                         <li>The <code>metrics.jmx.blacklist</code> and <code>metrics.jmx.whitelist</code> configurations were removed from the <code>org.apache.kafka.common.metrics.JmxReporter</code>
                             Please use <code>metrics.jmx.exclude</code> and <code>metrics.jmx.include</code> respectively instead.
@@ -144,7 +144,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Broker</b>
+                <li><a id="upgrade_400_notable_broker" href="#upgrade_400_notable_broker"><b>Broker</b></a>
                     <ul>
                         <li>The <code>delegation.token.master.key</code> configuration was removed.
                             Please use <code>delegation.token.secret.key</code> instead.
@@ -177,7 +177,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>MirrorMaker</b>
+                <li><a id="upgrade_400_notable_MirrorMaker" href="#upgrade_400_notable_MirrorMaker"><b>MirrorMaker</b></a>
                     <ul>
                         <li>The original MirrorMaker (MM1) and related classes were removed. Please use the Connect-based
                             MirrorMaker (MM2), as described in the <a href="/{{version}}/documentation/#georeplication">Geo-Replication section.</a>.
@@ -199,7 +199,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Tools</b>
+                <li><a id="upgrade_400_notable_tools" href="#upgrade_400_notable_tools"><b>Tools</b></a>
                     <ul>
                         <li>The <code>kafka.common.MessageReader</code> class was removed. Please use the
                             <a href="/{{version}}/javadoc/org/apache/kafka/tools/api/RecordReader.html"><code>org.apache.kafka.tools.api.RecordReader</code></a>
@@ -250,7 +250,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Connect</b>
+                <li><a id="upgrade_400_notable_connect" href="#upgrade_400_notable_connect"><b>Connect</b></a>
                     <ul>
                         <li>The <code>whitelist</code> and <code>blacklist</code> configurations were removed from the <code>org.apache.kafka.connect.transforms.ReplaceField</code> transformation.
                             Please use <code>include</code> and <code>exclude</code> respectively instead.
@@ -262,7 +262,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Consumer</b>
+                <li><a id="upgrade_400_notable_consumer" href="#upgrade_400_notable_consumer"><b>Consumer</b></a>
                     <ul>
                         <li>The <code>poll(long)</code> method was removed from the consumer. Please use <code>poll(Duration)</code> instead. Note that there is
                             a difference in behavior between the two methods. The <code>poll(Duration)</code> method does not block beyond the timeout awaiting
@@ -277,7 +277,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Producer</b>
+                <li><a id="upgrade_400_notable_producer" href="#upgrade_400_notable_producer"><b>Producer</b></a>
                     <ul>
                         <li>The <code>enable.idempotence</code> configuration will no longer automatically fall back when the <code>max.in.flight.requests.per.connection</code> value exceeds 5.
                         </li>
@@ -288,7 +288,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Admin client</b>
+                <li><a id="upgrade_400_notable_admin_client" href="#upgrade_400_notable_admin_client"><b>Admin client</b></a>
                     <ul>
                         <li>
                             The <code>alterConfigs</code> method was removed from the <code>org.apache.kafka.clients.admin.Admin</code>.
@@ -332,7 +332,7 @@
                         </li>
                     </ul>
                 </li>
-                <li><b>Kafka Streams</b>
+                <li><a id="upgrade_400_notable_kafka_streams" href="#upgrade_400_notable_kafka_streams"><b>Kafka Streams</b></a>
                     <ul>
                         <li>
 			    All public APIs, deprecated in Apache Kafka 3.6 or an earlier release, have been removed, with the exception of <code>JoinWindows.of()</code> and <code>JoinWindows#grace()</code>.