feat(release): release 1.5.5 (#2761)

Signed-off-by: Robin Han <hanxvdovehx@gmail.com>

.github/CODEOWNERS

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-* @superhx @SCNieh @ShadowySpirits @Chillax-0v0
+* @superhx @Gezi-lzq @1sonofqiu @woshigaopp

.github/workflows/build_automq.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2.9.0
       - name: Checkstyle
-        run: ./gradlew --build-cache rat checkstyleMain checkstyleTest
+        run: ./gradlew --build-cache rat checkstyleMain checkstyleTest spotlessJavaCheck
   spotbugs:
     name: "Spotbugs"
     runs-on: ${{ matrix.os }}

.github/workflows/docker-bitnami-release.yaml

@@ -0,0 +1,67 @@
+name: Docker Bitnami Release
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
+
+
+jobs:
+  docker-release:
+    name: Docker Image Release
+    strategy:
+      matrix:
+        platform: [ "ubuntu-24.04" ]
+        jdk: ["17"]
+    runs-on: ${{ matrix.platform }}
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up JDK ${{ matrix.jdk }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: "zulu"
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2.12.0
+      - name: Get project version
+        id: get_project_version
+        run: |
+          project_version=$(./gradlew properties | grep "version:" | awk '{print $2}')
+          echo "PROJECT_VERSION=${project_version}" >> $GITHUB_OUTPUT
+
+      - name: Build TarGz
+        run: |
+          ./gradlew -Pprefix=automq-${{ github.ref_name }}_ --build-cache --refresh-dependencies clean releaseTarGz
+
+      # docker image release
+      - name: Cp TarGz to Docker Path
+        run: |
+          cp ./core/build/distributions/automq-${{ github.ref_name }}_kafka-${{ steps.get_project_version.outputs.PROJECT_VERSION }}.tgz ./container/bitnami
+      - name: Determine Image Tags
+        id: image_tags
+        run: |
+          echo "tags=${{ secrets.DOCKERHUB_USERNAME }}/automq:${{ github.ref_name }}-bitnami" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_READ_WRITE_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./container/bitnami
+          push: true
+          tags: ${{ steps.image_tags.outputs.tags }}
+          platforms: linux/amd64,linux/arm64

.github/workflows/docker-release.yml

@@ -1,6 +1,7 @@
 name: Docker Release
 
 on:
+  workflow_dispatch:
   push:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
@@ -12,7 +13,7 @@ jobs:
     name: Docker Image Release
     strategy:
       matrix:
-        platform: [ "ubuntu-22.04" ]
+        platform: [ "ubuntu-24.04" ]
         jdk: ["17"]
     runs-on: ${{ matrix.platform }}
     permissions:
@@ -69,4 +70,4 @@ jobs:
           context: ./docker
           push: true
           tags: ${{ steps.image_tags.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64

.github/workflows/docker_build_and_test.yml

@@ -46,7 +46,7 @@ jobs:
       run: |
         python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }}
     - name: Run CVE scan
-      uses: aquasecurity/trivy-action@master
+      uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
       with:
         image-ref: 'kafka/test:test'
         format: 'table'

.github/workflows/docker_official_image_build_and_test.yml

@@ -45,7 +45,7 @@ jobs:
       run: |
         python docker_official_image_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }}
     - name: Run CVE scan
-      uses: aquasecurity/trivy-action@master
+      uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
       with:
         image-ref: 'kafka/test:test'
         format: 'table'

.github/workflows/docker_promote.yml

@@ -31,11 +31,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
     - name: Login to Docker Hub
-      uses: docker/login-action@v3
+      uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
       with:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker_rc_release.yml

@@ -47,11 +47,11 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r docker/requirements.txt
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
     - name: Login to Docker Hub
-      uses: docker/login-action@v3
+      uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
       with:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker_scan.yml

@@ -29,7 +29,7 @@ jobs:
         supported_image_tag: ['latest', '3.7.0']
     steps:
       - name: Run CVE scan
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
         if: always()
         with:
           image-ref: apache/kafka:${{ matrix.supported_image_tag }}

.github/workflows/github-release.yml

@@ -30,32 +30,39 @@ jobs:
         uses: gradle/gradle-build-action@v2.12.0
 
       - name: Build TarGz
+        id: build-targz 
         run: |
           ./gradlew -Pprefix=automq-${{ github.ref_name }}_ --build-cache --refresh-dependencies clean releaseTarGz
           mkdir -p core/build/distributions/latest
+          LATEST_TAG=$(git tag --sort=-v:refname | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)
+          echo "LATEST_TAG=$LATEST_TAG"
+          IS_LATEST="false"
+          if [ "$LATEST_TAG" == "${{ github.ref_name }}" ]; then
+              IS_LATEST=true
+          fi
+          echo "IS_LATEST=$IS_LATEST" >> $GITHUB_OUTPUT
           for file in core/build/distributions/automq-*.tgz; do
               if [[ ! "$file" =~ site-docs ]]; then
-                  echo "Find latest tgz file: $file"
-                  cp "$file" core/build/distributions/latest/automq-kafka-latest.tgz
-                  break
+                  if [ "$IS_LATEST" = "true" ]; then
+                      echo "Find latest tgz file: $file"
+                      cp "$file" core/build/distributions/latest/automq-kafka-latest.tgz
+                  fi
               else
                   echo "Skip and remove site-docs file: $file"
                   rm "$file"
               fi
           done
 
-      - uses: jakejarvis/s3-sync-action@master
-        name: s3-upload-latest
-        if: ${{ github.repository_owner == 'AutoMQ' }}
+      - uses: tvrcgo/oss-action@master
+        name: upload-latest
+        if: ${{ github.repository_owner == 'AutoMQ' && steps.build-targz.outputs.IS_LATEST == 'true' }}
         with:
-          args: --follow-symlinks --delete
-        env:
-          AWS_S3_BUCKET:  ${{ secrets.AWS_CN_PROD_BUCKET }}
-          AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_CN_PROD_AK }}
-          AWS_SECRET_ACCESS_KEY:  ${{ secrets.AWS_CN_PROD_SK }}
-          AWS_REGION: 'cn-northwest-1'
-          SOURCE_DIR: 'core/build/distributions/latest'
-          DEST_DIR: 'community_edition/artifacts'
+          bucket:  ${{ secrets.UPLOAD_BUCKET }}
+          key-id:  ${{ secrets.UPLOAD_BUCKET_AK }}
+          key-secret:  ${{ secrets.UPLOAD_BUCKET_SK }}
+          region: 'oss-cn-hangzhou'
+          assets: |
+            core/build/distributions/latest/automq-kafka-latest.tgz:community_edition/artifacts/automq-kafka-latest.tgz
 
       - name: GitHub Release
         uses: softprops/action-gh-release@v1

.github/workflows/spark-iceberg-docker-build.yml

@@ -0,0 +1,31 @@
+name: Spark Iceberg image
+
+on:
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_READ_WRITE_TOKEN }}
+
+      - name: Build and Push
+        uses: docker/build-push-action@v6
+        with:
+          context: docker/table_topic/spark_iceberg/
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: automqinc/spark-iceberg:latest

.gitignore

@@ -62,3 +62,7 @@ storage/kafka-tiered-storage/
 docker/test/report_*.html
 kafka.Kafka
 __pycache__
+
+# Ignore bin folder generated by the build, but exclude the one in the root
+bin/
+!/bin/

.idea/copyright/BSL.xml

@@ -1,6 +0,0 @@
-<component name="CopyrightManager">
-  <copyright>
-    <option name="notice" value="Copyright 2024, AutoMQ HK Limited.&#10;&#10;The use of this file is governed by the Business Source License,&#10;as detailed in the file &quot;/LICENSE.S3Stream&quot; included in this repository.&#10;&#10;As of the Change Date specified in that file, in accordance with&#10;the Business Source License, use of this software will be governed&#10;by the Apache License, Version 2.0" />
-    <option name="myName" value="BSL" />
-  </copyright>
-</component>

.idea/copyright/profiles_settings.xml

@@ -1,7 +0,0 @@
-<component name="CopyrightManager">
-  <settings default="BSL">
-    <module2copyright>
-      <element module="All" copyright="BSL" />
-    </module2copyright>
-  </settings>
-</component>

LICENSE

@@ -1,29 +1,202 @@
-Copyright (c) 2023-2024 AutoMQ HK Limited.
 
-this software are licensed as follows:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-1. Apache Kafka Source and Dependency Licensing:
-   All code in this repository that is forked from Apache Kafka and its
-   dependencies will continue to be licensed under the original Apache Kafka
-   open source license. For detailed licensing information regarding Apache
-   Kafka and its dependencies, please refer to the files under the "/licenses/"
-   folder in this repository.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-2. S3Stream Component Licensing:
-   The S3Stream component added to this project (specifically referring to all
-   files under the "/S3Stream/" directory) is licensed under a revised Business
-   Source License (BSL) by AutoMQ HK Limited, with the specific terms available
-   in the /LICENSE.S3Stream file in this repository. Any dependencies used by
-   the S3Stream component are subject to their respective open source licenses.
+   1. Definitions.
 
-3. File-Level License Precedence:
-   For each file in this repository, if the license is explicitly specified in
-   the header of the file, the license stated in the file header shall prevail.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE.S3Stream

@@ -1,96 +0,0 @@
-License text copyright © 2023 MariaDB plc, All Rights Reserved.
-"Business Source License" is a trademark of MariaDB plc.
-
-
-Parameters
-
-Licensor:             AutoMQ HK Limited.
-Licensed Work:        AutoMQ Version 1.1.2 or later. The Licensed Work is (c) 2024
-                      AutoMQ HK Limited.
-Additional Use Grant: You may make production use of the Licensed Work, provided
-                      Your use does not include offering the Licensed Work to third
-                      parties on a hosted or embedded basis in order to compete with
-                      AutoMQ's paid version(s) of the Licensed Work. For purposes
-                      of this license:
-
-                      A "competitive offering" is a Product that is offered to third
-                      parties on a paid basis, including through paid support
-                      arrangements, that significantly overlaps with the capabilities
-                      of AutoMQ's paid version(s) of the Licensed Work. If Your
-                      Product is not a competitive offering when You first make it
-                      generally available, it will not become a competitive offering
-                      later due to AutoMQ releasing a new version of the Licensed
-                      Work with additional capabilities. In addition, Products that
-                      are not provided on a paid basis are not competitive.
-
-                      "Product" means software that is offered to end users to manage
-                      in their own environments or offered as a service on a hosted
-                      basis.
-
-                      "Embedded" means including the source code or executable code
-                      from the Licensed Work in a competitive offering. "Embedded"
-                      also means packaging the competitive offering in such a way
-                      that the Licensed Work must be accessed or downloaded for the
-                      competitive offering to operate.
-
-                      Hosting or using the Licensed Work(s) for internal purposes
-                      within an organization is not considered a competitive
-                      offering. AutoMQ considers your organization to include all
-                      of your affiliates under common control.
-
-                      For binding interpretive guidance on using AutoMQ products
-                      under the Business Source License, please visit our FAQ.
-                      (https://www.automq.com/license-faq)
-Change Date:          Change date is four years from release date.
-                      Please see https://github.com/AutoMQ/automq/releases for exact dates
-Change License:       Apache License, Version 2.0
-                      URL: https://www.apache.org/licenses/LICENSE-2.0
-
-
-For information about alternative licensing arrangements for the Licensed Work,
-please contact licensing@automq.com.
-
-Notice
-
-Business Source License 1.1
-
-Terms
-
-The Licensor hereby grants you the right to copy, modify, create derivative
-works, redistribute, and make non-production use of the Licensed Work. The
-Licensor may make an Additional Use Grant, above, permitting limited production use.
-
-Effective on the Change Date, or the fourth anniversary of the first publicly
-available distribution of a specific version of the Licensed Work under this
-License, whichever comes first, the Licensor hereby grants you rights under
-the terms of the Change License, and the rights granted in the paragraph
-above terminate.
-
-If your use of the Licensed Work does not comply with the requirements
-currently in effect as described in this License, you must purchase a
-commercial license from the Licensor, its affiliated entities, or authorized
-resellers, or you must refrain from using the Licensed Work.
-
-All copies of the original and modified Licensed Work, and derivative works
-of the Licensed Work, are subject to this License. This License applies
-separately for each version of the Licensed Work and the Change Date may vary
-for each version of the Licensed Work released by Licensor.
-
-You must conspicuously display this License on each original or modified copy
-of the Licensed Work. If you receive the Licensed Work in original or
-modified form from a third party, the terms and conditions set forth in this
-License apply to your use of that work.
-
-Any use of the Licensed Work in violation of this License will automatically
-terminate your rights under this License for the current and all other
-versions of the Licensed Work.
-
-This License does not grant you any right in any trademark or logo of
-Licensor or its affiliates (provided that you may use a trademark or logo of
-Licensor as expressly required by this License).
-
-TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
-AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
-EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
-TITLE.

NOTICE

@@ -1,5 +1,5 @@
 AutoMQ NOTICE
-Copyright 2023-2024, AutoMQ HK Limited.
+Copyright 2023-2025, AutoMQ HK Limited.
 
 ---------------------------
 Apache Kafka NOTICE

NOTICE-binary

@@ -1,5 +1,5 @@
 AutoMQ Binary NOTICE
-Copyright 2023-2024, AutoMQ HK Limited.
+Copyright 2023-2025, AutoMQ HK Limited.
 
 ---------------------------
 Apache Kafka Binary NOTICE

automq-shell/src/main/java/com/automq/shell/AutoMQApplication.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell;

automq-shell/src/main/java/com/automq/shell/AutoMQCLI.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell;

automq-shell/src/main/java/com/automq/shell/commands/cluster/Cluster.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.commands.cluster;

automq-shell/src/main/java/com/automq/shell/commands/cluster/Create.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.commands.cluster;

automq-shell/src/main/java/com/automq/shell/commands/cluster/Deploy.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.commands.cluster;
@@ -102,9 +110,11 @@ public class Deploy implements Callable<Integer> {
             String globalAccessKey = null;
             String globalSecretKey = null;
             for (Env env : topo.getGlobal().getEnvs()) {
-                if ("KAFKA_S3_ACCESS_KEY".equals(env.getName())) {
+                if ("KAFKA_S3_ACCESS_KEY".equals(env.getName()) ||
+                    "AWS_ACCESS_KEY_ID".equals(env.getName())) {
                     globalAccessKey = env.getValue();
-                } else if ("KAFKA_S3_SECRET_KEY".equals(env.getName())) {
+                } else if ("KAFKA_S3_SECRET_KEY".equals(env.getName()) ||
+                            "AWS_SECRET_ACCESS_KEY".equals(env.getName())) {
                     globalSecretKey = env.getValue();
                 }
             }
@@ -159,6 +169,7 @@ public class Deploy implements Callable<Integer> {
         sb.append("--override cluster.id=").append(topo.getGlobal().getClusterId()).append(" ");
         sb.append("--override node.id=").append(node.getNodeId()).append(" ");
         sb.append("--override controller.quorum.voters=").append(getQuorumVoters(topo)).append(" ");
+        sb.append("--override controller.quorum.bootstrap.servers=").append(getBootstrapServers(topo)).append(" ");
         sb.append("--override advertised.listeners=").append("PLAINTEXT://").append(node.getHost()).append(":9092").append(" ");
     }
 
@@ -181,4 +192,14 @@ public class Deploy implements Callable<Integer> {
             .map(node -> node.getNodeId() + "@" + node.getHost() + ":9093")
             .collect(Collectors.joining(","));
     }
+
+    private static String getBootstrapServers(ClusterTopology topo) {
+        List<Node> nodes = topo.getControllers();
+        if (!(nodes.size() == 1 || nodes.size() == 3)) {
+            throw new IllegalArgumentException("Only support 1 or 3 controllers");
+        }
+        return nodes.stream()
+            .map(node -> node.getHost() + ":9093")
+            .collect(Collectors.joining(","));
+    }
 }

automq-shell/src/main/java/com/automq/shell/commands/cluster/Describe.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.commands.cluster;

automq-shell/src/main/java/com/automq/shell/constant/ServerConfigKey.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.automq.shell.constant;
 

automq-shell/src/main/java/com/automq/shell/log/LogRecorder.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.log;

automq-shell/src/main/java/com/automq/shell/log/LogUploader.java

@@ -1,17 +1,26 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.log;
 
 import com.automq.shell.AutoMQApplication;
+import com.automq.shell.util.Utils;
 import com.automq.stream.s3.operator.ObjectStorage;
 import com.automq.stream.s3.operator.ObjectStorage.ObjectInfo;
 import com.automq.stream.s3.operator.ObjectStorage.ObjectPath;
@@ -204,7 +213,7 @@ public class LogUploader implements LogRecorder {
 
                             try {
                                 String objectKey = getObjectKey();
-                                objectStorage.write(WriteOptions.DEFAULT, objectKey, uploadBuffer.retainedSlice().asReadOnly()).get();
+                                objectStorage.write(WriteOptions.DEFAULT, objectKey, Utils.compress(uploadBuffer.slice().asReadOnly())).get();
                                 break;
                             } catch (Exception e) {
                                 e.printStackTrace(System.err);

automq-shell/src/main/java/com/automq/shell/log/S3LogConfig.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.log;

automq-shell/src/main/java/com/automq/shell/log/S3RollingFileAppender.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.log;

automq-shell/src/main/java/com/automq/shell/metrics/PrometheusUtils.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.metrics;

automq-shell/src/main/java/com/automq/shell/metrics/S3MetricsConfig.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.metrics;

automq-shell/src/main/java/com/automq/shell/metrics/S3MetricsExporter.java

@@ -1,20 +1,30 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.metrics;
 
+import com.automq.shell.util.Utils;
 import com.automq.stream.s3.operator.ObjectStorage;
 import com.automq.stream.s3.operator.ObjectStorage.ObjectInfo;
 import com.automq.stream.s3.operator.ObjectStorage.ObjectPath;
 import com.automq.stream.s3.operator.ObjectStorage.WriteOptions;
+import com.automq.stream.utils.Threads;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -152,12 +162,16 @@ public class S3MetricsExporter implements MetricExporter {
                             CompletableFuture.allOf(deleteFutures).join();
                         }
                     }
-
-                    Thread.sleep(Duration.ofMinutes(1).toMillis());
+                    if (Threads.sleep(Duration.ofMinutes(1).toMillis())) {
+                        break;
+                    }
                 } catch (InterruptedException e) {
                     break;
                 } catch (Exception e) {
                     LOGGER.error("Cleanup s3 metrics failed", e);
+                    if (Threads.sleep(Duration.ofMinutes(1).toMillis())) {
+                        break;
+                    }
                 }
             }
         }
@@ -242,7 +256,7 @@ public class S3MetricsExporter implements MetricExporter {
         synchronized (uploadBuffer) {
             if (uploadBuffer.readableBytes() > 0) {
                 try {
-                    objectStorage.write(WriteOptions.DEFAULT, getObjectKey(), uploadBuffer.retainedSlice().asReadOnly()).get();
+                    objectStorage.write(WriteOptions.DEFAULT, getObjectKey(), Utils.compress(uploadBuffer.slice().asReadOnly())).get();
                 } catch (Exception e) {
                     LOGGER.error("Failed to upload metrics to s3", e);
                     return CompletableResultCode.ofFailure();

automq-shell/src/main/java/com/automq/shell/model/ClusterTopology.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.model;

automq-shell/src/main/java/com/automq/shell/model/EndpointProtocol.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.automq.shell.model;
 

automq-shell/src/main/java/com/automq/shell/model/Env.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.model;

automq-shell/src/main/java/com/automq/shell/model/Global.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.model;

automq-shell/src/main/java/com/automq/shell/model/Node.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.model;

automq-shell/src/main/java/com/automq/shell/model/S3Url.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.automq.shell.model;
 

automq-shell/src/main/java/com/automq/shell/model/StreamTags.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.model;

automq-shell/src/main/java/com/automq/shell/stream/ClientKVClient.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.stream;

automq-shell/src/main/java/com/automq/shell/stream/ClientStreamManager.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.stream;

automq-shell/src/main/java/com/automq/shell/util/CLIUtils.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.automq.shell.util;

automq-shell/src/main/java/com/automq/shell/util/S3PropUtil.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.automq.shell.util;
 

automq-shell/src/main/java/com/automq/shell/util/Utils.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.automq.shell.util;
+
+import com.automq.stream.s3.ByteBufAlloc;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import io.netty.buffer.ByteBuf;
+
+public class Utils {
+
+    public static ByteBuf compress(ByteBuf input) throws IOException {
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+        GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);
+
+        byte[] buffer = new byte[input.readableBytes()];
+        input.readBytes(buffer);
+        gzipOutputStream.write(buffer);
+        gzipOutputStream.close();
+
+        ByteBuf compressed = ByteBufAlloc.byteBuffer(byteArrayOutputStream.size());
+        compressed.writeBytes(byteArrayOutputStream.toByteArray());
+        return compressed;
+    }
+
+    public static ByteBuf decompress(ByteBuf input) throws IOException {
+        byte[] compressedData = new byte[input.readableBytes()];
+        input.readBytes(compressedData);
+        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData);
+        GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream);
+
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+        byte[] buffer = new byte[1024];
+        int bytesRead;
+        while ((bytesRead = gzipInputStream.read(buffer)) != -1) {
+            byteArrayOutputStream.write(buffer, 0, bytesRead);
+        }
+
+        gzipInputStream.close();
+        byteArrayOutputStream.close();
+
+        byte[] uncompressedData = byteArrayOutputStream.toByteArray();
+        ByteBuf output = ByteBufAlloc.byteBuffer(uncompressedData.length);
+        output.writeBytes(uncompressedData);
+        return output;
+    }
+}

automq-shell/src/main/resources/template/topo.yaml

@@ -9,10 +9,12 @@ global:
   config: |
     s3.data.buckets=0@s3://xxx_bucket?region=us-east-1
     s3.ops.buckets=1@s3://xxx_bucket?region=us-east-1
+    s3.wal.path=0@s3://xxx_bucket?region=us-east-1
+    log.dirs=/root/kraft-logs
   envs:
-    - name: KAFKA_S3_ACCESS_KEY
+    - name: AWS_ACCESS_KEY_ID
       value: 'xxxxx'
-    - name: KAFKA_S3_SECRET_KEY
+    - name: AWS_SECRET_ACCESS_KEY
       value: 'xxxxx'
 controllers:
   # The controllers default are combined nodes which roles are controller and broker.

automq-shell/src/test/java/com/automq/shell/util/UtilsTest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.automq.shell.util;
+
+import com.automq.stream.s3.ByteBufAlloc;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import io.netty.buffer.ByteBuf;
+
+@Timeout(60)
+@Tag("S3Unit")
+public class UtilsTest {
+
+    @Test
+    public void testCompression() {
+        String testStr = "This is a test string";
+        ByteBuf input = ByteBufAlloc.byteBuffer(testStr.length());
+        input.writeBytes(testStr.getBytes());
+        try {
+            ByteBuf compressed = Utils.compress(input);
+            ByteBuf decompressed = Utils.decompress(compressed);
+            String decompressedStr = decompressed.toString(io.netty.util.CharsetUtil.UTF_8);
+            System.out.printf("Original: %s, Decompressed: %s\n", testStr, decompressedStr);
+            Assertions.assertEquals(testStr, decompressedStr);
+        } catch (Exception e) {
+            Assertions.fail("Exception occurred during compression/decompression: " + e.getMessage());
+        }
+    }
+}

bin/kafka-server-start.sh

@@ -47,7 +47,7 @@ if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
 fi
 
 if [ "x$KAFKA_OPTS" = "x" ]; then
-    export KAFKA_OPTS="-Dio.netty.allocator.maxOrder=11"
+    export KAFKA_OPTS="-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -Dio.netty.allocator.maxOrder=11"
 fi
 
 EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'}

build.gradle

@@ -44,7 +44,9 @@ plugins {
   // be dropped from gradle/resources/dependencycheck-suppressions.xml
   id "com.github.spotbugs" version '5.1.3' apply false
   id 'org.scoverage' version '8.0.3' apply false
-  id 'io.github.goooler.shadow' version '8.1.3' apply false
+  // Updating the shadow plugin version to 8.1.1 causes issue with signing and publishing the shadowed
+  // artifacts - see https://github.com/johnrengelman/shadow/issues/901
+  id 'com.github.johnrengelman.shadow' version '8.1.0' apply false
   //  Spotless 6.13.0 has issue with Java 21 (see https://github.com/diffplug/spotless/pull/1920), and Spotless 6.14.0+ requires JRE 11
   //  We are going to drop JDK8 support. Hence, the spotless is upgrade to newest version and be applied only if the build env is compatible with JDK 11.
   //  spotless 6.15.0+ has issue in runtime with JDK8 even through we define it with `apply:false`. see https://github.com/diffplug/spotless/issues/2156 for more details
@@ -128,6 +130,9 @@ allprojects {
 
   repositories {
     mavenCentral()
+    maven {
+      url = uri("https://packages.confluent.io/maven/")
+    }
   }
 
   dependencyUpdates {
@@ -147,6 +152,10 @@ allprojects {
   }
 
   configurations.all {
+    // Globally exclude commons-logging and logback to ensure a single logging implementation (reload4j)
+    exclude group: "commons-logging", module: "commons-logging"
+    exclude group: "ch.qos.logback", module: "logback-classic"
+    exclude group: "ch.qos.logback", module: "logback-core"
     // zinc is the Scala incremental compiler, it has a configuration for its own dependencies
     // that are unrelated to the project dependencies, we should not change them
     if (name != "zinc") {
@@ -162,8 +171,8 @@ allprojects {
           // ZooKeeper (potentially older and containing CVEs)
           libs.nettyHandler,
           libs.nettyTransportNativeEpoll,
-	  // be explicit about the reload4j version instead of relying on the transitive versions
-	  libs.reload4j
+          // be explicit about the reload4j version instead of relying on the transitive versions
+          libs.reload4j
         )
       }
     }
@@ -295,7 +304,7 @@ subprojects {
           if (!shouldPublishWithShadow) {
             from components.java
           } else {
-            apply plugin: 'io.github.goooler.shadow'
+            apply plugin: 'com.github.johnrengelman.shadow'
             project.shadow.component(mavenJava)
 
             // Fix for avoiding inclusion of runtime dependencies marked as 'shadow' in MANIFEST Class-Path.
@@ -728,7 +737,7 @@ subprojects {
       jacoco {
         toolVersion = versions.jacoco
       }
-      
+
       jacocoTestReport {
         dependsOn tasks.test
         sourceSets sourceSets.main
@@ -752,8 +761,8 @@ subprojects {
     skipProjects = [ ":jmh-benchmarks", ":trogdor" ]
     skipConfigurations = [ "zinc" ]
   }
-  //  the task `removeUnusedImports` is implemented by google-java-format, 
-  //  and unfortunately the google-java-format version used by spotless 6.14.0 can't work with JDK 21. 
+  //  the task `removeUnusedImports` is implemented by google-java-format,
+  //  and unfortunately the google-java-format version used by spotless 6.14.0 can't work with JDK 21.
   //  Hence, we apply spotless tasks only if the env is either JDK11 or JDK17
   if ((JavaVersion.current().isJava11() || (JavaVersion.current() == JavaVersion.VERSION_17))) {
     apply plugin: 'com.diffplug.spotless'
@@ -835,6 +844,7 @@ project(':server') {
 
   dependencies {
     implementation project(':clients')
+    implementation project(':metadata')
     implementation project(':server-common')
     implementation project(':storage')
     implementation project(':group-coordinator')
@@ -944,6 +954,7 @@ project(':core') {
     implementation libs.scalaReflect
     implementation libs.scalaLogging
     implementation libs.slf4jApi
+    implementation libs.commonsIo // ZooKeeper dependency. Do not use, this is going away.
     implementation(libs.zookeeper) {
       // Dropwizard Metrics are required by ZooKeeper as of v3.6.0,
       // but the library should *not* be used in Kafka code
@@ -965,6 +976,8 @@ project(':core') {
     implementation libs.guava
     implementation libs.slf4jBridge
     implementation libs.slf4jReload4j
+    // The `jcl-over-slf4j` library is used to redirect JCL logging to SLF4J.
+    implementation libs.jclOverSlf4j
 
     implementation libs.opentelemetryJava8
     implementation libs.opentelemetryOshi
@@ -976,6 +989,71 @@ project(':core') {
     implementation libs.opentelemetryJmx
     implementation libs.awsSdkAuth
 
+    // table topic start
+    implementation ("org.apache.avro:avro:${versions.avro}")
+    implementation ("org.apache.avro:avro-protobuf:${versions.avro}")
+    implementation('com.google.protobuf:protobuf-java:3.25.5')
+    implementation ("org.apache.iceberg:iceberg-core:${versions.iceberg}")
+    implementation ("org.apache.iceberg:iceberg-api:${versions.iceberg}")
+    implementation ("org.apache.iceberg:iceberg-data:${versions.iceberg}")
+    implementation ("org.apache.iceberg:iceberg-parquet:${versions.iceberg}")
+    implementation ("org.apache.iceberg:iceberg-common:${versions.iceberg}")
+    implementation ("org.apache.iceberg:iceberg-aws:${versions.iceberg}")
+    implementation ("software.amazon.awssdk:glue:${versions.awsSdk}")
+    implementation ("software.amazon.awssdk:s3tables:${versions.awsSdk}")
+    implementation 'software.amazon.s3tables:s3-tables-catalog-for-iceberg:0.1.0'
+
+    implementation ('org.apache.hadoop:hadoop-common:3.4.1') {
+      exclude group: 'org.eclipse.jetty', module: '*'
+      exclude group: 'com.sun.jersey', module: '*'
+    }
+    // for hadoop common
+    implementation ("org.eclipse.jetty:jetty-webapp:${versions.jetty}")
+
+    implementation (libs.kafkaAvroSerializer) {
+      exclude group: 'org.apache.kafka', module: 'kafka-clients'
+    }
+
+    // > hive ext start
+    implementation 'org.apache.iceberg:iceberg-hive-metastore:1.6.1'
+    implementation('org.apache.hive:hive-metastore:3.1.3') {
+      // Remove useless dependencies (copy from iceberg-kafka-connect)
+      exclude group: "org.apache.avro", module: "avro"
+      exclude group: "org.slf4j", module: "slf4j-log4j12"
+      exclude group: "org.pentaho" // missing dependency
+      exclude group: "org.apache.hbase"
+      exclude group: "org.apache.logging.log4j"
+      exclude group: "co.cask.tephra"
+      exclude group: "com.google.code.findbugs", module: "jsr305"
+      exclude group: "org.eclipse.jetty.aggregate", module: "jetty-all"
+      exclude group: "org.eclipse.jetty.orbit", module: "javax.servlet"
+      exclude group: "org.apache.parquet", module: "parquet-hadoop-bundle"
+      exclude group: "com.tdunning", module: "json"
+      exclude group: "javax.transaction", module: "transaction-api"
+      exclude group: "com.zaxxer", module: "HikariCP"
+      exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-common"
+      exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-applicationhistoryservice"
+      exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-resourcemanager"
+      exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-web-proxy"
+      exclude group: "org.apache.hive", module: "hive-service-rpc"
+      exclude group: "com.github.joshelser", module: "dropwizard-metrics-hadoop-metrics2-reporter"
+    }
+    implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.4.1') {
+      exclude group: 'com.sun.jersey', module: '*'
+      exclude group: 'com.sun.jersey.contribs', module: '*'
+      exclude group: 'com.github.pjfanning', module: 'jersey-json'
+    }
+    // > hive ext end
+
+    // > Protobuf ext start
+    //   Wire Runtime for schema handling
+    implementation ("com.squareup.wire:wire-schema:${versions.wire}")
+    implementation ("com.squareup.wire:wire-runtime:${versions.wire}")
+    implementation 'com.google.api.grpc:proto-google-common-protos:2.52.0'
+    // > Protobuf ext end
+
+    // table topic end
+
     implementation(libs.oshi) {
       exclude group: 'org.slf4j', module: '*'
     }
@@ -990,6 +1068,7 @@ project(':core') {
     testImplementation project(':storage:storage-api').sourceSets.test.output
     testImplementation project(':server').sourceSets.test.output
     testImplementation libs.bcpkix
+    testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
     testImplementation libs.mockitoCore
     testImplementation libs.guava
     testImplementation(libs.apacheda) {
@@ -1160,7 +1239,6 @@ project(':core') {
     from(project.file("$rootDir/docker/docker-compose.yaml")) { into "docker/" }
     from(project.file("$rootDir/docker/telemetry")) { into "docker/telemetry/" }
     from(project.file("$rootDir/LICENSE")) { into "" }
-    from(project.file("$rootDir/LICENSE.S3Stream")) { into "" }
     from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
     from(configurations.runtimeClasspath) { into("libs/") }
     from(configurations.archives.artifacts.files) { into("libs/") }
@@ -1201,6 +1279,38 @@ project(':core') {
     from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") }
     duplicatesStrategy 'exclude'
   }
+  
+  // AutoMQ inject start
+  tasks.create(name: "releaseE2ETar", dependsOn: [configurations.archives.artifacts, 'copyDependantTestLibs'], type: Tar) {
+    def prefix = project.findProperty('prefix') ?: ''
+    archiveBaseName = "${prefix}kafka"
+
+    into "${prefix}kafka-${archiveVersion.get()}"
+    compression = Compression.GZIP
+    from(project.file("$rootDir/bin")) { into "bin/" }
+    from(project.file("$rootDir/config")) { into "config/" }
+    from(project.file("$rootDir/licenses")) { into "licenses/" }
+    from(project.file("$rootDir/docker/docker-compose.yaml")) { into "docker/" }
+    from(project.file("$rootDir/docker/telemetry")) { into "docker/telemetry/" }
+    from(project.file("$rootDir/LICENSE")) { into "" }
+    from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
+    from(configurations.runtimeClasspath) { into("libs/") }
+    from(configurations.archives.artifacts.files) { into("libs/") }
+    from(project.siteDocsTar) { into("site-docs/") }
+
+    // Include main and test jars from all subprojects
+    rootProject.subprojects.each { subproject ->
+      if (subproject.tasks.findByName('jar')) {
+        from(subproject.tasks.named('jar')) { into('libs/') }
+      }
+      if (subproject.tasks.findByName('testJar')) {
+        from(subproject.tasks.named('testJar')) { into('libs/') }
+      }
+      from(subproject.configurations.runtimeClasspath) { into('libs/') }
+    }
+    duplicatesStrategy 'exclude'
+  }
+  // AutoMQ inject end
 
   jar {
     dependsOn('copyDependantLibs')
@@ -1220,7 +1330,7 @@ project(':core') {
     //By default gradle does not handle test dependencies between the sub-projects
     //This line is to include clients project test jar to dependant-testlibs
     from (project(':clients').testJar ) { "$buildDir/dependant-testlibs" }
-    // log4j-appender is not in core dependencies, 
+    // log4j-appender is not in core dependencies,
     // so we add it to dependant-testlibs to avoid ClassNotFoundException in running kafka_log4j_appender.py
     from (project(':log4j-appender').jar ) { "$buildDir/dependant-testlibs" }
     duplicatesStrategy 'exclude'
@@ -1253,6 +1363,7 @@ project(':core') {
   }
 }
 
+
 project(':metadata') {
   base {
     archivesName = "kafka-metadata"
@@ -1480,7 +1591,7 @@ project(':transaction-coordinator') {
     implementation project(':clients')
     generator project(':generator')
   }
-  
+
   sourceSets {
     main {
       java {
@@ -1571,6 +1682,7 @@ project(':clients') {
     implementation libs.snappy
     implementation libs.slf4jApi
     implementation libs.opentelemetryProto
+    implementation libs.protobuf
 
     // libraries which should be added as runtime dependencies in generated pom.xml should be defined here:
     shadowed libs.zstd
@@ -1755,6 +1867,7 @@ project(':raft') {
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
     testImplementation libs.jqwik
+    testImplementation libs.hamcrest
 
     testRuntimeOnly libs.slf4jReload4j
     testRuntimeOnly libs.junitPlatformLanucher
@@ -1845,7 +1958,12 @@ project(':server-common') {
     implementation libs.jacksonDatabind
     implementation libs.pcollections
     implementation libs.opentelemetrySdk
+
+    // AutoMQ inject start
     implementation project(':s3stream')
+    implementation libs.commonLang
+    // AutoMQ inject end
+
 
     testImplementation project(':clients')
     testImplementation project(':clients').sourceSets.test.output
@@ -2140,11 +2258,12 @@ project(':s3stream') {
     implementation 'commons-codec:commons-codec:1.17.0'
     implementation 'org.hdrhistogram:HdrHistogram:2.2.2'
     implementation 'software.amazon.awssdk.crt:aws-crt:0.30.8'
+    implementation 'com.ibm.async:asyncutil:0.1.0'
 
     testImplementation 'org.slf4j:slf4j-simple:2.0.9'
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0'
-    testImplementation 'org.mockito:mockito-core:5.5.0'
-    testImplementation 'org.mockito:mockito-junit-jupiter:5.5.0'
+    testImplementation libs.junitJupiter
+    testImplementation libs.mockitoCore
+    testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
     testImplementation 'org.awaitility:awaitility:4.2.1'
   }
 
@@ -2217,21 +2336,15 @@ project(':tools') {
   }
 
   dependencies {
-    implementation (project(':clients')){
-      exclude group: 'org.slf4j', module: '*'
-    }
-    implementation (project(':server-common')){
-      exclude group: 'org.slf4j', module: '*'
-    }
-    implementation (project(':log4j-appender')){
-      exclude group: 'org.slf4j', module: '*'
-    }
-    implementation project(':automq-shell')
-
+    implementation project(':clients')
+    implementation project(':metadata')
     implementation project(':storage')
+    implementation project(':server')
+    implementation project(':server-common')
     implementation project(':connect:runtime')
     implementation project(':tools:tools-api')
     implementation project(':transaction-coordinator')
+    implementation project(':group-coordinator')
     implementation libs.argparse4j
     implementation libs.jacksonDatabind
     implementation libs.jacksonDataformatCsv
@@ -2243,6 +2356,16 @@ project(':tools') {
     implementation libs.hdrHistogram
     implementation libs.spotbugsAnnotations
 
+    // AutoMQ inject start
+    implementation project(':automq-shell')
+    implementation libs.guava
+    implementation (libs.kafkaAvroSerializer) {
+      exclude group: 'org.apache.kafka', module: 'kafka-clients'
+    }
+    implementation libs.bucket4j
+    implementation libs.oshi
+    // AutoMQ inject end
+
     // for SASL/OAUTHBEARER JWT validation
     implementation (libs.jose4j){
       exclude group: 'org.slf4j', module: '*'
@@ -2279,7 +2402,7 @@ project(':tools') {
     testImplementation project(':connect:runtime')
     testImplementation project(':connect:runtime').sourceSets.test.output
     testImplementation project(':storage:storage-api').sourceSets.main.output
-    testImplementation project(':group-coordinator')
+    testImplementation project(':storage').sourceSets.test.output
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
     testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
@@ -2577,6 +2700,7 @@ project(':streams') {
             ':streams:upgrade-system-tests-35:test',
             ':streams:upgrade-system-tests-36:test',
             ':streams:upgrade-system-tests-37:test',
+            ':streams:upgrade-system-tests-38:test',
             ':streams:examples:test'
     ]
   )
@@ -3076,9 +3200,24 @@ project(':streams:upgrade-system-tests-37') {
   }
 }
 
+project(':streams:upgrade-system-tests-38') {
+  base {
+    archivesName = "kafka-streams-upgrade-system-tests-38"
+  }
+
+  dependencies {
+    testImplementation libs.kafkaStreams_38
+    testRuntimeOnly libs.junitJupiter
+  }
+
+  systemTestLibs {
+    dependsOn testJar
+  }
+}
+
 project(':jmh-benchmarks') {
 
-  apply plugin: 'io.github.goooler.shadow'
+  apply plugin: 'com.github.johnrengelman.shadow'
 
   shadowJar {
     archiveBaseName = 'kafka-jmh-benchmarks'

chart/bitnami/README.md

@@ -0,0 +1,62 @@
+# AutoMQ
+
+[AutoMQ](https://www.automq.com/) is a cloud-native alternative to Kafka by decoupling durability to cloud storage services like S3. 10x Cost-Effective. No Cross-AZ Traffic Cost. Autoscale in seconds. Single-digit ms latency.
+This Helm chart simplifies the deployment of AutoMQ into your Kubernetes cluster using the Software model.
+
+## Prerequisites
+### Install Helm chart
+Install Helm chart and version v3.8.0+
+[Helm chart quickstart](https://helm.sh/zh/docs/intro/quickstart/)
+```shell
+helm version
+```
+### Using the Bitnami Helm repository
+AutoMQ is fully compatible with Bitnami's Helm Charts, so you can customize your AutoMQ Kubernetes cluster based on the relevant values.yaml of Bitnami.
+[Bitnami Helm Charts](https://github.com/bitnami/charts)
+
+## Quickstart
+### Setup a Kubernetes Cluster
+The quickest way to set up a Kubernetes cluster to install Bitnami Charts is by following the "Bitnami Get Started" guides for the different services:
+
+[Get Started with Bitnami Charts using the Amazon Elastic Container Service for Kubernetes (EKS)](https://docs.bitnami.com/kubernetes/get-started-eks/)
+
+
+### Installing the AutoMQ with Bitnami Chart
+
+As an alternative to supplying the configuration parameters as arguments, you can create a supplemental YAML file containing your specific config parameters. Any parameters not specified in this file will default to those set in [values.yaml](values.yaml).
+
+1. Create an empty `automq-values.yaml` file
+2. Edit the file with your specific parameters:
+
+    You can refer to the [demo-values.yaml](/chart/bitnami/demo-values.yaml)  based on the bitnami [values.yaml](https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml)
+    we provided for deploying AutoMQ on AWS across 3 Availability Zones using m7g.xlarge instances (4 vCPUs, 16GB Mem, 156MiB/s network bandwidth).
+
+
+    You need to replace the bucket configurations in the placeholders $｛｝, such as ops-bucket, data-bucket, region, endpoint, access-key/secret-key.
+
+3. Install or upgrade the AutoMQ Helm chart using your custom yaml file:
+
+    we recommend using the `--version` [31.x.x (31.1.0 ~ 31.5.0)](https://artifacthub.io/packages/helm/bitnami/kafka) bitnami helm chart while installing AutoMQ.
+
+```shell
+helm install automq-release oci://registry-1.docker.io/bitnamicharts/kafka -f demo-values.yaml --version 31.5.0 --namespace automq --create-namespace
+```
+
+### Upgrading
+
+To upgrade the deployment:
+
+```shell
+helm repo update
+helm upgrade automq-release oci://registry-1.docker.io/bitnamicharts/kafka -f demo-values.yaml --version 31.5.0 --namespace automq --create-namespace
+```
+
+### Uninstalling the Chart
+
+To uninstall/delete the deployment:
+
+```shell
+helm uninstall automq-release --namespace automq
+```
+
+This command removes all the Kubernetes components associated with the chart and deletes the release.

chart/bitnami/demo-values.yaml

@@ -0,0 +1,141 @@
+global:
+  security:
+    allowInsecureImages: true
+image:
+  registry: automqinc
+  repository: automq
+  tag: 1.5.5-bitnami
+  pullPolicy: Always
+extraEnvVars:
+  - name: AWS_ACCESS_KEY_ID
+    value: "${access-key}"
+  - name: AWS_SECRET_ACCESS_KEY
+    value: "${secret-key}"
+controller:
+  replicaCount: 3
+  resources:
+    requests:
+      cpu: "3000m"
+      memory: "12Gi"
+    limits:
+      cpu: "4000m"
+      memory: "16Gi"
+  heapOpts: -Xmx6g -Xms6g -XX:MaxDirectMemorySize=6g -XX:MetaspaceSize=96m
+  extraConfig: |
+    elasticstream.enable=true
+    autobalancer.client.auth.sasl.mechanism=PLAIN
+    autobalancer.client.auth.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="inter_broker_user" password="interbroker-password-placeholder" user_inter_broker_user="interbroker-password-placeholder";
+    autobalancer.client.auth.security.protocol=SASL_PLAINTEXT
+    autobalancer.client.listener.name=INTERNAL
+    s3.wal.cache.size=2147483648
+    s3.block.cache.size=1073741824
+    s3.stream.allocator.policy=POOLED_DIRECT
+    s3.network.baseline.bandwidth=245366784
+    # Replace the following with your bucket config
+    s3.ops.buckets=1@s3://${ops-bucket}?region=${region}&endpoint=${endpoint}
+    s3.data.buckets=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+    s3.wal.path=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+    automq.zonerouter.channels=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchExpressions:
+              - key: app.kubernetes.io/instance
+                operator: In
+                # your helm release name
+                values:
+                  - automq-release
+              - key: app.kubernetes.io/component
+                operator: In
+                values:
+                  - controller-eligible
+                  - broker
+          topologyKey: kubernetes.io/hostname
+#    ---     nodeAffinity recommended   ---
+#    nodeAffinity:
+#      requiredDuringSchedulingIgnoredDuringExecution:
+#        nodeSelectorTerms:
+#          - matchExpressions:
+#              - key: "${your-node-label-key}"
+#                operator: In
+#                values:
+#                  - "${your-node-label-value}"
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: topology.kubernetes.io/zone
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          app.kubernetes.io/component: controller-eligible
+  tolerations:
+    - key: "dedicated"
+      operator: "Equal"
+      value: "automq"
+      effect: "NoSchedule"
+  persistence:
+    size: 20Gi
+
+broker:
+  replicaCount: 3
+  resources:
+    requests:
+      cpu: "3000m"
+      memory: "12Gi"
+    limits:
+      cpu: "4000m"
+      memory: "16Gi"
+  heapOpts: -Xmx6g -Xms6g -XX:MaxDirectMemorySize=6g -XX:MetaspaceSize=96m
+  extraConfig: |
+    elasticstream.enable=true
+    autobalancer.client.auth.sasl.mechanism=PLAIN
+    autobalancer.client.auth.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="inter_broker_user" password="interbroker-password-placeholder" user_inter_broker_user="interbroker-password-placeholder";
+    autobalancer.client.auth.security.protocol=SASL_PLAINTEXT
+    autobalancer.client.listener.name=INTERNAL
+    s3.wal.cache.size=2147483648
+    s3.block.cache.size=1073741824
+    s3.stream.allocator.policy=POOLED_DIRECT
+    s3.network.baseline.bandwidth=245366784
+    # Replace the following with your bucket config
+    s3.ops.buckets=1@s3://${ops-bucket}?region=${region}&endpoint=${endpoint}
+    s3.data.buckets=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+    s3.wal.path=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+    automq.zonerouter.channels=0@s3://${data-bucket}?region=${region}&endpoint=${endpoint}
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchExpressions:
+              - key: app.kubernetes.io/instance
+                operator: In
+                # your helm release name
+                values:
+                  - automq-release
+              - key: app.kubernetes.io/component
+                operator: In
+                values:
+                  - controller-eligible
+                  - broker
+          topologyKey: kubernetes.io/hostname
+#    ---     nodeAffinity recommended   ---
+#    nodeAffinity:
+#      requiredDuringSchedulingIgnoredDuringExecution:
+#        nodeSelectorTerms:
+#          - matchExpressions:
+#              - key: "${your-node-label-key}"
+#                operator: In
+#                values:
+#                  - "${your-node-label-value}"
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: topology.kubernetes.io/zone
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          app.kubernetes.io/component: broker
+  tolerations:
+    - key: "dedicated"
+      operator: "Equal"
+      value: "automq"
+      effect: "NoSchedule"
+brokerRackAssignment: aws-az

checkstyle/import-control-metadata.xml

@@ -182,6 +182,10 @@
         <subpackage name="migration">
             <allow pkg="org.apache.kafka.controller" />
         </subpackage>
+        <subpackage name="storage">
+            <allow pkg="org.apache.kafka.common.internals" />
+            <allow pkg="org.apache.kafka.snapshot" />
+        </subpackage>
         <subpackage name="util">
             <allow class="org.apache.kafka.common.compress.Compression" exact-match="true" />
         </subpackage>

checkstyle/import-control-server.xml

@@ -80,6 +80,8 @@
   <allow pkg="org.apache.kafka.raft" />
 
   <subpackage name="server">
+    <allow pkg="org.apache.kafka.server" />
+    <allow pkg="org.apache.kafka.image" />
     <subpackage name="metrics">
       <allow class="org.apache.kafka.server.authorizer.AuthorizableRequestContext" />
       <allow pkg="org.apache.kafka.server.telemetry" />

checkstyle/import-control-storage.xml

@@ -83,6 +83,11 @@
         <allow pkg="org.apache.kafka.coordinator.transaction"/>
     </subpackage>
 
+    <subpackage name="storage.log">
+        <allow pkg="org.apache.kafka.server" />
+        <allow pkg="com.yammer.metrics" />
+    </subpackage>
+
     <!-- START OF TIERED STORAGE INTEGRATION TEST IMPORT DEPENDENCIES -->
     <subpackage name="tiered.storage">
         <allow pkg="scala" />

checkstyle/import-control.xml

@@ -49,6 +49,7 @@
 
   <subpackage name="common">
     <allow class="org.apache.kafka.clients.consumer.ConsumerRecord" exact-match="true" />
+    <allow class="org.apache.kafka.clients.NodeApiVersions" exact-match="true" />
     <allow class="org.apache.kafka.common.message.ApiMessageType" exact-match="true" />
     <disallow pkg="org.apache.kafka.clients" />
     <allow pkg="org.apache.kafka.common" exact-match="true" />
@@ -76,7 +77,10 @@
       <allow pkg="net.jpountz.xxhash" />
       <allow pkg="org.xerial.snappy" />
       <allow pkg="org.apache.kafka.common.compress" />
-      <allow class="org.apache.kafka.common.record.CompressionType" exact-match="true" />
+      <allow class="org.apache.kafka.common.record.CompressionType" />
+      <allow class="org.apache.kafka.common.record.CompressionType.GZIP" />
+      <allow class="org.apache.kafka.common.record.CompressionType.LZ4" />
+      <allow class="org.apache.kafka.common.record.CompressionType.ZSTD" />
       <allow class="org.apache.kafka.common.record.RecordBatch" exact-match="true" />
     </subpackage>
 
@@ -150,6 +154,7 @@
     </subpackage>
 
     <subpackage name="record">
+      <allow class="org.apache.kafka.common.config.ConfigDef.Range.between" exact-match="true" />
       <allow pkg="org.apache.kafka.common.compress" />
       <allow pkg="org.apache.kafka.common.header" />
       <allow pkg="org.apache.kafka.common.record" />
@@ -278,12 +283,16 @@
 
   <subpackage name="tools">
     <allow pkg="org.apache.kafka.common"/>
+    <allow pkg="org.apache.kafka.metadata.properties" />
+    <allow pkg="org.apache.kafka.network" />
     <allow pkg="org.apache.kafka.server.util" />
     <allow pkg="kafka.admin" />
     <allow pkg="kafka.server" />
     <allow pkg="org.apache.kafka.storage.internals" />
     <allow pkg="org.apache.kafka.server.config" />
     <allow pkg="org.apache.kafka.server.common" />
+    <allow pkg="org.apache.kafka.server.log.remote.metadata.storage" />
+    <allow pkg="org.apache.kafka.server.log.remote.storage" />
     <allow pkg="org.apache.kafka.clients" />
     <allow pkg="org.apache.kafka.clients.admin" />
     <allow pkg="org.apache.kafka.clients.producer" />
@@ -301,6 +310,7 @@
     <allow pkg="kafka.utils" />
     <allow pkg="scala.collection" />
     <allow pkg="org.apache.kafka.coordinator.transaction" />
+    <allow pkg="org.apache.kafka.coordinator.group" />
 
     <subpackage name="consumer">
       <allow pkg="org.apache.kafka.tools"/>

checkstyle/suppressions.xml

@@ -39,7 +39,7 @@
     <suppress checks="(NPathComplexity|ClassFanOutComplexity|CyclomaticComplexity|ClassDataAbstractionCoupling|FinalLocalVariable|LocalVariableName|MemberName|ParameterName|MethodLength|JavaNCSS|AvoidStarImport)"
               files="core[\\/]src[\\/](generated|generated-test)[\\/].+.java$"/>
     <suppress checks="NPathComplexity" files="(ClusterTestExtensions|KafkaApisBuilder|SharePartition).java"/>
-    <suppress checks="NPathComplexity|ClassFanOutComplexity|ClassDataAbstractionCoupling" files="(RemoteLogManager|RemoteLogManagerTest).java"/>
+    <suppress checks="NPathComplexity|ClassFanOutComplexity|ClassDataAbstractionCoupling|JavaNCSS" files="(RemoteLogManager|RemoteLogManagerTest).java"/>
     <suppress checks="MethodLength" files="RemoteLogManager.java"/>
     <suppress checks="ClassFanOutComplexity" files="RemoteLogManagerTest.java"/>
     <suppress checks="MethodLength"
@@ -190,11 +190,11 @@
 
     <!-- Raft -->
     <suppress checks="NPathComplexity"
-              files="RecordsIterator.java"/>
+              files="(DynamicVoter|RecordsIterator).java"/>
 
     <!-- Streams -->
     <suppress checks="ClassFanOutComplexity"
-              files="(KafkaStreams|KStreamImpl|KTableImpl|InternalTopologyBuilder|StreamsPartitionAssignor|StreamThread|IQv2StoreIntegrationTest|KStreamImplTest|RocksDBStore).java"/>
+              files="(KafkaStreams|KStreamImpl|KTableImpl|InternalTopologyBuilder|StreamsPartitionAssignor|StreamThread|IQv2StoreIntegrationTest|KStreamImplTest|RocksDBStore|StreamTask).java"/>
 
     <suppress checks="MethodLength"
               files="KTableImpl.java"/>
@@ -326,7 +326,7 @@
     <suppress checks="(ParameterNumber|ClassDataAbstractionCoupling)"
               files="(QuorumController).java"/>
     <suppress checks="(CyclomaticComplexity|NPathComplexity)"
-              files="(PartitionRegistration|PartitionChangeBuilder).java"/>
+              files="(PartitionRegistration|PartitionChangeBuilder|ScramParser).java"/>
     <suppress checks="CyclomaticComplexity"
               files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|QuorumController|ReplicationControlManager|KRaftMigrationDriver|ClusterControlManager|MetaPropertiesEnsemble).java"/>
     <suppress checks="NPathComplexity"
@@ -372,7 +372,7 @@
     <suppress checks="CyclomaticComplexity"
               files="(S3StreamsMetadataImage|S3StreamMetricsManager|BlockCache|StreamReader|S3MetricsExporter|PrometheusUtils).java"/>
     <suppress checks="NPathComplexity"
-              files="(StreamControlManager|S3StreamsMetadataImage|CompactionManagerTest|S3StreamMetricsManager|CompactionManager|BlockCache|DefaultS3BlockCache|StreamReader|S3Utils|AnomalyDetector|Recreate|ForceClose|QuorumController).java"/>
+              files="(StreamControlManager|S3StreamsMetadataImage|CompactionManagerTest|S3StreamMetricsManager|CompactionManager|BlockCache|DefaultS3BlockCache|StreamReader|S3Utils|AnomalyDetector|Recreate|ForceClose|QuorumController|AbstractObjectStorage).java"/>
     <suppress checks="MethodLength"
               files="(S3StreamMetricsManager|BlockWALServiceTest).java"/>
     <suppress id="dontUseSystemExit"

clients/src/main/java/org/apache/kafka/clients/admin/AddRaftVoterOptions.java

@@ -18,9 +18,21 @@ package org.apache.kafka.clients.admin;
 
 import org.apache.kafka.common.annotation.InterfaceStability;
 
+import java.util.Optional;
+
 /**
  * Options for {@link Admin#addRaftVoter}.
  */
 @InterfaceStability.Stable
 public class AddRaftVoterOptions extends AbstractOptions<AddRaftVoterOptions> {
+    private Optional<String> clusterId = Optional.empty();
+
+    public AddRaftVoterOptions setClusterId(Optional<String> clusterId) {
+        this.clusterId = clusterId;
+        return this;
+    }
+
+    public Optional<String> clusterId() {
+        return clusterId;
+    }
 }

clients/src/main/java/org/apache/kafka/clients/admin/Admin.java

@@ -1729,6 +1729,16 @@ public interface Admin extends AutoCloseable {
      * @return {@link GetNodesResult}
      */
     GetNodesResult getNodes(Collection<Integer> nodeIdList, GetNodesOptions options);
+
+    /**
+     * Update consumer group
+     *
+     * @param groupId group id
+     * @param groupSpec {@link UpdateGroupSpec}
+     * @param options {@link UpdateGroupOptions}
+     * @return {@link UpdateGroupResult}
+     */
+    UpdateGroupResult updateGroup(String groupId, UpdateGroupSpec groupSpec, UpdateGroupOptions options);
     // AutoMQ inject end
 
     /**

clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java

@@ -314,5 +314,11 @@ public class ForwardingAdmin implements Admin {
     public GetNodesResult getNodes(Collection<Integer> nodeIdList, GetNodesOptions options) {
         return delegate.getNodes(nodeIdList, options);
     }
+
+    @Override
+    public UpdateGroupResult updateGroup(String groupId, UpdateGroupSpec groupSpec, UpdateGroupOptions options) {
+        return delegate.updateGroup(groupId, groupSpec, options);
+    }
+
     // AutoMQ inject end
 }

clients/src/main/java/org/apache/kafka/clients/admin/GetNodesOptions.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kafka.clients.admin;

clients/src/main/java/org/apache/kafka/clients/admin/GetNodesResult.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kafka.clients.admin;

clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java

@@ -56,6 +56,7 @@ import org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler;
 import org.apache.kafka.clients.admin.internals.ListOffsetsHandler;
 import org.apache.kafka.clients.admin.internals.ListTransactionsHandler;
 import org.apache.kafka.clients.admin.internals.RemoveMembersFromConsumerGroupHandler;
+import org.apache.kafka.clients.admin.internals.UpdateGroupHandler;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
 import org.apache.kafka.common.Cluster;
@@ -241,6 +242,7 @@ import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
 import org.apache.kafka.common.requests.MetadataRequest;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.RemoveRaftVoterRequest;
+import org.apache.kafka.common.requests.RemoveRaftVoterResponse;
 import org.apache.kafka.common.requests.RenewDelegationTokenRequest;
 import org.apache.kafka.common.requests.RenewDelegationTokenResponse;
 import org.apache.kafka.common.requests.UnregisterBrokerRequest;
@@ -1202,16 +1204,27 @@ public class KafkaAdminClient extends AdminClient {
             long pollTimeout = Long.MAX_VALUE;
             log.trace("Trying to choose nodes for {} at {}", pendingCalls, now);
 
-            Iterator<Call> pendingIter = pendingCalls.iterator();
-            while (pendingIter.hasNext()) {
-                Call call = pendingIter.next();
+            List<Call> toRemove = new ArrayList<>();
+            // Using pendingCalls.size() to get the list size before the for-loop to avoid infinite loop.
+            // If call.fail keeps adding the call to pendingCalls,
+            // the loop like for (int i = 0; i < pendingCalls.size(); i++) can't stop.
+            int pendingSize = pendingCalls.size();
+            // pendingCalls could be modified in this loop,
+            // hence using for-loop instead of iterator to avoid ConcurrentModificationException.
+            for (int i = 0; i < pendingSize; i++) {
+                Call call = pendingCalls.get(i);
                 // If the call is being retried, await the proper backoff before finding the node
                 if (now < call.nextAllowedTryMs) {
                     pollTimeout = Math.min(pollTimeout, call.nextAllowedTryMs - now);
                 } else if (maybeDrainPendingCall(call, now)) {
-                    pendingIter.remove();
+                    toRemove.add(call);
                 }
             }
+
+            // Use remove instead of removeAll to avoid delete all matched elements
+            for (Call call : toRemove) {
+                pendingCalls.remove(call);
+            }
             return pollTimeout;
         }
 
@@ -4701,6 +4714,8 @@ public class KafkaAdminClient extends AdminClient {
                         setPort(endpoint.port())));
                 return new AddRaftVoterRequest.Builder(
                    new AddRaftVoterRequestData().
+                       setClusterId(options.clusterId().orElse(null)).
+                       setTimeoutMs(timeoutMs).
                        setVoterId(voterId) .
                        setVoterDirectoryId(voterDirectoryId).
                        setListeners(listeners));
@@ -4745,13 +4760,14 @@ public class KafkaAdminClient extends AdminClient {
             RemoveRaftVoterRequest.Builder createRequest(int timeoutMs) {
                 return new RemoveRaftVoterRequest.Builder(
                     new RemoveRaftVoterRequestData().
+                        setClusterId(options.clusterId().orElse(null)).
                         setVoterId(voterId) .
                         setVoterDirectoryId(voterDirectoryId));
             }
 
             @Override
             void handleResponse(AbstractResponse response) {
-                AddRaftVoterResponse addResponse = (AddRaftVoterResponse) response;
+                RemoveRaftVoterResponse addResponse = (RemoveRaftVoterResponse) response;
                 if (addResponse.data().errorCode() != Errors.NONE.code()) {
                     ApiError error = new ApiError(
                             addResponse.data().errorCode(),
@@ -4857,6 +4873,14 @@ public class KafkaAdminClient extends AdminClient {
         return new GetNodesResult(future);
     }
 
+    @Override
+    public UpdateGroupResult updateGroup(String groupId, UpdateGroupSpec groupSpec, UpdateGroupOptions options) {
+        SimpleAdminApiFuture<CoordinatorKey, Void> future = UpdateGroupHandler.newFuture(groupId);
+        UpdateGroupHandler handler = new UpdateGroupHandler(groupId, groupSpec, logContext);
+        invokeDriver(handler, future, options.timeoutMs);
+        return new UpdateGroupResult(future.get(CoordinatorKey.byGroupId(groupId)));
+    }
+
     private <K, V> void invokeDriver(
         AdminApiHandler<K, V> handler,
         AdminApiFuture<K, V> future,
@@ -4931,6 +4955,10 @@ public class KafkaAdminClient extends AdminClient {
             return ListOffsetsRequest.EARLIEST_TIMESTAMP;
         } else if (offsetSpec instanceof OffsetSpec.MaxTimestampSpec) {
             return ListOffsetsRequest.MAX_TIMESTAMP;
+        } else if (offsetSpec instanceof OffsetSpec.EarliestLocalSpec) {
+            return ListOffsetsRequest.EARLIEST_LOCAL_TIMESTAMP;
+        } else if (offsetSpec instanceof OffsetSpec.LatestTieredSpec) {
+            return ListOffsetsRequest.LATEST_TIERED_TIMESTAMP;
         }
         return ListOffsetsRequest.LATEST_TIMESTAMP;
     }

clients/src/main/java/org/apache/kafka/clients/admin/NodeMetadata.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kafka.clients.admin;

clients/src/main/java/org/apache/kafka/clients/admin/OffsetSpec.java

@@ -26,6 +26,8 @@ public class OffsetSpec {
     public static class EarliestSpec extends OffsetSpec { }
     public static class LatestSpec extends OffsetSpec { }
     public static class MaxTimestampSpec extends OffsetSpec { }
+    public static class EarliestLocalSpec extends OffsetSpec { }
+    public static class LatestTieredSpec extends OffsetSpec { }
     public static class TimestampSpec extends OffsetSpec {
         private final long timestamp;
 
@@ -70,4 +72,23 @@ public class OffsetSpec {
         return new MaxTimestampSpec();
     }
 
+    /**
+     * Used to retrieve the local log start offset.
+     * Local log start offset is the offset of a log above which reads
+     * are guaranteed to be served from the disk of the leader broker.
+     * <br/>
+     * Note: When tiered Storage is not enabled, it behaves the same as retrieving the earliest timestamp offset.
+     */
+    public static OffsetSpec earliestLocal() {
+        return new EarliestLocalSpec();
+    }
+
+    /**
+     * Used to retrieve the highest offset of data stored in remote storage.
+     * <br/>
+     * Note: When tiered storage is not enabled, we will return unknown offset.
+     */
+    public static OffsetSpec latestTiered() {
+        return new LatestTieredSpec();
+    }
 }

clients/src/main/java/org/apache/kafka/clients/admin/RaftVoterEndpoint.java

@@ -91,10 +91,8 @@ public class RaftVoterEndpoint {
 
     @Override
     public String toString() {
-        return "RaftVoterEndpoint" +
-            "(name=" + name +
-            ", host=" + host +
-            ", port=" + port +
-            ")";
+        // enclose IPv6 hosts in square brackets for readability
+        String hostString = host.contains(":") ? "[" + host + "]" : host;
+        return name + "://" + hostString + ":" + port;
     }
 }

clients/src/main/java/org/apache/kafka/clients/admin/RemoveRaftVoterOptions.java

@@ -18,9 +18,21 @@ package org.apache.kafka.clients.admin;
 
 import org.apache.kafka.common.annotation.InterfaceStability;
 
+import java.util.Optional;
+
 /**
  * Options for {@link Admin#removeRaftVoter}.
  */
 @InterfaceStability.Stable
 public class RemoveRaftVoterOptions extends AbstractOptions<RemoveRaftVoterOptions> {
+    private Optional<String> clusterId = Optional.empty();
+
+    public RemoveRaftVoterOptions setClusterId(Optional<String> clusterId) {
+        this.clusterId = clusterId;
+        return this;
+    }
+
+    public Optional<String> clusterId() {
+        return clusterId;
+    }
 }

clients/src/main/java/org/apache/kafka/clients/admin/UpdateGroupOptions.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+public class UpdateGroupOptions extends AbstractOptions<UpdateGroupOptions> {
+}

clients/src/main/java/org/apache/kafka/clients/admin/UpdateGroupResult.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import org.apache.kafka.common.KafkaFuture;
+
+public class UpdateGroupResult extends AbstractOptions<UpdateGroupResult> {
+    private final KafkaFuture<Void> future;
+
+    UpdateGroupResult(final KafkaFuture<Void> future) {
+        this.future = future;
+    }
+
+    /**
+     * Return a future which succeeds if all the feature updates succeed.
+     */
+    public KafkaFuture<Void> all() {
+        return future;
+    }
+}

clients/src/main/java/org/apache/kafka/clients/admin/UpdateGroupSpec.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.clients.admin;
+
+import java.util.Objects;
+
+public class UpdateGroupSpec {
+    private String linkId;
+    private boolean promoted;
+
+    public UpdateGroupSpec linkId(String linkId) {
+        this.linkId = linkId;
+        return this;
+    }
+
+    public UpdateGroupSpec promoted(boolean promoted) {
+        this.promoted = promoted;
+        return this;
+    }
+
+    public String linkId() {
+        return linkId;
+    }
+
+    public boolean promoted() {
+        return promoted;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        UpdateGroupSpec spec = (UpdateGroupSpec) o;
+        return promoted == spec.promoted && Objects.equals(linkId, spec.linkId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(linkId, promoted);
+    }
+
+    @Override
+    public String toString() {
+        return "UpdateGroupsSpec{" +
+            "linkId='" + linkId + '\'' +
+            ", promoted=" + promoted +
+            '}';
+    }
+}

clients/src/main/java/org/apache/kafka/clients/admin/internals/ListOffsetsHandler.java

@@ -93,9 +93,20 @@ public final class ListOffsetsHandler extends Batched<TopicPartition, ListOffset
             .stream()
             .anyMatch(key -> offsetTimestampsByPartition.get(key) == ListOffsetsRequest.MAX_TIMESTAMP);
 
-        return ListOffsetsRequest.Builder
-            .forConsumer(true, options.isolationLevel(), supportsMaxTimestamp)
-            .setTargetTimes(new ArrayList<>(topicsByName.values()));
+        boolean requireEarliestLocalTimestamp = keys
+                .stream()
+                .anyMatch(key -> offsetTimestampsByPartition.get(key) == ListOffsetsRequest.EARLIEST_LOCAL_TIMESTAMP);
+
+        boolean requireTieredStorageTimestamp = keys
+            .stream()
+            .anyMatch(key -> offsetTimestampsByPartition.get(key) == ListOffsetsRequest.LATEST_TIERED_TIMESTAMP);
+
+        return ListOffsetsRequest.Builder.forConsumer(true,
+                        options.isolationLevel(),
+                        supportsMaxTimestamp,
+                        requireEarliestLocalTimestamp,
+                        requireTieredStorageTimestamp)
+                .setTargetTimes(new ArrayList<>(topicsByName.values()));
     }
 
     @Override

clients/src/main/java/org/apache/kafka/clients/admin/internals/UpdateGroupHandler.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin.internals;
+
+import org.apache.kafka.clients.admin.UpdateGroupSpec;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.message.AutomqUpdateGroupRequestData;
+import org.apache.kafka.common.message.AutomqUpdateGroupResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
+import org.apache.kafka.common.requests.s3.AutomqUpdateGroupRequest;
+import org.apache.kafka.common.requests.s3.AutomqUpdateGroupResponse;
+import org.apache.kafka.common.utils.LogContext;
+
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.singleton;
+
+public class UpdateGroupHandler extends AdminApiHandler.Batched<CoordinatorKey, Void> {
+    private final CoordinatorKey groupId;
+    private final UpdateGroupSpec groupSpec;
+    private final Logger logger;
+    private final AdminApiLookupStrategy<CoordinatorKey> lookupStrategy;
+
+    public UpdateGroupHandler(
+        String groupId,
+        UpdateGroupSpec groupSpec,
+        LogContext logContext
+    ) {
+        this.groupId = CoordinatorKey.byGroupId(groupId);
+        this.groupSpec = groupSpec;
+        this.logger = logContext.logger(UpdateGroupHandler.class);
+        this.lookupStrategy = new CoordinatorStrategy(CoordinatorType.GROUP, logContext);
+    }
+
+    @Override
+    public String apiName() {
+        return "updateGroup";
+    }
+
+    @Override
+    public AdminApiLookupStrategy<CoordinatorKey> lookupStrategy() {
+        return lookupStrategy;
+    }
+
+    public static AdminApiFuture.SimpleAdminApiFuture<CoordinatorKey, Void> newFuture(
+        String groupId
+    ) {
+        return AdminApiFuture.forKeys(Collections.singleton(CoordinatorKey.byGroupId(groupId)));
+    }
+
+    private void validateKeys(Set<CoordinatorKey> groupIds) {
+        if (!groupIds.equals(singleton(groupId))) {
+            throw new IllegalArgumentException("Received unexpected group ids " + groupIds +
+                " (expected only " + singleton(groupId) + ")");
+        }
+    }
+
+    @Override
+    public AutomqUpdateGroupRequest.Builder buildBatchedRequest(
+        int coordinatorId,
+        Set<CoordinatorKey> groupIds
+    ) {
+        validateKeys(groupIds);
+        return new AutomqUpdateGroupRequest.Builder(
+            new AutomqUpdateGroupRequestData()
+                .setLinkId(groupSpec.linkId())
+                .setGroupId(this.groupId.idValue)
+                .setPromoted(groupSpec.promoted())
+        );
+    }
+
+    @Override
+    public ApiResult<CoordinatorKey, Void> handleResponse(
+        Node coordinator,
+        Set<CoordinatorKey> groupIds,
+        AbstractResponse abstractResponse
+    ) {
+        validateKeys(groupIds);
+
+        final Map<CoordinatorKey, Void> completed = new HashMap<>();
+        final Map<CoordinatorKey, Throwable> failed = new HashMap<>();
+        final List<CoordinatorKey> groupsToUnmap = new ArrayList<>();
+        AutomqUpdateGroupResponse response = (AutomqUpdateGroupResponse) abstractResponse;
+        AutomqUpdateGroupResponseData data = response.data();
+        Errors error = Errors.forCode(data.errorCode());
+        if (error != Errors.NONE) {
+            handleError(
+                CoordinatorKey.byGroupId(data.groupId()),
+                error,
+                data.errorMessage(),
+                failed,
+                groupsToUnmap
+            );
+        } else {
+            completed.put(groupId, null);
+        }
+        return new ApiResult<>(completed, failed, groupsToUnmap);
+    }
+
+    private void handleError(
+        CoordinatorKey groupId,
+        Errors error,
+        String errorMsg,
+        Map<CoordinatorKey, Throwable> failed,
+        List<CoordinatorKey> groupsToUnmap
+    ) {
+        switch (error) {
+            case COORDINATOR_NOT_AVAILABLE:
+            case NOT_COORDINATOR:
+                // If the coordinator is unavailable or there was a coordinator change, then we unmap
+                // the key so that we retry the `FindCoordinator` request
+                logger.debug("`{}` request for group id {} returned error {}. " +
+                    "Will attempt to find the coordinator again and retry.", apiName(), groupId.idValue, error);
+                groupsToUnmap.add(groupId);
+                break;
+            default:
+                logger.error("`{}` request for group id {} failed due to unexpected error {}.", apiName(), groupId.idValue, error);
+                failed.put(groupId, error.exception(errorMsg));
+        }
+    }
+}

clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java

@@ -1233,7 +1233,7 @@ public class AsyncKafkaConsumer<K, V> implements ConsumerDelegate<K, V> {
         wakeupTrigger.disableWakeups();
 
         final Timer closeTimer = time.timer(timeout);
-        clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(timeout.toMillis()));
+        clientTelemetryReporter.ifPresent(ClientTelemetryReporter::initiateClose);
         closeTimer.update();
         // Prepare shutting down the network thread
         swallow(log, Level.ERROR, "Failed to release assignment before closing consumer",

clients/src/main/java/org/apache/kafka/clients/consumer/internals/ClassicKafkaConsumer.java

@@ -1130,7 +1130,7 @@ public class ClassicKafkaConsumer<K, V> implements ConsumerDelegate<K, V> {
         AtomicReference<Throwable> firstException = new AtomicReference<>();
 
         final Timer closeTimer = createTimerForRequest(timeout);
-        clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(timeout.toMillis()));
+        clientTelemetryReporter.ifPresent(ClientTelemetryReporter::initiateClose);
         closeTimer.update();
         // Close objects with a timeout. The timeout is required because the coordinator & the fetcher send requests to
         // the server in the process of closing which may not respect the overall timeout defined for closing the

clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java

@@ -1457,15 +1457,16 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
             if (responseError != Errors.NONE) {
                 log.debug("Offset fetch failed: {}", responseError.message());
 
-                if (responseError == Errors.COORDINATOR_LOAD_IN_PROGRESS) {
-                    // just retry
-                    future.raise(responseError);
-                } else if (responseError == Errors.NOT_COORDINATOR) {
+                if (responseError == Errors.COORDINATOR_NOT_AVAILABLE ||
+                    responseError == Errors.NOT_COORDINATOR) {
                     // re-discover the coordinator and retry
                     markCoordinatorUnknown(responseError);
                     future.raise(responseError);
                 } else if (responseError == Errors.GROUP_AUTHORIZATION_FAILED) {
                     future.raise(GroupAuthorizationException.forGroupId(rebalanceConfig.groupId));
+                } else if (responseError.exception() instanceof RetriableException) {
+                    // retry
+                    future.raise(responseError);
                 } else {
                     future.raise(new KafkaException("Unexpected error in fetch offset response: " + responseError.message()));
                 }

clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcher.java

@@ -391,7 +391,7 @@ public class OffsetFetcher {
                                                                   final Map<TopicPartition, ListOffsetsPartition> timestampsToSearch,
                                                                   boolean requireTimestamp) {
         ListOffsetsRequest.Builder builder = ListOffsetsRequest.Builder
-                .forConsumer(requireTimestamp, isolationLevel, false)
+                .forConsumer(requireTimestamp, isolationLevel)
                 .setTargetTimes(ListOffsetsRequest.toListOffsetsTopics(timestampsToSearch));
 
         log.debug("Sending ListOffsetRequest {} to broker {}", builder, node);

clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java

@@ -337,7 +337,7 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
             boolean requireTimestamps,
             List<NetworkClientDelegate.UnsentRequest> unsentRequests) {
         ListOffsetsRequest.Builder builder = ListOffsetsRequest.Builder
-                .forConsumer(requireTimestamps, isolationLevel, false)
+                .forConsumer(requireTimestamps, isolationLevel)
                 .setTargetTimes(ListOffsetsRequest.toListOffsetsTopics(targetTimes));
 
         log.debug("Creating ListOffset request {} for broker {} to reset positions", builder,

clients/src/main/java/org/apache/kafka/clients/consumer/internals/ShareConsumerImpl.java

@@ -828,7 +828,7 @@ public class ShareConsumerImpl<K, V> implements ShareConsumerDelegate<K, V> {
         wakeupTrigger.disableWakeups();
 
         final Timer closeTimer = time.timer(timeout);
-        clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(timeout.toMillis()));
+        clientTelemetryReporter.ifPresent(ClientTelemetryReporter::initiateClose);
         closeTimer.update();
 
         // Prepare shutting down the network thread

clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java

@@ -1392,6 +1392,9 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
             } else {
                 // Try to close gracefully.
                 final Timer closeTimer = time.timer(timeout);
+                clientTelemetryReporter.ifPresent(ClientTelemetryReporter::initiateClose);
+                closeTimer.update();
+
                 if (this.sender != null) {
                     this.sender.initiateClose();
                     closeTimer.update();
@@ -1406,7 +1409,6 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
                         closeTimer.update();
                     }
                 }
-                clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(closeTimer.remainingMs()));
             }
         }
 

clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java

@@ -19,9 +19,6 @@ package org.apache.kafka.clients.producer;
 import org.apache.kafka.clients.ClientDnsLookup;
 import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.MetadataRecoveryStrategy;
-import org.apache.kafka.common.compress.GzipCompression;
-import org.apache.kafka.common.compress.Lz4Compression;
-import org.apache.kafka.common.compress.ZstdCompression;
 import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
@@ -381,9 +378,9 @@ public class ProducerConfig extends AbstractConfig {
                                         Importance.LOW,
                                         ACKS_DOC)
                                 .define(COMPRESSION_TYPE_CONFIG, Type.STRING, CompressionType.NONE.name, in(Utils.enumOptions(CompressionType.class)), Importance.HIGH, COMPRESSION_TYPE_DOC)
-                                .define(COMPRESSION_GZIP_LEVEL_CONFIG, Type.INT, GzipCompression.DEFAULT_LEVEL, new GzipCompression.LevelValidator(), Importance.MEDIUM, COMPRESSION_GZIP_LEVEL_DOC)
-                                .define(COMPRESSION_LZ4_LEVEL_CONFIG, Type.INT, Lz4Compression.DEFAULT_LEVEL, between(Lz4Compression.MIN_LEVEL, Lz4Compression.MAX_LEVEL), Importance.MEDIUM, COMPRESSION_LZ4_LEVEL_DOC)
-                                .define(COMPRESSION_ZSTD_LEVEL_CONFIG, Type.INT, ZstdCompression.DEFAULT_LEVEL, between(ZstdCompression.MIN_LEVEL, ZstdCompression.MAX_LEVEL), Importance.MEDIUM, COMPRESSION_ZSTD_LEVEL_DOC)
+                                .define(COMPRESSION_GZIP_LEVEL_CONFIG, Type.INT, CompressionType.GZIP.defaultLevel(), CompressionType.GZIP.levelValidator(), Importance.MEDIUM, COMPRESSION_GZIP_LEVEL_DOC)
+                                .define(COMPRESSION_LZ4_LEVEL_CONFIG, Type.INT, CompressionType.LZ4.defaultLevel(), CompressionType.LZ4.levelValidator(), Importance.MEDIUM, COMPRESSION_LZ4_LEVEL_DOC)
+                                .define(COMPRESSION_ZSTD_LEVEL_CONFIG, Type.INT, CompressionType.ZSTD.defaultLevel(), CompressionType.ZSTD.levelValidator(), Importance.MEDIUM, COMPRESSION_ZSTD_LEVEL_DOC)
                                 .define(BATCH_SIZE_CONFIG, Type.INT, 16384, atLeast(0), Importance.MEDIUM, BATCH_SIZE_DOC)
                                 .define(PARTITIONER_ADPATIVE_PARTITIONING_ENABLE_CONFIG, Type.BOOLEAN, true, Importance.LOW, PARTITIONER_ADPATIVE_PARTITIONING_ENABLE_DOC)
                                 .define(PARTITIONER_AVAILABILITY_TIMEOUT_MS_CONFIG, Type.LONG, 0, atLeast(0), Importance.LOW, PARTITIONER_AVAILABILITY_TIMEOUT_MS_DOC)

clients/src/main/java/org/apache/kafka/common/compress/GzipCompression.java

@@ -17,8 +17,6 @@
 package org.apache.kafka.common.compress;
 
 import org.apache.kafka.common.KafkaException;
-import org.apache.kafka.common.config.ConfigDef;
-import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.record.CompressionType;
 import org.apache.kafka.common.utils.BufferSupplier;
 import org.apache.kafka.common.utils.ByteBufferInputStream;
@@ -30,14 +28,11 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Objects;
-import java.util.zip.Deflater;
 import java.util.zip.GZIPInputStream;
 
-public class GzipCompression implements Compression {
+import static org.apache.kafka.common.record.CompressionType.GZIP;
 
-    public static final int MIN_LEVEL = Deflater.BEST_SPEED;
-    public static final int MAX_LEVEL = Deflater.BEST_COMPRESSION;
-    public static final int DEFAULT_LEVEL = Deflater.DEFAULT_COMPRESSION;
+public class GzipCompression implements Compression {
 
     private final int level;
 
@@ -47,7 +42,7 @@ public class GzipCompression implements Compression {
 
     @Override
     public CompressionType type() {
-        return CompressionType.GZIP;
+        return GZIP;
     }
 
     @Override
@@ -101,10 +96,10 @@ public class GzipCompression implements Compression {
     }
 
     public static class Builder implements Compression.Builder<GzipCompression> {
-        private int level = DEFAULT_LEVEL;
+        private int level = GZIP.defaultLevel();
 
         public Builder level(int level) {
-            if ((level < MIN_LEVEL || MAX_LEVEL < level) && level != DEFAULT_LEVEL) {
+            if ((level < GZIP.minLevel() || GZIP.maxLevel() < level) && level != GZIP.defaultLevel()) {
                 throw new IllegalArgumentException("gzip doesn't support given compression level: " + level);
             }
 
@@ -117,22 +112,4 @@ public class GzipCompression implements Compression {
             return new GzipCompression(level);
         }
     }
-
-    public static class LevelValidator implements ConfigDef.Validator {
-
-        @Override
-        public void ensureValid(String name, Object o) {
-            if (o == null)
-                throw new ConfigException(name, null, "Value must be non-null");
-            int level = ((Number) o).intValue();
-            if (level > MAX_LEVEL || (level < MIN_LEVEL && level != DEFAULT_LEVEL)) {
-                throw new ConfigException(name, o, "Value must be between " + MIN_LEVEL + " and " + MAX_LEVEL + " or equal to " + DEFAULT_LEVEL);
-            }
-        }
-
-        @Override
-        public String toString() {
-            return "[" + MIN_LEVEL + ",...," + MAX_LEVEL + "] or " + DEFAULT_LEVEL;
-        }
-    }
 }

clients/src/main/java/org/apache/kafka/common/compress/Lz4BlockOutputStream.java

@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.compress;
 
+import org.apache.kafka.common.record.CompressionType;
 import org.apache.kafka.common.utils.ByteUtils;
 
 import net.jpountz.lz4.LZ4Compressor;
@@ -75,7 +76,7 @@ public final class Lz4BlockOutputStream extends OutputStream {
          *
          * For backward compatibility, Lz4BlockOutputStream uses fastCompressor with default compression level but, with the other level, it uses highCompressor.
          */
-        compressor = level == Lz4Compression.DEFAULT_LEVEL ? LZ4Factory.fastestInstance().fastCompressor() : LZ4Factory.fastestInstance().highCompressor(level);
+        compressor = level == CompressionType.LZ4.defaultLevel() ? LZ4Factory.fastestInstance().fastCompressor() : LZ4Factory.fastestInstance().highCompressor(level);
         checksum = XXHashFactory.fastestInstance().hash32();
         this.useBrokenFlagDescriptorChecksum = useBrokenFlagDescriptorChecksum;
         bd = new BD(blockSize);

clients/src/main/java/org/apache/kafka/common/compress/Lz4Compression.java

@@ -28,13 +28,9 @@ import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
-public class Lz4Compression implements Compression {
+import static org.apache.kafka.common.record.CompressionType.LZ4;
 
-    // These values come from net.jpountz.lz4.LZ4Constants
-    // We may need to update them if the lz4 library changes these values.
-    public static final int MIN_LEVEL = 1;
-    public static final int MAX_LEVEL = 17;
-    public static final int DEFAULT_LEVEL = 9;
+public class Lz4Compression implements Compression {
 
     private final int level;
 
@@ -44,7 +40,7 @@ public class Lz4Compression implements Compression {
 
     @Override
     public CompressionType type() {
-        return CompressionType.LZ4;
+        return LZ4;
     }
 
     @Override
@@ -89,10 +85,10 @@ public class Lz4Compression implements Compression {
     }
 
     public static class Builder implements Compression.Builder<Lz4Compression> {
-        private int level = DEFAULT_LEVEL;
+        private int level = LZ4.defaultLevel();
 
         public Builder level(int level) {
-            if (level < MIN_LEVEL || MAX_LEVEL < level) {
+            if (level < LZ4.minLevel() || LZ4.maxLevel() < level) {
                 throw new IllegalArgumentException("lz4 doesn't support given compression level: " + level);
             }
 

clients/src/main/java/org/apache/kafka/common/compress/ZstdCompression.java

@@ -26,7 +26,6 @@ import org.apache.kafka.common.utils.ChunkedBytesStream;
 
 import com.github.luben.zstd.BufferPool;
 import com.github.luben.zstd.RecyclingBufferPool;
-import com.github.luben.zstd.Zstd;
 import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
 import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
 
@@ -37,11 +36,9 @@ import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
-public class ZstdCompression implements Compression {
+import static org.apache.kafka.common.record.CompressionType.ZSTD;
 
-    public static final int MIN_LEVEL = Zstd.minCompressionLevel();
-    public static final int MAX_LEVEL = Zstd.maxCompressionLevel();
-    public static final int DEFAULT_LEVEL = Zstd.defaultCompressionLevel();
+public class ZstdCompression implements Compression {
 
     private final int level;
 
@@ -51,7 +48,7 @@ public class ZstdCompression implements Compression {
 
     @Override
     public CompressionType type() {
-        return CompressionType.ZSTD;
+        return ZSTD;
     }
 
     @Override
@@ -125,10 +122,10 @@ public class ZstdCompression implements Compression {
     }
 
     public static class Builder implements Compression.Builder<ZstdCompression> {
-        private int level = DEFAULT_LEVEL;
+        private int level = ZSTD.defaultLevel();
 
         public Builder level(int level) {
-            if (MAX_LEVEL < level || level < MIN_LEVEL) {
+            if (level < ZSTD.minLevel() || ZSTD.maxLevel() < level) {
                 throw new IllegalArgumentException("zstd doesn't support given compression level: " + level);
             }
 

clients/src/main/java/org/apache/kafka/common/config/TopicConfig.java

@@ -81,38 +81,37 @@ public class TopicConfig {
 
     public static final String REMOTE_LOG_STORAGE_ENABLE_CONFIG = "remote.storage.enable";
     public static final String REMOTE_LOG_STORAGE_ENABLE_DOC = "To enable tiered storage for a topic, set this configuration as true. " +
-            "You can not disable this config once it is enabled. It will be provided in future versions.";
+        "You can not disable this config once it is enabled. It will be provided in future versions.";
 
     public static final String LOCAL_LOG_RETENTION_MS_CONFIG = "local.retention.ms";
     public static final String LOCAL_LOG_RETENTION_MS_DOC = "The number of milliseconds to keep the local log segment before it gets deleted. " +
-            "Default value is -2, it represents `retention.ms` value is to be used. The effective value should always be less than or equal " +
-            "to `retention.ms` value.";
+        "Default value is -2, it represents `retention.ms` value is to be used. The effective value should always be less than or equal " +
+        "to `retention.ms` value.";
 
     public static final String LOCAL_LOG_RETENTION_BYTES_CONFIG = "local.retention.bytes";
     public static final String LOCAL_LOG_RETENTION_BYTES_DOC = "The maximum size of local log segments that can grow for a partition before it " +
-            "deletes the old segments. Default value is -2, it represents `retention.bytes` value to be used. The effective value should always be " +
-            "less than or equal to `retention.bytes` value.";
+        "deletes the old segments. Default value is -2, it represents `retention.bytes` value to be used. The effective value should always be " +
+        "less than or equal to `retention.bytes` value.";
 
-    public static final String REMOTE_LOG_DISABLE_POLICY_RETAIN = "retain";
-    public static final String REMOTE_LOG_DISABLE_POLICY_DELETE = "delete";
+    public static final String REMOTE_LOG_COPY_DISABLE_CONFIG = "remote.log.copy.disable";
+    public static final String REMOTE_LOG_COPY_DISABLE_DOC = "Determines whether tiered data for a topic should become read only," +
+        " and no more data uploading on a topic. Once this config is set to true, the local retention configuration " +
+        "(i.e. local.retention.ms/bytes) becomes irrelevant, and all data expiration follows the topic-wide retention configuration" +
+        "(i.e. retention.ms/bytes).";
 
-    public static final String REMOTE_LOG_DISABLE_POLICY_CONFIG = "remote.log.disable.policy";
-
-    public static final String REMOTE_LOG_DISABLE_POLICY_DOC = String.format("Determines whether tiered data for a topic should be retained or " +
-            "deleted after tiered storage disablement on a topic. The two valid options are \"%s\" and \"%s\". If %s is " +
-            "selected then all data in remote will be kept post-disablement and will only be deleted when it breaches expiration " +
-            "thresholds. If %s is selected then the data will be made inaccessible immediately by advancing the log start offset and will be " +
-            "deleted asynchronously.", REMOTE_LOG_DISABLE_POLICY_RETAIN, REMOTE_LOG_DISABLE_POLICY_DELETE,
-            REMOTE_LOG_DISABLE_POLICY_RETAIN, REMOTE_LOG_DISABLE_POLICY_DELETE);
+    public static final String REMOTE_LOG_DELETE_ON_DISABLE_CONFIG = "remote.log.delete.on.disable";
+    public static final String REMOTE_LOG_DELETE_ON_DISABLE_DOC = "Determines whether tiered data for a topic should be " +
+        "deleted after tiered storage is disabled on a topic. This configuration should be enabled when trying to " +
+        "set `remote.storage.enable` from true to false";
 
     public static final String MAX_MESSAGE_BYTES_CONFIG = "max.message.bytes";
     public static final String MAX_MESSAGE_BYTES_DOC =
         "The largest record batch size allowed by Kafka (after compression if compression is enabled). " +
-        "If this is increased and there are consumers older than 0.10.2, the consumers' fetch " +
-        "size must also be increased so that they can fetch record batches this large. " +
-        "In the latest message format version, records are always grouped into batches for efficiency. " +
-        "In previous message format versions, uncompressed records are not grouped into batches and this " +
-        "limit only applies to a single record in that case.";
+            "If this is increased and there are consumers older than 0.10.2, the consumers' fetch " +
+            "size must also be increased so that they can fetch record batches this large. " +
+            "In the latest message format version, records are always grouped into batches for efficiency. " +
+            "In previous message format versions, uncompressed records are not grouped into batches and this " +
+            "limit only applies to a single record in that case.";
 
     public static final String INDEX_INTERVAL_BYTES_CONFIG = "index.interval.bytes";
     public static final String INDEX_INTERVAL_BYTES_DOC = "This setting controls how frequently " +
@@ -166,7 +165,9 @@ public class TopicConfig {
     public static final String UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG = "unclean.leader.election.enable";
     public static final String UNCLEAN_LEADER_ELECTION_ENABLE_DOC = "Indicates whether to enable replicas " +
         "not in the ISR set to be elected as leader as a last resort, even though doing so may result in data " +
-        "loss.";
+        "loss.<p>Note: In KRaft mode, when enabling this config dynamically, it needs to wait for the unclean leader election" +
+        "thread to trigger election periodically (default is 5 minutes). Please run `kafka-leader-election.sh` with `unclean` option " +
+        "to trigger the unclean leader election immediately if needed.</p>";
 
     public static final String MIN_IN_SYNC_REPLICAS_CONFIG = "min.insync.replicas";
     public static final String MIN_IN_SYNC_REPLICAS_DOC = "When a producer sets acks to \"all\" (or \"-1\"), " +
@@ -256,4 +257,32 @@ public class TopicConfig {
         "broker will not perform down-conversion for consumers expecting an older message format. The broker responds " +
         "with <code>UNSUPPORTED_VERSION</code> error for consume requests from such older clients. This configuration" +
         "does not apply to any message format conversion that might be required for replication to followers.";
+
+    // AutoMQ inject start
+    public static final String TABLE_TOPIC_ENABLE_CONFIG = "automq.table.topic.enable";
+    public static final String TABLE_TOPIC_ENABLE_DOC = "The configuration controls whether enable table topic";
+    public static final String TABLE_TOPIC_COMMIT_INTERVAL_CONFIG = "automq.table.topic.commit.interval.ms";
+    public static final String TABLE_TOPIC_COMMIT_INTERVAL_DOC = "The table topic commit interval(ms)";
+    public static final String TABLE_TOPIC_NAMESPACE_CONFIG = "automq.table.topic.namespace";
+    public static final String TABLE_TOPIC_NAMESPACE_DOC = "The table topic table namespace";
+    public static final String TABLE_TOPIC_SCHEMA_TYPE_CONFIG = "automq.table.topic.schema.type";
+    public static final String TABLE_TOPIC_SCHEMA_TYPE_DOC = "The table topic schema type, support schemaless, schema";
+    public static final String TABLE_TOPIC_ID_COLUMNS_CONFIG = "automq.table.topic.id.columns";
+    public static final String TABLE_TOPIC_ID_COLUMNS_DOC = "The primary key, comma-separated list of columns that identify a row in tables."
+        + "ex. [region, name]";
+    public static final String TABLE_TOPIC_PARTITION_BY_CONFIG = "automq.table.topic.partition.by";
+    public static final String TABLE_TOPIC_PARTITION_BY_DOC = "The partition fields of the table. ex. [bucket(name), month(timestamp)]";
+    public static final String TABLE_TOPIC_UPSERT_ENABLE_CONFIG = "automq.table.topic.upsert.enable";
+    public static final String TABLE_TOPIC_UPSERT_ENABLE_DOC = "The configuration controls whether enable table topic upsert";
+    public static final String TABLE_TOPIC_CDC_FIELD_CONFIG = "automq.table.topic.cdc.field";
+    public static final String TABLE_TOPIC_CDC_FIELD_DOC = "The name of the field containing the CDC operation, I, U, or D";
+
+    public static final String KAFKA_LINKS_ID_CONFIG = "automq.kafka.links.id";
+    public static final String KAFKA_LINKS_ID_DOC = "The unique id of a kafka link";
+    public static final String KAFKA_LINKS_TOPIC_START_TIME_CONFIG = "automq.kafka.links.topic.start.time";
+    public static final String KAFKA_LINKS_TOPIC_START_TIME_DOC = "The offset to start replicate from. Valid values: -1 (latest), -2 (earliest), positive value (timestamp)";
+    public static final String KAFKA_LINKS_TOPIC_STATE_CONFIG = "automq.kafka.links.topic.state";
+    public static final String KAFKA_LINKS_TOPIC_STATE_DOC = "The state of the topic that's in linking";
+    // AutoMQ inject end
+
 }

clients/src/main/java/org/apache/kafka/common/errors/s3/NodeLockedException.java

@@ -1,4 +1,6 @@
 /*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -14,16 +16,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.raft.errors;
 
-/**
- * Indicates that an append operation cannot be completed because it would have resulted in an
- * unexpected base offset.
- */
-public class UnexpectedBaseOffsetException extends RaftException {
-    private static final long serialVersionUID = 1L;
+package org.apache.kafka.common.errors.s3;
 
-    public UnexpectedBaseOffsetException(String s) {
-        super(s);
+import org.apache.kafka.common.errors.ApiException;
+
+public class NodeLockedException extends ApiException {
+
+    public NodeLockedException(String message) {
+        super(message);
     }
+
 }

clients/src/main/java/org/apache/kafka/common/errors/s3/ObjectNotCommittedException.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2025, AutoMQ HK Limited.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.errors.s3;
+
+import org.apache.kafka.common.errors.ApiException;
+
+public class ObjectNotCommittedException extends ApiException {
+    public ObjectNotCommittedException(String message) {
+        super(message);
+    }
+}

clients/src/main/java/org/apache/kafka/common/errors/s3/UnregisterNodeWithOpenStreamsException.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kafka.common.errors.s3;

clients/src/main/java/org/apache/kafka/common/internals/Topic.java

@@ -30,7 +30,13 @@ public class Topic {
     public static final String TRANSACTION_STATE_TOPIC_NAME = "__transaction_state";
     public static final String SHARE_GROUP_STATE_TOPIC_NAME = "__share_group_state";
     public static final String CLUSTER_METADATA_TOPIC_NAME = "__cluster_metadata";
+
+    // AutoMQ inject start
     public static final String AUTO_BALANCER_METRICS_TOPIC_NAME = "__auto_balancer_metrics";
+    public static final String TABLE_TOPIC_CONTROL_TOPIC_NAME = "__automq_table_control";
+    public static final String TABLE_TOPIC_DATA_TOPIC_NAME = "__automq_table_data";
+    // AutoMQ inject end
+
     public static final TopicPartition CLUSTER_METADATA_TOPIC_PARTITION = new TopicPartition(
         CLUSTER_METADATA_TOPIC_NAME,
         0

clients/src/main/java/org/apache/kafka/common/metrics/KafkaMetric.java

@@ -129,4 +129,13 @@ public final class KafkaMetric implements Metric {
             this.config = config;
         }
     }
+
+    // AutoMQ inject start
+    /**
+     * A public method to expose the {@link #measurableValue} method.
+     */
+    public double measurableValueV2(long timeMs) {
+        return measurableValue(timeMs);
+    }
+    // AutoMQ inject end
 }

clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java

@@ -123,8 +123,8 @@ public enum ApiKeys {
     SHARE_GROUP_DESCRIBE(ApiMessageType.SHARE_GROUP_DESCRIBE),
     SHARE_FETCH(ApiMessageType.SHARE_FETCH),
     SHARE_ACKNOWLEDGE(ApiMessageType.SHARE_ACKNOWLEDGE),
-    ADD_RAFT_VOTER(ApiMessageType.ADD_RAFT_VOTER),
-    REMOVE_RAFT_VOTER(ApiMessageType.REMOVE_RAFT_VOTER),
+    ADD_RAFT_VOTER(ApiMessageType.ADD_RAFT_VOTER, false, RecordBatch.MAGIC_VALUE_V0, true),
+    REMOVE_RAFT_VOTER(ApiMessageType.REMOVE_RAFT_VOTER, false, RecordBatch.MAGIC_VALUE_V0, true),
     UPDATE_RAFT_VOTER(ApiMessageType.UPDATE_RAFT_VOTER),
     INITIALIZE_SHARE_GROUP_STATE(ApiMessageType.INITIALIZE_SHARE_GROUP_STATE, true),
     READ_SHARE_GROUP_STATE(ApiMessageType.READ_SHARE_GROUP_STATE, true),
@@ -148,9 +148,11 @@ public enum ApiKeys {
     AUTOMQ_REGISTER_NODE(ApiMessageType.AUTOMQ_REGISTER_NODE, false, false),
     AUTOMQ_GET_NODES(ApiMessageType.AUTOMQ_GET_NODES, false, true),
     AUTOMQ_ZONE_ROUTER(ApiMessageType.AUTOMQ_ZONE_ROUTER, false, false),
+    AUTOMQ_GET_PARTITION_SNAPSHOT(ApiMessageType.AUTOMQ_GET_PARTITION_SNAPSHOT, false, false),
 
     GET_NEXT_NODE_ID(ApiMessageType.GET_NEXT_NODE_ID, false, true),
-    DESCRIBE_STREAMS(ApiMessageType.DESCRIBE_STREAMS, false, true);
+    DESCRIBE_STREAMS(ApiMessageType.DESCRIBE_STREAMS, false, true),
+    AUTOMQ_UPDATE_GROUP(ApiMessageType.AUTOMQ_UPDATE_GROUP);
     // AutoMQ for Kafka inject end
 
     private static final Map<ApiMessageType.ListenerType, EnumSet<ApiKeys>> APIS_BY_LISTENER =
@@ -307,7 +309,7 @@ public enum ApiKeys {
         b.append("<th>Key</th>\n");
         b.append("</tr>");
         clientApis().stream()
-            .filter(apiKey -> !apiKey.messageType.latestVersionUnstable())
+            .filter(apiKey -> apiKey.toApiVersion(false).isPresent())
             .forEach(apiKey -> {
                 b.append("<tr>\n");
                 b.append("<td>");

clients/src/main/java/org/apache/kafka/common/protocol/Errors.java

@@ -151,6 +151,8 @@ import org.apache.kafka.common.errors.s3.KeyExistException;
 import org.apache.kafka.common.errors.s3.NodeEpochExpiredException;
 import org.apache.kafka.common.errors.s3.NodeEpochNotExistException;
 import org.apache.kafka.common.errors.s3.NodeFencedException;
+import org.apache.kafka.common.errors.s3.NodeLockedException;
+import org.apache.kafka.common.errors.s3.ObjectNotCommittedException;
 import org.apache.kafka.common.errors.s3.ObjectNotExistException;
 import org.apache.kafka.common.errors.s3.OffsetNotMatchedException;
 import org.apache.kafka.common.errors.s3.RedundantOperationException;
@@ -435,6 +437,8 @@ public enum Errors {
     KEY_EXIST(512, "The key already exists.", KeyExistException::new),
     KEY_NOT_EXIST(513, "The key does not exist.", ObjectNotExistException::new),
     NODE_FENCED(514, "The node is fenced.", NodeFencedException::new),
+    NODE_LOCKED(515, "The node is locked", NodeLockedException::new),
+    OBJECT_NOT_COMMITED(516, "The object is not commited.", ObjectNotCommittedException::new),
     STREAM_INNER_ERROR(599, "The stream inner error.", StreamInnerErrorException::new),
     // AutoMQ inject end
 

clients/src/main/java/org/apache/kafka/common/record/AbstractLegacyRecordBatch.java

@@ -499,6 +499,11 @@ public abstract class AbstractLegacyRecordBatch extends AbstractRecordBatch impl
             throw new UnsupportedOperationException("Magic versions prior to 2 do not support partition leader epoch");
         }
 
+        @Override
+        public void setProducerId(long producerId) {
+            throw new UnsupportedOperationException("Magic versions prior to 2 do not support producer id");
+        }
+
         private void setTimestampAndUpdateCrc(TimestampType timestampType, long timestamp) {
             byte attributes = LegacyRecord.computeAttributes(magic(), compressionType(), timestampType);
             buffer.put(LOG_OVERHEAD + LegacyRecord.ATTRIBUTES_OFFSET, attributes);

clients/src/main/java/org/apache/kafka/common/record/CompressionType.java

@@ -16,6 +16,13 @@
  */
 package org.apache.kafka.common.record;
 
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.ConfigException;
+
+import java.util.zip.Deflater;
+
+import static org.apache.kafka.common.config.ConfigDef.Range.between;
+
 /**
  * The compression type to use
  */
@@ -23,7 +30,46 @@ public enum CompressionType {
     NONE((byte) 0, "none", 1.0f),
 
     // Shipped with the JDK
-    GZIP((byte) 1, "gzip", 1.0f),
+    GZIP((byte) 1, "gzip", 1.0f) {
+        public static final int MIN_LEVEL = Deflater.BEST_SPEED;
+        public static final int MAX_LEVEL = Deflater.BEST_COMPRESSION;
+        public static final int DEFAULT_LEVEL = Deflater.DEFAULT_COMPRESSION;
+
+        @Override
+        public int defaultLevel() {
+            return DEFAULT_LEVEL;
+        }
+
+        @Override
+        public int maxLevel() {
+            return MAX_LEVEL;
+        }
+
+        @Override
+        public int minLevel() {
+            return MIN_LEVEL;
+        }
+
+        @Override
+        public ConfigDef.Validator levelValidator() {
+            return new ConfigDef.Validator() {
+                @Override
+                public void ensureValid(String name, Object o) {
+                    if (o == null)
+                        throw new ConfigException(name, null, "Value must be non-null");
+                    int level = ((Number) o).intValue();
+                    if (level > MAX_LEVEL || (level < MIN_LEVEL && level != DEFAULT_LEVEL)) {
+                        throw new ConfigException(name, o, "Value must be between " + MIN_LEVEL + " and " + MAX_LEVEL + " or equal to " + DEFAULT_LEVEL);
+                    }
+                }
+
+                @Override
+                public String toString() {
+                    return "[" + MIN_LEVEL + ",...," + MAX_LEVEL + "] or " + DEFAULT_LEVEL;
+                }
+            };
+        }
+    },
 
     // We should only load classes from a given compression library when we actually use said compression library. This
     // is because compression libraries include native code for a set of platforms and we want to avoid errors
@@ -31,8 +77,65 @@ public enum CompressionType {
     // To ensure this, we only reference compression library code from classes that are only invoked when actual usage
     // happens.
     SNAPPY((byte) 2, "snappy", 1.0f),
-    LZ4((byte) 3, "lz4", 1.0f),
-    ZSTD((byte) 4, "zstd", 1.0f);
+    LZ4((byte) 3, "lz4", 1.0f) {
+        // These values come from net.jpountz.lz4.LZ4Constants
+        // We may need to update them if the lz4 library changes these values.
+        private static final int MIN_LEVEL = 1;
+        private static final int MAX_LEVEL = 17;
+        private static final int DEFAULT_LEVEL = 9;
+
+        @Override
+        public int defaultLevel() {
+            return DEFAULT_LEVEL;
+        }
+
+        @Override
+        public int maxLevel() {
+            return MAX_LEVEL;
+        }
+
+        @Override
+        public int minLevel() {
+            return MIN_LEVEL;
+        }
+
+        @Override
+        public ConfigDef.Validator levelValidator() {
+            return between(MIN_LEVEL, MAX_LEVEL);
+        }
+    },
+    ZSTD((byte) 4, "zstd", 1.0f) {
+        // These values come from the zstd library. We don't use the Zstd.minCompressionLevel(),
+        // Zstd.maxCompressionLevel() and Zstd.defaultCompressionLevel() methods to not load the Zstd library
+        // while parsing configuration.
+        // See ZSTD_minCLevel in https://github.com/facebook/zstd/blob/dev/lib/compress/zstd_compress.c#L6987
+        // and ZSTD_TARGETLENGTH_MAX https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L1249
+        private static final int MIN_LEVEL = -131072;
+        // See ZSTD_MAX_CLEVEL in https://github.com/facebook/zstd/blob/dev/lib/compress/clevels.h#L19
+        private static final int MAX_LEVEL = 22;
+        // See ZSTD_CLEVEL_DEFAULT in https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L129
+        private static final int DEFAULT_LEVEL = 3;
+
+        @Override
+        public int defaultLevel() {
+            return DEFAULT_LEVEL;
+        }
+
+        @Override
+        public int maxLevel() {
+            return MAX_LEVEL;
+        }
+
+        @Override
+        public int minLevel() {
+            return MIN_LEVEL;
+        }
+
+        @Override
+        public ConfigDef.Validator levelValidator() {
+            return between(MIN_LEVEL, MAX_LEVEL);
+        }
+    };
 
     // compression type is represented by two bits in the attributes field of the record batch header, so `byte` is
     // large enough
@@ -78,6 +181,22 @@ public enum CompressionType {
             throw new IllegalArgumentException("Unknown compression name: " + name);
     }
 
+    public int defaultLevel() {
+        throw new UnsupportedOperationException("Compression levels are not defined for this compression type: " + name);
+    }
+
+    public int maxLevel() {
+        throw new UnsupportedOperationException("Compression levels are not defined for this compression type: " + name);
+    }
+
+    public int minLevel() {
+        throw new UnsupportedOperationException("Compression levels are not defined for this compression type: " + name);
+    }
+
+    public ConfigDef.Validator levelValidator() {
+        throw new UnsupportedOperationException("Compression levels are not defined for this compression type: " + name);
+    }
+
     @Override
     public String toString() {
         return name;

clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java

@@ -190,6 +190,16 @@ public class DefaultRecordBatch extends AbstractRecordBatch implements MutableRe
         return buffer.getLong(PRODUCER_ID_OFFSET);
     }
 
+    @Override
+    public void setProducerId(long producerId) {
+        if (producerId() == producerId) {
+            return;
+        }
+        buffer.putLong(PRODUCER_ID_OFFSET, producerId);
+        long crc = computeChecksum();
+        ByteUtils.writeUnsignedInt(buffer, CRC_OFFSET, crc);
+    }
+
     @Override
     public short producerEpoch() {
         return buffer.getShort(PRODUCER_EPOCH_OFFSET);

clients/src/main/java/org/apache/kafka/common/record/MutableRecordBatch.java

@@ -65,4 +65,13 @@ public interface MutableRecordBatch extends RecordBatch {
      * @return The closeable iterator
      */
     CloseableIterator<Record> skipKeyValueIterator(BufferSupplier bufferSupplier);
+
+    // AutoMQ injection start
+
+    /**
+     * Set the producer id for this batch of records.
+     * @param producerId The producer id to use
+     */
+    void setProducerId(long producerId);
+    // AutoMQ injection end
 }

clients/src/main/java/org/apache/kafka/common/record/PooledRecords.java

@@ -1,12 +1,20 @@
 /*
- * Copyright 2024, AutoMQ HK Limited.
+ * Copyright 2025, AutoMQ HK Limited.
  *
- * The use of this file is governed by the Business Source License,
- * as detailed in the file "/LICENSE.S3Stream" included in this repository.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- * As of the Change Date specified in that file, in accordance with
- * the Business Source License, use of this software will be governed
- * by the Apache License, Version 2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kafka.common.record;

clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java

@@ -24,7 +24,9 @@ import org.apache.kafka.common.protocol.MessageUtil;
 import org.apache.kafka.common.protocol.ObjectSerializationCache;
 import org.apache.kafka.common.protocol.SendBuilder;
 import org.apache.kafka.common.requests.s3.AutomqGetNodesRequest;
+import org.apache.kafka.common.requests.s3.AutomqGetPartitionSnapshotRequest;
 import org.apache.kafka.common.requests.s3.AutomqRegisterNodeRequest;
+import org.apache.kafka.common.requests.s3.AutomqUpdateGroupRequest;
 import org.apache.kafka.common.requests.s3.AutomqZoneRouterRequest;
 import org.apache.kafka.common.requests.s3.CloseStreamsRequest;
 import org.apache.kafka.common.requests.s3.CommitStreamObjectRequest;
@@ -375,10 +377,14 @@ public abstract class AbstractRequest implements AbstractRequestResponse {
                 return AutomqGetNodesRequest.parse(buffer, apiVersion);
             case AUTOMQ_ZONE_ROUTER:
                 return AutomqZoneRouterRequest.parse(buffer, apiVersion);
+            case AUTOMQ_GET_PARTITION_SNAPSHOT:
+                return AutomqGetPartitionSnapshotRequest.parse(buffer, apiVersion);
             case GET_NEXT_NODE_ID:
                 return GetNextNodeIdRequest.parse(buffer, apiVersion);
             case DESCRIBE_STREAMS:
                 return DescribeStreamsRequest.parse(buffer, apiVersion);
+            case AUTOMQ_UPDATE_GROUP:
+                return AutomqUpdateGroupRequest.parse(buffer, apiVersion);
             // AutoMQ for Kafka inject end
 
             case SHARE_GROUP_HEARTBEAT:

clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java

@@ -22,7 +22,9 @@ import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.MessageUtil;
 import org.apache.kafka.common.protocol.SendBuilder;
 import org.apache.kafka.common.requests.s3.AutomqGetNodesResponse;
+import org.apache.kafka.common.requests.s3.AutomqGetPartitionSnapshotResponse;
 import org.apache.kafka.common.requests.s3.AutomqRegisterNodeResponse;
+import org.apache.kafka.common.requests.s3.AutomqUpdateGroupResponse;
 import org.apache.kafka.common.requests.s3.AutomqZoneRouterResponse;
 import org.apache.kafka.common.requests.s3.CloseStreamsResponse;
 import org.apache.kafka.common.requests.s3.CommitStreamObjectResponse;
@@ -312,10 +314,14 @@ public abstract class AbstractResponse implements AbstractRequestResponse {
                 return AutomqGetNodesResponse.parse(responseBuffer, version);
             case AUTOMQ_ZONE_ROUTER:
                 return AutomqZoneRouterResponse.parse(responseBuffer, version);
+            case AUTOMQ_GET_PARTITION_SNAPSHOT:
+                return AutomqGetPartitionSnapshotResponse.parse(responseBuffer, version);
             case GET_NEXT_NODE_ID:
                 return GetNextNodeIdResponse.parse(responseBuffer, version);
             case DESCRIBE_STREAMS:
                 return DescribeStreamsResponse.parse(responseBuffer, version);
+            case AUTOMQ_UPDATE_GROUP:
+                return AutomqUpdateGroupResponse.parse(responseBuffer, version);
 
             // AutoMQ for Kafka inject end
             case SHARE_GROUP_HEARTBEAT:

clients/src/main/java/org/apache/kafka/common/requests/AddRaftVoterResponse.java

@@ -46,7 +46,7 @@ public class AddRaftVoterResponse extends AbstractResponse {
 
     @Override
     public void maybeSetThrottleTimeMs(int throttleTimeMs) {
-        // not supported
+        data.setThrottleTimeMs(throttleTimeMs);
     }
 
     @Override

clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java

@@ -32,26 +32,39 @@ public class ApiVersionsRequest extends AbstractRequest {
     public static class Builder extends AbstractRequest.Builder<ApiVersionsRequest> {
         private static final String DEFAULT_CLIENT_SOFTWARE_NAME = "apache-kafka-java";
 
-        private static final ApiVersionsRequestData DATA = new ApiVersionsRequestData()
+        private static final ApiVersionsRequestData DEFAULT_DATA = new ApiVersionsRequestData()
             .setClientSoftwareName(DEFAULT_CLIENT_SOFTWARE_NAME)
             .setClientSoftwareVersion(AppInfoParser.getVersion());
 
+        private final ApiVersionsRequestData data;
+
         public Builder() {
-            super(ApiKeys.API_VERSIONS);
+            this(DEFAULT_DATA,
+                ApiKeys.API_VERSIONS.oldestVersion(),
+                ApiKeys.API_VERSIONS.latestVersion());
         }
 
         public Builder(short version) {
-            super(ApiKeys.API_VERSIONS, version);
+            this(DEFAULT_DATA, version, version);
+        }
+
+        public Builder(
+            ApiVersionsRequestData data,
+            short oldestAllowedVersion,
+            short latestAllowedVersion
+        ) {
+            super(ApiKeys.API_VERSIONS, oldestAllowedVersion, latestAllowedVersion);
+            this.data = data.duplicate();
         }
 
         @Override
         public ApiVersionsRequest build(short version) {
-            return new ApiVersionsRequest(DATA, version);
+            return new ApiVersionsRequest(data, version);
         }
 
         @Override
         public String toString() {
-            return DATA.toString();
+            return data.toString();
         }
     }
 

clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java

@@ -289,19 +289,17 @@ public class ApiVersionsResponse extends AbstractResponse {
         SupportedFeatureKeyCollection converted = new SupportedFeatureKeyCollection();
         for (Map.Entry<String, SupportedVersionRange> feature : latestSupportedFeatures.features().entrySet()) {
             final SupportedVersionRange versionRange = feature.getValue();
-            final SupportedFeatureKey key = new SupportedFeatureKey();
-            key.setName(feature.getKey());
             if (alterV0 && versionRange.min() == 0) {
                 // Some older clients will have deserialization problems if a feature's
                 // minimum supported level is 0. Therefore, when preparing ApiVersionResponse
-                // at versions less than 4, we must set the minimum version for these features
-                // to 1 rather than 0. See KAFKA-17011 for details.
-                key.setMinVersion((short) 1);
+                // at versions less than 4, we must omit these features. See KAFKA-17492.
             } else {
+                final SupportedFeatureKey key = new SupportedFeatureKey();
+                key.setName(feature.getKey());
                 key.setMinVersion(versionRange.min());
+                key.setMaxVersion(versionRange.max());
+                converted.add(key);
             }
-            key.setMaxVersion(versionRange.max());
-            converted.add(key);
         }
 
         return converted;