Compare commits
8 Commits
main
...
kvrecord_e
| Author | SHA1 | Date |
|---|---|---|
1sonofqiu
|
546cf76696 | |
|
1sonofqiu
|
9013068388 | |
|
1sonofqiu
|
36a804bb1d | |
|
1sonofqiu
|
28045f41a7 | |
|
1sonofqiu
|
e709fb9cbd | |
|
1sonofqiu
|
ab077eca85 | |
|
1sonofqiu
|
9a3b426edd | |
|
1sonofqiu
|
0ef0bed34a |
|
|
@ -13,4 +13,4 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
* @superhx @Gezi-lzq @1sonofqiu @woshigaopp
|
||||
* @superhx @SCNieh @Chillax-0v0 @Gezi-lzq
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
name: AutoMQ Kafka Docker Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'AutoMQ Version Tag'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
workflows: ["GitHub Release"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
env:
|
||||
KAFKA_VERSION: "3.9.0"
|
||||
|
||||
jobs:
|
||||
automq-kafka-release:
|
||||
name: AutoMQ Kafka Docker Image Release
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [ "ubuntu-24.04" ]
|
||||
jdk: [ "17" ]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get release tag
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.tag }}" ]]; then
|
||||
TAG="${{ github.event.inputs.tag }}"
|
||||
# use the latest tag if not specified
|
||||
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
TAG=$(git ls-remote --tags https://github.com/AutoMQ/automq.git | grep -v '\^{}' | tail -1 | sed 's/.*refs\/tags\///')
|
||||
else
|
||||
TAG="${{ github.event.workflow_run.head_branch }}"
|
||||
fi
|
||||
|
||||
AUTOMQ_URL="https://github.com/AutoMQ/automq/releases/download/${TAG}/automq-${TAG}_kafka-${KAFKA_VERSION}.tgz"
|
||||
|
||||
{
|
||||
echo "AUTOMQ_VERSION=${TAG}-kafka"
|
||||
echo "AUTOMQ_URL=${AUTOMQ_URL}"
|
||||
} >> $GITHUB_ENV
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_READ_WRITE_TOKEN }}
|
||||
|
||||
- name: Build and Push Docker Image
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
.venv/bin/pip install setuptools
|
||||
|
||||
cd docker
|
||||
python3 docker_release.py \
|
||||
${{ secrets.DOCKERHUB_USERNAME }}/automq:${AUTOMQ_VERSION} \
|
||||
--kafka-url ${AUTOMQ_URL}
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
name: Docker Strimzi Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'AutoMQ Version Tag'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
workflows: ["GitHub Release"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
env:
|
||||
KAFKA_VERSION: "3.9.0"
|
||||
STRIMZI_REPO: "https://github.com/AutoMQ/strimzi-kafka-operator.git"
|
||||
STRIMZI_BRANCH: "main"
|
||||
|
||||
jobs:
|
||||
strimzi-release:
|
||||
name: Strimzi Image Release
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [ "ubuntu-24.04" ]
|
||||
jdk: ["17"]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get release tag
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.tag }}" ]]; then
|
||||
TAG="${{ github.event.inputs.tag }}"
|
||||
# use the latest tag if not specified
|
||||
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
TAG=$(git ls-remote --tags https://github.com/AutoMQ/automq.git | grep -v '\^{}' | tail -1 | sed 's/.*refs\/tags\///')
|
||||
else
|
||||
TAG="${{ github.event.workflow_run.head_branch }}"
|
||||
fi
|
||||
|
||||
AUTOMQ_URL="https://github.com/AutoMQ/automq/releases/download/${TAG}/automq-${TAG}_kafka-${KAFKA_VERSION}.tgz"
|
||||
|
||||
{
|
||||
echo "AUTOMQ_VERSION=${TAG}"
|
||||
echo "AUTOMQ_URL=${AUTOMQ_URL}"
|
||||
} >> $GITHUB_ENV
|
||||
|
||||
- name: Set up JDK ${{ matrix.jdk }}
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: ${{ matrix.jdk }}
|
||||
distribution: "zulu"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_READ_WRITE_TOKEN }}
|
||||
|
||||
- name: Build AutoMQ Strimzi Image
|
||||
run: |
|
||||
git clone --depth 1 --branch "${{ env.STRIMZI_BRANCH }}" "${{ env.STRIMZI_REPO }}" strimzi
|
||||
cd strimzi
|
||||
|
||||
chmod +x ./tools/automq/build-automq-image.sh
|
||||
./tools/automq/build-automq-image.sh \
|
||||
"${{ env.AUTOMQ_VERSION }}" \
|
||||
"${{ env.AUTOMQ_URL }}" \
|
||||
"${{ env.KAFKA_VERSION }}" \
|
||||
"${{ secrets.DOCKERHUB_USERNAME }}" \
|
||||
"automq"
|
||||
|
|
@ -57,14 +57,12 @@ jobs:
|
|||
run: ./tests/docker/run_tests.sh
|
||||
env:
|
||||
ESK_TEST_YML: ${{ inputs.test-yaml }}
|
||||
_DUCKTAPE_OPTIONS: "--deflake 4"
|
||||
shell: bash
|
||||
- name: Run E2E tests with path
|
||||
if: ${{ inputs.test-path != '' }}
|
||||
run: ./tests/docker/run_tests.sh
|
||||
env:
|
||||
TC_PATHS: ${{ inputs.test-path }}
|
||||
_DUCKTAPE_OPTIONS: "--deflake 4"
|
||||
shell: bash
|
||||
- name: Extract results
|
||||
id: extract-results
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
name: Nightly Extra E2E tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 16 * * *'
|
||||
|
||||
jobs:
|
||||
benchmarks_e2e:
|
||||
name: "Run benchmarks E2E Tests"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "benchmarks"
|
||||
test-path: "tests/kafkatest/benchmarks"
|
||||
runner: "e2e"
|
||||
connect_e2e_1:
|
||||
name: "Run connect E2E Tests 1"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect1"
|
||||
test-yaml: "tests/suites/connect_test_suite1.yml"
|
||||
runner: "e2e"
|
||||
connect_e2e_2:
|
||||
name: "Run connect E2E Tests 2"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect2"
|
||||
test-yaml: "tests/suites/connect_test_suite2.yml"
|
||||
runner: "e2e"
|
||||
connect_e2e_3:
|
||||
name: "Run connect E2E Tests 3"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect3"
|
||||
test-yaml: "tests/suites/connect_test_suite3.yml"
|
||||
runner: "e2e"
|
||||
streams_e2e:
|
||||
name: "Run streams E2E Tests"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "streams"
|
||||
test-path: "tests/kafkatest/tests/streams"
|
||||
runner: "e2e"
|
||||
e2e_summary:
|
||||
name: "E2E Tests Summary"
|
||||
runs-on: "e2e"
|
||||
if: ${{ always() && github.repository_owner == 'AutoMQ' }}
|
||||
needs: [ benchmarks_e2e, connect_e2e_1, connect_e2e_2, connect_e2e_3, streams_e2e ]
|
||||
steps:
|
||||
- name: Report results
|
||||
run: python3 tests/report_e2e_results.py
|
||||
env:
|
||||
CURRENT_REPO: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
WEB_HOOK_URL: ${{ secrets.E2E_REPORT_WEB_HOOK_URL }}
|
||||
DATA_MAP: "{\"benchmarks_e2e\": ${{ toJSON(needs.benchmarks_e2e.outputs) }}, \"connect_e2e_1\": ${{ toJSON(needs.connect_e2e_1.outputs) }}, \"connect_e2e_2\": ${{ toJSON(needs.connect_e2e_2.outputs) }}, \"connect_e2e_3\": ${{ toJSON(needs.connect_e2e_3.outputs) }}, \"streams_e2e\": ${{ toJSON(needs.streams_e2e.outputs) }}}"
|
||||
REPORT_TITLE_PREFIX: "Extra"
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
name: Nightly E2E tests
|
||||
name: Nightly Main E2E tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 16 1,7,14,21,28 * *'
|
||||
- cron: '0 16 * * *'
|
||||
|
||||
jobs:
|
||||
main_e2e_1:
|
||||
|
|
@ -45,51 +45,11 @@ jobs:
|
|||
suite-id: "main5"
|
||||
test-path: "tests/kafkatest/automq"
|
||||
runner: "e2e"
|
||||
benchmarks_e2e:
|
||||
name: "Run benchmarks E2E Tests"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "benchmarks"
|
||||
test-path: "tests/kafkatest/benchmarks"
|
||||
runner: "e2e"
|
||||
connect_e2e_1:
|
||||
name: "Run connect E2E Tests 1"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect1"
|
||||
test-yaml: "tests/suites/connect_test_suite1.yml"
|
||||
runner: "e2e"
|
||||
connect_e2e_2:
|
||||
name: "Run connect E2E Tests 2"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect2"
|
||||
test-yaml: "tests/suites/connect_test_suite2.yml"
|
||||
runner: "e2e"
|
||||
connect_e2e_3:
|
||||
name: "Run connect E2E Tests 3"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "connect3"
|
||||
test-yaml: "tests/suites/connect_test_suite3.yml"
|
||||
runner: "e2e"
|
||||
streams_e2e:
|
||||
name: "Run streams E2E Tests"
|
||||
uses: ./.github/workflows/e2e-run.yml
|
||||
if: ${{ github.repository_owner == 'AutoMQ' }}
|
||||
with:
|
||||
suite-id: "streams"
|
||||
test-path: "tests/kafkatest/tests/streams"
|
||||
runner: "e2e"
|
||||
e2e_summary:
|
||||
runs-on: "e2e"
|
||||
name: "E2E Tests Summary"
|
||||
if: ${{ always() && github.repository_owner == 'AutoMQ' }}
|
||||
needs: [ main_e2e_1, main_e2e_2, main_e2e_3, main_e2e_4, main_e2e_5, benchmarks_e2e, connect_e2e_1, connect_e2e_2, connect_e2e_3, streams_e2e ]
|
||||
needs: [ main_e2e_1, main_e2e_2, main_e2e_3, main_e2e_4, main_e2e_5 ]
|
||||
steps:
|
||||
- name: Report results
|
||||
run: python3 tests/report_e2e_results.py
|
||||
|
|
@ -97,5 +57,5 @@ jobs:
|
|||
CURRENT_REPO: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
WEB_HOOK_URL: ${{ secrets.E2E_REPORT_WEB_HOOK_URL }}
|
||||
DATA_MAP: "{\"main_e2e_1\": ${{ toJSON(needs.main_e2e_1.outputs) }}, \"main_e2e_2\": ${{ toJSON(needs.main_e2e_2.outputs) }}, \"main_e2e_3\": ${{ toJSON(needs.main_e2e_3.outputs) }}, \"main_e2e_4\": ${{ toJSON(needs.main_e2e_4.outputs) }}, \"main_e2e_5\": ${{ toJSON(needs.main_e2e_5.outputs) }}, \"benchmarks_e2e\": ${{ toJSON(needs.benchmarks_e2e.outputs) }}, \"connect_e2e_1\": ${{ toJSON(needs.connect_e2e_1.outputs) }}, \"connect_e2e_2\": ${{ toJSON(needs.connect_e2e_2.outputs) }}, \"connect_e2e_3\": ${{ toJSON(needs.connect_e2e_3.outputs) }}, \"streams_e2e\": ${{ toJSON(needs.streams_e2e.outputs) }}}"
|
||||
DATA_MAP: "{\"main_e2e_1\": ${{ toJSON(needs.main_e2e_1.outputs) }}, \"main_e2e_2\": ${{ toJSON(needs.main_e2e_2.outputs) }}, \"main_e2e_3\": ${{ toJSON(needs.main_e2e_3.outputs) }}, \"main_e2e_4\": ${{ toJSON(needs.main_e2e_4.outputs) }}, \"main_e2e_5\": ${{ toJSON(needs.main_e2e_5.outputs) }}}"
|
||||
REPORT_TITLE_PREFIX: "Main"
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Publish Maven Package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version to publish'
|
||||
required: true
|
||||
push:
|
||||
tags:
|
||||
- '[0-9]+.[0-9]+.[0-9]+'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
|
||||
|
||||
env:
|
||||
VERSION: ${{ github.event.inputs.version || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: "Publish to Github Packages"
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ ubuntu-22.04 ]
|
||||
jdk: [ 17 ]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Gradle wrapper validation
|
||||
uses: gradle/actions/wrapper-validation@v3
|
||||
- name: Set up JDK ${{ matrix.jdk }}
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: ${{ matrix.jdk }}
|
||||
distribution: "zulu"
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v4
|
||||
with:
|
||||
gradle-version: '8.10'
|
||||
- name: Publish
|
||||
run: |
|
||||
gradle publish -PmavenUrl='https://maven.pkg.github.com/AutoMQ/automq' \
|
||||
-PmavenUsername=${{ env.GITHUB_ACTOR }} -PmavenPassword=${{ secrets.GITHUB_TOKEN }} \
|
||||
-PskipSigning=true \
|
||||
-Pgroup=com.automq.automq -Pversion=${{ env.VERSION }}
|
||||
|
|
@ -9,14 +9,6 @@ or [Slack](https://join.slack.com/t/automq/shared_invite/zt-29h17vye9-thf31ebIVL
|
|||
Before getting started, please review AutoMQ's Code of Conduct. Everyone interacting in Slack or WeChat
|
||||
follow [Code of Conduct](CODE_OF_CONDUCT.md).
|
||||
|
||||
## Suggested Onboarding Path for New Contributors
|
||||
|
||||
If you are new to AutoMQ, it is recommended to first deploy and run AutoMQ using Docker as described in the README.
|
||||
This helps you quickly understand AutoMQ’s core concepts and behavior without local environment complexity.
|
||||
|
||||
After gaining familiarity, contributors who want to work on code can follow the steps in this guide to build and run AutoMQ locally.
|
||||
|
||||
|
||||
## Code Contributions
|
||||
|
||||
### Finding or Reporting Issues
|
||||
|
|
|
|||
121
README.md
121
README.md
|
|
@ -1,105 +1,50 @@
|
|||
# A Diskless Kafka® on S3, Offering 10x Cost Savings and Scaling in Seconds.
|
||||
# AutoMQ: A stateless Kafka® on S3, offering 10x cost savings and scaling in seconds.
|
||||
|
||||
<div align="center">
|
||||
<p align="center">
|
||||
📑  <a
|
||||
href="https://www.automq.com/docs/automq/what-is-automq/overview?utm_source=github_automq"
|
||||
href="https://docs.automq.com/docs/automq-opensource/HSiEwHVfdiO7rWk34vKcVvcvn2Z?utm_source=github"
|
||||
target="_blank"
|
||||
><b>Documentation</b></a>   
|
||||
🔥  <a
|
||||
href="https://www.automq.com/docs/automq-cloud/getting-started/install-byoc-environment/aws/install-env-from-marketplace?utm_source=github_automq"
|
||||
href="https://www.automq.com/docs/automq-cloud/getting-started/install-byoc-environment/aws/install-env-from-marketplace"
|
||||
target="_blank"
|
||||
><b>Free trial of AutoMQ on AWS</b></a>   
|
||||
</p>
|
||||
|
||||
[](https://www.linkedin.com/company/automq)
|
||||
[](https://go.automq.com/slack)
|
||||
[-yellow)](https://www.automq.com/blog/automq-vs-apache-kafka-a-real-aws-cloud-bill-comparison?utm_source=github_automq)
|
||||
[-orange)](https://www.automq.com/docs/automq/benchmarks/automq-vs-apache-kafka-benchmarks-and-cost?utm_source=github_automq)
|
||||
[](https://twitter.com/intent/follow?screen_name=AutoMQ_Lab)
|
||||
[](https://join.slack.com/t/automq/shared_invite/zt-29h17vye9-thf31ebIVL9oXuRdACnOIA)
|
||||
[-yellow)](https://www.automq.com/blog/automq-vs-apache-kafka-a-real-aws-cloud-bill-comparison)
|
||||
[-orange)](https://docs.automq.com/docs/automq-opensource/IJLQwnVROiS5cUkXfF0cuHnWnNd)
|
||||
[](https://gurubase.io/g/automq)
|
||||
[](https://deepwiki.com/AutoMQ/automq)
|
||||
|
||||
<a href="https://trendshift.io/repositories/9782" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9782" alt="AutoMQ%2Fautomq | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
|
||||
<img width="97%" alt="automq-solgan" src="https://github.com/user-attachments/assets/bdf6c5f5-7fe1-4004-8e15-54f1aa6bc32f" />
|
||||
|
||||
<a href="https://www.youtube.com/watch?v=IB8sh639Rsg" target="_blank">
|
||||
<img alt="Grab" src="https://github.com/user-attachments/assets/01668da4-3916-4f49-97af-18f91b25f8c1" width="19%" />
|
||||
</a>
|
||||
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="Avia" src="https://github.com/user-attachments/assets/d2845e1c-caf4-444a-93f0-97b13c9c8490" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="Tencent" src="https://github.com/user-attachments/assets/2bdd205f-38c1-4110-9af1-d4c782db3395" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="Honda" src="https://github.com/user-attachments/assets/ee65af29-8ee3-404b-bf81-a004fe0c327c" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="Trip" src="https://github.com/user-attachments/assets/0cb4ae63-6dc1-43dc-9416-625a08dca2e5" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="LG" src="https://github.com/user-attachments/assets/ed9e0f87-abc6-4552-977c-f342ecb105a0" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/blog/jdcom-automq-cubefs-trillion-scale-kafka-messaging" target="_blank">
|
||||
<img alt="JD" src="https://github.com/user-attachments/assets/a7a86d2c-66fa-4575-b181-6cf56a31f880" width="19%" />
|
||||
</a>
|
||||
|
||||
<a href="https://www.automq.com/blog/automq-help-geely-auto-solve-the-pain-points-of-kafka-elasticity-in-the-v2x-scenario" target="_blank">
|
||||
<img alt="Geely" src="https://github.com/user-attachments/assets/d61f7c51-0d80-4290-a428-a941441c7ec9" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/blog/dewu-builds-trillion-level-monitoring-system-based-on-automq" target="_blank">
|
||||
<img alt="Poizon" src="https://github.com/user-attachments/assets/45f4c642-0495-4bcc-9224-d2c5c2b2f0d5" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="Bitkub" src="https://github.com/user-attachments/assets/3b95cd26-973d-4405-9d2c-289c5807bb39" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="PalmPay" src="https://github.com/user-attachments/assets/b22f70f5-7553-4283-ac20-f034868b0121" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/blog/automq-vs-kafka-evaluation-and-comparison-by-little-red-book" target="_blank">
|
||||
<img alt="RedNote" src="https://github.com/user-attachments/assets/4a62f1f3-e171-4d58-9d7e-ebabad6f8e23" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/blog/xpeng-motors-reduces-costs-by-50-by-replacing-kafka-with-automq" target="_blank">
|
||||
<img alt="XPENG" src="https://github.com/user-attachments/assets/8b32c484-a4bf-4793-80d0-f454da254337" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="OPPO" src="https://github.com/user-attachments/assets/2b6d3cf0-ae54-4073-bc06-c6623e31c6d0" width="19%" />
|
||||
</a>
|
||||
<a href="https://www.automq.com/customer" target="_blank">
|
||||
<img alt="BambuLab" src="https://github.com/user-attachments/assets/d09ded1b-3696-49ac-b38f-d02f9598b3bb" width="19%" />
|
||||
</a>
|
||||
</div>
|
||||
## 👥 Big Companies Worldwide are Using AutoMQ
|
||||
> Here are some of our major customers worldwide using AutoMQ.
|
||||
|
||||
<img width="1151" alt="image" src="https://github.com/user-attachments/assets/a2668e5e-eebf-479a-b85a-9611de1b60c8" />
|
||||
|
||||
- [Grab: Driving Efficiency with AutoMQ in DataStreaming Platform](https://www.youtube.com/watch?v=IB8sh639Rsg)
|
||||
- [JD.com x AutoMQ x CubeFS: A Cost-Effective Journey](https://www.automq.com/blog/jdcom-automq-cubefs-trillion-scale-kafka-messaging?utm_source=github_automq)
|
||||
- [Palmpay Uses AutoMQ to Replace Kafka, Optimizing Costs by 50%+](https://www.automq.com/blog/palmpay-uses-automq-to-replace-kafka?utm_source=github_automq)
|
||||
- [AutoMQ help Geely Auto(Fortune Global 500) solve the pain points of Kafka elasticity in the V2X scenario](https://www.automq.com/blog/automq-help-geely-auto-solve-the-pain-points-of-kafka-elasticity-in-the-v2x-scenario?utm_source=github_automq)
|
||||
- [How Asia’s Quora Zhihu uses AutoMQ to reduce Kafka cost and maintenance complexity](https://www.automq.com/blog/how-asias-quora-zhihu-use-automq-to-reduce-kafka-cost-and-maintenance-complexity?utm_source=github_automq)
|
||||
- [XPENG Motors Reduces Costs by 50%+ by Replacing Kafka with AutoMQ](https://www.automq.com/blog/xpeng-motors-reduces-costs-by-50-by-replacing-kafka-with-automq?utm_source=github_automq)
|
||||
- [Asia's GOAT, Poizon uses AutoMQ Kafka to build observability platform for massive data(30 GB/s)](https://www.automq.com/blog/asiax27s-goat-poizon-uses-automq-kafka-to-build-a-new-generation-observability-platform-for-massive-data?utm_source=github_automq)
|
||||
- [AutoMQ Helps CaoCao Mobility Address Kafka Scalability During Holidays](https://www.automq.com/blog/automq-helps-caocao-mobility-address-kafka-scalability-issues-during-mid-autumn-and-national-day?utm_source=github_automq)
|
||||
|
||||
|
||||
### Prerequisites
|
||||
Before running AutoMQ locally, please ensure:
|
||||
- Docker version 20.x or later
|
||||
- Docker Compose v2
|
||||
- At least 4 GB RAM allocated to Docker
|
||||
- Ports 9092 and 9000 are available on your system
|
||||
- [JD.com x AutoMQ x CubeFS: A Cost-Effective Journey](https://www.automq.com/blog/jdcom-automq-cubefs-trillion-scale-kafka-messaging)
|
||||
- [Palmpay Uses AutoMQ to Replace Kafka, Optimizing Costs by 50%+](https://www.automq.com/blog/palmpay-uses-automq-to-replace-kafka)
|
||||
- [AutoMQ help Geely Auto(Fortune Global 500) solve the pain points of Kafka elasticity in the V2X scenario](https://www.automq.com/blog/automq-help-geely-auto-solve-the-pain-points-of-kafka-elasticity-in-the-v2x-scenario)
|
||||
- [How Asia’s Quora Zhihu uses AutoMQ to reduce Kafka cost and maintenance complexity](https://www.automq.com/blog/how-asias-quora-zhihu-use-automq-to-reduce-kafka-cost-and-maintenance-complexity)
|
||||
- [XPENG Motors Reduces Costs by 50%+ by Replacing Kafka with AutoMQ](https://www.automq.com/blog/xpeng-motors-reduces-costs-by-50-by-replacing-kafka-with-automq)
|
||||
- [Asia's GOAT, Poizon uses AutoMQ Kafka to build observability platform for massive data(30 GB/s)](https://www.automq.com/blog/asiax27s-goat-poizon-uses-automq-kafka-to-build-a-new-generation-observability-platform-for-massive-data)
|
||||
- [AutoMQ Helps CaoCao Mobility Address Kafka Scalability During Holidays](https://www.automq.com/blog/automq-helps-caocao-mobility-address-kafka-scalability-issues-during-mid-autumn-and-national-day)
|
||||
|
||||
## ⛄ Get started with AutoMQ
|
||||
|
||||
> [!Tip]
|
||||
> Deploying a production-ready AutoMQ cluster is challenging. This Quick Start is only for evaluating AutoMQ features and is not suitable for production use. For production deployment best practices, please [contact](https://www.automq.com/contact) our community for support.
|
||||
|
||||
The `docker/docker-compose.yaml` file provides a simple single-node setup for quick evaluation and development:
|
||||
```shell
|
||||
curl -O https://raw.githubusercontent.com/AutoMQ/automq/refs/tags/1.5.5/docker/docker-compose.yaml && docker compose -f docker-compose.yaml up -d
|
||||
docker compose -f docker/docker-compose.yaml up -d
|
||||
```
|
||||
This setup features a single AutoMQ node serving as both controller and broker, alongside MinIO for S3 storage. All services operate within a Docker bridge network called `automq_net`, allowing you to start a Kafka producer in this network to test AutoMQ:
|
||||
```shell
|
||||
|
|
@ -109,19 +54,19 @@ docker run --network automq_net automqinc/automq:latest /bin/bash -c \
|
|||
```
|
||||
After testing, you can destroy the setup with:
|
||||
```shell
|
||||
docker compose -f docker-compose.yaml down
|
||||
docker compose -f docker/docker-compose.yaml down
|
||||
```
|
||||
The `docker/docker-compose-cluster.yaml` file offers a more complex setup with three AutoMQ nodes, ideal for testing AutoMQ's cluster features, and can be run in the same way.
|
||||
|
||||
There are more deployment options available:
|
||||
- [Deploy Multi-Nodes Test Cluster on Docker](https://www.automq.com/docs/automq/getting-started/deploy-multi-nodes-test-cluster-on-docker?utm_source=github_automq)
|
||||
- [Deploy on Linux with 5 Nodes](https://www.automq.com/docs/automq/deployment/deploy-multi-nodes-cluster-on-linux?utm_source=github_automq)
|
||||
- [Deploy on Kubernetes](https://www.automq.com/docs/automq/deployment/deploy-multi-nodes-cluster-on-kubernetes?utm_source=github_automq)
|
||||
- [Try AutoMQ on AWS Marketplace (Two Weeks Free Trial)](https://docs.automq.com/automq-cloud/getting-started/install-byoc-environment/aws/install-env-from-marketplace?utm_source=github_automq)
|
||||
- [Deploy on Linux with 5 Nodes](https://www.automq.com/docs/automq/getting-started/cluster-deployment-on-linux)
|
||||
- [Deploy on Kubernetes (Enterprise only now, open source soon)](https://www.automq.com/docs/automq/getting-started/cluster-deployment-on-kubernetes)
|
||||
- [Run on Ceph / MinIO / CubeFS / HDFS](https://www.automq.com/docs/automq/deployment/overview)
|
||||
- [Try AutoMQ on AWS Marketplace (Two Weeks Free Trial)](https://docs.automq.com/automq-cloud/getting-started/install-byoc-environment/aws/install-env-from-marketplace)
|
||||
- [Try AutoMQ on Alibaba Cloud Marketplace (Two Weeks Free Trial)](https://market.aliyun.com/products/55530001/cmgj00065841.html)
|
||||
|
||||
## 🗞️ Newest Feature - Table Topic
|
||||
Table Topic is a new feature in AutoMQ that combines stream and table functionalities to unify streaming and data analysis. Currently, it supports Apache Iceberg and integrates with catalog services such as AWS Glue, HMS, and the Rest catalog. Additionally, it natively supports S3 tables, a new AWS product announced at the 2024 re:Invent. [Learn more](https://www.automq.com/blog/automq-table-topic-seamless-integration-with-s3-tables-and-iceberg?utm_source=github_automq).
|
||||
Table Topic is a new feature in AutoMQ that combines stream and table functionalities to unify streaming and data analysis. Currently, it supports Apache Iceberg and integrates with catalog services such as AWS Glue, HMS, and the Rest catalog. Additionally, it natively supports S3 tables, a new AWS product announced at the 2024 re:Invent. [Learn more](https://www.automq.com/blog/automq-table-topic-seamless-integration-with-s3-tables-and-iceberg).
|
||||
|
||||
|
||||
|
||||
|
|
@ -129,7 +74,7 @@ Table Topic is a new feature in AutoMQ that combines stream and table functional
|
|||
AutoMQ is a stateless Kafka alternative that runs on S3 or any S3-compatible storage, such as MinIO. It is designed to address two major issues of Apache Kafka. First, Kafka clusters are difficult to scale out or in due to the stateful nature of its brokers. Data movement is required, and even reassigning partitions between brokers is a complex process. Second, hosting Kafka in the cloud can be prohibitively expensive. You face high costs for EBS storage, cross-AZ traffic, and significant over-provisioning due to Kafka's limited scalability.
|
||||
|
||||
Here are some key highlights of AutoMQ that make it an ideal choice to replace your Apache Kafka cluster, whether in the cloud or on-premise, as long as you have S3-compatible storage:
|
||||
- **Cost effective**: The first true cloud-native streaming storage system, designed for optimal cost and efficiency on the cloud. Refer to [this report](https://www.automq.com/docs/automq/benchmarks/cost-effective-automq-vs-apache-kafka?utm_source=github_automq) to see how we cut Apache Kafka billing by 90% on the cloud.
|
||||
- **Cost effective**: The first true cloud-native streaming storage system, designed for optimal cost and efficiency on the cloud. Refer to [this report](https://www.automq.com/docs/automq/benchmarks/cost-effective-automq-vs-apache-kafka) to see how we cut Apache Kafka billing by 90% on the cloud.
|
||||
- **High Reliability**: Leverage object storage service to achieve zero RPO, RTO in seconds and 99.999999999% durability.
|
||||
- **Zero Cross-AZ Traffic**: By using cloud object storage as the priority storage solution, AutoMQ eliminates cross-AZ traffic costs on AWS and GCP. In traditional Kafka setups, over 80% of costs arise from cross-AZ traffic, including producer, consumer, and replication sides.
|
||||
- **Serverless**:
|
||||
|
|
@ -138,9 +83,9 @@ Here are some key highlights of AutoMQ that make it an ideal choice to replace y
|
|||
- Infinite scalable: Utilize cloud object storage as the primary storage solution, eliminating concerns about storage capacity.
|
||||
- **Manage-less**: The built-in auto-balancer component automatically schedules partitions and network traffic between brokers, eliminating manual partition reassignment.
|
||||
- **High performance**:
|
||||
- High throughput: Leverage pre-fetching, batch processing, and parallel technologies to maximize the capabilities of cloud object storage. Refer to the [AutoMQ Performance White Paper](https://www.automq.com/docs/automq/benchmarks/automq-vs-apache-kafka-benchmarks-and-cost?utm_source=github_automq) to see how we achieve this.
|
||||
- Low Latency: AutoMQ defaults to running on S3 directly, resulting in hundreds of milliseconds of latency. The enterprise version offers single-digit millisecond latency. [Contact us](https://www.automq.com/contact?utm_source=github_automq) for more details.
|
||||
- **Built-in Metrics Export**: Natively export Prometheus and OpenTelemetry metrics, supporting both push and pull. Ditch inefficient JMX and monitor your cluster with modern tools. Refer to [full metrics list](https://www.automq.com/docs/automq/observability/metrics?utm_source=github_automq) provided by AutoMQ.
|
||||
- High throughput: Leverage pre-fetching, batch processing, and parallel technologies to maximize the capabilities of cloud object storage. Refer to the [AutoMQ Performance White Paper](https://www.automq.com/docs/automq/benchmarks/benchmark-automq-vs-apache-kafka) to see how we achieve this.
|
||||
- Low Latency: AutoMQ defaults to running on S3 directly, resulting in hundreds of milliseconds of latency. The enterprise version offers single-digit millisecond latency. [Contact us](https://www.automq.com/contact) for more details.
|
||||
- **Built-in Metrics Export**: Natively export Prometheus and OpenTelemetry metrics, supporting both push and pull. Ditch inefficient JMX and monitor your cluster with modern tools. Refer to [full metrics list](https://www.automq.com/docs/automq/observability/metrics) provided by AutoMQ.
|
||||
- **100% Kafka Compatible**: Fully compatible with Apache Kafka, offering all features with greater cost-effectiveness and operational efficiency.
|
||||
|
||||
## ✨Architecture
|
||||
|
|
@ -154,7 +99,7 @@ Regarding the architecture of AutoMQ, it is fundamentally different from Kafka.
|
|||
- Auto Balancer: a component that automatically balances traffic and partitions between brokers, eliminating the need for manual reassignment. Unlike Kafka, this built-in feature removes the need for cruise control.
|
||||
- Rack-aware Router: Kafka has long faced cross-AZ traffic fees on AWS and GCP. Our shared storage architecture addresses this by using a rack-aware router to provide clients in different AZs with specific partition metadata, avoiding cross-AZ fees while exchanging data through object storage.
|
||||
|
||||
For more on AutoMQ's architecture, visit [AutoMQ Architecture](https://www.automq.com/docs/automq/architecture/overview?utm_source=github_automq) or explore the source code directly.
|
||||
For more on AutoMQ's architecture, visit [AutoMQ Architecture](https://docs.automq.com/automq/architecture/overview) or explore the source code directly.
|
||||
|
||||
## 🌟 Stay Ahead
|
||||
Star AutoMQ on GitHub for instant updates on new releases.
|
||||
|
|
@ -163,7 +108,7 @@ Star AutoMQ on GitHub for instant updates on new releases.
|
|||
## 💬 Community
|
||||
You can join the following groups or channels to discuss or ask questions about AutoMQ:
|
||||
- Ask questions or report a bug by [GitHub Issues](https://github.com/AutoMQ/automq/issues)
|
||||
- Discuss about AutoMQ or Kafka by [Slack](https://go.automq.com/slack) or [Wechat Group](docs/images/automq-wechat.png)
|
||||
- Discuss about AutoMQ or Kafka by [Slack](https://join.slack.com/t/automq/shared_invite/zt-29h17vye9-thf31ebIVL9oXuRdACnOIA) or [Wechat Group](docs/images/automq-wechat.png)
|
||||
|
||||
|
||||
## 👥 How to contribute
|
||||
|
|
@ -172,9 +117,9 @@ To contribute to AutoMQ please see [Code of Conduct](CODE_OF_CONDUCT.md) and [Co
|
|||
We have a list of [good first issues](https://github.com/AutoMQ/automq/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) that help you to get started, gain experience, and get familiar with our contribution process.
|
||||
|
||||
## 👍 AutoMQ Enterprise Edition
|
||||
The enterprise edition of AutoMQ offers a robust, user-friendly control plane for seamless cluster management, with enhanced availability and observability over the open-source version. Additionally, we offer [Kafka Linking](https://www.automq.com/solutions/kafka-linking?utm_source=github_automq) for zero-downtime migration from any Kafka-compatible cluster to AutoMQ.
|
||||
The enterprise edition of AutoMQ offers a robust, user-friendly control plane for seamless cluster management, with enhanced availability and observability over the open-source version. Additionally, we offer [Kafka Linking](https://www.automq.com/solutions/kafka-linking) for zero-downtime migration from any Kafka-compatible cluster to AutoMQ.
|
||||
|
||||
[Contact us](https://www.automq.com/contact?utm_source=github_automq) for more information about the AutoMQ enterprise edition, and we'll gladly assist with your free trial.
|
||||
[Contact us](https://www.automq.com/contact) for more information about the AutoMQ enterprise edition, and we'll gladly assist with your free trial.
|
||||
|
||||
## 📜 License
|
||||
AutoMQ is under the Apache 2.0 license. See the [LICENSE](https://github.com/AutoMQ/automq/blob/main/LICENSE) file for details.
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
# AutoMQ Log Uploader Module
|
||||
|
||||
This module provides asynchronous S3 log upload capability based on Log4j 1.x. Other submodules only need to depend on this module and configure it simply to synchronize logs to object storage. Core components:
|
||||
|
||||
- `com.automq.log.S3RollingFileAppender`: Extends `RollingFileAppender`, pushes log events to the uploader while writing to local files.
|
||||
- `com.automq.log.uploader.LogUploader`: Asynchronously buffers, compresses, and uploads logs; supports configuration switches and periodic cleanup.
|
||||
- `com.automq.log.uploader.S3LogConfig`: Interface that abstracts the configuration required for uploading. Implementations must provide cluster ID, node ID, object storage instance, and leadership status.
|
||||
|
||||
## Quick Integration
|
||||
|
||||
1. Add dependency in your module's `build.gradle`:
|
||||
```groovy
|
||||
implementation project(':automq-log-uploader')
|
||||
```
|
||||
2. Implement or provide an `S3LogConfig` instance and configure the appender:
|
||||
|
||||
```java
|
||||
// Set up the S3LogConfig through your application
|
||||
S3LogConfig config = // your S3LogConfig implementation
|
||||
S3RollingFileAppender.setup(config);
|
||||
```
|
||||
3. Reference the Appender in `log4j.properties`:
|
||||
|
||||
```properties
|
||||
log4j.appender.s3_uploader=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.s3_uploader.File=logs/server.log
|
||||
log4j.appender.s3_uploader.MaxFileSize=100MB
|
||||
log4j.appender.s3_uploader.MaxBackupIndex=10
|
||||
log4j.appender.s3_uploader.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.s3_uploader.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
```
|
||||
|
||||
## S3LogConfig Interface
|
||||
|
||||
The `S3LogConfig` interface provides the configuration needed for log uploading:
|
||||
|
||||
```java
|
||||
public interface S3LogConfig {
|
||||
boolean isEnabled(); // Whether S3 upload is enabled
|
||||
String clusterId(); // Cluster identifier
|
||||
int nodeId(); // Node identifier
|
||||
ObjectStorage objectStorage(); // S3 object storage instance
|
||||
boolean isLeader(); // Whether this node should upload logs
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
The upload schedule can be overridden by environment variables:
|
||||
|
||||
- `AUTOMQ_OBSERVABILITY_UPLOAD_INTERVAL`: Maximum upload interval (milliseconds).
|
||||
- `AUTOMQ_OBSERVABILITY_CLEANUP_INTERVAL`: Retention period (milliseconds), old objects earlier than this time will be cleaned up.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
### Leader Selection
|
||||
|
||||
The log uploader relies on the `S3LogConfig.isLeader()` method to determine whether the current node should upload logs and perform cleanup tasks. This avoids multiple nodes in a cluster simultaneously executing these operations.
|
||||
|
||||
### Object Storage Path
|
||||
|
||||
Logs are uploaded to object storage following this path pattern:
|
||||
```
|
||||
automq/logs/{clusterId}/{nodeId}/{hour}/{uuid}
|
||||
```
|
||||
|
||||
Where:
|
||||
- `clusterId` and `nodeId` come from the S3LogConfig
|
||||
- `hour` is the timestamp hour for log organization
|
||||
- `uuid` is a unique identifier for each log batch
|
||||
|
||||
## Usage Example
|
||||
|
||||
Complete example of using the log uploader:
|
||||
|
||||
```java
|
||||
import com.automq.log.S3RollingFileAppender;
|
||||
import com.automq.log.uploader.S3LogConfig;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
|
||||
// Implement S3LogConfig
|
||||
public class MyS3LogConfig implements S3LogConfig {
|
||||
@Override
|
||||
public boolean isEnabled() {
|
||||
return true; // Enable S3 upload
|
||||
}
|
||||
|
||||
@Override
|
||||
public String clusterId() {
|
||||
return "my-cluster";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nodeId() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() {
|
||||
// Return your ObjectStorage instance
|
||||
return myObjectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLeader() {
|
||||
// Return true if this node should upload logs
|
||||
return isCurrentNodeLeader();
|
||||
}
|
||||
}
|
||||
|
||||
// Setup and use
|
||||
S3LogConfig config = new MyS3LogConfig();
|
||||
S3RollingFileAppender.setup(config);
|
||||
|
||||
// Configure Log4j to use the appender
|
||||
// The appender will now automatically upload logs to S3
|
||||
```
|
||||
|
||||
## Lifecycle Management
|
||||
|
||||
Remember to properly shutdown the log uploader when your application terminates:
|
||||
|
||||
```java
|
||||
// During application shutdown
|
||||
S3RollingFileAppender.shutdown();
|
||||
```
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.log;
|
||||
|
||||
import com.automq.log.uploader.LogRecorder;
|
||||
import com.automq.log.uploader.LogUploader;
|
||||
import com.automq.log.uploader.S3LogConfig;
|
||||
|
||||
import org.apache.log4j.RollingFileAppender;
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class S3RollingFileAppender extends RollingFileAppender {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(S3RollingFileAppender.class);
|
||||
private static final Object INIT_LOCK = new Object();
|
||||
|
||||
private static volatile LogUploader logUploaderInstance;
|
||||
private static volatile S3LogConfig s3LogConfig;
|
||||
|
||||
public S3RollingFileAppender() {
|
||||
super();
|
||||
}
|
||||
|
||||
public static void setup(S3LogConfig config) {
|
||||
s3LogConfig = config;
|
||||
synchronized (INIT_LOCK) {
|
||||
if (logUploaderInstance != null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (s3LogConfig == null) {
|
||||
LOGGER.error("No s3LogConfig available; S3 log upload remains disabled.");
|
||||
throw new RuntimeException("S3 log configuration is missing.");
|
||||
}
|
||||
if (!s3LogConfig.isEnabled() || s3LogConfig.objectStorage() == null) {
|
||||
LOGGER.warn("S3 log upload is disabled by configuration.");
|
||||
return;
|
||||
}
|
||||
|
||||
LogUploader uploader = new LogUploader();
|
||||
uploader.start(s3LogConfig);
|
||||
logUploaderInstance = uploader;
|
||||
LOGGER.info("S3RollingFileAppender initialized successfully using s3LogConfig {}.", s3LogConfig.getClass().getName());
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to initialize S3RollingFileAppender", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void shutdown() {
|
||||
if (logUploaderInstance != null) {
|
||||
synchronized (INIT_LOCK) {
|
||||
if (logUploaderInstance != null) {
|
||||
try {
|
||||
logUploaderInstance.close();
|
||||
logUploaderInstance = null;
|
||||
LOGGER.info("S3RollingFileAppender log uploader closed successfully.");
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to close S3RollingFileAppender log uploader", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void subAppend(LoggingEvent event) {
|
||||
super.subAppend(event);
|
||||
if (!closed && logUploaderInstance != null) {
|
||||
LogRecorder.LogEvent logEvent = new LogRecorder.LogEvent(
|
||||
event.getTimeStamp(),
|
||||
event.getLevel().toString(),
|
||||
event.getLoggerName(),
|
||||
event.getRenderedMessage(),
|
||||
event.getThrowableStrRep());
|
||||
|
||||
try {
|
||||
logEvent.validate();
|
||||
logUploaderInstance.append(logEvent);
|
||||
} catch (IllegalArgumentException e) {
|
||||
errorHandler.error("Failed to validate and append log event", e, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.log.uploader.util;
|
||||
|
||||
import com.automq.stream.s3.ByteBufAlloc;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
|
||||
public class Utils {
|
||||
|
||||
private Utils() {
|
||||
}
|
||||
|
||||
public static ByteBuf compress(ByteBuf input) throws IOException {
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
|
||||
byte[] buffer = new byte[input.readableBytes()];
|
||||
input.readBytes(buffer);
|
||||
gzipOutputStream.write(buffer);
|
||||
}
|
||||
|
||||
ByteBuf compressed = ByteBufAlloc.byteBuffer(byteArrayOutputStream.size());
|
||||
compressed.writeBytes(byteArrayOutputStream.toByteArray());
|
||||
return compressed;
|
||||
}
|
||||
|
||||
public static ByteBuf decompress(ByteBuf input) throws IOException {
|
||||
byte[] compressedData = new byte[input.readableBytes()];
|
||||
input.readBytes(compressedData);
|
||||
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData);
|
||||
|
||||
try (GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream);
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
byte[] buffer = new byte[1024];
|
||||
int bytesRead;
|
||||
while ((bytesRead = gzipInputStream.read(buffer)) != -1) {
|
||||
byteArrayOutputStream.write(buffer, 0, bytesRead);
|
||||
}
|
||||
|
||||
byte[] uncompressedData = byteArrayOutputStream.toByteArray();
|
||||
ByteBuf output = ByteBufAlloc.byteBuffer(uncompressedData.length);
|
||||
output.writeBytes(uncompressedData);
|
||||
return output;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,459 +0,0 @@
|
|||
# AutoMQ automq-metrics Module
|
||||
|
||||
## Module Structure
|
||||
|
||||
```
|
||||
com.automq.opentelemetry/
|
||||
├── AutoMQTelemetryManager.java # Main management class for initialization and lifecycle
|
||||
├── TelemetryConstants.java # Constants definition
|
||||
├── common/
|
||||
│ ├── OTLPCompressionType.java # OTLP compression types
|
||||
│ └── OTLPProtocol.java # OTLP protocol types
|
||||
├── exporter/
|
||||
│ ├── MetricsExporter.java # Exporter interface
|
||||
│ ├── MetricsExportConfig.java # Export configuration
|
||||
│ ├── MetricsExporterProvider.java # Exporter factory provider
|
||||
│ ├── MetricsExporterType.java # Exporter type enumeration
|
||||
│ ├── MetricsExporterURI.java # URI parser for exporters
|
||||
│ ├── OTLPMetricsExporter.java # OTLP exporter implementation
|
||||
│ ├── PrometheusMetricsExporter.java # Prometheus exporter implementation
|
||||
│ └── s3/ # S3 metrics exporter implementation
|
||||
│ ├── CompressionUtils.java # Utility for data compression
|
||||
│ ├── PrometheusUtils.java # Utilities for Prometheus format
|
||||
│ ├── S3MetricsExporter.java # S3 metrics exporter implementation
|
||||
│ └── S3MetricsExporterAdapter.java # Adapter to handle S3 metrics export
|
||||
└── yammer/
|
||||
├── DeltaHistogram.java # Delta histogram implementation
|
||||
├── OTelMetricUtils.java # OpenTelemetry metrics utilities
|
||||
├── YammerMetricsProcessor.java # Yammer metrics processor
|
||||
└── YammerMetricsReporter.java # Yammer metrics reporter
|
||||
```
|
||||
|
||||
The AutoMQ OpenTelemetry module is a telemetry data collection and export component based on OpenTelemetry SDK, specifically designed for AutoMQ Kafka. This module provides unified telemetry data management capabilities, supporting the collection of JVM metrics, JMX metrics, and Yammer metrics, and can export data to Prometheus, OTLP-compatible backend systems, or S3-compatible storage.
|
||||
|
||||
## Core Features
|
||||
|
||||
### 1. Metrics Collection
|
||||
- **JVM Metrics**: Automatically collect JVM runtime metrics including CPU, memory pools, garbage collection, threads, etc.
|
||||
- **JMX Metrics**: Define and collect JMX Bean metrics through configuration files
|
||||
- **Yammer Metrics**: Bridge existing Kafka Yammer metrics system to OpenTelemetry
|
||||
|
||||
### 2. Multiple Exporter Support
|
||||
- **Prometheus**: Expose metrics in Prometheus format through HTTP server
|
||||
- **OTLP**: Support both gRPC and HTTP/Protobuf protocols for exporting to OTLP backends
|
||||
- **S3**: Export metrics to S3-compatible object storage systems
|
||||
|
||||
### 3. Flexible Configuration
|
||||
- Support parameter settings through Properties configuration files
|
||||
- Configurable export intervals, compression methods, timeout values, etc.
|
||||
- Support metric cardinality limits to control memory usage
|
||||
|
||||
## Module Structure
|
||||
|
||||
```
|
||||
com.automq.opentelemetry/
|
||||
├── AutoMQTelemetryManager.java # Main management class for initialization and lifecycle
|
||||
├── TelemetryConfig.java # Configuration management class
|
||||
├── TelemetryConstants.java # Constants definition
|
||||
├── common/
|
||||
│ └── MetricsUtils.java # Metrics utility class
|
||||
├── exporter/
|
||||
│ ├── MetricsExporter.java # Exporter interface
|
||||
│ ├── MetricsExporterURI.java # URI parser
|
||||
│ <20><><EFBFBD>── OTLPMetricsExporter.java # OTLP exporter implementation
|
||||
│ ├── PrometheusMetricsExporter.java # Prometheus exporter implementation
|
||||
│ └── s3/ # S3 metrics exporter implementation
|
||||
│ ├── CompressionUtils.java # Utility for data compression
|
||||
│ ├── PrometheusUtils.java # Utilities for Prometheus format
|
||||
│ ├── S3MetricsConfig.java # Configuration interface
|
||||
│ ├── S3MetricsExporter.java # S3 metrics exporter implementation
|
||||
│ ├── S3MetricsExporterAdapter.java # Adapter to handle S3 metrics export
|
||||
│ ├── LeaderNodeSelector.java # Interface for node selection logic
|
||||
│ └── LeaderNodeSelectors.java # Factory for node selector implementations
|
||||
└── yammer/
|
||||
├── DeltaHistogram.java # Delta histogram implementation
|
||||
├── OTelMetricUtils.java # OpenTelemetry metrics utilities
|
||||
├── YammerMetricsProcessor.java # Yammer metrics processor
|
||||
└── YammerMetricsReporter.java # Yammer metrics reporter
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Basic Usage
|
||||
|
||||
```java
|
||||
import com.automq.opentelemetry.AutoMQTelemetryManager;
|
||||
import com.automq.opentelemetry.exporter.MetricsExportConfig;
|
||||
|
||||
// Implement MetricsExportConfig
|
||||
public class MyMetricsExportConfig implements MetricsExportConfig {
|
||||
@Override
|
||||
public String clusterId() { return "my-cluster"; }
|
||||
|
||||
@Override
|
||||
public boolean isLeader() { return true; }
|
||||
|
||||
@Override
|
||||
public int nodeId() { return 1; }
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() {
|
||||
// Return your object storage instance for S3 exports
|
||||
return myObjectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Pair<String, String>> baseLabels() {
|
||||
return Arrays.asList(
|
||||
Pair.of("environment", "production"),
|
||||
Pair.of("region", "us-east-1")
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intervalMs() { return 60000; } // 60 seconds
|
||||
}
|
||||
|
||||
// Create export configuration
|
||||
MetricsExportConfig config = new MyMetricsExportConfig();
|
||||
|
||||
// Initialize telemetry manager singleton
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"prometheus://localhost:9090", // exporter URI
|
||||
"automq-kafka", // service name
|
||||
"broker-1", // instance ID
|
||||
config // export config
|
||||
);
|
||||
|
||||
// Start Yammer metrics reporting (optional)
|
||||
MetricsRegistry yammerRegistry = // Get Kafka's Yammer registry
|
||||
manager.startYammerMetricsReporter(yammerRegistry);
|
||||
|
||||
// Application running...
|
||||
|
||||
// Shutdown telemetry system
|
||||
AutoMQTelemetryManager.shutdownInstance();
|
||||
```
|
||||
|
||||
### 2. Get Meter Instance
|
||||
|
||||
```java
|
||||
// Get the singleton instance
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.getInstance();
|
||||
|
||||
// Get Meter for custom metrics
|
||||
Meter meter = manager.getMeter();
|
||||
|
||||
// Create custom metrics
|
||||
LongCounter requestCounter = meter
|
||||
.counterBuilder("http_requests_total")
|
||||
.setDescription("Total number of HTTP requests")
|
||||
.build();
|
||||
|
||||
requestCounter.add(1, Attributes.of(AttributeKey.stringKey("method"), "GET"));
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Basic Configuration
|
||||
|
||||
Configuration is provided through the `MetricsExportConfig` interface and constructor parameters:
|
||||
|
||||
| Parameter | Description | Example |
|
||||
|-----------|-------------|---------|
|
||||
| `exporterUri` | Metrics exporter URI | `prometheus://localhost:9090` |
|
||||
| `serviceName` | Service name for telemetry | `automq-kafka` |
|
||||
| `instanceId` | Unique service instance ID | `broker-1` |
|
||||
| `config` | MetricsExportConfig implementation | See example above |
|
||||
|
||||
### Exporter Configuration
|
||||
|
||||
All configuration is done through the `MetricsExportConfig` interface and constructor parameters. Export intervals, compression settings, and other options are controlled through:
|
||||
|
||||
1. **Exporter URI**: Determines the export destination and protocol
|
||||
2. **MetricsExportConfig**: Provides cluster information, intervals, and base labels
|
||||
3. **Constructor parameters**: Service name and instance ID
|
||||
|
||||
#### Prometheus Exporter
|
||||
```java
|
||||
// Use prometheus:// URI scheme
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"prometheus://localhost:9090",
|
||||
"automq-kafka",
|
||||
"broker-1",
|
||||
config
|
||||
);
|
||||
```
|
||||
|
||||
#### OTLP Exporter
|
||||
```java
|
||||
// Use otlp:// URI scheme with optional query parameters
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"otlp://localhost:4317?protocol=grpc&compression=gzip&timeout=30000",
|
||||
"automq-kafka",
|
||||
"broker-1",
|
||||
config
|
||||
);
|
||||
```
|
||||
|
||||
#### S3 Metrics Exporter
|
||||
```java
|
||||
// Use s3:// URI scheme
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"s3://access-key:secret-key@my-bucket.s3.amazonaws.com",
|
||||
"automq-kafka",
|
||||
"broker-1",
|
||||
config // config.clusterId(), nodeId(), isLeader() used for S3 export
|
||||
);
|
||||
```
|
||||
|
||||
Example usage with S3 exporter:
|
||||
|
||||
```java
|
||||
// Implementation for S3 export configuration
|
||||
public class S3MetricsExportConfig implements MetricsExportConfig {
|
||||
private final ObjectStorage objectStorage;
|
||||
|
||||
public S3MetricsExportConfig(ObjectStorage objectStorage) {
|
||||
this.objectStorage = objectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String clusterId() { return "my-kafka-cluster"; }
|
||||
|
||||
@Override
|
||||
public boolean isLeader() {
|
||||
// Only one node in the cluster should return true
|
||||
return isCurrentNodeLeader();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nodeId() { return 1; }
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() { return objectStorage; }
|
||||
|
||||
@Override
|
||||
public List<Pair<String, String>> baseLabels() {
|
||||
return Arrays.asList(Pair.of("environment", "production"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intervalMs() { return 60000; }
|
||||
}
|
||||
|
||||
// Initialize telemetry manager with S3 export
|
||||
ObjectStorage objectStorage = // Create your object storage instance
|
||||
MetricsExportConfig config = new S3MetricsExportConfig(objectStorage);
|
||||
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"s3://access-key:secret-key@my-bucket.s3.amazonaws.com",
|
||||
"automq-kafka",
|
||||
"broker-1",
|
||||
config
|
||||
);
|
||||
|
||||
// Application running...
|
||||
|
||||
// Shutdown telemetry system
|
||||
AutoMQTelemetryManager.shutdownInstance();
|
||||
```
|
||||
|
||||
### JMX Metrics Configuration
|
||||
|
||||
Define JMX metrics collection rules through YAML configuration files:
|
||||
|
||||
```java
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
exporterUri, serviceName, instanceId, config
|
||||
);
|
||||
|
||||
// Set JMX config paths after initialization
|
||||
manager.setJmxConfigPaths("/jmx-config.yaml,/kafka-jmx.yaml");
|
||||
```
|
||||
|
||||
#### Configuration File Requirements
|
||||
|
||||
1. **Directory Requirements**:
|
||||
- Configuration files must be placed in the project's classpath (e.g., `src/main/resources` directory)
|
||||
- Support subdirectory structure, e.g., `/config/jmx-metrics.yaml`
|
||||
|
||||
2. **Path Format**:
|
||||
- Paths must start with `/` to indicate starting from classpath root
|
||||
- Multiple configuration files separated by commas
|
||||
|
||||
3. **File Format**:
|
||||
- Use YAML format (`.yaml` or `.yml` extension)
|
||||
- Filenames can be customized, meaningful names are recommended
|
||||
|
||||
#### Recommended Directory Structure
|
||||
|
||||
```
|
||||
src/main/resources/
|
||||
├── jmx-kafka-broker.yaml # Kafka Broker metrics configuration
|
||||
├── jmx-kafka-consumer.yaml # Kafka Consumer metrics configuration
|
||||
├── jmx-kafka-producer.yaml # Kafka Producer metrics configuration
|
||||
└── config/
|
||||
├── custom-jmx.yaml # Custom JMX metrics configuration
|
||||
└── third-party-jmx.yaml # Third-party component JMX configuration
|
||||
```
|
||||
|
||||
JMX configuration file example (`jmx-config.yaml`):
|
||||
```yaml
|
||||
rules:
|
||||
- bean: kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec
|
||||
metricAttribute:
|
||||
name: kafka_server_broker_topic_messages_in_per_sec
|
||||
description: Messages in per second
|
||||
unit: "1/s"
|
||||
attributes:
|
||||
- name: topic
|
||||
value: topic
|
||||
```
|
||||
|
||||
## Supported Metric Types
|
||||
|
||||
### 1. JVM Metrics
|
||||
- Memory usage (heap memory, non-heap memory, memory pools)
|
||||
- CPU usage
|
||||
- Garbage collection statistics
|
||||
- Thread states
|
||||
|
||||
### 2. Kafka Metrics
|
||||
Through Yammer metrics bridging, supports the following types of Kafka metrics:
|
||||
- `BytesInPerSec` - Bytes input per second
|
||||
- `BytesOutPerSec` - Bytes output per second
|
||||
- `Size` - Log size (for identifying idle partitions)
|
||||
|
||||
### 3. Custom Metrics
|
||||
Support creating custom metrics through OpenTelemetry API:
|
||||
- Counter
|
||||
- Gauge
|
||||
- Histogram
|
||||
- UpDownCounter
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Production Environment Configuration
|
||||
|
||||
```java
|
||||
public class ProductionMetricsConfig implements MetricsExportConfig {
|
||||
@Override
|
||||
public String clusterId() { return "production-cluster"; }
|
||||
|
||||
@Override
|
||||
public boolean isLeader() {
|
||||
// Implement your leader election logic
|
||||
return isCurrentNodeController();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nodeId() { return getCurrentNodeId(); }
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() {
|
||||
return productionObjectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Pair<String, String>> baseLabels() {
|
||||
return Arrays.asList(
|
||||
Pair.of("environment", "production"),
|
||||
Pair.of("region", System.getenv("AWS_REGION")),
|
||||
Pair.of("version", getApplicationVersion())
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intervalMs() { return 60000; } // 1 minute
|
||||
}
|
||||
|
||||
// Initialize for production
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"prometheus://0.0.0.0:9090", // Or S3 URI for object storage export
|
||||
"automq-kafka",
|
||||
System.getenv("HOSTNAME"),
|
||||
new ProductionMetricsConfig()
|
||||
);
|
||||
```
|
||||
|
||||
### 2. Development Environment Configuration
|
||||
|
||||
```java
|
||||
public class DevelopmentMetricsConfig implements MetricsExportConfig {
|
||||
@Override
|
||||
public String clusterId() { return "dev-cluster"; }
|
||||
|
||||
@Override
|
||||
public boolean isLeader() { return true; } // Single node in dev
|
||||
|
||||
@Override
|
||||
public int nodeId() { return 1; }
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() { return null; } // Not needed for OTLP
|
||||
|
||||
@Override
|
||||
public List<Pair<String, String>> baseLabels() {
|
||||
return Arrays.asList(Pair.of("environment", "development"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intervalMs() { return 10000; } // 10 seconds for faster feedback
|
||||
}
|
||||
|
||||
// Initialize for development
|
||||
AutoMQTelemetryManager manager = AutoMQTelemetryManager.initializeInstance(
|
||||
"otlp://localhost:4317",
|
||||
"automq-kafka-dev",
|
||||
"local-dev",
|
||||
new DevelopmentMetricsConfig()
|
||||
);
|
||||
```
|
||||
|
||||
### 3. Resource Management
|
||||
- Set appropriate metric cardinality limits to avoid memory leaks
|
||||
- Call `shutdown()` method when application closes to release resources
|
||||
- Monitor exporter health status
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Metrics not exported**
|
||||
- Check if exporter URI passed to `initializeInstance()` is correct
|
||||
- Verify target endpoint is reachable
|
||||
- Check error messages in logs
|
||||
- Ensure `MetricsExportConfig.intervalMs()` returns reasonable value
|
||||
|
||||
2. **JMX metrics missing**
|
||||
- Confirm JMX configuration file path set via `setJmxConfigPaths()` is correct
|
||||
- Check YAML configuration file format
|
||||
- Verify JMX Bean exists
|
||||
- Ensure files are in classpath
|
||||
|
||||
3. **High memory usage**
|
||||
- Implement cardinality limits in your `MetricsExportConfig`
|
||||
- Check for high cardinality labels in `baseLabels()`
|
||||
- Consider increasing export interval via `intervalMs()`
|
||||
|
||||
### Logging Configuration
|
||||
|
||||
Enable debug logging for more information using your logging framework configuration (e.g., logback.xml, log4j2.xml):
|
||||
|
||||
```xml
|
||||
<!-- For Logback -->
|
||||
<logger name="com.automq.opentelemetry" level="DEBUG" />
|
||||
<logger name="io.opentelemetry" level="INFO" />
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Java 8+
|
||||
- OpenTelemetry SDK 1.30+
|
||||
- Apache Commons Lang3
|
||||
- SLF4J logging framework
|
||||
|
||||
## License
|
||||
|
||||
This module is open source under the Apache License 2.0.
|
||||
|
|
@ -1,330 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry;
|
||||
|
||||
import com.automq.opentelemetry.exporter.MetricsExportConfig;
|
||||
import com.automq.opentelemetry.exporter.MetricsExporter;
|
||||
import com.automq.opentelemetry.exporter.MetricsExporterURI;
|
||||
import com.automq.opentelemetry.yammer.YammerMetricsReporter;
|
||||
import com.yammer.metrics.core.MetricsRegistry;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.bridge.SLF4JBridgeHandler;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import io.opentelemetry.api.OpenTelemetry;
|
||||
import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator;
|
||||
import io.opentelemetry.api.common.Attributes;
|
||||
import io.opentelemetry.api.common.AttributesBuilder;
|
||||
import io.opentelemetry.api.metrics.Meter;
|
||||
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
|
||||
import io.opentelemetry.context.propagation.ContextPropagators;
|
||||
import io.opentelemetry.context.propagation.TextMapPropagator;
|
||||
import io.opentelemetry.instrumentation.jmx.engine.JmxMetricInsight;
|
||||
import io.opentelemetry.instrumentation.jmx.engine.MetricConfiguration;
|
||||
import io.opentelemetry.instrumentation.jmx.yaml.RuleParser;
|
||||
import io.opentelemetry.instrumentation.runtimemetrics.java8.Cpu;
|
||||
import io.opentelemetry.instrumentation.runtimemetrics.java8.GarbageCollector;
|
||||
import io.opentelemetry.instrumentation.runtimemetrics.java8.MemoryPools;
|
||||
import io.opentelemetry.instrumentation.runtimemetrics.java8.Threads;
|
||||
import io.opentelemetry.sdk.OpenTelemetrySdk;
|
||||
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
|
||||
import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
|
||||
import io.opentelemetry.sdk.metrics.export.MetricReader;
|
||||
import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;
|
||||
import io.opentelemetry.sdk.resources.Resource;
|
||||
|
||||
/**
|
||||
* The main manager for AutoMQ telemetry.
|
||||
* This class is responsible for initializing, configuring, and managing the lifecycle of all
|
||||
* telemetry components, including the OpenTelemetry SDK, metric exporters, and various metric sources.
|
||||
*/
|
||||
public class AutoMQTelemetryManager {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AutoMQTelemetryManager.class);
|
||||
|
||||
// Singleton instance support
|
||||
private static volatile AutoMQTelemetryManager instance;
|
||||
private static final Object LOCK = new Object();
|
||||
|
||||
private final String exporterUri;
|
||||
private final String serviceName;
|
||||
private final String instanceId;
|
||||
private final MetricsExportConfig metricsExportConfig;
|
||||
private final List<MetricReader> metricReaders = new ArrayList<>();
|
||||
private final List<AutoCloseable> autoCloseableList;
|
||||
private OpenTelemetrySdk openTelemetrySdk;
|
||||
private YammerMetricsReporter yammerReporter;
|
||||
|
||||
private int metricCardinalityLimit = TelemetryConstants.DEFAULT_METRIC_CARDINALITY_LIMIT;
|
||||
private String jmxConfigPath;
|
||||
|
||||
/**
|
||||
* Constructs a new Telemetry Manager with the given configuration.
|
||||
*
|
||||
* @param exporterUri The metrics exporter URI.
|
||||
* @param serviceName The service name to be used in telemetry data.
|
||||
* @param instanceId The unique instance ID for this service instance.
|
||||
* @param metricsExportConfig The metrics configuration.
|
||||
*/
|
||||
public AutoMQTelemetryManager(String exporterUri, String serviceName, String instanceId, MetricsExportConfig metricsExportConfig) {
|
||||
this.exporterUri = exporterUri;
|
||||
this.serviceName = serviceName;
|
||||
this.instanceId = instanceId;
|
||||
this.metricsExportConfig = metricsExportConfig;
|
||||
this.autoCloseableList = new ArrayList<>();
|
||||
// Redirect JUL from OpenTelemetry SDK to SLF4J for unified logging
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the singleton instance of AutoMQTelemetryManager.
|
||||
* Returns null if no instance has been initialized.
|
||||
*
|
||||
* @return the singleton instance, or null if not initialized
|
||||
*/
|
||||
public static AutoMQTelemetryManager getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the singleton instance with the given configuration.
|
||||
* This method should be called before any other components try to access the instance.
|
||||
*
|
||||
* @param exporterUri The metrics exporter URI.
|
||||
* @param serviceName The service name to be used in telemetry data.
|
||||
* @param instanceId The unique instance ID for this service instance.
|
||||
* @param metricsExportConfig The metrics configuration.
|
||||
* @return the initialized singleton instance
|
||||
*/
|
||||
public static AutoMQTelemetryManager initializeInstance(String exporterUri, String serviceName, String instanceId, MetricsExportConfig metricsExportConfig) {
|
||||
if (instance == null) {
|
||||
synchronized (LOCK) {
|
||||
if (instance == null) {
|
||||
AutoMQTelemetryManager newInstance = new AutoMQTelemetryManager(exporterUri, serviceName, instanceId, metricsExportConfig);
|
||||
newInstance.init();
|
||||
instance = newInstance;
|
||||
LOGGER.info("AutoMQTelemetryManager singleton instance initialized");
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shuts down the singleton instance and releases all resources.
|
||||
*/
|
||||
public static void shutdownInstance() {
|
||||
if (instance != null) {
|
||||
synchronized (LOCK) {
|
||||
if (instance != null) {
|
||||
instance.shutdown();
|
||||
instance = null;
|
||||
LOGGER.info("AutoMQTelemetryManager singleton instance shutdown");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the telemetry system. This method sets up the OpenTelemetry SDK,
|
||||
* configures exporters, and registers JVM and JMX metrics.
|
||||
*/
|
||||
public void init() {
|
||||
SdkMeterProvider meterProvider = buildMeterProvider();
|
||||
|
||||
this.openTelemetrySdk = OpenTelemetrySdk.builder()
|
||||
.setMeterProvider(meterProvider)
|
||||
.setPropagators(ContextPropagators.create(TextMapPropagator.composite(
|
||||
W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance())))
|
||||
.buildAndRegisterGlobal();
|
||||
|
||||
// Register JVM and JMX metrics
|
||||
registerJvmMetrics(openTelemetrySdk);
|
||||
registerJmxMetrics(openTelemetrySdk);
|
||||
|
||||
LOGGER.info("AutoMQ Telemetry Manager initialized successfully.");
|
||||
}
|
||||
|
||||
private SdkMeterProvider buildMeterProvider() {
|
||||
String hostName;
|
||||
try {
|
||||
hostName = InetAddress.getLocalHost().getHostName();
|
||||
} catch (UnknownHostException e) {
|
||||
hostName = "unknown-host";
|
||||
}
|
||||
AttributesBuilder attrsBuilder = Attributes.builder()
|
||||
.put(TelemetryConstants.SERVICE_NAME_KEY, serviceName)
|
||||
.put(TelemetryConstants.SERVICE_INSTANCE_ID_KEY, instanceId)
|
||||
.put(TelemetryConstants.HOST_NAME_KEY, hostName)
|
||||
// Add attributes for Prometheus compatibility
|
||||
.put(TelemetryConstants.PROMETHEUS_JOB_KEY, serviceName)
|
||||
.put(TelemetryConstants.PROMETHEUS_INSTANCE_KEY, instanceId);
|
||||
|
||||
for (Pair<String, String> label : metricsExportConfig.baseLabels()) {
|
||||
attrsBuilder.put(label.getKey(), label.getValue());
|
||||
}
|
||||
|
||||
Resource resource = Resource.getDefault().merge(Resource.create(attrsBuilder.build()));
|
||||
SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder().setResource(resource);
|
||||
|
||||
// Configure exporters from URI
|
||||
MetricsExporterURI exporterURI = buildMetricsExporterURI(exporterUri, metricsExportConfig);
|
||||
for (MetricsExporter exporter : exporterURI.getMetricsExporters()) {
|
||||
MetricReader reader = exporter.asMetricReader();
|
||||
metricReaders.add(reader);
|
||||
SdkMeterProviderUtil.registerMetricReaderWithCardinalitySelector(meterProviderBuilder, reader,
|
||||
instrumentType -> metricCardinalityLimit);
|
||||
}
|
||||
|
||||
return meterProviderBuilder.build();
|
||||
}
|
||||
|
||||
protected MetricsExporterURI buildMetricsExporterURI(String exporterUri, MetricsExportConfig metricsExportConfig) {
|
||||
return MetricsExporterURI.parse(exporterUri, metricsExportConfig);
|
||||
}
|
||||
|
||||
private void registerJvmMetrics(OpenTelemetry openTelemetry) {
|
||||
autoCloseableList.addAll(MemoryPools.registerObservers(openTelemetry));
|
||||
autoCloseableList.addAll(Cpu.registerObservers(openTelemetry));
|
||||
autoCloseableList.addAll(GarbageCollector.registerObservers(openTelemetry));
|
||||
autoCloseableList.addAll(Threads.registerObservers(openTelemetry));
|
||||
LOGGER.info("JVM metrics registered.");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"NP_LOAD_OF_KNOWN_NULL_VALUE", "RCN_REDUNDANT_NULLCHECK_OF_NULL_VALUE"})
|
||||
private void registerJmxMetrics(OpenTelemetry openTelemetry) {
|
||||
List<String> jmxConfigPaths = getJmxConfigPaths();
|
||||
if (jmxConfigPaths.isEmpty()) {
|
||||
LOGGER.info("No JMX metric config paths provided, skipping JMX metrics registration.");
|
||||
return;
|
||||
}
|
||||
|
||||
JmxMetricInsight jmxMetricInsight = JmxMetricInsight.createService(openTelemetry, metricsExportConfig.intervalMs());
|
||||
MetricConfiguration metricConfig = new MetricConfiguration();
|
||||
|
||||
for (String path : jmxConfigPaths) {
|
||||
try (InputStream ins = this.getClass().getResourceAsStream(path)) {
|
||||
if (ins == null) {
|
||||
LOGGER.error("JMX config file not found in classpath: {}", path);
|
||||
continue;
|
||||
}
|
||||
RuleParser parser = RuleParser.get();
|
||||
parser.addMetricDefsTo(metricConfig, ins, path);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to parse JMX config file: {}", path, e);
|
||||
}
|
||||
}
|
||||
|
||||
jmxMetricInsight.start(metricConfig);
|
||||
// JmxMetricInsight doesn't implement Closeable, but we can create a wrapper
|
||||
|
||||
LOGGER.info("JMX metrics registered with config paths: {}", jmxConfigPaths);
|
||||
}
|
||||
|
||||
public List<String> getJmxConfigPaths() {
|
||||
if (StringUtils.isEmpty(jmxConfigPath)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return Stream.of(jmxConfigPath.split(","))
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts reporting metrics from a given Yammer MetricsRegistry.
|
||||
*
|
||||
* @param registry The Yammer registry to bridge metrics from.
|
||||
*/
|
||||
public void startYammerMetricsReporter(MetricsRegistry registry) {
|
||||
if (this.openTelemetrySdk == null) {
|
||||
throw new IllegalStateException("TelemetryManager is not initialized. Call init() first.");
|
||||
}
|
||||
if (registry == null) {
|
||||
LOGGER.warn("Yammer MetricsRegistry is null, skipping reporter start.");
|
||||
return;
|
||||
}
|
||||
this.yammerReporter = new YammerMetricsReporter(registry);
|
||||
this.yammerReporter.start(getMeter());
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
autoCloseableList.forEach(autoCloseable -> {
|
||||
try {
|
||||
autoCloseable.close();
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to close auto closeable", e);
|
||||
}
|
||||
});
|
||||
metricReaders.forEach(metricReader -> {
|
||||
metricReader.forceFlush();
|
||||
try {
|
||||
metricReader.close();
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Failed to close metric reader", e);
|
||||
}
|
||||
});
|
||||
if (openTelemetrySdk != null) {
|
||||
openTelemetrySdk.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get YammerMetricsReporter instance.
|
||||
*
|
||||
* @return The YammerMetricsReporter instance.
|
||||
*/
|
||||
public YammerMetricsReporter getYammerReporter() {
|
||||
return this.yammerReporter;
|
||||
}
|
||||
|
||||
public void setMetricCardinalityLimit(int limit) {
|
||||
this.metricCardinalityLimit = limit;
|
||||
}
|
||||
|
||||
public void setJmxConfigPaths(String jmxConfigPaths) {
|
||||
this.jmxConfigPath = jmxConfigPaths;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the default meter from the initialized OpenTelemetry SDK.
|
||||
*
|
||||
* @return The meter instance.
|
||||
*/
|
||||
public Meter getMeter() {
|
||||
if (this.openTelemetrySdk == null) {
|
||||
throw new IllegalStateException("TelemetryManager is not initialized. Call init() first.");
|
||||
}
|
||||
return this.openTelemetrySdk.getMeter(TelemetryConstants.TELEMETRY_SCOPE_NAME);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry;
|
||||
|
||||
import io.opentelemetry.api.common.AttributeKey;
|
||||
|
||||
/**
|
||||
* Constants for telemetry, including configuration keys, attribute keys, and default values.
|
||||
*/
|
||||
public class TelemetryConstants {
|
||||
|
||||
//################################################################
|
||||
// Service and Resource Attributes
|
||||
//################################################################
|
||||
public static final String SERVICE_NAME_KEY = "service.name";
|
||||
public static final String SERVICE_INSTANCE_ID_KEY = "service.instance.id";
|
||||
public static final String HOST_NAME_KEY = "host.name";
|
||||
public static final String TELEMETRY_SCOPE_NAME = "automq_for_kafka";
|
||||
|
||||
/**
|
||||
* The cardinality limit for any single metric.
|
||||
*/
|
||||
public static final String METRIC_CARDINALITY_LIMIT_KEY = "automq.telemetry.metric.cardinality.limit";
|
||||
public static final int DEFAULT_METRIC_CARDINALITY_LIMIT = 20000;
|
||||
|
||||
//################################################################
|
||||
// Prometheus specific Attributes, for compatibility
|
||||
//################################################################
|
||||
public static final String PROMETHEUS_JOB_KEY = "job";
|
||||
public static final String PROMETHEUS_INSTANCE_KEY = "instance";
|
||||
|
||||
//################################################################
|
||||
// Custom Kafka-related Attribute Keys
|
||||
//################################################################
|
||||
public static final AttributeKey<Long> START_OFFSET_KEY = AttributeKey.longKey("startOffset");
|
||||
public static final AttributeKey<Long> END_OFFSET_KEY = AttributeKey.longKey("endOffset");
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter;
|
||||
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Configuration interface for metrics exporter.
|
||||
*/
|
||||
public interface MetricsExportConfig {
|
||||
|
||||
/**
|
||||
* Get the cluster ID.
|
||||
* @return The cluster ID.
|
||||
*/
|
||||
String clusterId();
|
||||
|
||||
/**
|
||||
* Check if the current node is a primary node for metrics upload.
|
||||
* @return True if the current node should upload metrics, false otherwise.
|
||||
*/
|
||||
boolean isLeader();
|
||||
|
||||
/**
|
||||
* Get the node ID.
|
||||
* @return The node ID.
|
||||
*/
|
||||
int nodeId();
|
||||
|
||||
/**
|
||||
* Get the object storage instance.
|
||||
* @return The object storage instance.
|
||||
*/
|
||||
ObjectStorage objectStorage();
|
||||
|
||||
/**
|
||||
* Get the base labels to include in all metrics.
|
||||
* @return The base labels.
|
||||
*/
|
||||
List<Pair<String, String>> baseLabels();
|
||||
|
||||
/**
|
||||
* Get the interval in milliseconds for metrics export.
|
||||
* @return The interval in milliseconds.
|
||||
*/
|
||||
int intervalMs();
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Service Provider Interface that allows extending the available metrics exporters
|
||||
* without modifying the core AutoMQ OpenTelemetry module.
|
||||
*/
|
||||
public interface MetricsExporterProvider {
|
||||
|
||||
/**
|
||||
* @param scheme exporter scheme (e.g. "rw")
|
||||
* @return true if this provider can create an exporter for the supplied scheme
|
||||
*/
|
||||
boolean supports(String scheme);
|
||||
|
||||
/**
|
||||
* Creates a metrics exporter for the provided URI.
|
||||
*
|
||||
* @param config metrics configuration
|
||||
* @param uri original exporter URI
|
||||
* @param queryParameters parsed query parameters from the URI
|
||||
* @return a MetricsExporter instance, or {@code null} if unable to create one
|
||||
*/
|
||||
MetricsExporter create(MetricsExportConfig config, URI uri, Map<String, List<String>> queryParameters);
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter;
|
||||
|
||||
import com.automq.opentelemetry.common.OTLPCompressionType;
|
||||
import com.automq.opentelemetry.common.OTLPProtocol;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
/**
|
||||
* Parses the exporter URI and creates the corresponding MetricsExporter instances.
|
||||
*/
|
||||
public class MetricsExporterURI {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MetricsExporterURI.class);
|
||||
|
||||
private static final List<MetricsExporterProvider> PROVIDERS;
|
||||
|
||||
static {
|
||||
List<MetricsExporterProvider> providers = new ArrayList<>();
|
||||
ServiceLoader.load(MetricsExporterProvider.class).forEach(providers::add);
|
||||
PROVIDERS = Collections.unmodifiableList(providers);
|
||||
if (!PROVIDERS.isEmpty()) {
|
||||
LOGGER.info("Loaded {} telemetry exporter providers", PROVIDERS.size());
|
||||
}
|
||||
}
|
||||
|
||||
private final List<MetricsExporter> metricsExporters;
|
||||
|
||||
private MetricsExporterURI(List<MetricsExporter> metricsExporters) {
|
||||
this.metricsExporters = metricsExporters != null ? metricsExporters : new ArrayList<>();
|
||||
}
|
||||
|
||||
public List<MetricsExporter> getMetricsExporters() {
|
||||
return metricsExporters;
|
||||
}
|
||||
|
||||
public static MetricsExporterURI parse(String uriStr, MetricsExportConfig config) {
|
||||
LOGGER.info("Parsing metrics exporter URI: {}", uriStr);
|
||||
if (StringUtils.isBlank(uriStr)) {
|
||||
LOGGER.info("Metrics exporter URI is not configured, no metrics will be exported.");
|
||||
return new MetricsExporterURI(Collections.emptyList());
|
||||
}
|
||||
|
||||
// Support multiple exporters separated by comma
|
||||
String[] exporterUris = uriStr.split(",");
|
||||
if (exporterUris.length == 0) {
|
||||
return new MetricsExporterURI(Collections.emptyList());
|
||||
}
|
||||
|
||||
List<MetricsExporter> exporters = new ArrayList<>();
|
||||
for (String uri : exporterUris) {
|
||||
if (StringUtils.isBlank(uri)) {
|
||||
continue;
|
||||
}
|
||||
MetricsExporter exporter = parseExporter(config, uri.trim());
|
||||
if (exporter != null) {
|
||||
exporters.add(exporter);
|
||||
}
|
||||
}
|
||||
return new MetricsExporterURI(exporters);
|
||||
}
|
||||
|
||||
public static MetricsExporter parseExporter(MetricsExportConfig config, String uriStr) {
|
||||
try {
|
||||
URI uri = new URI(uriStr);
|
||||
String type = uri.getScheme();
|
||||
if (StringUtils.isBlank(type)) {
|
||||
LOGGER.error("Invalid metrics exporter URI: {}, exporter scheme is missing", uriStr);
|
||||
throw new IllegalArgumentException("Invalid metrics exporter URI: " + uriStr);
|
||||
}
|
||||
|
||||
Map<String, List<String>> queries = parseQueryParameters(uri);
|
||||
return parseExporter(config, type, queries, uri);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Parse metrics exporter URI {} failed", uriStr, e);
|
||||
throw new IllegalArgumentException("Invalid metrics exporter URI: " + uriStr, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static MetricsExporter parseExporter(MetricsExportConfig config, String type, Map<String, List<String>> queries, URI uri) {
|
||||
MetricsExporterType exporterType = MetricsExporterType.fromString(type);
|
||||
switch (exporterType) {
|
||||
case PROMETHEUS:
|
||||
return buildPrometheusExporter(config, queries, uri);
|
||||
case OTLP:
|
||||
return buildOtlpExporter(config, queries, uri);
|
||||
case OPS:
|
||||
return buildS3MetricsExporter(config, uri);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
MetricsExporterProvider provider = findProvider(type);
|
||||
if (provider != null) {
|
||||
MetricsExporter exporter = provider.create(config, uri, queries);
|
||||
if (exporter != null) {
|
||||
return exporter;
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.warn("Unsupported metrics exporter type: {}", type);
|
||||
return null;
|
||||
}
|
||||
|
||||
private static MetricsExporter buildPrometheusExporter(MetricsExportConfig config, Map<String, List<String>> queries, URI uri) {
|
||||
// Use query parameters if available, otherwise fall back to URI authority or config defaults
|
||||
String host = getStringFromQuery(queries, "host", uri.getHost());
|
||||
if (StringUtils.isBlank(host)) {
|
||||
host = "localhost";
|
||||
}
|
||||
|
||||
int port = uri.getPort();
|
||||
if (port <= 0) {
|
||||
String portStr = getStringFromQuery(queries, "port", null);
|
||||
if (StringUtils.isNotBlank(portStr)) {
|
||||
try {
|
||||
port = Integer.parseInt(portStr);
|
||||
} catch (NumberFormatException e) {
|
||||
LOGGER.warn("Invalid port in query parameters: {}, using default", portStr);
|
||||
port = 9090;
|
||||
}
|
||||
} else {
|
||||
port = 9090;
|
||||
}
|
||||
}
|
||||
|
||||
return new PrometheusMetricsExporter(host, port, config.baseLabels());
|
||||
}
|
||||
|
||||
private static MetricsExporter buildOtlpExporter(MetricsExportConfig config, Map<String, List<String>> queries, URI uri) {
|
||||
// Get endpoint from query parameters or construct from URI
|
||||
String endpoint = getStringFromQuery(queries, "endpoint", null);
|
||||
if (StringUtils.isBlank(endpoint)) {
|
||||
endpoint = uri.getScheme() + "://" + uri.getAuthority();
|
||||
}
|
||||
|
||||
// Get protocol from query parameters or config
|
||||
String protocol = getStringFromQuery(queries, "protocol", OTLPProtocol.GRPC.getProtocol());
|
||||
|
||||
// Get compression from query parameters or config
|
||||
String compression = getStringFromQuery(queries, "compression", OTLPCompressionType.NONE.getType());
|
||||
|
||||
return new OTLPMetricsExporter(config.intervalMs(), endpoint, protocol, compression);
|
||||
}
|
||||
|
||||
private static MetricsExporter buildS3MetricsExporter(MetricsExportConfig config, URI uri) {
|
||||
LOGGER.info("Creating S3 metrics exporter from URI: {}", uri);
|
||||
if (config.objectStorage() == null) {
|
||||
LOGGER.warn("No object storage configured, skip s3 metrics exporter creation.");
|
||||
return null;
|
||||
}
|
||||
// Create the S3MetricsExporterAdapter with appropriate configuration
|
||||
return new com.automq.opentelemetry.exporter.s3.S3MetricsExporterAdapter(config);
|
||||
}
|
||||
|
||||
private static Map<String, List<String>> parseQueryParameters(URI uri) {
|
||||
Map<String, List<String>> queries = new HashMap<>();
|
||||
String query = uri.getQuery();
|
||||
if (StringUtils.isNotBlank(query)) {
|
||||
String[] pairs = query.split("&");
|
||||
for (String pair : pairs) {
|
||||
String[] keyValue = pair.split("=", 2);
|
||||
if (keyValue.length == 2) {
|
||||
String key = keyValue[0];
|
||||
String value = keyValue[1];
|
||||
queries.computeIfAbsent(key, k -> new ArrayList<>()).add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return queries;
|
||||
}
|
||||
|
||||
private static String getStringFromQuery(Map<String, List<String>> queries, String key, String defaultValue) {
|
||||
List<String> values = queries.get(key);
|
||||
if (values != null && !values.isEmpty()) {
|
||||
return values.get(0);
|
||||
}
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
private static MetricsExporterProvider findProvider(String scheme) {
|
||||
for (MetricsExporterProvider provider : PROVIDERS) {
|
||||
try {
|
||||
if (provider.supports(scheme)) {
|
||||
return provider;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Telemetry exporter provider {} failed to evaluate support for scheme {}", provider.getClass().getName(), scheme, e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter.s3;
|
||||
|
||||
import com.automq.stream.s3.ByteBufAlloc;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
|
||||
/**
|
||||
* Utility class for data compression and decompression.
|
||||
*/
|
||||
public class CompressionUtils {
|
||||
|
||||
/**
|
||||
* Compress a ByteBuf using GZIP.
|
||||
*
|
||||
* @param input The input ByteBuf to compress.
|
||||
* @return A new ByteBuf containing the compressed data.
|
||||
* @throws IOException If an I/O error occurs during compression.
|
||||
*/
|
||||
public static ByteBuf compress(ByteBuf input) throws IOException {
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);
|
||||
|
||||
byte[] buffer = new byte[input.readableBytes()];
|
||||
input.readBytes(buffer);
|
||||
gzipOutputStream.write(buffer);
|
||||
gzipOutputStream.close();
|
||||
|
||||
ByteBuf compressed = ByteBufAlloc.byteBuffer(byteArrayOutputStream.size());
|
||||
compressed.writeBytes(byteArrayOutputStream.toByteArray());
|
||||
return compressed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress a GZIP-compressed ByteBuf.
|
||||
*
|
||||
* @param input The compressed ByteBuf to decompress.
|
||||
* @return A new ByteBuf containing the decompressed data.
|
||||
* @throws IOException If an I/O error occurs during decompression.
|
||||
*/
|
||||
public static ByteBuf decompress(ByteBuf input) throws IOException {
|
||||
byte[] compressedData = new byte[input.readableBytes()];
|
||||
input.readBytes(compressedData);
|
||||
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData);
|
||||
GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream);
|
||||
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[1024];
|
||||
int bytesRead;
|
||||
while ((bytesRead = gzipInputStream.read(buffer)) != -1) {
|
||||
byteArrayOutputStream.write(buffer, 0, bytesRead);
|
||||
}
|
||||
|
||||
gzipInputStream.close();
|
||||
byteArrayOutputStream.close();
|
||||
|
||||
byte[] uncompressedData = byteArrayOutputStream.toByteArray();
|
||||
ByteBuf output = ByteBufAlloc.byteBuffer(uncompressedData.length);
|
||||
output.writeBytes(uncompressedData);
|
||||
return output;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,276 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter.s3;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Utility class for Prometheus metric and label naming.
|
||||
*/
|
||||
public class PrometheusUtils {
|
||||
private static final String TOTAL_SUFFIX = "_total";
|
||||
|
||||
/**
|
||||
* Get the Prometheus unit from the OpenTelemetry unit.
|
||||
*
|
||||
* @param unit The OpenTelemetry unit.
|
||||
* @return The Prometheus unit.
|
||||
*/
|
||||
public static String getPrometheusUnit(String unit) {
|
||||
if (unit.contains("{")) {
|
||||
return "";
|
||||
}
|
||||
switch (unit) {
|
||||
// Time
|
||||
case "d":
|
||||
return "days";
|
||||
case "h":
|
||||
return "hours";
|
||||
case "min":
|
||||
return "minutes";
|
||||
case "s":
|
||||
return "seconds";
|
||||
case "ms":
|
||||
return "milliseconds";
|
||||
case "us":
|
||||
return "microseconds";
|
||||
case "ns":
|
||||
return "nanoseconds";
|
||||
// Bytes
|
||||
case "By":
|
||||
return "bytes";
|
||||
case "KiBy":
|
||||
return "kibibytes";
|
||||
case "MiBy":
|
||||
return "mebibytes";
|
||||
case "GiBy":
|
||||
return "gibibytes";
|
||||
case "TiBy":
|
||||
return "tibibytes";
|
||||
case "KBy":
|
||||
return "kilobytes";
|
||||
case "MBy":
|
||||
return "megabytes";
|
||||
case "GBy":
|
||||
return "gigabytes";
|
||||
case "TBy":
|
||||
return "terabytes";
|
||||
// SI
|
||||
case "m":
|
||||
return "meters";
|
||||
case "V":
|
||||
return "volts";
|
||||
case "A":
|
||||
return "amperes";
|
||||
case "J":
|
||||
return "joules";
|
||||
case "W":
|
||||
return "watts";
|
||||
case "g":
|
||||
return "grams";
|
||||
// Misc
|
||||
case "Cel":
|
||||
return "celsius";
|
||||
case "Hz":
|
||||
return "hertz";
|
||||
case "1":
|
||||
return "";
|
||||
case "%":
|
||||
return "percent";
|
||||
// Rate units (per second)
|
||||
case "1/s":
|
||||
return "per_second";
|
||||
case "By/s":
|
||||
return "bytes_per_second";
|
||||
case "KiBy/s":
|
||||
return "kibibytes_per_second";
|
||||
case "MiBy/s":
|
||||
return "mebibytes_per_second";
|
||||
case "GiBy/s":
|
||||
return "gibibytes_per_second";
|
||||
case "KBy/s":
|
||||
return "kilobytes_per_second";
|
||||
case "MBy/s":
|
||||
return "megabytes_per_second";
|
||||
case "GBy/s":
|
||||
return "gigabytes_per_second";
|
||||
// Rate units (per minute)
|
||||
case "1/min":
|
||||
return "per_minute";
|
||||
case "By/min":
|
||||
return "bytes_per_minute";
|
||||
// Rate units (per hour)
|
||||
case "1/h":
|
||||
return "per_hour";
|
||||
case "By/h":
|
||||
return "bytes_per_hour";
|
||||
// Rate units (per day)
|
||||
case "1/d":
|
||||
return "per_day";
|
||||
case "By/d":
|
||||
return "bytes_per_day";
|
||||
default:
|
||||
return unit;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a metric name to a Prometheus-compatible name.
|
||||
*
|
||||
* @param name The original metric name.
|
||||
* @param unit The metric unit.
|
||||
* @param isCounter Whether the metric is a counter.
|
||||
* @param isGauge Whether the metric is a gauge.
|
||||
* @return The Prometheus-compatible metric name.
|
||||
*/
|
||||
public static String mapMetricsName(String name, String unit, boolean isCounter, boolean isGauge) {
|
||||
// Replace "." into "_"
|
||||
name = name.replaceAll("\\.", "_");
|
||||
|
||||
String prometheusUnit = getPrometheusUnit(unit);
|
||||
boolean shouldAppendUnit = StringUtils.isNotBlank(prometheusUnit) && !name.contains(prometheusUnit);
|
||||
|
||||
// append prometheus unit if not null or empty.
|
||||
// unit should be appended before type suffix
|
||||
if (shouldAppendUnit) {
|
||||
name = name + "_" + prometheusUnit;
|
||||
}
|
||||
|
||||
// trim counter's _total suffix so the unit is placed before it.
|
||||
if (isCounter && name.endsWith(TOTAL_SUFFIX)) {
|
||||
name = name.substring(0, name.length() - TOTAL_SUFFIX.length());
|
||||
}
|
||||
|
||||
// replace _total suffix, or add if it wasn't already present.
|
||||
if (isCounter) {
|
||||
name = name + TOTAL_SUFFIX;
|
||||
}
|
||||
|
||||
// special case - gauge with intelligent Connect metric handling
|
||||
if ("1".equals(unit) && isGauge && !name.contains("ratio")) {
|
||||
if (isConnectMetric(name)) {
|
||||
// For Connect metrics, use improved logic to avoid misleading _ratio suffix
|
||||
if (shouldAddRatioSuffixForConnect(name)) {
|
||||
name = name + "_ratio";
|
||||
}
|
||||
} else {
|
||||
// For other metrics, maintain original behavior
|
||||
name = name + "_ratio";
|
||||
}
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a label name to a Prometheus-compatible name.
|
||||
*
|
||||
* @param name The original label name.
|
||||
* @return The Prometheus-compatible label name.
|
||||
*/
|
||||
public static String mapLabelName(String name) {
|
||||
if (StringUtils.isBlank(name)) {
|
||||
return "";
|
||||
}
|
||||
return name.replaceAll("\\.", "_");
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a metric name is related to Kafka Connect.
|
||||
*
|
||||
* @param name The metric name to check.
|
||||
* @return true if it's a Connect metric, false otherwise.
|
||||
*/
|
||||
private static boolean isConnectMetric(String name) {
|
||||
String lowerName = name.toLowerCase(Locale.ROOT);
|
||||
return lowerName.contains("kafka_connector_") ||
|
||||
lowerName.contains("kafka_task_") ||
|
||||
lowerName.contains("kafka_worker_") ||
|
||||
lowerName.contains("kafka_connect_") ||
|
||||
lowerName.contains("kafka_source_task_") ||
|
||||
lowerName.contains("kafka_sink_task_") ||
|
||||
lowerName.contains("connector_metrics") ||
|
||||
lowerName.contains("task_metrics") ||
|
||||
lowerName.contains("worker_metrics") ||
|
||||
lowerName.contains("source_task_metrics") ||
|
||||
lowerName.contains("sink_task_metrics");
|
||||
}
|
||||
|
||||
/**
|
||||
* Intelligently determine if a Connect metric should have a _ratio suffix.
|
||||
* This method avoids adding misleading _ratio suffixes to count-based metrics.
|
||||
*
|
||||
* @param name The metric name to check.
|
||||
* @return true if _ratio suffix should be added, false otherwise.
|
||||
*/
|
||||
private static boolean shouldAddRatioSuffixForConnect(String name) {
|
||||
String lowerName = name.toLowerCase(Locale.ROOT);
|
||||
|
||||
if (hasRatioRelatedWords(lowerName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isCountMetric(lowerName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isRatioMetric(lowerName);
|
||||
}
|
||||
|
||||
private static boolean hasRatioRelatedWords(String lowerName) {
|
||||
return lowerName.contains("ratio") || lowerName.contains("percent") ||
|
||||
lowerName.contains("rate") || lowerName.contains("fraction");
|
||||
}
|
||||
|
||||
private static boolean isCountMetric(String lowerName) {
|
||||
return hasBasicCountKeywords(lowerName) || hasConnectCountKeywords(lowerName) ||
|
||||
hasStatusCountKeywords(lowerName);
|
||||
}
|
||||
|
||||
private static boolean hasBasicCountKeywords(String lowerName) {
|
||||
return lowerName.contains("count") || lowerName.contains("num") ||
|
||||
lowerName.contains("size") || lowerName.contains("total") ||
|
||||
lowerName.contains("active") || lowerName.contains("current");
|
||||
}
|
||||
|
||||
private static boolean hasConnectCountKeywords(String lowerName) {
|
||||
return lowerName.contains("partition") || lowerName.contains("task") ||
|
||||
lowerName.contains("connector") || lowerName.contains("seq_no") ||
|
||||
lowerName.contains("seq_num") || lowerName.contains("attempts");
|
||||
}
|
||||
|
||||
private static boolean hasStatusCountKeywords(String lowerName) {
|
||||
return lowerName.contains("success") || lowerName.contains("failure") ||
|
||||
lowerName.contains("errors") || lowerName.contains("retries") ||
|
||||
lowerName.contains("skipped") || lowerName.contains("running") ||
|
||||
lowerName.contains("paused") || lowerName.contains("failed") ||
|
||||
lowerName.contains("destroyed");
|
||||
}
|
||||
|
||||
private static boolean isRatioMetric(String lowerName) {
|
||||
return lowerName.contains("utilization") ||
|
||||
lowerName.contains("usage") ||
|
||||
lowerName.contains("load") ||
|
||||
lowerName.contains("efficiency") ||
|
||||
lowerName.contains("hit_rate") ||
|
||||
lowerName.contains("miss_rate");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter.s3;
|
||||
|
||||
import com.automq.opentelemetry.exporter.MetricsExportConfig;
|
||||
import com.automq.opentelemetry.exporter.MetricsExporter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import io.opentelemetry.sdk.metrics.export.MetricReader;
|
||||
import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
|
||||
|
||||
/**
|
||||
* An adapter class that implements the MetricsExporter interface and uses S3MetricsExporter
|
||||
* for actual metrics exporting functionality.
|
||||
*/
|
||||
public class S3MetricsExporterAdapter implements MetricsExporter {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(S3MetricsExporterAdapter.class);
|
||||
|
||||
private final MetricsExportConfig metricsExportConfig;
|
||||
|
||||
/**
|
||||
* Creates a new S3MetricsExporterAdapter.
|
||||
*
|
||||
* @param metricsExportConfig The configuration for the S3 metrics exporter.
|
||||
*/
|
||||
public S3MetricsExporterAdapter(MetricsExportConfig metricsExportConfig) {
|
||||
this.metricsExportConfig = metricsExportConfig;
|
||||
LOGGER.info("S3MetricsExporterAdapter initialized with labels :{}", metricsExportConfig.baseLabels());
|
||||
}
|
||||
|
||||
@Override
|
||||
public MetricReader asMetricReader() {
|
||||
// Create and start the S3MetricsExporter
|
||||
S3MetricsExporter s3MetricsExporter = new S3MetricsExporter(metricsExportConfig);
|
||||
s3MetricsExporter.start();
|
||||
|
||||
// Create and return the periodic metric reader
|
||||
return PeriodicMetricReader.builder(s3MetricsExporter)
|
||||
.setInterval(Duration.ofMillis(metricsExportConfig.intervalMs()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -18,8 +18,7 @@ dependencies {
|
|||
compileOnly libs.awsSdkAuth
|
||||
implementation libs.reload4j
|
||||
implementation libs.nettyBuffer
|
||||
implementation project(':automq-metrics')
|
||||
implementation project(':automq-log-uploader')
|
||||
implementation libs.opentelemetrySdk
|
||||
implementation libs.jacksonDatabind
|
||||
implementation libs.jacksonYaml
|
||||
implementation libs.commonLang
|
||||
|
|
@ -66,4 +65,4 @@ jar {
|
|||
manifest {
|
||||
attributes 'Main-Class': 'com.automq.shell.AutoMQCLI'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -110,11 +110,9 @@ public class Deploy implements Callable<Integer> {
|
|||
String globalAccessKey = null;
|
||||
String globalSecretKey = null;
|
||||
for (Env env : topo.getGlobal().getEnvs()) {
|
||||
if ("KAFKA_S3_ACCESS_KEY".equals(env.getName()) ||
|
||||
"AWS_ACCESS_KEY_ID".equals(env.getName())) {
|
||||
if ("KAFKA_S3_ACCESS_KEY".equals(env.getName())) {
|
||||
globalAccessKey = env.getValue();
|
||||
} else if ("KAFKA_S3_SECRET_KEY".equals(env.getName()) ||
|
||||
"AWS_SECRET_ACCESS_KEY".equals(env.getName())) {
|
||||
} else if ("KAFKA_S3_SECRET_KEY".equals(env.getName())) {
|
||||
globalSecretKey = env.getValue();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.log.uploader;
|
||||
package com.automq.shell.log;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
|
@ -17,9 +17,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.log.uploader;
|
||||
package com.automq.shell.log;
|
||||
|
||||
import com.automq.log.uploader.util.Utils;
|
||||
import com.automq.shell.AutoMQApplication;
|
||||
import com.automq.shell.util.Utils;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
import com.automq.stream.s3.operator.ObjectStorage.ObjectInfo;
|
||||
import com.automq.stream.s3.operator.ObjectStorage.ObjectPath;
|
||||
|
|
@ -54,14 +55,12 @@ public class LogUploader implements LogRecorder {
|
|||
|
||||
public static final int DEFAULT_MAX_QUEUE_SIZE = 64 * 1024;
|
||||
public static final int DEFAULT_BUFFER_SIZE = 16 * 1024 * 1024;
|
||||
public static final int UPLOAD_INTERVAL = System.getenv("AUTOMQ_OBSERVABILITY_UPLOAD_INTERVAL") != null
|
||||
? Integer.parseInt(System.getenv("AUTOMQ_OBSERVABILITY_UPLOAD_INTERVAL"))
|
||||
: 60 * 1000;
|
||||
public static final int CLEANUP_INTERVAL = System.getenv("AUTOMQ_OBSERVABILITY_CLEANUP_INTERVAL") != null
|
||||
? Integer.parseInt(System.getenv("AUTOMQ_OBSERVABILITY_CLEANUP_INTERVAL"))
|
||||
: 2 * 60 * 1000;
|
||||
public static final int UPLOAD_INTERVAL = System.getenv("AUTOMQ_OBSERVABILITY_UPLOAD_INTERVAL") != null ? Integer.parseInt(System.getenv("AUTOMQ_OBSERVABILITY_UPLOAD_INTERVAL")) : 60 * 1000;
|
||||
public static final int CLEANUP_INTERVAL = System.getenv("AUTOMQ_OBSERVABILITY_CLEANUP_INTERVAL") != null ? Integer.parseInt(System.getenv("AUTOMQ_OBSERVABILITY_CLEANUP_INTERVAL")) : 2 * 60 * 1000;
|
||||
public static final int MAX_JITTER_INTERVAL = 60 * 1000;
|
||||
|
||||
private static final LogUploader INSTANCE = new LogUploader();
|
||||
|
||||
private final BlockingQueue<LogEvent> queue = new LinkedBlockingQueue<>(DEFAULT_MAX_QUEUE_SIZE);
|
||||
private final ByteBuf uploadBuffer = Unpooled.directBuffer(DEFAULT_BUFFER_SIZE);
|
||||
private final Random random = new Random();
|
||||
|
|
@ -72,42 +71,16 @@ public class LogUploader implements LogRecorder {
|
|||
|
||||
private volatile S3LogConfig config;
|
||||
|
||||
private volatile CompletableFuture<Void> startFuture;
|
||||
private ObjectStorage objectStorage;
|
||||
private Thread uploadThread;
|
||||
private Thread cleanupThread;
|
||||
|
||||
public LogUploader() {
|
||||
private LogUploader() {
|
||||
}
|
||||
|
||||
public synchronized void start(S3LogConfig config) {
|
||||
if (this.config != null) {
|
||||
LOGGER.warn("LogUploader is already started.");
|
||||
return;
|
||||
}
|
||||
this.config = config;
|
||||
if (!config.isEnabled() || config.objectStorage() == null) {
|
||||
LOGGER.warn("LogUploader is disabled due to configuration.");
|
||||
closed = true;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.objectStorage = config.objectStorage();
|
||||
this.uploadThread = new Thread(new UploadTask());
|
||||
this.uploadThread.setName("log-uploader-upload-thread");
|
||||
this.uploadThread.setDaemon(true);
|
||||
this.uploadThread.start();
|
||||
|
||||
this.cleanupThread = new Thread(new CleanupTask());
|
||||
this.cleanupThread.setName("log-uploader-cleanup-thread");
|
||||
this.cleanupThread.setDaemon(true);
|
||||
this.cleanupThread.start();
|
||||
|
||||
LOGGER.info("LogUploader started successfully.");
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to start LogUploader", e);
|
||||
closed = true;
|
||||
}
|
||||
public static LogUploader getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
public void close() throws InterruptedException {
|
||||
|
|
@ -124,15 +97,63 @@ public class LogUploader implements LogRecorder {
|
|||
|
||||
@Override
|
||||
public boolean append(LogEvent event) {
|
||||
if (!closed) {
|
||||
if (!closed && couldUpload()) {
|
||||
return queue.offer(event);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean couldUpload() {
|
||||
initConfiguration();
|
||||
boolean enabled = config != null && config.isEnabled() && config.objectStorage() != null;
|
||||
|
||||
if (enabled) {
|
||||
initUploadComponent();
|
||||
}
|
||||
|
||||
return enabled && startFuture != null && startFuture.isDone();
|
||||
}
|
||||
|
||||
private void initConfiguration() {
|
||||
if (config == null) {
|
||||
synchronized (this) {
|
||||
if (config == null) {
|
||||
config = AutoMQApplication.getBean(S3LogConfig.class);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void initUploadComponent() {
|
||||
if (startFuture == null) {
|
||||
synchronized (this) {
|
||||
if (startFuture == null) {
|
||||
startFuture = CompletableFuture.runAsync(() -> {
|
||||
try {
|
||||
objectStorage = config.objectStorage();
|
||||
uploadThread = new Thread(new UploadTask());
|
||||
uploadThread.setName("log-uploader-upload-thread");
|
||||
uploadThread.setDaemon(true);
|
||||
uploadThread.start();
|
||||
|
||||
cleanupThread = new Thread(new CleanupTask());
|
||||
cleanupThread.setName("log-uploader-cleanup-thread");
|
||||
cleanupThread.setDaemon(true);
|
||||
cleanupThread.start();
|
||||
|
||||
startFuture.complete(null);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Initialize log uploader failed", e);
|
||||
}
|
||||
}, command -> new Thread(command).start());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class UploadTask implements Runnable {
|
||||
|
||||
private String formatTimestampInMillis(long timestamp) {
|
||||
public String formatTimestampInMillis(long timestamp) {
|
||||
return ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault())
|
||||
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS Z"));
|
||||
}
|
||||
|
|
@ -144,6 +165,7 @@ public class LogUploader implements LogRecorder {
|
|||
long now = System.currentTimeMillis();
|
||||
LogEvent event = queue.poll(1, TimeUnit.SECONDS);
|
||||
if (event != null) {
|
||||
// DateTime Level [Logger] Message \n stackTrace
|
||||
StringBuilder logLine = new StringBuilder()
|
||||
.append(formatTimestampInMillis(event.timestampMillis()))
|
||||
.append(" ")
|
||||
|
|
@ -182,22 +204,25 @@ public class LogUploader implements LogRecorder {
|
|||
|
||||
private void upload(long now) {
|
||||
if (uploadBuffer.readableBytes() > 0) {
|
||||
try {
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
if (objectStorage == null) {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
String objectKey = getObjectKey();
|
||||
objectStorage.write(WriteOptions.DEFAULT, objectKey, Utils.compress(uploadBuffer.slice().asReadOnly())).get();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to upload logs, will retry", e);
|
||||
Thread.sleep(1000);
|
||||
if (couldUpload()) {
|
||||
try {
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
if (objectStorage == null) {
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
String objectKey = getObjectKey();
|
||||
objectStorage.write(WriteOptions.DEFAULT, objectKey, Utils.compress(uploadBuffer.slice().asReadOnly())).get();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace(System.err);
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
//ignore
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
uploadBuffer.clear();
|
||||
lastUploadTimestamp = now;
|
||||
|
|
@ -212,11 +237,12 @@ public class LogUploader implements LogRecorder {
|
|||
public void run() {
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
try {
|
||||
if (closed || !config.isLeader()) {
|
||||
if (closed || !config.isActiveController()) {
|
||||
Thread.sleep(Duration.ofMinutes(1).toMillis());
|
||||
continue;
|
||||
}
|
||||
long expiredTime = System.currentTimeMillis() - CLEANUP_INTERVAL;
|
||||
|
||||
List<ObjectInfo> objects = objectStorage.list(String.format("automq/logs/%s", config.clusterId())).join();
|
||||
|
||||
if (!objects.isEmpty()) {
|
||||
|
|
@ -226,6 +252,7 @@ public class LogUploader implements LogRecorder {
|
|||
.collect(Collectors.toList());
|
||||
|
||||
if (!keyList.isEmpty()) {
|
||||
// Some of s3 implements allow only 1000 keys per request.
|
||||
CompletableFuture<?>[] deleteFutures = Lists.partition(keyList, 1000)
|
||||
.stream()
|
||||
.map(objectStorage::delete)
|
||||
|
|
@ -233,6 +260,7 @@ public class LogUploader implements LogRecorder {
|
|||
CompletableFuture.allOf(deleteFutures).join();
|
||||
}
|
||||
}
|
||||
|
||||
Thread.sleep(Duration.ofMinutes(1).toMillis());
|
||||
} catch (InterruptedException e) {
|
||||
break;
|
||||
|
|
@ -247,4 +275,5 @@ public class LogUploader implements LogRecorder {
|
|||
String hour = LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyyMMddHH"));
|
||||
return String.format("automq/logs/%s/%s/%s/%s", config.clusterId(), config.nodeId(), hour, UUID.randomUUID());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,18 +17,19 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.log.uploader;
|
||||
package com.automq.shell.log;
|
||||
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
|
||||
public interface S3LogConfig {
|
||||
|
||||
boolean isEnabled();
|
||||
|
||||
boolean isActiveController();
|
||||
|
||||
String clusterId();
|
||||
|
||||
int nodeId();
|
||||
|
||||
ObjectStorage objectStorage();
|
||||
|
||||
boolean isLeader();
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.shell.log;
|
||||
|
||||
import org.apache.log4j.RollingFileAppender;
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
|
||||
public class S3RollingFileAppender extends RollingFileAppender {
|
||||
private final LogUploader logUploader = LogUploader.getInstance();
|
||||
|
||||
@Override
|
||||
protected void subAppend(LoggingEvent event) {
|
||||
super.subAppend(event);
|
||||
if (!closed) {
|
||||
LogRecorder.LogEvent logEvent = new LogRecorder.LogEvent(
|
||||
event.getTimeStamp(),
|
||||
event.getLevel().toString(),
|
||||
event.getLoggerName(),
|
||||
event.getRenderedMessage(),
|
||||
event.getThrowableStrRep());
|
||||
|
||||
try {
|
||||
logEvent.validate();
|
||||
} catch (IllegalArgumentException e) {
|
||||
// Drop invalid log event
|
||||
errorHandler.error("Failed to validate log event", e, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
logUploader.append(logEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.shell.metrics;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class PrometheusUtils {
|
||||
private static final String TOTAL_SUFFIX = "_total";
|
||||
|
||||
public static String getPrometheusUnit(String unit) {
|
||||
if (unit.contains("{")) {
|
||||
return "";
|
||||
}
|
||||
switch (unit) {
|
||||
// Time
|
||||
case "d":
|
||||
return "days";
|
||||
case "h":
|
||||
return "hours";
|
||||
case "min":
|
||||
return "minutes";
|
||||
case "s":
|
||||
return "seconds";
|
||||
case "ms":
|
||||
return "milliseconds";
|
||||
case "us":
|
||||
return "microseconds";
|
||||
case "ns":
|
||||
return "nanoseconds";
|
||||
// Bytes
|
||||
case "By":
|
||||
return "bytes";
|
||||
case "KiBy":
|
||||
return "kibibytes";
|
||||
case "MiBy":
|
||||
return "mebibytes";
|
||||
case "GiBy":
|
||||
return "gibibytes";
|
||||
case "TiBy":
|
||||
return "tibibytes";
|
||||
case "KBy":
|
||||
return "kilobytes";
|
||||
case "MBy":
|
||||
return "megabytes";
|
||||
case "GBy":
|
||||
return "gigabytes";
|
||||
case "TBy":
|
||||
return "terabytes";
|
||||
// SI
|
||||
case "m":
|
||||
return "meters";
|
||||
case "V":
|
||||
return "volts";
|
||||
case "A":
|
||||
return "amperes";
|
||||
case "J":
|
||||
return "joules";
|
||||
case "W":
|
||||
return "watts";
|
||||
case "g":
|
||||
return "grams";
|
||||
// Misc
|
||||
case "Cel":
|
||||
return "celsius";
|
||||
case "Hz":
|
||||
return "hertz";
|
||||
case "1":
|
||||
return "";
|
||||
case "%":
|
||||
return "percent";
|
||||
default:
|
||||
return unit;
|
||||
}
|
||||
}
|
||||
|
||||
public static String mapMetricsName(String name, String unit, boolean isCounter, boolean isGauge) {
|
||||
// Replace "." into "_"
|
||||
name = name.replaceAll("\\.", "_");
|
||||
|
||||
String prometheusUnit = getPrometheusUnit(unit);
|
||||
boolean shouldAppendUnit = StringUtils.isNotBlank(prometheusUnit) && !name.contains(prometheusUnit);
|
||||
|
||||
// append prometheus unit if not null or empty.
|
||||
// unit should be appended before type suffix
|
||||
if (shouldAppendUnit) {
|
||||
name = name + "_" + prometheusUnit;
|
||||
}
|
||||
|
||||
// trim counter's _total suffix so the unit is placed before it.
|
||||
if (isCounter && name.endsWith(TOTAL_SUFFIX)) {
|
||||
name = name.substring(0, name.length() - TOTAL_SUFFIX.length());
|
||||
}
|
||||
|
||||
// replace _total suffix, or add if it wasn't already present.
|
||||
if (isCounter) {
|
||||
name = name + TOTAL_SUFFIX;
|
||||
}
|
||||
// special case - gauge
|
||||
if (unit.equals("1") && isGauge && !name.contains("ratio")) {
|
||||
name = name + "_ratio";
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
public static String mapLabelName(String name) {
|
||||
if (StringUtils.isBlank(name)) {
|
||||
return "";
|
||||
}
|
||||
return name.replaceAll("\\.", "_");
|
||||
}
|
||||
}
|
||||
|
|
@ -17,15 +17,23 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package kafka.automq.table.process;
|
||||
package com.automq.shell.metrics;
|
||||
|
||||
import kafka.automq.table.process.exception.ConverterException;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface Converter {
|
||||
public interface S3MetricsConfig {
|
||||
|
||||
ConversionResult convert(String topic, ByteBuffer buffer) throws ConverterException;
|
||||
String clusterId();
|
||||
|
||||
boolean isActiveController();
|
||||
|
||||
int nodeId();
|
||||
|
||||
ObjectStorage objectStorage();
|
||||
|
||||
List<Pair<String, String>> baseLabels();
|
||||
}
|
||||
|
|
@ -17,9 +17,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.automq.opentelemetry.exporter.s3;
|
||||
package com.automq.shell.metrics;
|
||||
|
||||
import com.automq.opentelemetry.exporter.MetricsExportConfig;
|
||||
import com.automq.shell.util.Utils;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
import com.automq.stream.s3.operator.ObjectStorage.ObjectInfo;
|
||||
import com.automq.stream.s3.operator.ObjectStorage.ObjectPath;
|
||||
|
|
@ -60,9 +60,6 @@ import io.opentelemetry.sdk.metrics.data.HistogramPointData;
|
|||
import io.opentelemetry.sdk.metrics.data.MetricData;
|
||||
import io.opentelemetry.sdk.metrics.export.MetricExporter;
|
||||
|
||||
/**
|
||||
* An S3 metrics exporter that uploads metrics data to S3 buckets.
|
||||
*/
|
||||
public class S3MetricsExporter implements MetricExporter {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(S3MetricsExporter.class);
|
||||
|
||||
|
|
@ -71,13 +68,13 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
public static final int MAX_JITTER_INTERVAL = 60 * 1000;
|
||||
public static final int DEFAULT_BUFFER_SIZE = 16 * 1024 * 1024;
|
||||
|
||||
private final MetricsExportConfig config;
|
||||
private final S3MetricsConfig config;
|
||||
private final Map<String, String> defaultTagMap = new HashMap<>();
|
||||
|
||||
private final ByteBuf uploadBuffer = Unpooled.directBuffer(DEFAULT_BUFFER_SIZE);
|
||||
private static final Random RANDOM = new Random();
|
||||
private final Random random = new Random();
|
||||
private volatile long lastUploadTimestamp = System.currentTimeMillis();
|
||||
private volatile long nextUploadInterval = UPLOAD_INTERVAL + RANDOM.nextInt(MAX_JITTER_INTERVAL);
|
||||
private volatile long nextUploadInterval = UPLOAD_INTERVAL + random.nextInt(MAX_JITTER_INTERVAL);
|
||||
|
||||
private final ObjectStorage objectStorage;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
|
@ -86,12 +83,7 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
private final Thread uploadThread;
|
||||
private final Thread cleanupThread;
|
||||
|
||||
/**
|
||||
* Creates a new S3MetricsExporter.
|
||||
*
|
||||
* @param config The configuration for the S3 metrics exporter.
|
||||
*/
|
||||
public S3MetricsExporter(MetricsExportConfig config) {
|
||||
public S3MetricsExporter(S3MetricsConfig config) {
|
||||
this.config = config;
|
||||
this.objectStorage = config.objectStorage();
|
||||
|
||||
|
|
@ -109,9 +101,6 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
cleanupThread.setDaemon(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the exporter threads.
|
||||
*/
|
||||
public void start() {
|
||||
uploadThread.start();
|
||||
cleanupThread.start();
|
||||
|
|
@ -150,7 +139,7 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
public void run() {
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
try {
|
||||
if (closed || !config.isLeader()) {
|
||||
if (closed || !config.isActiveController()) {
|
||||
Thread.sleep(Duration.ofMinutes(1).toMillis());
|
||||
continue;
|
||||
}
|
||||
|
|
@ -173,11 +162,16 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
CompletableFuture.allOf(deleteFutures).join();
|
||||
}
|
||||
}
|
||||
Threads.sleep(Duration.ofMinutes(1).toMillis());
|
||||
if (Threads.sleep(Duration.ofMinutes(1).toMillis())) {
|
||||
break;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Cleanup s3 metrics failed", e);
|
||||
if (Threads.sleep(Duration.ofMinutes(1).toMillis())) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -262,13 +256,13 @@ public class S3MetricsExporter implements MetricExporter {
|
|||
synchronized (uploadBuffer) {
|
||||
if (uploadBuffer.readableBytes() > 0) {
|
||||
try {
|
||||
objectStorage.write(WriteOptions.DEFAULT, getObjectKey(), CompressionUtils.compress(uploadBuffer.slice().asReadOnly())).get();
|
||||
objectStorage.write(WriteOptions.DEFAULT, getObjectKey(), Utils.compress(uploadBuffer.slice().asReadOnly())).get();
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to upload metrics to s3", e);
|
||||
return CompletableResultCode.ofFailure();
|
||||
} finally {
|
||||
lastUploadTimestamp = System.currentTimeMillis();
|
||||
nextUploadInterval = UPLOAD_INTERVAL + RANDOM.nextInt(MAX_JITTER_INTERVAL);
|
||||
nextUploadInterval = UPLOAD_INTERVAL + random.nextInt(MAX_JITTER_INTERVAL);
|
||||
uploadBuffer.clear();
|
||||
}
|
||||
}
|
||||
|
|
@ -37,7 +37,9 @@ import org.apache.kafka.common.requests.s3.GetKVsRequest;
|
|||
import org.apache.kafka.common.requests.s3.PutKVsRequest;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
|
||||
import com.automq.shell.metrics.S3MetricsExporter;
|
||||
import com.automq.stream.api.KeyValue;
|
||||
import com.automq.stream.api.KeyValue.ValueAndEpoch;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -47,7 +49,7 @@ import java.util.List;
|
|||
import java.util.Objects;
|
||||
|
||||
public class ClientKVClient {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ClientKVClient.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(S3MetricsExporter.class);
|
||||
|
||||
private final NetworkClient networkClient;
|
||||
private final Node bootstrapServer;
|
||||
|
|
@ -94,6 +96,32 @@ public class ClientKVClient {
|
|||
throw code.exception();
|
||||
}
|
||||
|
||||
public ValueAndEpoch getKV(String key, String namespace) throws IOException {
|
||||
|
||||
if (LOGGER.isTraceEnabled()) {
|
||||
LOGGER.trace("[ClientKVClient]: Get KV: {} Namespace: {}", key, namespace);
|
||||
}
|
||||
|
||||
GetKVsRequestData data = new GetKVsRequestData()
|
||||
.setGetKeyRequests(List.of(new GetKVsRequestData.GetKVRequest().setKey(key).setNamespace(namespace)));
|
||||
|
||||
long now = Time.SYSTEM.milliseconds();
|
||||
ClientRequest clientRequest = networkClient.newClientRequest(String.valueOf(bootstrapServer.id()),
|
||||
new GetKVsRequest.Builder(data), now, true, 3000, null);
|
||||
|
||||
ClientResponse response = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, Time.SYSTEM);
|
||||
GetKVsResponseData responseData = (GetKVsResponseData) response.responseBody().data();
|
||||
|
||||
Errors code = Errors.forCode(responseData.errorCode());
|
||||
if (Objects.requireNonNull(code) == Errors.NONE) {
|
||||
return ValueAndEpoch.of(
|
||||
responseData.getKVResponses().get(0).value(),
|
||||
responseData.getKVResponses().get(0).epoch());
|
||||
}
|
||||
|
||||
throw code.exception();
|
||||
}
|
||||
|
||||
public KeyValue.Value putKV(String key, byte[] value) throws IOException {
|
||||
long now = Time.SYSTEM.milliseconds();
|
||||
|
||||
|
|
@ -118,6 +146,32 @@ public class ClientKVClient {
|
|||
throw code.exception();
|
||||
}
|
||||
|
||||
public ValueAndEpoch putKV(String key, byte[] value, String namespace, long epoch) throws IOException {
|
||||
long now = Time.SYSTEM.milliseconds();
|
||||
|
||||
if (LOGGER.isTraceEnabled()) {
|
||||
LOGGER.trace("[ClientKVClient]: put KV: {}", key);
|
||||
}
|
||||
|
||||
PutKVsRequestData data = new PutKVsRequestData()
|
||||
.setPutKVRequests(List.of(new PutKVsRequestData.PutKVRequest().setKey(key).setValue(value).setNamespace(namespace).setEpoch(epoch)));
|
||||
|
||||
ClientRequest clientRequest = networkClient.newClientRequest(String.valueOf(bootstrapServer.id()),
|
||||
new PutKVsRequest.Builder(data), now, true, 3000, null);
|
||||
|
||||
ClientResponse response = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, Time.SYSTEM);
|
||||
PutKVsResponseData responseData = (PutKVsResponseData) response.responseBody().data();
|
||||
|
||||
Errors code = Errors.forCode(responseData.errorCode());
|
||||
if (Objects.requireNonNull(code) == Errors.NONE) {
|
||||
return ValueAndEpoch.of(
|
||||
responseData.putKVResponses().get(0).value(),
|
||||
responseData.putKVResponses().get(0).epoch());
|
||||
}
|
||||
|
||||
throw code.exception();
|
||||
}
|
||||
|
||||
public KeyValue.Value deleteKV(String key) throws IOException {
|
||||
long now = Time.SYSTEM.milliseconds();
|
||||
|
||||
|
|
@ -141,4 +195,30 @@ public class ClientKVClient {
|
|||
|
||||
throw code.exception();
|
||||
}
|
||||
|
||||
public ValueAndEpoch deleteKV(String key, String namespace, long epoch) throws IOException {
|
||||
long now = Time.SYSTEM.milliseconds();
|
||||
|
||||
if (LOGGER.isTraceEnabled()) {
|
||||
LOGGER.trace("[ClientKVClient]: Delete KV: {}", key);
|
||||
}
|
||||
|
||||
DeleteKVsRequestData data = new DeleteKVsRequestData()
|
||||
.setDeleteKVRequests(List.of(new DeleteKVsRequestData.DeleteKVRequest().setKey(key).setNamespace(namespace).setEpoch(epoch)));
|
||||
|
||||
ClientRequest clientRequest = networkClient.newClientRequest(String.valueOf(bootstrapServer.id()),
|
||||
new DeleteKVsRequest.Builder(data), now, true, 3000, null);
|
||||
|
||||
ClientResponse response = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, Time.SYSTEM);
|
||||
DeleteKVsResponseData responseData = (DeleteKVsResponseData) response.responseBody().data();
|
||||
|
||||
Errors code = Errors.forCode(responseData.errorCode());
|
||||
if (Objects.requireNonNull(code) == Errors.NONE) {
|
||||
return ValueAndEpoch.of(
|
||||
responseData.deleteKVResponses().get(0).value(),
|
||||
responseData.deleteKVResponses().get(0).epoch());
|
||||
}
|
||||
|
||||
throw code.exception();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,5 +42,4 @@ case $COMMAND in
|
|||
;;
|
||||
esac
|
||||
|
||||
export KAFKA_CONNECT_MODE=true
|
||||
exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectDistributed "$@"
|
||||
|
|
|
|||
|
|
@ -42,5 +42,4 @@ case $COMMAND in
|
|||
;;
|
||||
esac
|
||||
|
||||
export KAFKA_CONNECT_MODE=true
|
||||
exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectStandalone "$@"
|
||||
|
|
|
|||
|
|
@ -40,23 +40,7 @@ should_include_file() {
|
|||
fi
|
||||
file=$1
|
||||
if [ -z "$(echo "$file" | grep -E "$regex")" ] ; then
|
||||
# If Connect mode is enabled, apply additional filtering
|
||||
if [ "$KAFKA_CONNECT_MODE" = "true" ]; then
|
||||
# Skip if file doesn't exist
|
||||
[ ! -f "$file" ] && return 1
|
||||
|
||||
# Exclude heavy dependencies that Connect doesn't need
|
||||
case "$file" in
|
||||
*hadoop*) return 1 ;;
|
||||
*hive*) return 1 ;;
|
||||
*iceberg*) return 1 ;;
|
||||
*avro*) return 1 ;;
|
||||
*parquet*) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
|
|
|||
216
build.gradle
216
build.gradle
|
|
@ -53,7 +53,7 @@ plugins {
|
|||
|
||||
ext {
|
||||
gradleVersion = versions.gradle
|
||||
minJavaVersion = 17
|
||||
minJavaVersion = 11
|
||||
buildVersionFileName = "kafka-version.properties"
|
||||
|
||||
defaultMaxHeapSize = "2g"
|
||||
|
|
@ -150,10 +150,6 @@ allprojects {
|
|||
}
|
||||
|
||||
configurations.all {
|
||||
// Globally exclude commons-logging and logback to ensure a single logging implementation (reload4j)
|
||||
exclude group: "commons-logging", module: "commons-logging"
|
||||
exclude group: "ch.qos.logback", module: "logback-classic"
|
||||
exclude group: "ch.qos.logback", module: "logback-core"
|
||||
// zinc is the Scala incremental compiler, it has a configuration for its own dependencies
|
||||
// that are unrelated to the project dependencies, we should not change them
|
||||
if (name != "zinc") {
|
||||
|
|
@ -264,10 +260,7 @@ subprojects {
|
|||
options.compilerArgs << "-Xlint:-rawtypes"
|
||||
options.compilerArgs << "-Xlint:-serial"
|
||||
options.compilerArgs << "-Xlint:-try"
|
||||
// AutoMQ inject start
|
||||
// TODO: remove me, when upgrade to 4.x
|
||||
// options.compilerArgs << "-Werror"
|
||||
// AutoMQ inject start
|
||||
options.compilerArgs << "-Werror"
|
||||
|
||||
// --release is the recommended way to select the target release, but it's only supported in Java 9 so we also
|
||||
// set --source and --target via `sourceCompatibility` and `targetCompatibility` a couple of lines below
|
||||
|
|
@ -838,13 +831,6 @@ tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" })
|
|||
|
||||
tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {}
|
||||
|
||||
// OpenTelemetry related tasks
|
||||
tasks.create(name: "jarOpenTelemetry", dependsOn: ":opentelemetry:jar") {}
|
||||
|
||||
tasks.create(name: "testOpenTelemetry", dependsOn: ":opentelemetry:test") {}
|
||||
|
||||
tasks.create(name: "buildOpenTelemetry", dependsOn: [":opentelemetry:jar", ":opentelemetry:test"]) {}
|
||||
|
||||
project(':server') {
|
||||
base {
|
||||
archivesName = "kafka-server"
|
||||
|
|
@ -945,8 +931,6 @@ project(':core') {
|
|||
implementation project(':storage')
|
||||
implementation project(':server')
|
||||
implementation project(':automq-shell')
|
||||
implementation project(':automq-metrics')
|
||||
implementation project(':automq-log-uploader')
|
||||
|
||||
implementation libs.argparse4j
|
||||
implementation libs.commonsValidator
|
||||
|
|
@ -984,9 +968,15 @@ project(':core') {
|
|||
implementation libs.guava
|
||||
implementation libs.slf4jBridge
|
||||
implementation libs.slf4jReload4j
|
||||
// The `jcl-over-slf4j` library is used to redirect JCL logging to SLF4J.
|
||||
implementation libs.jclOverSlf4j
|
||||
|
||||
implementation libs.opentelemetryJava8
|
||||
implementation libs.opentelemetryOshi
|
||||
implementation libs.opentelemetrySdk
|
||||
implementation libs.opentelemetrySdkMetrics
|
||||
implementation libs.opentelemetryExporterLogging
|
||||
implementation libs.opentelemetryExporterProm
|
||||
implementation libs.opentelemetryExporterOTLP
|
||||
implementation libs.opentelemetryJmx
|
||||
implementation libs.awsSdkAuth
|
||||
|
||||
// table topic start
|
||||
|
|
@ -999,7 +989,6 @@ project(':core') {
|
|||
implementation ("org.apache.iceberg:iceberg-parquet:${versions.iceberg}")
|
||||
implementation ("org.apache.iceberg:iceberg-common:${versions.iceberg}")
|
||||
implementation ("org.apache.iceberg:iceberg-aws:${versions.iceberg}")
|
||||
implementation ("org.apache.iceberg:iceberg-nessie:${versions.iceberg}")
|
||||
implementation ("software.amazon.awssdk:glue:${versions.awsSdk}")
|
||||
implementation ("software.amazon.awssdk:s3tables:${versions.awsSdk}")
|
||||
implementation 'software.amazon.s3tables:s3-tables-catalog-for-iceberg:0.1.0'
|
||||
|
|
@ -1015,37 +1004,6 @@ project(':core') {
|
|||
exclude group: 'org.apache.kafka', module: 'kafka-clients'
|
||||
}
|
||||
|
||||
// > hive ext start
|
||||
implementation 'org.apache.iceberg:iceberg-hive-metastore:1.6.1'
|
||||
implementation('org.apache.hive:hive-metastore:3.1.3') {
|
||||
// Remove useless dependencies (copy from iceberg-kafka-connect)
|
||||
exclude group: "org.apache.avro", module: "avro"
|
||||
exclude group: "org.slf4j", module: "slf4j-log4j12"
|
||||
exclude group: "org.pentaho" // missing dependency
|
||||
exclude group: "org.apache.hbase"
|
||||
exclude group: "org.apache.logging.log4j"
|
||||
exclude group: "co.cask.tephra"
|
||||
exclude group: "com.google.code.findbugs", module: "jsr305"
|
||||
exclude group: "org.eclipse.jetty.aggregate", module: "jetty-all"
|
||||
exclude group: "org.eclipse.jetty.orbit", module: "javax.servlet"
|
||||
exclude group: "org.apache.parquet", module: "parquet-hadoop-bundle"
|
||||
exclude group: "com.tdunning", module: "json"
|
||||
exclude group: "javax.transaction", module: "transaction-api"
|
||||
exclude group: "com.zaxxer", module: "HikariCP"
|
||||
exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-common"
|
||||
exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-applicationhistoryservice"
|
||||
exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-resourcemanager"
|
||||
exclude group: "org.apache.hadoop", module: "hadoop-yarn-server-web-proxy"
|
||||
exclude group: "org.apache.hive", module: "hive-service-rpc"
|
||||
exclude group: "com.github.joshelser", module: "dropwizard-metrics-hadoop-metrics2-reporter"
|
||||
}
|
||||
implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.4.1') {
|
||||
exclude group: 'com.sun.jersey', module: '*'
|
||||
exclude group: 'com.sun.jersey.contribs', module: '*'
|
||||
exclude group: 'com.github.pjfanning', module: 'jersey-json'
|
||||
}
|
||||
// > hive ext end
|
||||
|
||||
// > Protobuf ext start
|
||||
// Wire Runtime for schema handling
|
||||
implementation ("com.squareup.wire:wire-schema:${versions.wire}")
|
||||
|
|
@ -1069,7 +1027,6 @@ project(':core') {
|
|||
testImplementation project(':storage:storage-api').sourceSets.test.output
|
||||
testImplementation project(':server').sourceSets.test.output
|
||||
testImplementation libs.bcpkix
|
||||
testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
|
||||
testImplementation libs.mockitoCore
|
||||
testImplementation libs.guava
|
||||
testImplementation(libs.apacheda) {
|
||||
|
|
@ -1250,10 +1207,6 @@ project(':core') {
|
|||
from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") }
|
||||
from(project(':automq-shell').jar) { into("libs/") }
|
||||
from(project(':automq-shell').configurations.runtimeClasspath) { into("libs/") }
|
||||
from(project(':automq-metrics').jar) { into("libs/") }
|
||||
from(project(':automq-metrics').configurations.runtimeClasspath) { into("libs/") }
|
||||
from(project(':automq-log-uploader').jar) { into("libs/") }
|
||||
from(project(':automq-log-uploader').configurations.runtimeClasspath) { into("libs/") }
|
||||
from(project(':shell').jar) { into("libs/") }
|
||||
from(project(':shell').configurations.runtimeClasspath) { into("libs/") }
|
||||
from(project(':connect:api').jar) { into("libs/") }
|
||||
|
|
@ -1285,38 +1238,6 @@ project(':core') {
|
|||
duplicatesStrategy 'exclude'
|
||||
}
|
||||
|
||||
// AutoMQ inject start
|
||||
tasks.create(name: "releaseE2ETar", dependsOn: [configurations.archives.artifacts, 'copyDependantTestLibs'], type: Tar) {
|
||||
def prefix = project.findProperty('prefix') ?: ''
|
||||
archiveBaseName = "${prefix}kafka"
|
||||
|
||||
into "${prefix}kafka-${archiveVersion.get()}"
|
||||
compression = Compression.GZIP
|
||||
from(project.file("$rootDir/bin")) { into "bin/" }
|
||||
from(project.file("$rootDir/config")) { into "config/" }
|
||||
from(project.file("$rootDir/licenses")) { into "licenses/" }
|
||||
from(project.file("$rootDir/docker/docker-compose.yaml")) { into "docker/" }
|
||||
from(project.file("$rootDir/docker/telemetry")) { into "docker/telemetry/" }
|
||||
from(project.file("$rootDir/LICENSE")) { into "" }
|
||||
from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
|
||||
from(configurations.runtimeClasspath) { into("libs/") }
|
||||
from(configurations.archives.artifacts.files) { into("libs/") }
|
||||
from(project.siteDocsTar) { into("site-docs/") }
|
||||
|
||||
// Include main and test jars from all subprojects
|
||||
rootProject.subprojects.each { subproject ->
|
||||
if (subproject.tasks.findByName('jar')) {
|
||||
from(subproject.tasks.named('jar')) { into('libs/') }
|
||||
}
|
||||
if (subproject.tasks.findByName('testJar')) {
|
||||
from(subproject.tasks.named('testJar')) { into('libs/') }
|
||||
}
|
||||
from(subproject.configurations.runtimeClasspath) { into('libs/') }
|
||||
}
|
||||
duplicatesStrategy 'exclude'
|
||||
}
|
||||
// AutoMQ inject end
|
||||
|
||||
jar {
|
||||
dependsOn('copyDependantLibs')
|
||||
}
|
||||
|
|
@ -1391,7 +1312,6 @@ project(':metadata') {
|
|||
implementation libs.guava
|
||||
implementation libs.awsSdkAuth
|
||||
implementation project(':s3stream')
|
||||
implementation ("org.apache.avro:avro:${versions.avro}")
|
||||
|
||||
implementation libs.jacksonDatabind
|
||||
implementation libs.jacksonJDK8Datatypes
|
||||
|
|
@ -2264,10 +2184,10 @@ project(':s3stream') {
|
|||
implementation 'software.amazon.awssdk.crt:aws-crt:0.30.8'
|
||||
implementation 'com.ibm.async:asyncutil:0.1.0'
|
||||
|
||||
testImplementation 'org.slf4j:slf4j-simple:1.7.36'
|
||||
testImplementation libs.junitJupiter
|
||||
testImplementation libs.mockitoCore
|
||||
testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
|
||||
testImplementation 'org.slf4j:slf4j-simple:2.0.9'
|
||||
testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0'
|
||||
testImplementation 'org.mockito:mockito-core:5.5.0'
|
||||
testImplementation 'org.mockito:mockito-junit-jupiter:5.5.0'
|
||||
testImplementation 'org.awaitility:awaitility:4.2.1'
|
||||
}
|
||||
|
||||
|
|
@ -2334,107 +2254,6 @@ project(':tools:tools-api') {
|
|||
}
|
||||
}
|
||||
|
||||
project(':automq-metrics') {
|
||||
archivesBaseName = "automq-metrics"
|
||||
|
||||
checkstyle {
|
||||
configProperties = checkstyleConfigProperties("import-control-server.xml")
|
||||
}
|
||||
|
||||
configurations {
|
||||
all {
|
||||
exclude group: 'io.opentelemetry', module: 'opentelemetry-exporter-sender-okhttp'
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// OpenTelemetry core dependencies
|
||||
api libs.opentelemetryJava8
|
||||
api libs.opentelemetryOshi
|
||||
api libs.opentelemetrySdk
|
||||
api libs.opentelemetrySdkMetrics
|
||||
api libs.opentelemetryExporterLogging
|
||||
api libs.opentelemetryExporterProm
|
||||
api libs.opentelemetryExporterOTLP
|
||||
api libs.opentelemetryExporterSenderJdk
|
||||
api libs.opentelemetryJmx
|
||||
|
||||
// Logging dependencies
|
||||
api libs.slf4jApi
|
||||
api libs.slf4jBridge // 添加 SLF4J Bridge 依赖
|
||||
api libs.reload4j
|
||||
|
||||
api libs.commonLang
|
||||
|
||||
// Yammer metrics (for integration)
|
||||
api 'com.yammer.metrics:metrics-core:2.2.0'
|
||||
|
||||
implementation(project(':s3stream')) {
|
||||
exclude(group: 'io.opentelemetry', module: '*')
|
||||
exclude(group: 'io.opentelemetry.instrumentation', module: '*')
|
||||
exclude(group: 'io.opentelemetry.proto', module: '*')
|
||||
exclude(group: 'io.netty', module: 'netty-tcnative-boringssl-static')
|
||||
exclude(group: 'com.github.jnr', module: '*')
|
||||
exclude(group: 'org.aspectj', module: '*')
|
||||
exclude(group: 'net.java.dev.jna', module: '*')
|
||||
exclude(group: 'net.sourceforge.argparse4j', module: '*')
|
||||
exclude(group: 'com.bucket4j', module: '*')
|
||||
exclude(group: 'com.yammer.metrics', module: '*')
|
||||
exclude(group: 'com.github.spotbugs', module: '*')
|
||||
exclude(group: 'org.apache.kafka.shaded', module: '*')
|
||||
}
|
||||
implementation libs.nettyBuffer
|
||||
implementation libs.jacksonDatabind
|
||||
implementation libs.guava
|
||||
implementation project(':clients')
|
||||
|
||||
// Test dependencies
|
||||
testImplementation libs.junitJupiter
|
||||
testImplementation libs.mockitoCore
|
||||
testImplementation libs.slf4jReload4j
|
||||
|
||||
testRuntimeOnly libs.junitPlatformLanucher
|
||||
|
||||
implementation('io.opentelemetry:opentelemetry-sdk:1.40.0')
|
||||
implementation("io.opentelemetry.semconv:opentelemetry-semconv:1.25.0-alpha")
|
||||
implementation("io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:2.6.0-alpha")
|
||||
implementation('com.google.protobuf:protobuf-java:3.25.5')
|
||||
implementation('org.xerial.snappy:snappy-java:1.1.10.5')
|
||||
}
|
||||
|
||||
clean.doFirst {
|
||||
delete "$buildDir/kafka/"
|
||||
}
|
||||
|
||||
javadoc {
|
||||
enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
project(':automq-log-uploader') {
|
||||
archivesBaseName = "automq-log-uploader"
|
||||
|
||||
checkstyle {
|
||||
configProperties = checkstyleConfigProperties("import-control-server.xml")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api project(':s3stream')
|
||||
|
||||
implementation project(':clients')
|
||||
implementation libs.reload4j
|
||||
implementation libs.slf4jApi
|
||||
implementation libs.slf4jBridge
|
||||
implementation libs.nettyBuffer
|
||||
implementation libs.guava
|
||||
implementation libs.commonLang
|
||||
}
|
||||
|
||||
javadoc {
|
||||
enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
project(':tools') {
|
||||
base {
|
||||
archivesName = "kafka-tools"
|
||||
|
|
@ -2456,9 +2275,7 @@ project(':tools') {
|
|||
exclude group: 'org.apache.kafka', module: 'kafka-clients'
|
||||
}
|
||||
implementation libs.bucket4j
|
||||
implementation (libs.oshi){
|
||||
exclude group: 'org.slf4j', module: '*'
|
||||
}
|
||||
implementation libs.oshi
|
||||
// AutoMQ inject end
|
||||
|
||||
implementation project(':storage')
|
||||
|
|
@ -3542,8 +3359,6 @@ project(':connect:runtime') {
|
|||
api project(':clients')
|
||||
api project(':connect:json')
|
||||
api project(':connect:transforms')
|
||||
api project(':automq-metrics')
|
||||
api project(':automq-log-uploader')
|
||||
|
||||
implementation libs.slf4jApi
|
||||
implementation libs.reload4j
|
||||
|
|
@ -3552,7 +3367,6 @@ project(':connect:runtime') {
|
|||
implementation libs.jacksonJaxrsJsonProvider
|
||||
implementation libs.jerseyContainerServlet
|
||||
implementation libs.jerseyHk2
|
||||
implementation libs.jaxrsApi
|
||||
implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
|
||||
implementation libs.activation // Jersey dependency that was available in the JDK before Java 9
|
||||
implementation libs.jettyServer
|
||||
|
|
|
|||
|
|
@ -378,6 +378,5 @@
|
|||
<suppress id="dontUseSystemExit"
|
||||
files="(BenchTool|S3Utils|AutoMQCLI).java"/>
|
||||
<suppress checks="ClassDataAbstractionCoupling" files="(StreamControlManagerTest|ControllerStreamManager).java"/>
|
||||
<suppress files="core[\/]src[\/]test[\/]java[\/]kafka[\/]automq[\/]table[\/]process[\/]proto[\/].*\.java$" checks=".*"/>
|
||||
|
||||
</suppressions>
|
||||
|
|
|
|||
|
|
@ -1739,6 +1739,48 @@ public interface Admin extends AutoCloseable {
|
|||
* @return {@link UpdateGroupResult}
|
||||
*/
|
||||
UpdateGroupResult updateGroup(String groupId, UpdateGroupSpec groupSpec, UpdateGroupOptions options);
|
||||
|
||||
GetNamespacedKVResult getNamespacedKV(
|
||||
Optional<Set<TopicPartition>> partitions,
|
||||
String namespace,
|
||||
String key,
|
||||
GetNamespacedKVOptions options
|
||||
);
|
||||
|
||||
/**
|
||||
* Put a key-value pair in the namespaced KV store.
|
||||
* @param partitions
|
||||
* @param namespace
|
||||
* @param key
|
||||
* @param value
|
||||
* @param options
|
||||
* @return
|
||||
*/
|
||||
PutNamespacedKVResult putNamespacedKV(
|
||||
Optional<Set<TopicPartition>> partitions,
|
||||
String namespace,
|
||||
String key,
|
||||
String value,
|
||||
PutNamespacedKVOptions options
|
||||
);
|
||||
|
||||
/**
|
||||
* Delete a key-value pair in the namespaced KV store.
|
||||
* @param partitions
|
||||
* @param namespace
|
||||
* @param key
|
||||
* @param options
|
||||
* @return
|
||||
*/
|
||||
DeleteNamespacedKVResult deleteNamespacedKV(
|
||||
Optional<Set<TopicPartition>> partitions,
|
||||
String namespace,
|
||||
String key,
|
||||
DeleteNamespacedKVOptions options
|
||||
);
|
||||
|
||||
|
||||
|
||||
// AutoMQ inject end
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
public class DeleteNamespacedKVOptions extends AbstractOptions<DeleteNamespacedKVOptions> {
|
||||
|
||||
private long ifMatchEpoch = 0L;
|
||||
|
||||
public DeleteNamespacedKVOptions ifMatchEpoch(long epoch) {
|
||||
this.ifMatchEpoch = epoch;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long ifMatchEpoch() {
|
||||
return ifMatchEpoch;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
import org.apache.kafka.common.KafkaFuture;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class DeleteNamespacedKVResult extends AbstractOptions<DeleteNamespacedKVResult> {
|
||||
|
||||
private final Map<TopicPartition, KafkaFuture<Void>> futures;
|
||||
|
||||
public DeleteNamespacedKVResult(Map<TopicPartition, KafkaFuture<Void>> futures) {
|
||||
this.futures = futures;
|
||||
}
|
||||
|
||||
public KafkaFuture<Map<TopicPartition, KafkaFuture<Void>>> all() {
|
||||
return KafkaFuture.completedFuture(futures);
|
||||
}
|
||||
}
|
||||
|
|
@ -320,5 +320,20 @@ public class ForwardingAdmin implements Admin {
|
|||
return delegate.updateGroup(groupId, groupSpec, options);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetNamespacedKVResult getNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, GetNamespacedKVOptions options) {
|
||||
return delegate.getNamespacedKV(partitions, namespace, key, options);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PutNamespacedKVResult putNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, String value, PutNamespacedKVOptions options) {
|
||||
return delegate.putNamespacedKV(partitions, namespace, key, value, options);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DeleteNamespacedKVResult deleteNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, DeleteNamespacedKVOptions options) {
|
||||
return delegate.deleteNamespacedKV(partitions, namespace, key, options);
|
||||
}
|
||||
|
||||
// AutoMQ inject end
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
public class GetNamespacedKVOptions extends AbstractOptions<GetNamespacedKVOptions> {
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
import org.apache.kafka.common.KafkaFuture;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.GetKVsResponseData.GetKVResponse;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
public class GetNamespacedKVResult {
|
||||
|
||||
private final Map<TopicPartition, KafkaFuture<GetKVResponse>> futures;
|
||||
|
||||
public GetNamespacedKVResult(Map<TopicPartition, KafkaFuture<GetKVResponse>> futures) {
|
||||
this.futures = futures;
|
||||
}
|
||||
|
||||
public KafkaFuture<Map<TopicPartition, KafkaFuture<GetKVResponse>>> all() throws ExecutionException, InterruptedException {
|
||||
return KafkaFuture.completedFuture(futures);
|
||||
}
|
||||
}
|
||||
|
|
@ -47,14 +47,19 @@ import org.apache.kafka.clients.admin.internals.AlterConsumerGroupOffsetsHandler
|
|||
import org.apache.kafka.clients.admin.internals.CoordinatorKey;
|
||||
import org.apache.kafka.clients.admin.internals.DeleteConsumerGroupOffsetsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DeleteConsumerGroupsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DeleteNamespacedKVHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DeleteRecordsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DescribeConsumerGroupsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DescribeProducersHandler;
|
||||
import org.apache.kafka.clients.admin.internals.DescribeTransactionsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.FenceProducersHandler;
|
||||
import org.apache.kafka.clients.admin.internals.GetNamespacedKVHandler;
|
||||
import org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.ListOffsetsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.ListTransactionsHandler;
|
||||
import org.apache.kafka.clients.admin.internals.NamespacedKVRecordsToGet;
|
||||
import org.apache.kafka.clients.admin.internals.NamespacedKVRecordsToPut;
|
||||
import org.apache.kafka.clients.admin.internals.PutNamespacedKVHandler;
|
||||
import org.apache.kafka.clients.admin.internals.RemoveMembersFromConsumerGroupHandler;
|
||||
import org.apache.kafka.clients.admin.internals.UpdateGroupHandler;
|
||||
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
|
||||
|
|
@ -132,6 +137,7 @@ import org.apache.kafka.common.message.DeleteAclsRequestData.DeleteAclsFilter;
|
|||
import org.apache.kafka.common.message.DeleteAclsResponseData;
|
||||
import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult;
|
||||
import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl;
|
||||
import org.apache.kafka.common.message.DeleteKVsRequestData;
|
||||
import org.apache.kafka.common.message.DeleteTopicsRequestData;
|
||||
import org.apache.kafka.common.message.DeleteTopicsRequestData.DeleteTopicState;
|
||||
import org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult;
|
||||
|
|
@ -152,6 +158,8 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData;
|
|||
import org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData.UserName;
|
||||
import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData;
|
||||
import org.apache.kafka.common.message.ExpireDelegationTokenRequestData;
|
||||
import org.apache.kafka.common.message.GetKVsRequestData;
|
||||
import org.apache.kafka.common.message.GetKVsResponseData;
|
||||
import org.apache.kafka.common.message.GetNextNodeIdRequestData;
|
||||
import org.apache.kafka.common.message.GetTelemetrySubscriptionsRequestData;
|
||||
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
|
||||
|
|
@ -160,6 +168,8 @@ import org.apache.kafka.common.message.ListGroupsRequestData;
|
|||
import org.apache.kafka.common.message.ListGroupsResponseData;
|
||||
import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
|
||||
import org.apache.kafka.common.message.MetadataRequestData;
|
||||
import org.apache.kafka.common.message.PutKVsRequestData;
|
||||
import org.apache.kafka.common.message.PutKVsResponseData;
|
||||
import org.apache.kafka.common.message.RemoveRaftVoterRequestData;
|
||||
import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
|
||||
import org.apache.kafka.common.message.UnregisterBrokerRequestData;
|
||||
|
|
@ -266,6 +276,7 @@ import org.apache.kafka.common.utils.Utils;
|
|||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Duration;
|
||||
|
|
@ -4866,6 +4877,86 @@ public class KafkaAdminClient extends AdminClient {
|
|||
return new UpdateGroupResult(future.get(CoordinatorKey.byGroupId(groupId)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetNamespacedKVResult getNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, GetNamespacedKVOptions options) {
|
||||
Set<TopicPartition> targetPartitions = partitions.orElseThrow(() ->
|
||||
new IllegalArgumentException("Partitions cannot be empty")
|
||||
);
|
||||
|
||||
NamespacedKVRecordsToGet.Builder recordsToGetBuilder = NamespacedKVRecordsToGet.newBuilder();
|
||||
for (TopicPartition tp : targetPartitions) {
|
||||
GetKVsRequestData.GetKVRequest kvRequest = new GetKVsRequestData.GetKVRequest()
|
||||
.setKey(key)
|
||||
.setNamespace(namespace);
|
||||
|
||||
recordsToGetBuilder.addRecord(tp, kvRequest);
|
||||
}
|
||||
|
||||
NamespacedKVRecordsToGet recordsToGet = recordsToGetBuilder.build();
|
||||
GetNamespacedKVHandler handler = new GetNamespacedKVHandler(logContext, recordsToGet);
|
||||
SimpleAdminApiFuture<TopicPartition, GetKVsResponseData.GetKVResponse> future = GetNamespacedKVHandler.newFuture(targetPartitions);
|
||||
|
||||
invokeDriver(handler, future, options.timeoutMs);
|
||||
|
||||
return new GetNamespacedKVResult(future.all());
|
||||
}
|
||||
|
||||
@Override
|
||||
public PutNamespacedKVResult putNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, String value, PutNamespacedKVOptions options) {
|
||||
Set<TopicPartition> targetPartitions = partitions.orElseThrow(() ->
|
||||
new IllegalArgumentException("Partitions cannot be empty")
|
||||
);
|
||||
|
||||
NamespacedKVRecordsToPut.Builder recordsToPutBuilder = NamespacedKVRecordsToPut.newBuilder();
|
||||
for (TopicPartition tp : targetPartitions) {
|
||||
PutKVsRequestData.PutKVRequest kvRequest = new PutKVsRequestData.PutKVRequest()
|
||||
.setKey(key)
|
||||
.setValue(value.getBytes(StandardCharsets.UTF_8))
|
||||
.setNamespace(namespace)
|
||||
.setOverwrite(options.overwrite())
|
||||
.setEpoch(options.ifMatchEpoch());
|
||||
|
||||
recordsToPutBuilder.addRecord(tp, kvRequest);
|
||||
}
|
||||
|
||||
NamespacedKVRecordsToPut recordsToPut = recordsToPutBuilder.build();
|
||||
|
||||
PutNamespacedKVHandler handler = new PutNamespacedKVHandler(logContext, recordsToPut);
|
||||
SimpleAdminApiFuture<TopicPartition, PutKVsResponseData.PutKVResponse> future = PutNamespacedKVHandler.newFuture(targetPartitions);
|
||||
|
||||
invokeDriver(handler, future, options.timeoutMs);
|
||||
|
||||
return new PutNamespacedKVResult(future.all());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DeleteNamespacedKVResult deleteNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, DeleteNamespacedKVOptions options) {
|
||||
|
||||
Set<TopicPartition> targetPartitions = partitions.orElseThrow(() ->
|
||||
new IllegalArgumentException("Partitions cannot be empty")
|
||||
);
|
||||
|
||||
NamespacedKVRecordsToDelete.Builder recordsToDeleteBuilder = NamespacedKVRecordsToDelete.newBuilder();
|
||||
for (TopicPartition tp : targetPartitions) {
|
||||
DeleteKVsRequestData.DeleteKVRequest kvRequest = new DeleteKVsRequestData.DeleteKVRequest()
|
||||
.setKey(key)
|
||||
.setNamespace(namespace)
|
||||
.setEpoch(options.ifMatchEpoch());
|
||||
|
||||
recordsToDeleteBuilder.addRecord(tp, kvRequest);
|
||||
}
|
||||
|
||||
NamespacedKVRecordsToDelete recordsToDelete = recordsToDeleteBuilder.build();
|
||||
|
||||
DeleteNamespacedKVHandler handler = new DeleteNamespacedKVHandler(logContext, recordsToDelete);
|
||||
SimpleAdminApiFuture<TopicPartition, Void> future = DeleteNamespacedKVHandler.newFuture(targetPartitions);
|
||||
|
||||
invokeDriver(handler, future, options.timeoutMs);
|
||||
|
||||
return new DeleteNamespacedKVResult(future.all());
|
||||
}
|
||||
|
||||
private <K, V> void invokeDriver(
|
||||
AdminApiHandler<K, V> handler,
|
||||
AdminApiFuture<K, V> future,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.DeleteKVsRequestData.DeleteKVRequest;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NamespacedKVRecordsToDelete {
|
||||
|
||||
private final Map<TopicPartition, List<DeleteKVRequest>> recordsByPartition;
|
||||
|
||||
public NamespacedKVRecordsToDelete(Map<TopicPartition, List<DeleteKVRequest>> recordsByPartition) {
|
||||
this.recordsByPartition = recordsByPartition;
|
||||
}
|
||||
|
||||
public static NamespacedKVRecordsToDelete.Builder newBuilder() {
|
||||
return new NamespacedKVRecordsToDelete.Builder();
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<DeleteKVRequest>> recordsByPartition() {
|
||||
return recordsByPartition;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private final Map<TopicPartition, List<DeleteKVRequest>> records = new HashMap<>();
|
||||
|
||||
public NamespacedKVRecordsToDelete.Builder addRecord(TopicPartition partition, DeleteKVRequest request) {
|
||||
records.computeIfAbsent(partition, k -> new ArrayList<>()).add(request);
|
||||
return this;
|
||||
}
|
||||
|
||||
public NamespacedKVRecordsToDelete build() {
|
||||
return new NamespacedKVRecordsToDelete(records);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
public class PutNamespacedKVOptions extends AbstractOptions<PutNamespacedKVOptions> {
|
||||
|
||||
private boolean overwrite = false;
|
||||
private long ifMatchEpoch = 0L;
|
||||
|
||||
public PutNamespacedKVOptions overwrite(boolean overwrite) {
|
||||
this.overwrite = overwrite;
|
||||
return this;
|
||||
}
|
||||
|
||||
public PutNamespacedKVOptions ifMatchEpoch(long epoch) {
|
||||
this.ifMatchEpoch = epoch;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean overwrite() {
|
||||
return overwrite;
|
||||
}
|
||||
public long ifMatchEpoch() {
|
||||
return ifMatchEpoch;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package org.apache.kafka.clients.admin;
|
||||
|
||||
import org.apache.kafka.common.KafkaFuture;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.PutKVsResponseData.PutKVResponse;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class PutNamespacedKVResult {
|
||||
|
||||
private final Map<TopicPartition, KafkaFuture<PutKVResponse>> futures;
|
||||
|
||||
public PutNamespacedKVResult(Map<TopicPartition, KafkaFuture<PutKVResponse>> futures) {
|
||||
this.futures = futures;
|
||||
}
|
||||
|
||||
|
||||
public KafkaFuture<Map<TopicPartition, KafkaFuture<PutKVResponse>>> all() {
|
||||
return KafkaFuture.completedFuture(futures);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package org.apache.kafka.clients.admin.internals;
|
||||
|
||||
import org.apache.kafka.clients.admin.NamespacedKVRecordsToDelete;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.DeleteKVsRequestData;
|
||||
import org.apache.kafka.common.message.DeleteKVsRequestData.DeleteKVRequest;
|
||||
import org.apache.kafka.common.message.DeleteKVsResponseData;
|
||||
import org.apache.kafka.common.protocol.Errors;
|
||||
import org.apache.kafka.common.requests.AbstractRequest;
|
||||
import org.apache.kafka.common.requests.AbstractResponse;
|
||||
import org.apache.kafka.common.requests.s3.DeleteKVsRequest;
|
||||
import org.apache.kafka.common.requests.s3.DeleteKVsResponse;
|
||||
import org.apache.kafka.common.utils.LogContext;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class DeleteNamespacedKVHandler extends AdminApiHandler.Batched<TopicPartition, Void> {
|
||||
|
||||
private final Logger logger;
|
||||
private final NamespacedKVRecordsToDelete recordsToDelete;
|
||||
private final AdminApiLookupStrategy<TopicPartition> lookupStrategy;
|
||||
|
||||
public DeleteNamespacedKVHandler(LogContext logContext, NamespacedKVRecordsToDelete recordsToDelete) {
|
||||
this.logger = logContext.logger(PutNamespacedKVHandler.class);
|
||||
this.recordsToDelete = recordsToDelete;
|
||||
this.lookupStrategy = new PartitionLeaderStrategy(logContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
AbstractRequest.Builder<?> buildBatchedRequest(int brokerId, Set<TopicPartition> partitions) {
|
||||
Map<TopicPartition, List<DeleteKVRequest>> filteredRecords = new HashMap<>();
|
||||
for (TopicPartition partition : partitions) {
|
||||
if (recordsToDelete.recordsByPartition().containsKey(partition)) {
|
||||
filteredRecords.put(partition, recordsToDelete.recordsByPartition().get(partition));
|
||||
}
|
||||
}
|
||||
|
||||
DeleteKVsRequestData requestData = new DeleteKVsRequestData();
|
||||
List<DeleteKVRequest> allRequests = new ArrayList<>();
|
||||
filteredRecords.values().forEach(allRequests::addAll);
|
||||
requestData.setDeleteKVRequests(allRequests);
|
||||
|
||||
return new DeleteKVsRequest.Builder(requestData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String apiName() {
|
||||
return "DeleteKVs";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ApiResult<TopicPartition, Void> handleResponse(Node broker, Set<TopicPartition> partitions, AbstractResponse response) {
|
||||
DeleteKVsResponse deleteResponse = (DeleteKVsResponse) response;
|
||||
DeleteKVsResponseData responseData = deleteResponse.data();
|
||||
final Map<TopicPartition, Void> completed = new HashMap<>();
|
||||
final Map<TopicPartition, Throwable> failed = new HashMap<>();
|
||||
|
||||
partitions.forEach(partition -> {
|
||||
Errors error = Errors.forCode(responseData.errorCode());
|
||||
if (error != Errors.NONE) {
|
||||
failed.put(partition, error.exception());
|
||||
} else {
|
||||
completed.put(partition, null);
|
||||
}
|
||||
});
|
||||
|
||||
return new ApiResult<>(completed, failed, Collections.emptyList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminApiLookupStrategy<TopicPartition> lookupStrategy() {
|
||||
return this.lookupStrategy;
|
||||
}
|
||||
|
||||
public static AdminApiFuture.SimpleAdminApiFuture<TopicPartition, Void> newFuture(
|
||||
Set<TopicPartition> partitions
|
||||
) {
|
||||
return AdminApiFuture.forKeys(new HashSet<>(partitions));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package org.apache.kafka.clients.admin.internals;
|
||||
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.GetKVsRequestData;
|
||||
import org.apache.kafka.common.message.GetKVsResponseData;
|
||||
import org.apache.kafka.common.message.GetKVsResponseData.GetKVResponse;
|
||||
import org.apache.kafka.common.protocol.Errors;
|
||||
import org.apache.kafka.common.requests.AbstractRequest;
|
||||
import org.apache.kafka.common.requests.AbstractResponse;
|
||||
import org.apache.kafka.common.requests.s3.GetKVsRequest;
|
||||
import org.apache.kafka.common.requests.s3.GetKVsResponse;
|
||||
import org.apache.kafka.common.utils.LogContext;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class GetNamespacedKVHandler extends AdminApiHandler.Batched<TopicPartition, GetKVResponse> {
|
||||
private final Logger logger;
|
||||
private final NamespacedKVRecordsToGet recordsToGet;
|
||||
private final AdminApiLookupStrategy<TopicPartition> lookupStrategy;
|
||||
private final List<TopicPartition> orderedPartitions;
|
||||
|
||||
public GetNamespacedKVHandler(LogContext logContext, NamespacedKVRecordsToGet recordsToGet) {
|
||||
this.logger = logContext.logger(PutNamespacedKVHandler.class);
|
||||
this.recordsToGet = recordsToGet;
|
||||
this.lookupStrategy = new PartitionLeaderStrategy(logContext);
|
||||
this.orderedPartitions = new ArrayList<>(recordsToGet.recordsByPartition().keySet());
|
||||
}
|
||||
|
||||
@Override
|
||||
AbstractRequest.Builder<?> buildBatchedRequest(int brokerId, Set<TopicPartition> partitions) {
|
||||
|
||||
GetKVsRequestData requestData = new GetKVsRequestData();
|
||||
for (TopicPartition tp : orderedPartitions) {
|
||||
if (partitions.contains(tp)) {
|
||||
requestData.getKeyRequests().addAll(
|
||||
recordsToGet.recordsByPartition().get(tp)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return new GetKVsRequest.Builder(requestData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String apiName() {
|
||||
return "GetKVs";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ApiResult<TopicPartition, GetKVResponse> handleResponse(Node broker, Set<TopicPartition> partitions, AbstractResponse response) {
|
||||
|
||||
GetKVsResponseData data = ((GetKVsResponse) response).data();
|
||||
final Map<TopicPartition, GetKVResponse> completed = new LinkedHashMap<>();
|
||||
final Map<TopicPartition, Throwable> failed = new HashMap<>();
|
||||
List<GetKVResponse> responses = data.getKVResponses();
|
||||
int responseIndex = 0;
|
||||
for (TopicPartition tp : orderedPartitions) {
|
||||
if (!partitions.contains(tp)) {
|
||||
continue;
|
||||
}
|
||||
if (responseIndex >= responses.size()) {
|
||||
failed.put(tp, new IllegalStateException("Missing response for partition"));
|
||||
continue;
|
||||
}
|
||||
GetKVResponse resp = responses.get(responseIndex++);
|
||||
if (resp.errorCode() == Errors.NONE.code()) {
|
||||
completed.put(tp, resp);
|
||||
} else {
|
||||
failed.put(tp, Errors.forCode(resp.errorCode()).exception());
|
||||
}
|
||||
}
|
||||
return new ApiResult<>(completed, failed, Collections.emptyList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminApiLookupStrategy<TopicPartition> lookupStrategy() {
|
||||
return this.lookupStrategy;
|
||||
}
|
||||
|
||||
public static AdminApiFuture.SimpleAdminApiFuture<TopicPartition, GetKVResponse> newFuture(
|
||||
Set<TopicPartition> partitions
|
||||
) {
|
||||
return AdminApiFuture.forKeys(new HashSet<>(partitions));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package org.apache.kafka.clients.admin.internals;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.GetKVsRequestData.GetKVRequest;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NamespacedKVRecordsToGet {
|
||||
|
||||
private final Map<TopicPartition, List<GetKVRequest>> recordsByPartition;
|
||||
|
||||
public NamespacedKVRecordsToGet(Map<TopicPartition, List<GetKVRequest>> recordsByPartition) {
|
||||
this.recordsByPartition = recordsByPartition;
|
||||
}
|
||||
|
||||
public static NamespacedKVRecordsToGet.Builder newBuilder() {
|
||||
return new NamespacedKVRecordsToGet.Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private final Map<TopicPartition, List<GetKVRequest>> records = new HashMap<>();
|
||||
public Builder addRecord(TopicPartition tp, GetKVRequest req) {
|
||||
records.computeIfAbsent(tp, k -> new ArrayList<>()).add(req);
|
||||
return this;
|
||||
}
|
||||
|
||||
public NamespacedKVRecordsToGet build() {
|
||||
return new NamespacedKVRecordsToGet(records);
|
||||
}
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<GetKVRequest>> recordsByPartition() {
|
||||
return recordsByPartition;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package org.apache.kafka.clients.admin.internals;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.PutKVsRequestData.PutKVRequest;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NamespacedKVRecordsToPut {
|
||||
|
||||
private final Map<TopicPartition, List<PutKVRequest>> recordsByPartition;
|
||||
|
||||
private NamespacedKVRecordsToPut(Map<TopicPartition, List<PutKVRequest>> recordsByPartition) {
|
||||
this.recordsByPartition = recordsByPartition;
|
||||
}
|
||||
|
||||
public static Builder newBuilder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<PutKVRequest>> recordsByPartition() {
|
||||
return recordsByPartition;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private final Map<TopicPartition, List<PutKVRequest>> records = new HashMap<>();
|
||||
|
||||
public Builder addRecord(TopicPartition partition, PutKVRequest request) {
|
||||
records.computeIfAbsent(partition, k -> new ArrayList<>()).add(request);
|
||||
return this;
|
||||
}
|
||||
|
||||
public NamespacedKVRecordsToPut build() {
|
||||
return new NamespacedKVRecordsToPut(records);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package org.apache.kafka.clients.admin.internals;
|
||||
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.message.PutKVsRequestData;
|
||||
import org.apache.kafka.common.message.PutKVsRequestData.PutKVRequest;
|
||||
import org.apache.kafka.common.message.PutKVsResponseData;
|
||||
import org.apache.kafka.common.message.PutKVsResponseData.PutKVResponse;
|
||||
import org.apache.kafka.common.protocol.Errors;
|
||||
import org.apache.kafka.common.requests.AbstractRequest;
|
||||
import org.apache.kafka.common.requests.AbstractResponse;
|
||||
import org.apache.kafka.common.requests.s3.PutKVsRequest;
|
||||
import org.apache.kafka.common.requests.s3.PutKVsResponse;
|
||||
import org.apache.kafka.common.utils.LogContext;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class PutNamespacedKVHandler extends AdminApiHandler.Batched<TopicPartition, PutKVResponse> {
|
||||
private final Logger logger;
|
||||
private final NamespacedKVRecordsToPut recordsToPut;
|
||||
private final AdminApiLookupStrategy<TopicPartition> lookupStrategy;
|
||||
private final List<TopicPartition> orderedPartitions;
|
||||
|
||||
public PutNamespacedKVHandler(LogContext logContext, NamespacedKVRecordsToPut recordsToPut) {
|
||||
this.logger = logContext.logger(PutNamespacedKVHandler.class);
|
||||
this.recordsToPut = recordsToPut;
|
||||
this.lookupStrategy = new PartitionLeaderStrategy(logContext);
|
||||
this.orderedPartitions = new ArrayList<>(recordsToPut.recordsByPartition().keySet());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AbstractRequest.Builder<?> buildBatchedRequest(int brokerId, Set<TopicPartition> partitions) {
|
||||
PutKVsRequestData requestData = new PutKVsRequestData();
|
||||
List<PutKVRequest> allPutRequests = orderedPartitions.stream()
|
||||
.filter(partitions::contains)
|
||||
.map(tp -> recordsToPut.recordsByPartition().get(tp))
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(Collection::stream).collect(Collectors.toList());
|
||||
|
||||
requestData.setPutKVRequests(allPutRequests);
|
||||
return new PutKVsRequest.Builder(requestData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String apiName() {
|
||||
return "PutKVs";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ApiResult<TopicPartition, PutKVResponse> handleResponse(Node broker, Set<TopicPartition> partitions, AbstractResponse response) {
|
||||
PutKVsResponseData responseData = ((PutKVsResponse) response).data();
|
||||
List<PutKVResponse> responses = responseData.putKVResponses();
|
||||
final Map<TopicPartition, PutKVResponse> completed = new LinkedHashMap<>();
|
||||
final Map<TopicPartition, Throwable> failed = new HashMap<>();
|
||||
int responseIndex = 0;
|
||||
for (TopicPartition tp : orderedPartitions) {
|
||||
if (!partitions.contains(tp)) {
|
||||
continue;
|
||||
}
|
||||
if (responseIndex >= responses.size()) {
|
||||
failed.put(tp, new IllegalStateException("Missing response for partition"));
|
||||
continue;
|
||||
}
|
||||
PutKVResponse resp = responses.get(responseIndex++);
|
||||
if (resp.errorCode() == Errors.NONE.code()) {
|
||||
completed.put(tp, resp);
|
||||
} else {
|
||||
failed.put(tp, Errors.forCode(resp.errorCode()).exception());
|
||||
}
|
||||
}
|
||||
return new ApiResult<>(completed, failed, Collections.emptyList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public AdminApiLookupStrategy<TopicPartition> lookupStrategy() {
|
||||
return this.lookupStrategy;
|
||||
}
|
||||
|
||||
public static AdminApiFuture.SimpleAdminApiFuture<TopicPartition, PutKVResponse> newFuture(
|
||||
Set<TopicPartition> partitions
|
||||
) {
|
||||
return AdminApiFuture.forKeys(new HashSet<>(partitions));
|
||||
}
|
||||
}
|
||||
|
|
@ -264,51 +264,8 @@ public class TopicConfig {
|
|||
public static final String TABLE_TOPIC_COMMIT_INTERVAL_DOC = "The table topic commit interval(ms)";
|
||||
public static final String TABLE_TOPIC_NAMESPACE_CONFIG = "automq.table.topic.namespace";
|
||||
public static final String TABLE_TOPIC_NAMESPACE_DOC = "The table topic table namespace";
|
||||
|
||||
public static final String TABLE_TOPIC_SCHEMA_TYPE_CONFIG = "automq.table.topic.schema.type";
|
||||
public static final String TABLE_TOPIC_SCHEMA_TYPE_DOC = "[DEPRECATED] The table topic schema type configuration. " +
|
||||
"This configuration is deprecated and will be removed in a future release. " +
|
||||
"Please use the new separate converter and transform configurations instead. " +
|
||||
"Supported values: 'schemaless' (maps to convert.value.type=raw, transform.value.type=none), " +
|
||||
"'schema' (maps to convert.value.type=by_schema_id, transform.value.type=flatten).";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_TYPE_CONFIG = "automq.table.topic.convert.value.type";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_TYPE_DOC = "How to parse Kafka record values. " +
|
||||
"Supported: 'raw', 'string', 'by_schema_id', 'by_latest_schema'. " +
|
||||
"Schema Registry URL required for 'by_schema_id' and 'by_latest_schema'.";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_TYPE_CONFIG = "automq.table.topic.convert.key.type";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_TYPE_DOC = "How to parse Kafka record keys. " +
|
||||
"Supported: 'raw', 'string', 'by_schema_id', 'by_latest_schema'. " +
|
||||
"Schema Registry URL required for 'by_schema_id' and 'by_latest_schema'.";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_BY_LATEST_SCHEMA_SUBJECT_CONFIG =
|
||||
"automq.table.topic.convert.value.by_latest_schema.subject";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_BY_LATEST_SCHEMA_SUBJECT_DOC =
|
||||
"Subject name to resolve the latest value schema from Schema Registry when using convert.value.type=by_latest_schema. " +
|
||||
"If not set, defaults to '<topic>-value'.";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_BY_LATEST_SCHEMA_MESSAGE_FULL_NAME_CONFIG =
|
||||
"automq.table.topic.convert.value.by_latest_schema.message.full.name";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_VALUE_BY_LATEST_SCHEMA_MESSAGE_FULL_NAME_DOC =
|
||||
"Fully-qualified message name for the latest value schema (if using Protobuf) when convert.value.type=by_latest_schema." +
|
||||
"If not set, uses the first message.";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_BY_LATEST_SCHEMA_SUBJECT_CONFIG =
|
||||
"automq.table.topic.convert.key.by_latest_schema.subject";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_BY_LATEST_SCHEMA_SUBJECT_DOC =
|
||||
"Subject name to resolve the latest key schema from Schema Registry when using convert.key.type=by_latest_schema. " +
|
||||
"If not set, defaults to '<topic>-key'.";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_BY_LATEST_SCHEMA_MESSAGE_FULL_NAME_CONFIG =
|
||||
"automq.table.topic.convert.key.by_latest_schema.message.full.name";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_CONVERT_KEY_BY_LATEST_SCHEMA_MESSAGE_FULL_NAME_DOC =
|
||||
"Fully-qualified message name for the latest key schema (if using Protobuf) when convert.key.type=by_latest_schema. " +
|
||||
"If not set, uses the first message.";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_TRANSFORM_VALUE_TYPE_CONFIG = "automq.table.topic.transform.value.type";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_TRANSFORM_VALUE_TYPE_DOC = "Transformation to apply to the record value after conversion. " +
|
||||
"Supported: 'none', 'flatten' (extract fields from structured records), " +
|
||||
"'flatten_debezium' (process Debezium CDC events). " +
|
||||
"Note: 'flatten_debezium' requires schema-based conversion.";
|
||||
|
||||
public static final String TABLE_TOPIC_SCHEMA_TYPE_DOC = "The table topic schema type, support schemaless, schema";
|
||||
public static final String TABLE_TOPIC_ID_COLUMNS_CONFIG = "automq.table.topic.id.columns";
|
||||
public static final String TABLE_TOPIC_ID_COLUMNS_DOC = "The primary key, comma-separated list of columns that identify a row in tables."
|
||||
+ "ex. [region, name]";
|
||||
|
|
@ -319,21 +276,6 @@ public class TopicConfig {
|
|||
public static final String TABLE_TOPIC_CDC_FIELD_CONFIG = "automq.table.topic.cdc.field";
|
||||
public static final String TABLE_TOPIC_CDC_FIELD_DOC = "The name of the field containing the CDC operation, I, U, or D";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_ERRORS_TOLERANCE_CONFIG = "automq.table.topic.errors.tolerance";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_ERRORS_TOLERANCE_DOC = "Configures the error handling strategy for table topic record processing. Valid values are <code>none</code>, <code>invalid_data</code>, and <code>all</code>.";
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_ENABLED_CONFIG = "automq.table.topic.expire.snapshot.enabled";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_ENABLED_DOC = "Enable/disable automatic snapshot expiration.";
|
||||
public static final boolean AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_ENABLED_DEFAULT = true;
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_OLDER_THAN_HOURS_CONFIG = "automq.table.topic.expire.snapshot.older.than.hours";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_OLDER_THAN_HOURS_DOC = "Set retention duration in hours.";
|
||||
public static final int AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_OLDER_THAN_HOURS_DEFAULT = 1;
|
||||
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_RETAIN_LAST_CONFIG = "automq.table.topic.expire.snapshot.retain.last";
|
||||
public static final String AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_RETAIN_LAST_DOC = "Minimum snapshots to retain.";
|
||||
public static final int AUTOMQ_TABLE_TOPIC_EXPIRE_SNAPSHOT_RETAIN_LAST_DEFAULT = 1;
|
||||
|
||||
public static final String KAFKA_LINKS_ID_CONFIG = "automq.kafka.links.id";
|
||||
public static final String KAFKA_LINKS_ID_DOC = "The unique id of a kafka link";
|
||||
public static final String KAFKA_LINKS_TOPIC_START_TIME_CONFIG = "automq.kafka.links.topic.start.time";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
package org.apache.kafka.common.errors;
|
||||
|
||||
public class InvalidKVRecordEpochException extends ApiException {
|
||||
|
||||
public InvalidKVRecordEpochException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -61,6 +61,7 @@ import org.apache.kafka.common.errors.InvalidConfigurationException;
|
|||
import org.apache.kafka.common.errors.InvalidFetchSessionEpochException;
|
||||
import org.apache.kafka.common.errors.InvalidFetchSizeException;
|
||||
import org.apache.kafka.common.errors.InvalidGroupIdException;
|
||||
import org.apache.kafka.common.errors.InvalidKVRecordEpochException;
|
||||
import org.apache.kafka.common.errors.InvalidPartitionsException;
|
||||
import org.apache.kafka.common.errors.InvalidPidMappingException;
|
||||
import org.apache.kafka.common.errors.InvalidPrincipalTypeException;
|
||||
|
|
@ -440,6 +441,7 @@ public enum Errors {
|
|||
NODE_LOCKED(515, "The node is locked", NodeLockedException::new),
|
||||
OBJECT_NOT_COMMITED(516, "The object is not commited.", ObjectNotCommittedException::new),
|
||||
STREAM_INNER_ERROR(599, "The stream inner error.", StreamInnerErrorException::new),
|
||||
INVALID_KV_RECORD_EPOCH(600, "The KV record epoch is invalid.", InvalidKVRecordEpochException::new),
|
||||
// AutoMQ inject end
|
||||
|
||||
INVALID_RECORD_STATE(121, "The record state is invalid. The acknowledgement of delivery could not be completed.", InvalidRecordStateException::new),
|
||||
|
|
|
|||
|
|
@ -39,22 +39,6 @@ public enum TimestampType {
|
|||
throw new NoSuchElementException("Invalid timestamp type " + name);
|
||||
}
|
||||
|
||||
public static TimestampType forId(int id) {
|
||||
switch (id) {
|
||||
case -1: {
|
||||
return NO_TIMESTAMP_TYPE;
|
||||
}
|
||||
case 0: {
|
||||
return CREATE_TIME;
|
||||
}
|
||||
case 1: {
|
||||
return LOG_APPEND_TIME;
|
||||
}
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid timestamp type " + id);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return name;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
"broker"
|
||||
],
|
||||
"name": "AutomqGetPartitionSnapshotRequest",
|
||||
"validVersions": "0-2",
|
||||
"validVersions": "0",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -34,18 +34,6 @@
|
|||
"type": "int32",
|
||||
"versions": "0+",
|
||||
"about": "The get session epoch, which is used for ordering requests in a session"
|
||||
},
|
||||
{
|
||||
"name": "RequestCommit",
|
||||
"type": "bool",
|
||||
"versions": "1+",
|
||||
"about": "Request commit the ConfirmWAL data to the main storage."
|
||||
},
|
||||
{
|
||||
"name": "Version",
|
||||
"type": "int16",
|
||||
"versions": "1+",
|
||||
"about": "The route request version"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
"apiKey": 516,
|
||||
"type": "response",
|
||||
"name": "AutomqGetPartitionSnapshotResponse",
|
||||
"validVersions": "0-2",
|
||||
"validVersions": "0",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{ "name": "ErrorCode", "type": "int16", "versions": "0+", "about": "The top level response error code" },
|
||||
|
|
@ -36,29 +36,9 @@
|
|||
{ "name": "StreamMetadata", "type": "[]StreamMetadata", "versions": "0+", "nullableVersions": "0+", "fields": [
|
||||
{ "name": "StreamId", "type": "int64", "versions": "0+", "about": "The streamId" },
|
||||
{ "name": "EndOffset", "type": "int64", "versions": "0+", "about": "The stream end offset" }
|
||||
]},
|
||||
{ "name": "LastTimestampOffset", "type": "TimestampOffsetData", "versions": "1+", "nullableVersions": "1+", "about": "The last segment's last time index" }
|
||||
]}
|
||||
]}
|
||||
]},
|
||||
{
|
||||
"name": "ConfirmWalEndOffset",
|
||||
"type": "bytes",
|
||||
"versions": "1+",
|
||||
"about": "The confirm WAL end offset."
|
||||
},
|
||||
{
|
||||
"name": "ConfirmWalConfig",
|
||||
"type": "string",
|
||||
"versions": "1+",
|
||||
"about": "The confirm WAL config."
|
||||
},
|
||||
{
|
||||
"name": "ConfirmWalDeltaData",
|
||||
"type": "bytes",
|
||||
"versions": "2+",
|
||||
"nullableVersions": "2+",
|
||||
"about": "The confirm WAL delta data between two end offsets. It's an optional field. If not present, the client should read the delta from WAL"
|
||||
}
|
||||
]}
|
||||
],
|
||||
"commonStructs": [
|
||||
{ "name": "LogMetadata", "versions": "0+", "fields": [
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
"broker"
|
||||
],
|
||||
"name": "AutomqZoneRouterRequest",
|
||||
"validVersions": "0-1",
|
||||
"validVersions": "0",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -28,18 +28,6 @@
|
|||
"type": "bytes",
|
||||
"versions": "0+",
|
||||
"about": "The router metadata"
|
||||
},
|
||||
{
|
||||
"name": "RouteEpoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "The route requests epoch"
|
||||
},
|
||||
{
|
||||
"name": "Version",
|
||||
"type": "int16",
|
||||
"versions": "1+",
|
||||
"about": "The route request version"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
"apiKey": 515,
|
||||
"type": "response",
|
||||
"name": "AutomqZoneRouterResponse",
|
||||
"validVersions": "0-1",
|
||||
"validVersions": "0",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
"broker"
|
||||
],
|
||||
"name": "DeleteKVsRequest",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -35,8 +35,20 @@
|
|||
"type": "string",
|
||||
"versions": "0+",
|
||||
"about": "Key is the key to delete"
|
||||
},
|
||||
{
|
||||
"name": "Namespace",
|
||||
"type": "string",
|
||||
"versions": "1+",
|
||||
"about": "Namespace"
|
||||
},
|
||||
{
|
||||
"name": "Epoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "Epoch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"apiKey": 511,
|
||||
"type": "response",
|
||||
"name": "DeleteKVsResponse",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -50,8 +50,15 @@
|
|||
"versions": "0+",
|
||||
"nullableVersions": "0+",
|
||||
"about": "Value"
|
||||
},
|
||||
{
|
||||
"name": "Epoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "Epoch"
|
||||
}
|
||||
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
"broker"
|
||||
],
|
||||
"name": "GetKVsRequest",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -35,8 +35,14 @@
|
|||
"type": "string",
|
||||
"versions": "0+",
|
||||
"about": "Key is the key to get"
|
||||
},
|
||||
{
|
||||
"name": "Namespace",
|
||||
"type": "string",
|
||||
"versions": "1+",
|
||||
"about": "Namespace"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"apiKey": 509,
|
||||
"type": "response",
|
||||
"name": "GetKVsResponse",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -50,8 +50,20 @@
|
|||
"versions": "0+",
|
||||
"nullableVersions": "0+",
|
||||
"about": "Value"
|
||||
},
|
||||
{
|
||||
"name": "Namespace",
|
||||
"type": "string",
|
||||
"versions": "1+",
|
||||
"about": "Namespace"
|
||||
},
|
||||
{
|
||||
"name": "Epoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "Epoch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
"broker"
|
||||
],
|
||||
"name": "PutKVsRequest",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -47,8 +47,20 @@
|
|||
"type": "bool",
|
||||
"versions": "0+",
|
||||
"about": "overwrite put kv"
|
||||
},
|
||||
{
|
||||
"name": "Namespace",
|
||||
"type": "string",
|
||||
"versions": "1+",
|
||||
"about": "Namespace"
|
||||
},
|
||||
{
|
||||
"name": "Epoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "Epoch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"apiKey": 510,
|
||||
"type": "response",
|
||||
"name": "PutKVsResponse",
|
||||
"validVersions": "0",
|
||||
"validVersions": "0-1",
|
||||
"flexibleVersions": "0+",
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -49,8 +49,14 @@
|
|||
"type": "bytes",
|
||||
"versions": "0+",
|
||||
"about": "Value"
|
||||
},
|
||||
{
|
||||
"name": "Epoch",
|
||||
"type": "int64",
|
||||
"versions": "1+",
|
||||
"about": "Epoch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult;
|
|||
import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult;
|
||||
import org.apache.kafka.common.message.FindCoordinatorRequestData;
|
||||
import org.apache.kafka.common.message.FindCoordinatorResponseData;
|
||||
import org.apache.kafka.common.message.GetKVsResponseData;
|
||||
import org.apache.kafka.common.message.GetTelemetrySubscriptionsResponseData;
|
||||
import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
|
||||
import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse;
|
||||
|
|
@ -141,6 +142,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResp
|
|||
import org.apache.kafka.common.message.OffsetFetchRequestData;
|
||||
import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestGroup;
|
||||
import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopics;
|
||||
import org.apache.kafka.common.message.PutKVsResponseData;
|
||||
import org.apache.kafka.common.message.UnregisterBrokerResponseData;
|
||||
import org.apache.kafka.common.message.WriteTxnMarkersResponseData;
|
||||
import org.apache.kafka.common.protocol.ApiKeys;
|
||||
|
|
@ -6948,7 +6950,7 @@ public class KafkaAdminClientTest {
|
|||
assertNotNull(result.descriptions().get(1).get());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDescribeReplicaLogDirsWithNonExistReplica() throws Exception {
|
||||
int brokerId = 0;
|
||||
|
|
@ -6967,7 +6969,7 @@ public class KafkaAdminClientTest {
|
|||
|
||||
DescribeReplicaLogDirsResult result = env.adminClient().describeReplicaLogDirs(asList(tpr1, tpr2));
|
||||
Map<TopicPartitionReplica, KafkaFuture<DescribeReplicaLogDirsResult.ReplicaLogDirInfo>> values = result.values();
|
||||
|
||||
|
||||
assertEquals(logDir, values.get(tpr1).get().getCurrentReplicaLogDir());
|
||||
assertNull(values.get(tpr1).get().getFutureReplicaLogDir());
|
||||
assertEquals(offsetLag, values.get(tpr1).get().getCurrentReplicaOffsetLag());
|
||||
|
|
|
|||
|
|
@ -1450,6 +1450,21 @@ public class MockAdminClient extends AdminClient {
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetNamespacedKVResult getNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, GetNamespacedKVOptions options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PutNamespacedKVResult putNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, String value, PutNamespacedKVOptions options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DeleteNamespacedKVResult deleteNamespacedKV(Optional<Set<TopicPartition>> partitions, String namespace, String key, DeleteNamespacedKVOptions options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
// AutoMQ inject end
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,8 +24,7 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
|||
# location of the log files (e.g. ${kafka.logs.dir}/connect.log). The `MaxFileSize` option specifies the maximum size of the log file,
|
||||
# and the `MaxBackupIndex` option specifies the number of backup files to keep.
|
||||
#
|
||||
log4j.appender.connectAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.connectAppender.configProviderClass=org.apache.kafka.connect.automq.log.ConnectS3LogConfigProvider
|
||||
log4j.appender.connectAppender=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.connectAppender.MaxFileSize=10MB
|
||||
log4j.appender.connectAppender.MaxBackupIndex=11
|
||||
log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log
|
||||
|
|
|
|||
|
|
@ -21,73 +21,70 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
|||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.logger.com.automq.log.S3RollingFileAppender=INFO, stdout
|
||||
log4j.additivity.com.automq.log.S3RollingFileAppender=false
|
||||
|
||||
log4j.appender.kafkaAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.kafkaAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.kafkaAppender.MaxFileSize=100MB
|
||||
log4j.appender.kafkaAppender.MaxBackupIndex=14
|
||||
log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
|
||||
log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.stateChangeAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.stateChangeAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.stateChangeAppender.MaxFileSize=10MB
|
||||
log4j.appender.stateChangeAppender.MaxBackupIndex=11
|
||||
log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
|
||||
log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.requestAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.requestAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.requestAppender.MaxFileSize=10MB
|
||||
log4j.appender.requestAppender.MaxBackupIndex=11
|
||||
log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
|
||||
log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.cleanerAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.cleanerAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.cleanerAppender.MaxFileSize=10MB
|
||||
log4j.appender.cleanerAppender.MaxBackupIndex=11
|
||||
log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
|
||||
log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.controllerAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.controllerAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.controllerAppender.MaxFileSize=100MB
|
||||
log4j.appender.controllerAppender.MaxBackupIndex=14
|
||||
log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
|
||||
log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.authorizerAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.authorizerAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.authorizerAppender.MaxFileSize=10MB
|
||||
log4j.appender.authorizerAppender.MaxBackupIndex=11
|
||||
log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
|
||||
log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.s3ObjectAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.s3ObjectAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.s3ObjectAppender.MaxFileSize=100MB
|
||||
log4j.appender.s3ObjectAppender.MaxBackupIndex=14
|
||||
log4j.appender.s3ObjectAppender.File=${kafka.logs.dir}/s3-object.log
|
||||
log4j.appender.s3ObjectAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.s3ObjectAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.s3StreamMetricsAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.s3StreamMetricsAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.s3StreamMetricsAppender.MaxFileSize=10MB
|
||||
log4j.appender.s3StreamMetricsAppender.MaxBackupIndex=11
|
||||
log4j.appender.s3StreamMetricsAppender.File=${kafka.logs.dir}/s3stream-metrics.log
|
||||
log4j.appender.s3StreamMetricsAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.s3StreamMetricsAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.s3StreamThreadPoolAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.s3StreamThreadPoolAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.s3StreamThreadPoolAppender.MaxFileSize=10MB
|
||||
log4j.appender.s3StreamThreadPoolAppender.MaxBackupIndex=11
|
||||
log4j.appender.s3StreamThreadPoolAppender.File=${kafka.logs.dir}/s3stream-threads.log
|
||||
log4j.appender.s3StreamThreadPoolAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.s3StreamThreadPoolAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
||||
|
||||
log4j.appender.autoBalancerAppender=com.automq.log.S3RollingFileAppender
|
||||
log4j.appender.autoBalancerAppender=com.automq.shell.log.S3RollingFileAppender
|
||||
log4j.appender.autoBalancerAppender.MaxFileSize=10MB
|
||||
log4j.appender.autoBalancerAppender.MaxBackupIndex=11
|
||||
log4j.appender.autoBalancerAppender.File=${kafka.logs.dir}/auto-balancer.log
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
log4j.rootLogger=ERROR, stdout, perfAppender
|
||||
log4j.rootLogger=INFO, stdout, perfAppender
|
||||
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
|
|
@ -26,15 +26,7 @@ log4j.appender.perfAppender.File=${kafka.logs.dir}/perf.log
|
|||
log4j.appender.perfAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.perfAppender.layout.ConversionPattern=%d -%5p [%15.15t] %m (%c#%M:%L)%n
|
||||
|
||||
log4j.appender.clientAppender=org.apache.log4j.RollingFileAppender
|
||||
log4j.appender.clientAppender.MaxFileSize=100MB
|
||||
log4j.appender.clientAppender.MaxBackupIndex=10
|
||||
log4j.appender.clientAppender.File=${kafka.logs.dir}/client.log
|
||||
log4j.appender.clientAppender.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.clientAppender.layout.ConversionPattern=%d -%5p [%15.15t] %m (%c#%M:%L)%n
|
||||
log4j.logger.org.apache.kafka=INFO, perfAppender
|
||||
log4j.additivity.org.apache.kafka=false
|
||||
|
||||
log4j.logger.org.apache.kafka.tools.automq=INFO, stdout, perfAppender
|
||||
log4j.additivity.org.apache.kafka.tools.automq=false
|
||||
|
||||
log4j.logger.org.apache.kafka.clients=INFO, clientAppender
|
||||
log4j.additivity.org.apache.kafka.clients=false
|
||||
|
|
|
|||
|
|
@ -1,221 +0,0 @@
|
|||
# Kafka Connect OpenTelemetry Metrics Integration
|
||||
|
||||
## Overview
|
||||
|
||||
This integration allows Kafka Connect to export metrics through the AutoMQ OpenTelemetry module, enabling unified observability across your Kafka ecosystem.
|
||||
|
||||
## Configuration
|
||||
|
||||
### 1. Enable the MetricsReporter
|
||||
|
||||
Add the following to your Kafka Connect configuration file (`connect-distributed.properties` or `connect-standalone.properties`):
|
||||
|
||||
```properties
|
||||
# Enable OpenTelemetry MetricsReporter
|
||||
metric.reporters=org.apache.kafka.connect.automq.metrics.OpenTelemetryMetricsReporter
|
||||
|
||||
# OpenTelemetry configuration
|
||||
opentelemetry.metrics.enabled=true
|
||||
opentelemetry.metrics.prefix=kafka.connect
|
||||
|
||||
# Optional: Filter metrics
|
||||
opentelemetry.metrics.include.pattern=.*connector.*|.*task.*|.*worker.*
|
||||
opentelemetry.metrics.exclude.pattern=.*jmx.*|.*debug.*
|
||||
```
|
||||
|
||||
### 2. AutoMQ Telemetry Configuration
|
||||
|
||||
Ensure the AutoMQ telemetry is properly configured. Add these properties to your application configuration:
|
||||
|
||||
```properties
|
||||
# Telemetry export configuration
|
||||
automq.telemetry.exporter.uri=prometheus://localhost:9090
|
||||
# or for OTLP: automq.telemetry.exporter.uri=otlp://localhost:4317
|
||||
|
||||
# Service identification
|
||||
service.name=kafka-connect
|
||||
service.instance.id=connect-worker-1
|
||||
|
||||
# Export settings
|
||||
automq.telemetry.exporter.interval.ms=30000
|
||||
automq.telemetry.metric.cardinality.limit=10000
|
||||
```
|
||||
|
||||
## S3 Log Upload
|
||||
|
||||
Kafka Connect bundles the AutoMQ log uploader so that worker logs can be streamed to S3 together with in-cluster cleanup. The uploader uses the connect-leader election mechanism by default and requires no additional configuration.
|
||||
|
||||
### Worker Configuration
|
||||
|
||||
Add the following properties to your worker configuration (ConfigMap, properties file, etc.):
|
||||
|
||||
```properties
|
||||
# Enable S3 log upload
|
||||
log.s3.enable=true
|
||||
log.s3.bucket=0@s3://your-log-bucket?region=us-east-1
|
||||
|
||||
# Optional overrides (defaults shown)
|
||||
log.s3.selector.type=connect-leader
|
||||
# Provide credentials if the bucket URI does not embed them
|
||||
# log.s3.access.key=...
|
||||
# log.s3.secret.key=...
|
||||
```
|
||||
|
||||
`log.s3.node.id` defaults to a hash of the pod hostname if not provided, ensuring objects are partitioned per worker.
|
||||
|
||||
### Log4j Integration
|
||||
|
||||
`config/connect-log4j.properties` has switched `connectAppender` to `com.automq.log.S3RollingFileAppender` and specifies `org.apache.kafka.connect.automq.log.ConnectS3LogConfigProvider` as the config provider. As long as you enable `log.s3.enable=true` and configure the bucket info in the worker config, log upload will be automatically initialized with the Connect process; if not set or returns `log.s3.enable=false`, the uploader remains disabled.
|
||||
|
||||
## Programmatic Usage
|
||||
|
||||
### 1. Initialize Telemetry Manager
|
||||
|
||||
```java
|
||||
import com.automq.opentelemetry.AutoMQTelemetryManager;
|
||||
import java.util.Properties;
|
||||
|
||||
// Initialize AutoMQ telemetry before starting Kafka Connect
|
||||
Properties telemetryProps = new Properties();
|
||||
telemetryProps.setProperty("automq.telemetry.exporter.uri", "prometheus://localhost:9090");
|
||||
telemetryProps.setProperty("service.name", "kafka-connect");
|
||||
telemetryProps.setProperty("service.instance.id", "worker-1");
|
||||
|
||||
// Initialize singleton instance
|
||||
AutoMQTelemetryManager.initializeInstance(telemetryProps);
|
||||
|
||||
// Now start Kafka Connect - it will automatically use the OpenTelemetryMetricsReporter
|
||||
```
|
||||
|
||||
### 2. Shutdown
|
||||
|
||||
```java
|
||||
// When shutting down your application
|
||||
AutoMQTelemetryManager.shutdownInstance();
|
||||
```
|
||||
|
||||
## Exported Metrics
|
||||
|
||||
The integration automatically converts Kafka Connect metrics to OpenTelemetry format:
|
||||
|
||||
### Metric Naming Convention
|
||||
- **Format**: `kafka.connect.{group}.{metric_name}`
|
||||
- **Example**: `kafka.connect.connector.task.batch.size.avg` → `kafka.connect.connector_task_batch_size_avg`
|
||||
|
||||
### Metric Types
|
||||
- **Counters**: Metrics containing "total", "count", "error", "failure"
|
||||
- **Gauges**: All other numeric metrics (rates, averages, sizes, etc.)
|
||||
|
||||
### Attributes
|
||||
Kafka metric tags are converted to OpenTelemetry attributes:
|
||||
- `connector` → `connector`
|
||||
- `task` → `task`
|
||||
- `worker-id` → `worker_id`
|
||||
- Plus standard attributes: `metric.group`, `service.name`, `service.instance.id`
|
||||
|
||||
## Example Metrics
|
||||
|
||||
Common Kafka Connect metrics that will be exported:
|
||||
|
||||
```
|
||||
# Connector metrics
|
||||
kafka.connect.connector.startup.attempts.total
|
||||
kafka.connect.connector.startup.success.total
|
||||
kafka.connect.connector.startup.failure.total
|
||||
|
||||
# Task metrics
|
||||
kafka.connect.connector.task.batch.size.avg
|
||||
kafka.connect.connector.task.batch.size.max
|
||||
kafka.connect.connector.task.offset.commit.avg.time.ms
|
||||
|
||||
# Worker metrics
|
||||
kafka.connect.worker.connector.count
|
||||
kafka.connect.worker.task.count
|
||||
kafka.connect.worker.connector.startup.attempts.total
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### OpenTelemetry MetricsReporter Options
|
||||
|
||||
| Property | Description | Default | Example |
|
||||
|----------|-------------|---------|---------|
|
||||
| `opentelemetry.metrics.enabled` | Enable/disable metrics export | `true` | `false` |
|
||||
| `opentelemetry.metrics.prefix` | Metric name prefix | `kafka.connect` | `my.connect` |
|
||||
| `opentelemetry.metrics.include.pattern` | Regex for included metrics | All metrics | `.*connector.*` |
|
||||
| `opentelemetry.metrics.exclude.pattern` | Regex for excluded metrics | None | `.*jmx.*` |
|
||||
|
||||
### AutoMQ Telemetry Options
|
||||
|
||||
| Property | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `automq.telemetry.exporter.uri` | Exporter endpoint | Empty |
|
||||
| `automq.telemetry.exporter.interval.ms` | Export interval | `60000` |
|
||||
| `automq.telemetry.metric.cardinality.limit` | Max metric cardinality | `20000` |
|
||||
|
||||
## Monitoring Examples
|
||||
|
||||
### Prometheus Queries
|
||||
|
||||
```promql
|
||||
# Connector count by worker
|
||||
kafka_connect_worker_connector_count
|
||||
|
||||
# Task failure rate
|
||||
rate(kafka_connect_connector_task_startup_failure_total[5m])
|
||||
|
||||
# Average batch processing time
|
||||
kafka_connect_connector_task_batch_size_avg
|
||||
|
||||
# Connector startup success rate
|
||||
rate(kafka_connect_connector_startup_success_total[5m]) /
|
||||
rate(kafka_connect_connector_startup_attempts_total[5m])
|
||||
```
|
||||
|
||||
### Grafana Dashboard
|
||||
|
||||
Common panels to create:
|
||||
|
||||
1. **Connector Health**: Count of running/failed connectors
|
||||
2. **Task Performance**: Batch size, processing time, throughput
|
||||
3. **Error Rates**: Failed startups, task failures
|
||||
4. **Resource Usage**: Combined with JVM metrics from AutoMQ telemetry
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Metrics not appearing**
|
||||
```
|
||||
Check logs for: "AutoMQTelemetryManager is not initialized"
|
||||
Solution: Ensure AutoMQTelemetryManager.initializeInstance() is called before Connect starts
|
||||
```
|
||||
|
||||
2. **High cardinality warnings**
|
||||
```
|
||||
Solution: Use include/exclude patterns to filter metrics
|
||||
```
|
||||
|
||||
3. **Missing dependencies**
|
||||
```
|
||||
Ensure connect-runtime depends on the opentelemetry module
|
||||
```
|
||||
|
||||
### Debug Logging
|
||||
|
||||
Enable debug logging to troubleshoot:
|
||||
|
||||
```properties
|
||||
log4j.logger.org.apache.kafka.connect.automq=DEBUG
|
||||
log4j.logger.com.automq.opentelemetry=DEBUG
|
||||
```
|
||||
|
||||
## Integration with Existing Monitoring
|
||||
|
||||
This integration works alongside:
|
||||
- Existing JMX metrics (not replaced)
|
||||
- Kafka broker metrics via AutoMQ telemetry
|
||||
- Application-specific metrics
|
||||
- Third-party monitoring tools
|
||||
|
||||
The OpenTelemetry integration provides a unified export path while preserving existing monitoring setups.
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.az;
|
||||
|
||||
import org.apache.kafka.clients.CommonClientConfigs;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public final class AzAwareClientConfigurator {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AzAwareClientConfigurator.class);
|
||||
|
||||
private AzAwareClientConfigurator() {
|
||||
}
|
||||
|
||||
public enum ClientFamily {
|
||||
PRODUCER,
|
||||
CONSUMER,
|
||||
ADMIN
|
||||
}
|
||||
|
||||
public static void maybeApplyAz(Map<String, Object> props, ClientFamily family, String roleDescriptor) {
|
||||
Optional<String> azOpt = AzMetadataProviderHolder.provider().availabilityZoneId();
|
||||
LOGGER.info("AZ-aware client.id configuration for role {}: resolved availability zone id '{}'",
|
||||
roleDescriptor, azOpt.orElse("unknown"));
|
||||
if (azOpt.isEmpty()) {
|
||||
LOGGER.info("Skipping AZ-aware client.id configuration for role {} as no availability zone id is available",
|
||||
roleDescriptor);
|
||||
return;
|
||||
}
|
||||
|
||||
String az = azOpt.get();
|
||||
|
||||
String encodedAz = URLEncoder.encode(az, StandardCharsets.UTF_8);
|
||||
String automqClientId;
|
||||
|
||||
if (props.containsKey(CommonClientConfigs.CLIENT_ID_CONFIG)) {
|
||||
Object currentId = props.get(CommonClientConfigs.CLIENT_ID_CONFIG);
|
||||
if (currentId instanceof String currentIdStr) {
|
||||
automqClientId = "automq_az=" + encodedAz + "&" + currentIdStr;
|
||||
} else {
|
||||
LOGGER.warn("client.id for role {} is not a string ({});",
|
||||
roleDescriptor, currentId.getClass().getName());
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
automqClientId = "automq_az=" + encodedAz;
|
||||
}
|
||||
props.put(CommonClientConfigs.CLIENT_ID_CONFIG, automqClientId);
|
||||
LOGGER.info("Applied AZ-aware client.id for role {} -> {}", roleDescriptor, automqClientId);
|
||||
|
||||
if (family == ClientFamily.CONSUMER) {
|
||||
LOGGER.info("Applying client.rack configuration for consumer role {} -> {}", roleDescriptor, az);
|
||||
Object rackValue = props.get(ConsumerConfig.CLIENT_RACK_CONFIG);
|
||||
if (rackValue == null || String.valueOf(rackValue).isBlank()) {
|
||||
props.put(ConsumerConfig.CLIENT_RACK_CONFIG, az);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void maybeApplyProducerAz(Map<String, Object> props, String roleDescriptor) {
|
||||
maybeApplyAz(props, ClientFamily.PRODUCER, roleDescriptor);
|
||||
}
|
||||
|
||||
public static void maybeApplyConsumerAz(Map<String, Object> props, String roleDescriptor) {
|
||||
maybeApplyAz(props, ClientFamily.CONSUMER, roleDescriptor);
|
||||
}
|
||||
|
||||
public static void maybeApplyAdminAz(Map<String, Object> props, String roleDescriptor) {
|
||||
maybeApplyAz(props, ClientFamily.ADMIN, roleDescriptor);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.az;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Pluggable provider for availability-zone metadata used to tune Kafka client configurations.
|
||||
*/
|
||||
public interface AzMetadataProvider {
|
||||
|
||||
/**
|
||||
* Configure the provider with the worker properties. Implementations may cache values extracted from the
|
||||
* configuration map. This method is invoked exactly once during worker bootstrap.
|
||||
*/
|
||||
default void configure(Map<String, String> workerProps) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the availability-zone identifier for the current node, if known.
|
||||
*/
|
||||
default Optional<String> availabilityZoneId() {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.az;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
public final class AzMetadataProviderHolder {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AzMetadataProviderHolder.class);
|
||||
private static final AzMetadataProvider DEFAULT_PROVIDER = new AzMetadataProvider() { };
|
||||
|
||||
private static volatile AzMetadataProvider provider = DEFAULT_PROVIDER;
|
||||
|
||||
private AzMetadataProviderHolder() {
|
||||
}
|
||||
|
||||
public static void initialize(Map<String, String> workerProps) {
|
||||
AzMetadataProvider selected = DEFAULT_PROVIDER;
|
||||
try {
|
||||
ServiceLoader<AzMetadataProvider> loader = ServiceLoader.load(AzMetadataProvider.class);
|
||||
for (AzMetadataProvider candidate : loader) {
|
||||
try {
|
||||
candidate.configure(workerProps);
|
||||
selected = candidate;
|
||||
LOGGER.info("Loaded AZ metadata provider: {}", candidate.getClass().getName());
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to initialize AZ metadata provider: {}", candidate.getClass().getName(), e);
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
LOGGER.warn("Failed to load AZ metadata providers", t);
|
||||
}
|
||||
provider = selected;
|
||||
}
|
||||
|
||||
public static AzMetadataProvider provider() {
|
||||
return provider;
|
||||
}
|
||||
|
||||
public static void setProviderForTest(AzMetadataProvider newProvider) {
|
||||
provider = newProvider != null ? newProvider : DEFAULT_PROVIDER;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.log;
|
||||
|
||||
import com.automq.log.S3RollingFileAppender;
|
||||
import com.automq.log.uploader.S3LogConfig;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Initializes the AutoMQ S3 log uploader for Kafka Connect.
|
||||
*/
|
||||
public final class ConnectLogUploader {
|
||||
private static Logger getLogger() {
|
||||
return LoggerFactory.getLogger(ConnectLogUploader.class);
|
||||
}
|
||||
|
||||
private ConnectLogUploader() {
|
||||
}
|
||||
|
||||
public static void initialize(Map<String, String> workerProps) {
|
||||
Properties props = new Properties();
|
||||
if (workerProps != null) {
|
||||
workerProps.forEach((k, v) -> {
|
||||
if (k != null && v != null) {
|
||||
props.put(k, v);
|
||||
}
|
||||
});
|
||||
}
|
||||
ConnectS3LogConfigProvider.initialize(props);
|
||||
S3LogConfig s3LogConfig = new ConnectS3LogConfigProvider().get();
|
||||
S3RollingFileAppender.setup(s3LogConfig);
|
||||
getLogger().info("Initialized Connect S3 log uploader context");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.log;
|
||||
|
||||
import org.apache.kafka.connect.automq.runtime.LeaderNodeSelector;
|
||||
import org.apache.kafka.connect.automq.runtime.RuntimeLeaderSelectorProvider;
|
||||
|
||||
import com.automq.log.uploader.S3LogConfig;
|
||||
import com.automq.stream.s3.operator.BucketURI;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
import com.automq.stream.s3.operator.ObjectStorageFactory;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ConnectS3LogConfig implements S3LogConfig {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ConnectS3LogConfig.class);
|
||||
|
||||
private final boolean enable;
|
||||
private final String clusterId;
|
||||
private final int nodeId;
|
||||
private final String bucketURI;
|
||||
private ObjectStorage objectStorage;
|
||||
private LeaderNodeSelector leaderNodeSelector;
|
||||
|
||||
|
||||
public ConnectS3LogConfig(boolean enable, String clusterId, int nodeId, String bucketURI) {
|
||||
this.enable = enable;
|
||||
this.clusterId = clusterId;
|
||||
this.nodeId = nodeId;
|
||||
this.bucketURI = bucketURI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEnabled() {
|
||||
return this.enable;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String clusterId() {
|
||||
return this.clusterId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nodeId() {
|
||||
return this.nodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized ObjectStorage objectStorage() {
|
||||
if (this.objectStorage != null) {
|
||||
return this.objectStorage;
|
||||
}
|
||||
if (StringUtils.isBlank(bucketURI)) {
|
||||
LOGGER.error("Mandatory log config bucketURI is not set.");
|
||||
return null;
|
||||
}
|
||||
|
||||
String normalizedBucket = bucketURI.trim();
|
||||
BucketURI logBucket = BucketURI.parse(normalizedBucket);
|
||||
this.objectStorage = ObjectStorageFactory.instance().builder(logBucket).threadPrefix("s3-log-uploader").build();
|
||||
return this.objectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLeader() {
|
||||
LeaderNodeSelector selector = leaderSelector();
|
||||
return selector != null && selector.isLeader();
|
||||
}
|
||||
|
||||
public LeaderNodeSelector leaderSelector() {
|
||||
if (leaderNodeSelector == null) {
|
||||
this.leaderNodeSelector = new RuntimeLeaderSelectorProvider().createSelector();
|
||||
}
|
||||
return leaderNodeSelector;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.log;
|
||||
|
||||
import com.automq.log.uploader.S3LogConfig;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* Provides S3 log uploader configuration for Kafka Connect workers.
|
||||
*/
|
||||
public class ConnectS3LogConfigProvider {
|
||||
private static Logger getLogger() {
|
||||
return LoggerFactory.getLogger(ConnectS3LogConfigProvider.class);
|
||||
}
|
||||
private static final AtomicReference<Properties> CONFIG = new AtomicReference<>();
|
||||
private static final long WAIT_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(10);
|
||||
private static final CountDownLatch INIT = new CountDownLatch(1);
|
||||
|
||||
public static void initialize(Properties workerProps) {
|
||||
try {
|
||||
if (workerProps == null) {
|
||||
CONFIG.set(null);
|
||||
return;
|
||||
}
|
||||
Properties copy = new Properties();
|
||||
for (Map.Entry<Object, Object> entry : workerProps.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
copy.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
CONFIG.set(copy);
|
||||
} finally {
|
||||
INIT.countDown();
|
||||
}
|
||||
getLogger().info("Initializing ConnectS3LogConfigProvider");
|
||||
}
|
||||
|
||||
public S3LogConfig get() {
|
||||
|
||||
try {
|
||||
if (!INIT.await(WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
getLogger().warn("S3 log uploader config not initialized within timeout; uploader disabled.");
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
getLogger().warn("Interrupted while waiting for S3 log uploader config; uploader disabled.");
|
||||
return null;
|
||||
}
|
||||
|
||||
Properties source = CONFIG.get();
|
||||
if (source == null) {
|
||||
getLogger().warn("S3 log upload configuration was not provided; uploader disabled.");
|
||||
return null;
|
||||
}
|
||||
|
||||
String bucketURI = source.getProperty(LogConfigConstants.LOG_S3_BUCKET_KEY);
|
||||
String clusterId = source.getProperty(LogConfigConstants.LOG_S3_CLUSTER_ID_KEY);
|
||||
String nodeIdStr = resolveNodeId(source);
|
||||
boolean enable = Boolean.parseBoolean(source.getProperty(LogConfigConstants.LOG_S3_ENABLE_KEY, "false"));
|
||||
return new ConnectS3LogConfig(enable, clusterId, Integer.parseInt(nodeIdStr), bucketURI);
|
||||
}
|
||||
|
||||
private String resolveNodeId(Properties workerProps) {
|
||||
String fromConfig = workerProps.getProperty(LogConfigConstants.LOG_S3_NODE_ID_KEY);
|
||||
if (!isBlank(fromConfig)) {
|
||||
return fromConfig.trim();
|
||||
}
|
||||
String env = System.getenv("CONNECT_NODE_ID");
|
||||
if (!isBlank(env)) {
|
||||
return env.trim();
|
||||
}
|
||||
String host = workerProps.getProperty("automq.log.s3.node.hostname");
|
||||
if (isBlank(host)) {
|
||||
try {
|
||||
host = InetAddress.getLocalHost().getHostName();
|
||||
} catch (Exception e) {
|
||||
host = System.getenv().getOrDefault("HOSTNAME", "0");
|
||||
}
|
||||
}
|
||||
return Integer.toString(host.hashCode() & Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
private boolean isBlank(String value) {
|
||||
return value == null || value.trim().isEmpty();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.log;
|
||||
|
||||
public class LogConfigConstants {
|
||||
public static final String LOG_S3_ENABLE_KEY = "log.s3.enable";
|
||||
|
||||
public static final String LOG_S3_BUCKET_KEY = "log.s3.bucket";
|
||||
|
||||
public static final String LOG_S3_CLUSTER_ID_KEY = "log.s3.cluster.id";
|
||||
|
||||
public static final String LOG_S3_NODE_ID_KEY = "log.s3.node.id";
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
package org.apache.kafka.connect.automq.metrics;
|
||||
|
||||
import org.apache.kafka.connect.automq.runtime.LeaderNodeSelector;
|
||||
import org.apache.kafka.connect.automq.runtime.RuntimeLeaderSelectorProvider;
|
||||
|
||||
import com.automq.opentelemetry.exporter.MetricsExportConfig;
|
||||
import com.automq.stream.s3.operator.BucketURI;
|
||||
import com.automq.stream.s3.operator.ObjectStorage;
|
||||
import com.automq.stream.s3.operator.ObjectStorageFactory;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ConnectMetricsExportConfig implements MetricsExportConfig {
|
||||
|
||||
private final BucketURI metricsBucket;
|
||||
private final String clusterId;
|
||||
private final int nodeId;
|
||||
private final int intervalMs;
|
||||
private final List<Pair<String, String>> baseLabels;
|
||||
private ObjectStorage objectStorage;
|
||||
private LeaderNodeSelector leaderNodeSelector;
|
||||
|
||||
|
||||
public ConnectMetricsExportConfig(String clusterId, int nodeId, BucketURI metricsBucket, List<Pair<String, String>> baseLabels, int intervalMs) {
|
||||
this.clusterId = clusterId;
|
||||
this.nodeId = nodeId;
|
||||
this.metricsBucket = metricsBucket;
|
||||
this.baseLabels = baseLabels;
|
||||
this.intervalMs = intervalMs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String clusterId() {
|
||||
return this.clusterId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLeader() {
|
||||
LeaderNodeSelector selector = leaderSelector();
|
||||
return selector != null && selector.isLeader();
|
||||
}
|
||||
|
||||
public LeaderNodeSelector leaderSelector() {
|
||||
if (leaderNodeSelector == null) {
|
||||
this.leaderNodeSelector = new RuntimeLeaderSelectorProvider().createSelector();
|
||||
}
|
||||
return leaderNodeSelector;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int nodeId() {
|
||||
return this.nodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectStorage objectStorage() {
|
||||
if (metricsBucket == null) {
|
||||
return null;
|
||||
}
|
||||
if (this.objectStorage == null) {
|
||||
this.objectStorage = ObjectStorageFactory.instance().builder(metricsBucket).threadPrefix("s3-metric").build();
|
||||
}
|
||||
return this.objectStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Pair<String, String>> baseLabels() {
|
||||
return this.baseLabels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intervalMs() {
|
||||
return this.intervalMs;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package org.apache.kafka.connect.automq.metrics;
|
||||
|
||||
public class MetricsConfigConstants {
|
||||
public static final String SERVICE_NAME_KEY = "service.name";
|
||||
public static final String SERVICE_INSTANCE_ID_KEY = "service.instance.id";
|
||||
public static final String S3_CLIENT_ID_KEY = "automq.telemetry.s3.cluster.id";
|
||||
/**
|
||||
* The URI for configuring metrics exporters. e.g. prometheus://localhost:9090, otlp://localhost:4317
|
||||
*/
|
||||
public static final String EXPORTER_URI_KEY = "automq.telemetry.exporter.uri";
|
||||
/**
|
||||
* The export interval in milliseconds.
|
||||
*/
|
||||
public static final String EXPORTER_INTERVAL_MS_KEY = "automq.telemetry.exporter.interval.ms";
|
||||
/**
|
||||
* The cardinality limit for any single metric.
|
||||
*/
|
||||
public static final String METRIC_CARDINALITY_LIMIT_KEY = "automq.telemetry.metric.cardinality.limit";
|
||||
public static final int DEFAULT_METRIC_CARDINALITY_LIMIT = 20000;
|
||||
|
||||
public static final String TELEMETRY_METRICS_BASE_LABELS_CONFIG = "automq.telemetry.metrics.base.labels";
|
||||
public static final String TELEMETRY_METRICS_BASE_LABELS_DOC = "The base labels that will be added to all metrics. The format is key1=value1,key2=value2.";
|
||||
|
||||
public static final String S3_BUCKET = "automq.telemetry.s3.bucket";
|
||||
public static final String S3_BUCKETS_DOC = "The buckets url with format 0@s3://$bucket?region=$region. \n" +
|
||||
"the full url format for s3 is 0@s3://$bucket?region=$region[&endpoint=$endpoint][&pathStyle=$enablePathStyle][&authType=$authType][&accessKey=$accessKey][&secretKey=$secretKey][&checksumAlgorithm=$checksumAlgorithm]" +
|
||||
"- pathStyle: true|false. The object storage access path style. When using MinIO, it should be set to true.\n" +
|
||||
"- authType: instance|static. When set to instance, it will use instance profile to auth. When set to static, it will get accessKey and secretKey from the url or from system environment KAFKA_S3_ACCESS_KEY/KAFKA_S3_SECRET_KEY.";
|
||||
|
||||
}
|
||||
|
|
@ -1,822 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.metrics;
|
||||
|
||||
import org.apache.kafka.common.MetricName;
|
||||
import org.apache.kafka.common.metrics.KafkaMetric;
|
||||
import org.apache.kafka.common.metrics.MetricsReporter;
|
||||
|
||||
import com.automq.opentelemetry.AutoMQTelemetryManager;
|
||||
import com.automq.stream.s3.operator.BucketURI;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import io.opentelemetry.api.common.Attributes;
|
||||
import io.opentelemetry.api.common.AttributesBuilder;
|
||||
import io.opentelemetry.api.metrics.Meter;
|
||||
import io.opentelemetry.api.metrics.ObservableDoubleCounter;
|
||||
import io.opentelemetry.api.metrics.ObservableDoubleGauge;
|
||||
import io.opentelemetry.api.metrics.ObservableLongCounter;
|
||||
|
||||
/**
|
||||
* A MetricsReporter implementation that bridges Kafka Connect metrics to OpenTelemetry.
|
||||
*
|
||||
* <p>This reporter integrates with the AutoMQ OpenTelemetry module to export Kafka Connect
|
||||
* metrics through various exporters (Prometheus, OTLP, etc.). It automatically converts
|
||||
* Kafka metrics to OpenTelemetry instruments based on metric types and provides proper
|
||||
* labeling and naming conventions.
|
||||
*
|
||||
* <p>Key features:
|
||||
* <ul>
|
||||
* <li>Automatic metric type detection and conversion</li>
|
||||
* <li>Support for gauges and counters using async observable instruments</li>
|
||||
* <li>Proper attribute mapping from Kafka metric tags</li>
|
||||
* <li>Integration with AutoMQ telemetry infrastructure</li>
|
||||
* <li>Configurable metric filtering</li>
|
||||
* <li>Real-time metric value updates through callbacks</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>Configuration options:
|
||||
* <ul>
|
||||
* <li>{@code opentelemetry.metrics.enabled} - Enable/disable OpenTelemetry metrics (default: true)</li>
|
||||
* <li>{@code opentelemetry.metrics.prefix} - Prefix for metric names (default: "kafka.connect")</li>
|
||||
* <li>{@code opentelemetry.metrics.include.pattern} - Regex pattern for included metrics</li>
|
||||
* <li>{@code opentelemetry.metrics.exclude.pattern} - Regex pattern for excluded metrics</li>
|
||||
* </ul>
|
||||
*/
|
||||
public class OpenTelemetryMetricsReporter implements MetricsReporter {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(OpenTelemetryMetricsReporter.class);
|
||||
|
||||
private static final String ENABLED_CONFIG = "opentelemetry.metrics.enabled";
|
||||
private static final String PREFIX_CONFIG = "opentelemetry.metrics.prefix";
|
||||
private static final String INCLUDE_PATTERN_CONFIG = "opentelemetry.metrics.include.pattern";
|
||||
private static final String EXCLUDE_PATTERN_CONFIG = "opentelemetry.metrics.exclude.pattern";
|
||||
|
||||
private static final String DEFAULT_PREFIX = "kafka";
|
||||
|
||||
private boolean enabled = true;
|
||||
private String metricPrefix = DEFAULT_PREFIX;
|
||||
private String includePattern = null;
|
||||
private String excludePattern = null;
|
||||
|
||||
private Meter meter;
|
||||
private final Map<String, AutoCloseable> observableHandles = new ConcurrentHashMap<>();
|
||||
private final Map<String, KafkaMetric> registeredMetrics = new ConcurrentHashMap<>();
|
||||
|
||||
public static void initializeTelemetry(Properties props) {
|
||||
String exportURIStr = props.getProperty(MetricsConfigConstants.EXPORTER_URI_KEY);
|
||||
String serviceName = props.getProperty(MetricsConfigConstants.SERVICE_NAME_KEY, "connect-default");
|
||||
String instanceId = props.getProperty(MetricsConfigConstants.SERVICE_INSTANCE_ID_KEY, "0");
|
||||
String clusterId = props.getProperty(MetricsConfigConstants.S3_CLIENT_ID_KEY, "cluster-default");
|
||||
int intervalMs = Integer.parseInt(props.getProperty(MetricsConfigConstants.EXPORTER_INTERVAL_MS_KEY, "60000"));
|
||||
BucketURI metricsBucket = getMetricsBucket(props);
|
||||
List<Pair<String, String>> baseLabels = getBaseLabels(props);
|
||||
|
||||
AutoMQTelemetryManager.initializeInstance(exportURIStr, serviceName, instanceId, new ConnectMetricsExportConfig(clusterId, Integer.parseInt(instanceId), metricsBucket, baseLabels, intervalMs));
|
||||
LOGGER.info("OpenTelemetryMetricsReporter initialized");
|
||||
}
|
||||
|
||||
private static BucketURI getMetricsBucket(Properties props) {
|
||||
String metricsBucket = props.getProperty(MetricsConfigConstants.S3_BUCKET, "");
|
||||
if (StringUtils.isNotBlank(metricsBucket)) {
|
||||
List<BucketURI> bucketList = BucketURI.parseBuckets(metricsBucket);
|
||||
if (!bucketList.isEmpty()) {
|
||||
return bucketList.get(0);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<Pair<String, String>> getBaseLabels(Properties props) {
|
||||
// This part is hard to abstract without a clear config pattern.
|
||||
// Assuming for now it's empty. The caller can extend this class
|
||||
// or the manager can have a method to add more labels.
|
||||
String baseLabels = props.getProperty(MetricsConfigConstants.TELEMETRY_METRICS_BASE_LABELS_CONFIG);
|
||||
if (StringUtils.isBlank(baseLabels)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<Pair<String, String>> labels = new ArrayList<>();
|
||||
for (String label : baseLabels.split(",")) {
|
||||
String[] kv = label.split("=");
|
||||
if (kv.length != 2) {
|
||||
continue;
|
||||
}
|
||||
labels.add(Pair.of(kv[0], kv[1]));
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(Map<String, ?> configs) {
|
||||
// Parse configuration
|
||||
Object enabledObj = configs.get(ENABLED_CONFIG);
|
||||
if (enabledObj != null) {
|
||||
enabled = Boolean.parseBoolean(enabledObj.toString());
|
||||
}
|
||||
|
||||
Object prefixObj = configs.get(PREFIX_CONFIG);
|
||||
if (prefixObj != null) {
|
||||
metricPrefix = prefixObj.toString();
|
||||
}
|
||||
|
||||
Object includeObj = configs.get(INCLUDE_PATTERN_CONFIG);
|
||||
if (includeObj != null) {
|
||||
includePattern = includeObj.toString();
|
||||
}
|
||||
|
||||
Object excludeObj = configs.get(EXCLUDE_PATTERN_CONFIG);
|
||||
if (excludeObj != null) {
|
||||
excludePattern = excludeObj.toString();
|
||||
}
|
||||
|
||||
LOGGER.info("OpenTelemetryMetricsReporter configured - enabled: {}, prefix: {}, include: {}, exclude: {}",
|
||||
enabled, metricPrefix, includePattern, excludePattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(List<KafkaMetric> metrics) {
|
||||
if (!enabled) {
|
||||
LOGGER.info("OpenTelemetryMetricsReporter is disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the OpenTelemetry meter from AutoMQTelemetryManager
|
||||
// This assumes the telemetry manager is already initialized
|
||||
meter = AutoMQTelemetryManager.getInstance().getMeter();
|
||||
if (meter == null) {
|
||||
LOGGER.warn("AutoMQTelemetryManager is not initialized, OpenTelemetry metrics will not be available");
|
||||
enabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Register initial metrics
|
||||
for (KafkaMetric metric : metrics) {
|
||||
registerMetric(metric);
|
||||
}
|
||||
|
||||
LOGGER.info("OpenTelemetryMetricsReporter initialized with {} metrics", metrics.size());
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Failed to initialize OpenTelemetryMetricsReporter", e);
|
||||
enabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void metricChange(KafkaMetric metric) {
|
||||
if (!enabled || meter == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
registerMetric(metric);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to register metric change for {}", metric.metricName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void metricRemoval(KafkaMetric metric) {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String metricKey = buildMetricKey(metric.metricName());
|
||||
closeHandle(metricKey);
|
||||
registeredMetrics.remove(metricKey);
|
||||
LOGGER.debug("Removed metric: {}", metricKey);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to remove metric {}", metric.metricName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (enabled) {
|
||||
// Close all observable handles to prevent memory leaks
|
||||
observableHandles.values().forEach(handle -> {
|
||||
try {
|
||||
handle.close();
|
||||
} catch (Exception e) {
|
||||
LOGGER.debug("Error closing observable handle", e);
|
||||
}
|
||||
});
|
||||
observableHandles.clear();
|
||||
registeredMetrics.clear();
|
||||
}
|
||||
LOGGER.info("OpenTelemetryMetricsReporter closed");
|
||||
}
|
||||
|
||||
private void registerMetric(KafkaMetric metric) {
|
||||
LOGGER.debug("OpenTelemetryMetricsReporter registering metric {}", metric.metricName());
|
||||
MetricName metricName = metric.metricName();
|
||||
String metricKey = buildMetricKey(metricName);
|
||||
|
||||
// Apply filtering
|
||||
if (!shouldIncludeMetric(metricKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if metric value is numeric at registration time
|
||||
Object testValue = safeMetricValue(metric);
|
||||
if (!(testValue instanceof Number)) {
|
||||
LOGGER.debug("Skipping non-numeric metric: {}", metricKey);
|
||||
return;
|
||||
}
|
||||
|
||||
Attributes attributes = buildAttributes(metricName);
|
||||
|
||||
// Close existing handle if present (for metric updates)
|
||||
closeHandle(metricKey);
|
||||
|
||||
// Register the metric for future access
|
||||
registeredMetrics.put(metricKey, metric);
|
||||
|
||||
// Determine metric type and register accordingly
|
||||
if (isCounterMetric(metricName)) {
|
||||
registerAsyncCounter(metricKey, metricName, metric, attributes, (Number) testValue);
|
||||
} else {
|
||||
registerAsyncGauge(metricKey, metricName, metric, attributes);
|
||||
}
|
||||
}
|
||||
|
||||
private void registerAsyncGauge(String metricKey, MetricName metricName, KafkaMetric metric, Attributes attributes) {
|
||||
try {
|
||||
String description = buildDescription(metricName);
|
||||
String unit = determineUnit(metricName);
|
||||
|
||||
ObservableDoubleGauge gauge = meter.gaugeBuilder(metricKey)
|
||||
.setDescription(description)
|
||||
.setUnit(unit)
|
||||
.buildWithCallback(measurement -> {
|
||||
Number value = (Number) safeMetricValue(metric);
|
||||
if (value != null) {
|
||||
measurement.record(value.doubleValue(), attributes);
|
||||
}
|
||||
});
|
||||
|
||||
observableHandles.put(metricKey, gauge);
|
||||
LOGGER.debug("Registered async gauge: {}", metricKey);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to register async gauge for {}", metricKey, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void registerAsyncCounter(String metricKey, MetricName metricName, KafkaMetric metric,
|
||||
Attributes attributes, Number initialValue) {
|
||||
try {
|
||||
String description = buildDescription(metricName);
|
||||
String unit = determineUnit(metricName);
|
||||
|
||||
// Use appropriate counter type based on initial value type
|
||||
if (initialValue instanceof Long || initialValue instanceof Integer) {
|
||||
ObservableLongCounter counter = meter.counterBuilder(metricKey)
|
||||
.setDescription(description)
|
||||
.setUnit(unit)
|
||||
.buildWithCallback(measurement -> {
|
||||
Number value = (Number) safeMetricValue(metric);
|
||||
if (value != null) {
|
||||
long longValue = value.longValue();
|
||||
if (longValue >= 0) {
|
||||
measurement.record(longValue, attributes);
|
||||
}
|
||||
}
|
||||
});
|
||||
observableHandles.put(metricKey, counter);
|
||||
} else {
|
||||
ObservableDoubleCounter counter = meter.counterBuilder(metricKey)
|
||||
.ofDoubles()
|
||||
.setDescription(description)
|
||||
.setUnit(unit)
|
||||
.buildWithCallback(measurement -> {
|
||||
Number value = (Number) safeMetricValue(metric);
|
||||
if (value != null) {
|
||||
double doubleValue = value.doubleValue();
|
||||
if (doubleValue >= 0) {
|
||||
measurement.record(doubleValue, attributes);
|
||||
}
|
||||
}
|
||||
});
|
||||
observableHandles.put(metricKey, counter);
|
||||
}
|
||||
|
||||
LOGGER.debug("Registered async counter: {}", metricKey);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("Failed to register async counter for {}", metricKey, e);
|
||||
}
|
||||
}
|
||||
|
||||
private Object safeMetricValue(KafkaMetric metric) {
|
||||
try {
|
||||
return metric.metricValue();
|
||||
} catch (Exception e) {
|
||||
LOGGER.debug("Failed to read metric value for {}", metric.metricName(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void closeHandle(String metricKey) {
|
||||
AutoCloseable handle = observableHandles.remove(metricKey);
|
||||
if (handle != null) {
|
||||
try {
|
||||
handle.close();
|
||||
} catch (Exception e) {
|
||||
LOGGER.debug("Error closing handle for {}", metricKey, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String buildMetricKey(MetricName metricName) {
|
||||
StringBuilder sb = new StringBuilder(metricPrefix);
|
||||
sb.append(".");
|
||||
|
||||
// Add group if present
|
||||
if (metricName.group() != null && !metricName.group().isEmpty()) {
|
||||
sb.append(metricName.group().replace("-", "_").toLowerCase(Locale.ROOT));
|
||||
sb.append(".");
|
||||
}
|
||||
|
||||
// Add name
|
||||
sb.append(metricName.name().replace("-", "_").toLowerCase(Locale.ROOT));
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private Attributes buildAttributes(MetricName metricName) {
|
||||
AttributesBuilder builder = Attributes.builder();
|
||||
|
||||
// Add metric tags as attributes
|
||||
Map<String, String> tags = metricName.tags();
|
||||
if (tags != null) {
|
||||
for (Map.Entry<String, String> entry : tags.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
String value = entry.getValue();
|
||||
if (key != null && value != null) {
|
||||
builder.put(sanitizeAttributeKey(key), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add standard attributes
|
||||
if (metricName.group() != null) {
|
||||
builder.put("metric.group", metricName.group());
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private String sanitizeAttributeKey(String key) {
|
||||
return key.replace("-", "_").replace(".", "_").toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String buildDescription(MetricName metricName) {
|
||||
StringBuilder description = new StringBuilder();
|
||||
description.append("Kafka Connect metric: ");
|
||||
|
||||
if (metricName.group() != null) {
|
||||
description.append(metricName.group()).append(" - ");
|
||||
}
|
||||
|
||||
description.append(metricName.name());
|
||||
|
||||
return description.toString();
|
||||
}
|
||||
|
||||
private String determineUnit(MetricName metricName) {
|
||||
String name = metricName.name().toLowerCase(Locale.ROOT);
|
||||
String group = metricName.group() != null ? metricName.group().toLowerCase(Locale.ROOT) : "";
|
||||
|
||||
if (isKafkaConnectMetric(group)) {
|
||||
return determineConnectMetricUnit(name);
|
||||
}
|
||||
|
||||
if (isTimeMetric(name)) {
|
||||
return determineTimeUnit(name);
|
||||
}
|
||||
|
||||
if (isBytesMetric(name)) {
|
||||
return determineBytesUnit(name);
|
||||
}
|
||||
|
||||
if (isRateMetric(name)) {
|
||||
return "1/s";
|
||||
}
|
||||
|
||||
if (isRatioOrPercentageMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
|
||||
if (isCountMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
|
||||
return "1";
|
||||
}
|
||||
|
||||
private boolean isCounterMetric(MetricName metricName) {
|
||||
String name = metricName.name().toLowerCase(Locale.ROOT);
|
||||
String group = metricName.group() != null ? metricName.group().toLowerCase(Locale.ROOT) : "";
|
||||
|
||||
if (isKafkaConnectMetric(group)) {
|
||||
return isConnectCounterMetric(name);
|
||||
}
|
||||
|
||||
if (isGaugeMetric(name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return hasCounterKeywords(name);
|
||||
}
|
||||
|
||||
private boolean isGaugeMetric(String name) {
|
||||
return hasRateOrAvgKeywords(name) || hasRatioOrPercentKeywords(name) ||
|
||||
hasMinMaxOrCurrentKeywords(name) || hasActiveOrSizeKeywords(name) ||
|
||||
hasTimeButNotTotal(name);
|
||||
}
|
||||
|
||||
private boolean hasRateOrAvgKeywords(String name) {
|
||||
return name.contains("rate") || name.contains("avg") || name.contains("mean");
|
||||
}
|
||||
|
||||
private boolean hasRatioOrPercentKeywords(String name) {
|
||||
return name.contains("ratio") || name.contains("percent") || name.contains("pct");
|
||||
}
|
||||
|
||||
private boolean hasMinMaxOrCurrentKeywords(String name) {
|
||||
return name.contains("max") || name.contains("min") || name.contains("current");
|
||||
}
|
||||
|
||||
private boolean hasActiveOrSizeKeywords(String name) {
|
||||
return name.contains("active") || name.contains("lag") || name.contains("size");
|
||||
}
|
||||
|
||||
private boolean hasTimeButNotTotal(String name) {
|
||||
return name.contains("time") && !name.contains("total");
|
||||
}
|
||||
|
||||
private boolean hasCounterKeywords(String name) {
|
||||
String[] parts = name.split("[._-]");
|
||||
for (String part : parts) {
|
||||
if (isCounterKeyword(part)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean isCounterKeyword(String part) {
|
||||
return isBasicCounterKeyword(part) || isAdvancedCounterKeyword(part);
|
||||
}
|
||||
|
||||
private boolean isBasicCounterKeyword(String part) {
|
||||
return "total".equals(part) || "count".equals(part) || "sum".equals(part) ||
|
||||
"attempts".equals(part);
|
||||
}
|
||||
|
||||
private boolean isAdvancedCounterKeyword(String part) {
|
||||
return "success".equals(part) || "failure".equals(part) ||
|
||||
"errors".equals(part) || "retries".equals(part) || "skipped".equals(part);
|
||||
}
|
||||
|
||||
private boolean isConnectCounterMetric(String name) {
|
||||
if (hasTotalBasedCounters(name)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (hasRecordCounters(name)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (hasActiveCountMetrics(name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean hasTotalBasedCounters(String name) {
|
||||
return hasBasicTotalCounters(name) || hasSuccessFailureCounters(name) ||
|
||||
hasErrorRetryCounters(name) || hasRequestCompletionCounters(name);
|
||||
}
|
||||
|
||||
private boolean hasBasicTotalCounters(String name) {
|
||||
return name.contains("total") || name.contains("attempts");
|
||||
}
|
||||
|
||||
private boolean hasSuccessFailureCounters(String name) {
|
||||
return (name.contains("success") && name.contains("total")) ||
|
||||
(name.contains("failure") && name.contains("total"));
|
||||
}
|
||||
|
||||
private boolean hasErrorRetryCounters(String name) {
|
||||
return name.contains("errors") || name.contains("retries") || name.contains("skipped");
|
||||
}
|
||||
|
||||
private boolean hasRequestCompletionCounters(String name) {
|
||||
return name.contains("requests") || name.contains("completions");
|
||||
}
|
||||
|
||||
private boolean hasRecordCounters(String name) {
|
||||
return hasRecordKeyword(name) && hasTotalOperation(name);
|
||||
}
|
||||
|
||||
private boolean hasRecordKeyword(String name) {
|
||||
return name.contains("record") || name.contains("records");
|
||||
}
|
||||
|
||||
private boolean hasTotalOperation(String name) {
|
||||
return hasPollWriteTotal(name) || hasReadSendTotal(name);
|
||||
}
|
||||
|
||||
private boolean hasPollWriteTotal(String name) {
|
||||
return name.contains("poll-total") || name.contains("write-total");
|
||||
}
|
||||
|
||||
private boolean hasReadSendTotal(String name) {
|
||||
return name.contains("read-total") || name.contains("send-total");
|
||||
}
|
||||
|
||||
private boolean hasActiveCountMetrics(String name) {
|
||||
return hasCountMetrics(name) || hasSequenceMetrics(name);
|
||||
}
|
||||
|
||||
private boolean hasCountMetrics(String name) {
|
||||
return hasActiveTaskCount(name) || hasConnectorCount(name) || hasStatusCount(name);
|
||||
}
|
||||
|
||||
private boolean hasActiveTaskCount(String name) {
|
||||
return name.contains("active-count") || name.contains("partition-count") ||
|
||||
name.contains("task-count");
|
||||
}
|
||||
|
||||
private boolean hasConnectorCount(String name) {
|
||||
return name.contains("connector-count") || name.contains("running-count");
|
||||
}
|
||||
|
||||
private boolean hasStatusCount(String name) {
|
||||
return name.contains("paused-count") || name.contains("failed-count");
|
||||
}
|
||||
|
||||
private boolean hasSequenceMetrics(String name) {
|
||||
return name.contains("seq-no") || name.contains("seq-num");
|
||||
}
|
||||
|
||||
private boolean isKafkaConnectMetric(String group) {
|
||||
return group.contains("connector") || group.contains("task") ||
|
||||
group.contains("connect") || group.contains("worker");
|
||||
}
|
||||
|
||||
private String determineConnectMetricUnit(String name) {
|
||||
String timeUnit = getTimeUnit(name);
|
||||
if (timeUnit != null) {
|
||||
return timeUnit;
|
||||
}
|
||||
|
||||
String countUnit = getCountUnit(name);
|
||||
if (countUnit != null) {
|
||||
return countUnit;
|
||||
}
|
||||
|
||||
String specialUnit = getSpecialUnit(name);
|
||||
if (specialUnit != null) {
|
||||
return specialUnit;
|
||||
}
|
||||
|
||||
return "1";
|
||||
}
|
||||
|
||||
private String getTimeUnit(String name) {
|
||||
if (isTimeBasedMetric(name)) {
|
||||
return "ms";
|
||||
}
|
||||
if (isTimestampMetric(name)) {
|
||||
return "ms";
|
||||
}
|
||||
if (isTimeSinceMetric(name)) {
|
||||
return "ms";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String getCountUnit(String name) {
|
||||
if (isSequenceOrCountMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
if (isLagMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
if (isTotalOrCounterMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String getSpecialUnit(String name) {
|
||||
if (isStatusOrMetadataMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
if (isConnectRateMetric(name)) {
|
||||
return "1/s";
|
||||
}
|
||||
if (isRatioMetric(name)) {
|
||||
return "1";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isTimeBasedMetric(String name) {
|
||||
return hasTimeMs(name) || hasCommitBatchTime(name);
|
||||
}
|
||||
|
||||
private boolean hasTimeMs(String name) {
|
||||
return name.endsWith("-time-ms") || name.endsWith("-avg-time-ms") ||
|
||||
name.endsWith("-max-time-ms");
|
||||
}
|
||||
|
||||
private boolean hasCommitBatchTime(String name) {
|
||||
return name.contains("commit-time") || name.contains("batch-time") ||
|
||||
name.contains("rebalance-time");
|
||||
}
|
||||
|
||||
private boolean isSequenceOrCountMetric(String name) {
|
||||
return hasSequenceNumbers(name) || hasCountSuffix(name);
|
||||
}
|
||||
|
||||
private boolean hasSequenceNumbers(String name) {
|
||||
return name.contains("seq-no") || name.contains("seq-num");
|
||||
}
|
||||
|
||||
private boolean hasCountSuffix(String name) {
|
||||
return name.endsWith("-count") || name.contains("task-count") ||
|
||||
name.contains("partition-count");
|
||||
}
|
||||
|
||||
private boolean isLagMetric(String name) {
|
||||
return name.contains("lag");
|
||||
}
|
||||
|
||||
private boolean isStatusOrMetadataMetric(String name) {
|
||||
return isStatusMetric(name) || hasProtocolLeaderMetrics(name) ||
|
||||
hasConnectorMetrics(name);
|
||||
}
|
||||
|
||||
private boolean isStatusMetric(String name) {
|
||||
return "status".equals(name) || name.contains("protocol");
|
||||
}
|
||||
|
||||
private boolean hasProtocolLeaderMetrics(String name) {
|
||||
return name.contains("leader-name");
|
||||
}
|
||||
|
||||
private boolean hasConnectorMetrics(String name) {
|
||||
return name.contains("connector-type") || name.contains("connector-class") ||
|
||||
name.contains("connector-version");
|
||||
}
|
||||
|
||||
private boolean isRatioMetric(String name) {
|
||||
return name.contains("ratio") || name.contains("percentage");
|
||||
}
|
||||
|
||||
private boolean isTotalOrCounterMetric(String name) {
|
||||
return hasTotalSum(name) || hasAttempts(name) || hasSuccessFailure(name) ||
|
||||
hasErrorsRetries(name);
|
||||
}
|
||||
|
||||
private boolean hasTotalSum(String name) {
|
||||
return name.contains("total") || name.contains("sum");
|
||||
}
|
||||
|
||||
private boolean hasAttempts(String name) {
|
||||
return name.contains("attempts");
|
||||
}
|
||||
|
||||
private boolean hasSuccessFailure(String name) {
|
||||
return name.contains("success") || name.contains("failure");
|
||||
}
|
||||
|
||||
private boolean hasErrorsRetries(String name) {
|
||||
return name.contains("errors") || name.contains("retries") || name.contains("skipped");
|
||||
}
|
||||
|
||||
private boolean isTimestampMetric(String name) {
|
||||
return name.contains("timestamp") || name.contains("epoch");
|
||||
}
|
||||
|
||||
private boolean isConnectRateMetric(String name) {
|
||||
return name.contains("rate") && !name.contains("ratio");
|
||||
}
|
||||
|
||||
private boolean isTimeSinceMetric(String name) {
|
||||
return name.contains("time-since-last") || name.contains("since-last");
|
||||
}
|
||||
|
||||
private boolean isTimeMetric(String name) {
|
||||
return hasTimeKeywords(name) && !hasTimeExclusions(name);
|
||||
}
|
||||
|
||||
private boolean hasTimeKeywords(String name) {
|
||||
return name.contains("time") || name.contains("latency") ||
|
||||
name.contains("duration");
|
||||
}
|
||||
|
||||
private boolean hasTimeExclusions(String name) {
|
||||
return name.contains("ratio") || name.contains("rate") ||
|
||||
name.contains("count") || name.contains("since-last");
|
||||
}
|
||||
|
||||
private String determineTimeUnit(String name) {
|
||||
if (name.contains("ms") || name.contains("millisecond")) {
|
||||
return "ms";
|
||||
} else if (name.contains("us") || name.contains("microsecond")) {
|
||||
return "us";
|
||||
} else if (name.contains("ns") || name.contains("nanosecond")) {
|
||||
return "ns";
|
||||
} else if (name.contains("s") && !name.contains("ms")) {
|
||||
return "s";
|
||||
} else {
|
||||
return "ms";
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isBytesMetric(String name) {
|
||||
return name.contains("byte") || name.contains("bytes") ||
|
||||
name.contains("size") && !name.contains("batch-size");
|
||||
}
|
||||
|
||||
private String determineBytesUnit(String name) {
|
||||
boolean isRate = name.contains("rate") || name.contains("per-sec") ||
|
||||
name.contains("persec") || name.contains("/s");
|
||||
return isRate ? "By/s" : "By";
|
||||
}
|
||||
|
||||
private boolean isRateMetric(String name) {
|
||||
return hasRateKeywords(name) && !hasExcludedKeywords(name);
|
||||
}
|
||||
|
||||
private boolean hasRateKeywords(String name) {
|
||||
return name.contains("rate") || name.contains("per-sec") ||
|
||||
name.contains("persec") || name.contains("/s");
|
||||
}
|
||||
|
||||
private boolean hasExcludedKeywords(String name) {
|
||||
return name.contains("byte") || name.contains("ratio");
|
||||
}
|
||||
|
||||
private boolean isRatioOrPercentageMetric(String name) {
|
||||
return hasPercentKeywords(name) || hasRatioKeywords(name);
|
||||
}
|
||||
|
||||
private boolean hasPercentKeywords(String name) {
|
||||
return name.contains("percent") || name.contains("pct");
|
||||
}
|
||||
|
||||
private boolean hasRatioKeywords(String name) {
|
||||
return name.contains("ratio");
|
||||
}
|
||||
|
||||
private boolean isCountMetric(String name) {
|
||||
return name.contains("count") || name.contains("total") ||
|
||||
name.contains("sum") || name.endsWith("-num");
|
||||
}
|
||||
|
||||
private boolean shouldIncludeMetric(String metricKey) {
|
||||
if (excludePattern != null && metricKey.matches(excludePattern)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (includePattern != null) {
|
||||
return metricKey.matches(includePattern);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.runtime;
|
||||
|
||||
/**
|
||||
* An interface for determining which node should be responsible for clean metrics.
|
||||
* This abstraction allows different implementations of clean node selection strategies.
|
||||
*/
|
||||
public interface LeaderNodeSelector {
|
||||
|
||||
/**
|
||||
* Determines if the current node should be responsible for clean metrics.
|
||||
*
|
||||
* @return true if the current node should clean metrics, false otherwise.
|
||||
*/
|
||||
boolean isLeader();
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.runtime;
|
||||
|
||||
/**
|
||||
* SPI interface for providing custom LeaderNodeSelector implementations.
|
||||
* Third-party libraries can implement this interface and register their implementations
|
||||
* using Java's ServiceLoader mechanism.
|
||||
*/
|
||||
public interface LeaderNodeSelectorProvider {
|
||||
|
||||
/**
|
||||
* Creates a new LeaderNodeSelector instance based on the provided configuration.
|
||||
*
|
||||
* @return A new LeaderNodeSelector instance
|
||||
* @throws Exception If the selector cannot be created
|
||||
*/
|
||||
LeaderNodeSelector createSelector() throws Exception;
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.runtime;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.function.BooleanSupplier;
|
||||
|
||||
/**
|
||||
* Stores runtime-provided suppliers that answer whether the current process
|
||||
* should act as the leader.
|
||||
*/
|
||||
public final class RuntimeLeaderRegistry {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(RuntimeLeaderRegistry.class);
|
||||
private static BooleanSupplier supplier = () -> false;
|
||||
|
||||
private RuntimeLeaderRegistry() {
|
||||
}
|
||||
|
||||
public static void register(BooleanSupplier supplier) {
|
||||
RuntimeLeaderRegistry.supplier = supplier;
|
||||
LOGGER.info("Registered runtime leader supplier for log metrics.");
|
||||
}
|
||||
|
||||
public static BooleanSupplier supplier() {
|
||||
return supplier;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
/*
|
||||
* Copyright 2025, AutoMQ HK Limited.
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kafka.connect.automq.runtime;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.BooleanSupplier;
|
||||
|
||||
public class RuntimeLeaderSelectorProvider implements LeaderNodeSelectorProvider {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(RuntimeLeaderSelectorProvider.class);
|
||||
|
||||
@Override
|
||||
public LeaderNodeSelector createSelector() {
|
||||
final AtomicBoolean missingLogged = new AtomicBoolean(false);
|
||||
final AtomicBoolean leaderLogged = new AtomicBoolean(false);
|
||||
|
||||
return () -> {
|
||||
BooleanSupplier current = org.apache.kafka.connect.automq.runtime.RuntimeLeaderRegistry.supplier();
|
||||
if (current == null) {
|
||||
if (missingLogged.compareAndSet(false, true)) {
|
||||
LOGGER.warn("leader supplier for key not yet available; treating node as follower until registration happens.");
|
||||
}
|
||||
if (leaderLogged.getAndSet(false)) {
|
||||
LOGGER.info("Node stepped down from leadership because supplier is unavailable.");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (missingLogged.get()) {
|
||||
missingLogged.set(false);
|
||||
LOGGER.info("leader supplier is now available.");
|
||||
}
|
||||
|
||||
try {
|
||||
boolean leader = current.getAsBoolean();
|
||||
if (leader) {
|
||||
if (!leaderLogged.getAndSet(true)) {
|
||||
LOGGER.info("Node became leader");
|
||||
}
|
||||
} else {
|
||||
if (leaderLogged.getAndSet(false)) {
|
||||
LOGGER.info("Node stepped down from leadership");
|
||||
}
|
||||
}
|
||||
return leader;
|
||||
} catch (RuntimeException e) {
|
||||
if (leaderLogged.getAndSet(false)) {
|
||||
LOGGER.info("Node stepped down from leadership due to supplier exception.");
|
||||
}
|
||||
LOGGER.warn("leader supplier threw exception. Treating as follower.", e);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -19,9 +19,6 @@ package org.apache.kafka.connect.cli;
|
|||
import org.apache.kafka.common.utils.Exit;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.apache.kafka.connect.automq.az.AzMetadataProviderHolder;
|
||||
import org.apache.kafka.connect.automq.log.ConnectLogUploader;
|
||||
import org.apache.kafka.connect.automq.metrics.OpenTelemetryMetricsReporter;
|
||||
import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy;
|
||||
import org.apache.kafka.connect.runtime.Connect;
|
||||
import org.apache.kafka.connect.runtime.Herder;
|
||||
|
|
@ -39,7 +36,6 @@ import java.net.URI;
|
|||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Common initialization logic for Kafka Connect, intended for use by command line utilities
|
||||
|
|
@ -49,9 +45,7 @@ import java.util.Properties;
|
|||
*/
|
||||
public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfig> {
|
||||
|
||||
private static Logger getLogger() {
|
||||
return LoggerFactory.getLogger(AbstractConnectCli.class);
|
||||
}
|
||||
private static final Logger log = LoggerFactory.getLogger(AbstractConnectCli.class);
|
||||
private final String[] args;
|
||||
private final Time time = Time.SYSTEM;
|
||||
|
||||
|
|
@ -89,6 +83,7 @@ public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfi
|
|||
*/
|
||||
public void run() {
|
||||
if (args.length < 1 || Arrays.asList(args).contains("--help")) {
|
||||
log.info("Usage: {}", usage());
|
||||
Exit.exit(1);
|
||||
}
|
||||
|
||||
|
|
@ -97,17 +92,6 @@ public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfi
|
|||
Map<String, String> workerProps = !workerPropsFile.isEmpty() ?
|
||||
Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
|
||||
String[] extraArgs = Arrays.copyOfRange(args, 1, args.length);
|
||||
|
||||
// AutoMQ inject start
|
||||
// Initialize S3 log uploader and OpenTelemetry with worker properties
|
||||
ConnectLogUploader.initialize(workerProps);
|
||||
AzMetadataProviderHolder.initialize(workerProps);
|
||||
|
||||
Properties telemetryProps = new Properties();
|
||||
telemetryProps.putAll(workerProps);
|
||||
OpenTelemetryMetricsReporter.initializeTelemetry(telemetryProps);
|
||||
// AutoMQ inject end
|
||||
|
||||
Connect<H> connect = startConnect(workerProps);
|
||||
processExtraArgs(connect, extraArgs);
|
||||
|
||||
|
|
@ -115,7 +99,7 @@ public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfi
|
|||
connect.awaitStop();
|
||||
|
||||
} catch (Throwable t) {
|
||||
getLogger().error("Stopping due to error", t);
|
||||
log.error("Stopping due to error", t);
|
||||
Exit.exit(2);
|
||||
}
|
||||
}
|
||||
|
|
@ -127,17 +111,17 @@ public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfi
|
|||
* @return a started instance of {@link Connect}
|
||||
*/
|
||||
public Connect<H> startConnect(Map<String, String> workerProps) {
|
||||
getLogger().info("Kafka Connect worker initializing ...");
|
||||
log.info("Kafka Connect worker initializing ...");
|
||||
long initStart = time.hiResClockMs();
|
||||
|
||||
WorkerInfo initInfo = new WorkerInfo();
|
||||
initInfo.logAll();
|
||||
|
||||
getLogger().info("Scanning for plugin classes. This might take a moment ...");
|
||||
log.info("Scanning for plugin classes. This might take a moment ...");
|
||||
Plugins plugins = new Plugins(workerProps);
|
||||
plugins.compareAndSwapWithDelegatingLoader();
|
||||
T config = createConfig(workerProps);
|
||||
getLogger().debug("Kafka cluster ID: {}", config.kafkaClusterId());
|
||||
log.debug("Kafka cluster ID: {}", config.kafkaClusterId());
|
||||
|
||||
RestClient restClient = new RestClient(config);
|
||||
|
||||
|
|
@ -154,11 +138,11 @@ public abstract class AbstractConnectCli<H extends Herder, T extends WorkerConfi
|
|||
H herder = createHerder(config, workerId, plugins, connectorClientConfigOverridePolicy, restServer, restClient);
|
||||
|
||||
final Connect<H> connect = new Connect<>(herder, restServer);
|
||||
getLogger().info("Kafka Connect worker initialization took {}ms", time.hiResClockMs() - initStart);
|
||||
log.info("Kafka Connect worker initialization took {}ms", time.hiResClockMs() - initStart);
|
||||
try {
|
||||
connect.start();
|
||||
} catch (Exception e) {
|
||||
getLogger().error("Failed to start Connect", e);
|
||||
log.error("Failed to start Connect", e);
|
||||
connect.stop();
|
||||
Exit.exit(3);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
package org.apache.kafka.connect.cli;
|
||||
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.connect.automq.runtime.RuntimeLeaderRegistry;
|
||||
import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy;
|
||||
import org.apache.kafka.connect.json.JsonConverter;
|
||||
import org.apache.kafka.connect.json.JsonConverterConfig;
|
||||
|
|
@ -40,7 +39,6 @@ import org.apache.kafka.connect.util.SharedTopicAdmin;
|
|||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.BooleanSupplier;
|
||||
|
||||
import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG;
|
||||
|
||||
|
|
@ -98,16 +96,10 @@ public class ConnectDistributed extends AbstractConnectCli<DistributedHerder, Di
|
|||
|
||||
// Pass the shared admin to the distributed herder as an additional AutoCloseable object that should be closed when the
|
||||
// herder is stopped. This is easier than having to track and own the lifecycle ourselves.
|
||||
DistributedHerder herder = new DistributedHerder(config, Time.SYSTEM, worker,
|
||||
return new DistributedHerder(config, Time.SYSTEM, worker,
|
||||
kafkaClusterId, statusBackingStore, configBackingStore,
|
||||
restServer.advertisedUrl().toString(), restClient, connectorClientConfigOverridePolicy,
|
||||
Collections.emptyList(), sharedAdmin);
|
||||
// AutoMQ for Kafka connect inject start
|
||||
BooleanSupplier leaderSupplier = herder::isLeaderInstance;
|
||||
RuntimeLeaderRegistry.register(leaderSupplier);
|
||||
// AutoMQ for Kafka connect inject end
|
||||
|
||||
return herder;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@ import org.apache.kafka.connect.runtime.distributed.DistributedHerder;
|
|||
import org.apache.kafka.connect.runtime.rest.ConnectRestServer;
|
||||
import org.apache.kafka.connect.runtime.rest.RestServer;
|
||||
|
||||
import com.automq.log.S3RollingFileAppender;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -117,9 +115,6 @@ public class Connect<H extends Herder> {
|
|||
try {
|
||||
startLatch.await();
|
||||
Connect.this.stop();
|
||||
// AutoMQ inject start
|
||||
S3RollingFileAppender.shutdown();
|
||||
// AutoMQ inject end
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Interrupted in shutdown hook while waiting for Kafka Connect startup to finish");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,6 @@ import org.apache.kafka.common.utils.ThreadUtils;
|
|||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.common.utils.Timer;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.apache.kafka.connect.automq.az.AzAwareClientConfigurator;
|
||||
import org.apache.kafka.connect.connector.ConnectRecord;
|
||||
import org.apache.kafka.connect.connector.Connector;
|
||||
import org.apache.kafka.connect.connector.Task;
|
||||
|
|
@ -842,10 +841,6 @@ public class Worker {
|
|||
connectorClientConfigOverridePolicy);
|
||||
producerProps.putAll(producerOverrides);
|
||||
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(producerProps, defaultClientId);
|
||||
// AutoMQ for Kafka inject end
|
||||
|
||||
return producerProps;
|
||||
}
|
||||
|
||||
|
|
@ -914,10 +909,6 @@ public class Worker {
|
|||
connectorClientConfigOverridePolicy);
|
||||
consumerProps.putAll(consumerOverrides);
|
||||
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyConsumerAz(consumerProps, defaultClientId);
|
||||
// AutoMQ for Kafka inject end
|
||||
|
||||
return consumerProps;
|
||||
}
|
||||
|
||||
|
|
@ -947,10 +938,6 @@ public class Worker {
|
|||
// Admin client-specific overrides in the worker config
|
||||
adminProps.putAll(config.originalsWithPrefix("admin."));
|
||||
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyAdminAz(adminProps, defaultClientId);
|
||||
// AutoMQ for Kafka inject end
|
||||
|
||||
// Connector-specified overrides
|
||||
Map<String, Object> adminOverrides =
|
||||
connectorClientConfigOverrides(connName, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX,
|
||||
|
|
|
|||
|
|
@ -1735,12 +1735,6 @@ public class DistributedHerder extends AbstractHerder implements Runnable {
|
|||
configBackingStore.putLoggerLevel(namespace, level);
|
||||
}
|
||||
|
||||
// AutoMQ inject start
|
||||
public boolean isLeaderInstance() {
|
||||
return isLeader();
|
||||
}
|
||||
// AutoMQ inject end
|
||||
|
||||
// Should only be called from work thread, so synchronization should not be needed
|
||||
protected boolean isLeader() {
|
||||
return assignment != null && member.memberId().equals(assignment.leader());
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import org.apache.kafka.common.serialization.StringSerializer;
|
|||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.common.utils.Timer;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.apache.kafka.connect.automq.az.AzAwareClientConfigurator;
|
||||
import org.apache.kafka.connect.data.Schema;
|
||||
import org.apache.kafka.connect.data.SchemaAndValue;
|
||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||
|
|
@ -441,9 +440,6 @@ public class KafkaConfigBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
Map<String, Object> result = new HashMap<>(baseProducerProps(workerConfig));
|
||||
|
||||
result.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId + "-leader");
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(result, "config-log-leader");
|
||||
// AutoMQ for Kafka inject end
|
||||
// Always require producer acks to all to ensure durable writes
|
||||
result.put(ProducerConfig.ACKS_CONFIG, "all");
|
||||
// We can set this to 5 instead of 1 without risking reordering because we are using an idempotent producer
|
||||
|
|
@ -777,17 +773,11 @@ public class KafkaConfigBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
|
||||
Map<String, Object> producerProps = new HashMap<>(baseProducerProps);
|
||||
producerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(producerProps, "config-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
|
||||
Map<String, Object> consumerProps = new HashMap<>(originals);
|
||||
consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
|
||||
consumerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyConsumerAz(consumerProps, "config-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
|
||||
if (config.exactlyOnceSourceEnabled()) {
|
||||
ConnectUtils.ensureProperty(
|
||||
|
|
@ -800,9 +790,6 @@ public class KafkaConfigBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
Map<String, Object> adminProps = new HashMap<>(originals);
|
||||
ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
|
||||
adminProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyAdminAz(adminProps, "config-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
|
||||
Map<String, Object> topicSettings = config instanceof DistributedConfig
|
||||
? ((DistributedConfig) config).configStorageTopicSettings()
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.kafka.common.errors.UnsupportedVersionException;
|
|||
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
|
||||
import org.apache.kafka.common.serialization.ByteArraySerializer;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.connect.automq.az.AzAwareClientConfigurator;
|
||||
import org.apache.kafka.connect.errors.ConnectException;
|
||||
import org.apache.kafka.connect.runtime.WorkerConfig;
|
||||
import org.apache.kafka.connect.runtime.distributed.DistributedConfig;
|
||||
|
|
@ -193,18 +192,12 @@ public class KafkaOffsetBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
// gets approved and scheduled for release.
|
||||
producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false");
|
||||
producerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(producerProps, "offset-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(producerProps, config, clusterId);
|
||||
|
||||
Map<String, Object> consumerProps = new HashMap<>(originals);
|
||||
consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
|
||||
consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
|
||||
consumerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyConsumerAz(consumerProps, "offset-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
|
||||
if (config.exactlyOnceSourceEnabled()) {
|
||||
ConnectUtils.ensureProperty(
|
||||
|
|
@ -216,9 +209,6 @@ public class KafkaOffsetBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
|
||||
Map<String, Object> adminProps = new HashMap<>(originals);
|
||||
adminProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyAdminAz(adminProps, "offset-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
|
||||
NewTopic topicDescription = newTopicDescription(topic, config);
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.kafka.common.serialization.StringDeserializer;
|
|||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.apache.kafka.common.utils.ThreadUtils;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.connect.automq.az.AzAwareClientConfigurator;
|
||||
import org.apache.kafka.connect.data.Schema;
|
||||
import org.apache.kafka.connect.data.SchemaAndValue;
|
||||
import org.apache.kafka.connect.data.SchemaBuilder;
|
||||
|
|
@ -184,25 +183,16 @@ public class KafkaStatusBackingStore extends KafkaTopicBasedBackingStore impleme
|
|||
// gets approved and scheduled for release.
|
||||
producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false"); // disable idempotence since retries is force to 0
|
||||
producerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(producerProps, "status-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(producerProps, config, clusterId);
|
||||
|
||||
Map<String, Object> consumerProps = new HashMap<>(originals);
|
||||
consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
|
||||
consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
|
||||
consumerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyConsumerAz(consumerProps, "status-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
|
||||
|
||||
Map<String, Object> adminProps = new HashMap<>(originals);
|
||||
adminProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
|
||||
// AutoMQ for Kafka inject start
|
||||
AzAwareClientConfigurator.maybeApplyAdminAz(adminProps, "status-log");
|
||||
// AutoMQ for Kafka inject end
|
||||
ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
|
||||
|
||||
Map<String, Object> topicSettings = config instanceof DistributedConfig
|
||||
|
|
|
|||
|
|
@ -1,115 +0,0 @@
|
|||
package org.apache.kafka.connect.automq;
|
||||
|
||||
import org.apache.kafka.clients.admin.AdminClientConfig;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.connect.automq.az.AzAwareClientConfigurator;
|
||||
import org.apache.kafka.connect.automq.az.AzMetadataProvider;
|
||||
import org.apache.kafka.connect.automq.az.AzMetadataProviderHolder;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
class AzAwareClientConfiguratorTest {
|
||||
|
||||
@AfterEach
|
||||
void resetProvider() {
|
||||
AzMetadataProviderHolder.setProviderForTest(null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDecorateProducerClientId() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new FixedAzProvider("us-east-1a"));
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-1");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(props, "producer-1");
|
||||
|
||||
assertEquals("automq_type=producer&automq_role=producer-1&automq_az=us-east-1a&producer-1",
|
||||
props.get(ProducerConfig.CLIENT_ID_CONFIG));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPreserveCustomClientIdInAzConfig() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new FixedAzProvider("us-east-1a"));
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.CLIENT_ID_CONFIG, "custom-id");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(props, "producer-1");
|
||||
|
||||
assertEquals("automq_type=producer&automq_role=producer-1&automq_az=us-east-1a&custom-id",
|
||||
props.get(ProducerConfig.CLIENT_ID_CONFIG));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldAssignRackForConsumers() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new FixedAzProvider("us-west-2c"));
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer-1");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyConsumerAz(props, "consumer-1");
|
||||
|
||||
assertEquals("us-west-2c", props.get(ConsumerConfig.CLIENT_RACK_CONFIG));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDecorateAdminClientId() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new FixedAzProvider("eu-west-1b"));
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(AdminClientConfig.CLIENT_ID_CONFIG, "admin-1");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyAdminAz(props, "admin-1");
|
||||
|
||||
assertEquals("automq_type=admin&automq_role=admin-1&automq_az=eu-west-1b&admin-1",
|
||||
props.get(AdminClientConfig.CLIENT_ID_CONFIG));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldLeaveClientIdWhenAzUnavailable() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new AzMetadataProvider() {
|
||||
@Override
|
||||
public Optional<String> availabilityZoneId() {
|
||||
return Optional.empty();
|
||||
}
|
||||
});
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-1");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(props, "producer-1");
|
||||
|
||||
assertEquals("producer-1", props.get(ProducerConfig.CLIENT_ID_CONFIG));
|
||||
assertFalse(props.containsKey(ConsumerConfig.CLIENT_RACK_CONFIG));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldEncodeSpecialCharactersInClientId() {
|
||||
AzMetadataProviderHolder.setProviderForTest(new FixedAzProvider("us-east-1a"));
|
||||
Map<String, Object> props = new HashMap<>();
|
||||
props.put(ProducerConfig.CLIENT_ID_CONFIG, "client-with-spaces & symbols");
|
||||
|
||||
AzAwareClientConfigurator.maybeApplyProducerAz(props, "test-role");
|
||||
|
||||
assertEquals("automq_type=producer&automq_role=test-role&automq_az=us-east-1a&client-with-spaces & symbols",
|
||||
props.get(ProducerConfig.CLIENT_ID_CONFIG));
|
||||
}
|
||||
|
||||
private static final class FixedAzProvider implements AzMetadataProvider {
|
||||
private final String az;
|
||||
|
||||
private FixedAzProvider(String az) {
|
||||
this.az = az;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> availabilityZoneId() {
|
||||
return Optional.ofNullable(az);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -87,28 +87,6 @@ index 717a36c21f..ea5eb74efb 100644
|
|||
USER 1001
|
||||
ENTRYPOINT [ "/opt/bitnami/scripts/kafka/entrypoint.sh" ]
|
||||
CMD [ "/opt/bitnami/scripts/kafka/run.sh" ]
|
||||
diff --git a/container/bitnami/prebuildfs/opt/bitnami/scripts/libbitnami.sh b/container/bitnami/prebuildfs/opt/bitnami/scripts/libbitnami.sh
|
||||
index 00d053b521..09e3d3084d 100644
|
||||
--- a/container/bitnami/prebuildfs/opt/bitnami/scripts/libbitnami.sh
|
||||
+++ b/container/bitnami/prebuildfs/opt/bitnami/scripts/libbitnami.sh
|
||||
@@ -42,12 +42,13 @@ print_welcome_page() {
|
||||
# None
|
||||
#########################
|
||||
print_image_welcome_page() {
|
||||
- local github_url="https://github.com/bitnami/containers"
|
||||
+ local docs_url="https://www.automq.com/docs/automq/deployment/deploy-multi-nodes-cluster-on-kubernetes"
|
||||
|
||||
info ""
|
||||
- info "${BOLD}Welcome to the Bitnami ${BITNAMI_APP_NAME} container${RESET}"
|
||||
- info "Subscribe to project updates by watching ${BOLD}${github_url}${RESET}"
|
||||
- info "Did you know there are enterprise versions of the Bitnami catalog? For enhanced secure software supply chain features, unlimited pulls from Docker, LTS support, or application customization, see Bitnami Premium or Tanzu Application Catalog. See https://www.arrow.com/globalecs/na/vendors/bitnami/ for more information."
|
||||
+ info "${BOLD}Welcome to the AutoMQ for Apache Kafka on Bitnami Container${RESET}"
|
||||
+ info "${BOLD}This image is compatible with Bitnami's container standards.${RESET}"
|
||||
+ info "Refer to the documentation for complete configuration and Kubernetes deployment guidelines:"
|
||||
+ info "${BOLD}${docs_url}${RESET}"
|
||||
info ""
|
||||
}
|
||||
|
||||
diff --git a/container/bitnami/rootfs/opt/bitnami/scripts/kafka/postunpack.sh b/container/bitnami/rootfs/opt/bitnami/scripts/kafka/postunpack.sh
|
||||
index 7255563236..673c84e721 100644
|
||||
--- a/container/bitnami/rootfs/opt/bitnami/scripts/kafka/postunpack.sh
|
||||
|
|
|
|||
|
|
@ -42,13 +42,12 @@ print_welcome_page() {
|
|||
# None
|
||||
#########################
|
||||
print_image_welcome_page() {
|
||||
local docs_url="https://www.automq.com/docs/automq/deployment/deploy-multi-nodes-cluster-on-kubernetes"
|
||||
local github_url="https://github.com/bitnami/containers"
|
||||
|
||||
info ""
|
||||
info "${BOLD}Welcome to the AutoMQ for Apache Kafka on Bitnami Container${RESET}"
|
||||
info "${BOLD}This image is compatible with Bitnami's container standards.${RESET}"
|
||||
info "Refer to the documentation for complete configuration and Kubernetes deployment guidelines:"
|
||||
info "${BOLD}${docs_url}${RESET}"
|
||||
info "${BOLD}Welcome to the Bitnami ${BITNAMI_APP_NAME} container${RESET}"
|
||||
info "Subscribe to project updates by watching ${BOLD}${github_url}${RESET}"
|
||||
info "Did you know there are enterprise versions of the Bitnami catalog? For enhanced secure software supply chain features, unlimited pulls from Docker, LTS support, or application customization, see Bitnami Premium or Tanzu Application Catalog. See https://www.arrow.com/globalecs/na/vendors/bitnami/ for more information."
|
||||
info ""
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ public class RawMetricTypes {
|
|||
public static final byte BROKER_MAX_PENDING_FETCH_LATENCY_MS = (byte) 5;
|
||||
public static final byte BROKER_METRIC_VERSION = (byte) 6;
|
||||
public static final Map<Byte, AbnormalMetric> ABNORMAL_METRICS = Map.of(
|
||||
// BROKER_APPEND_LATENCY_AVG_MS, new AbnormalLatency(100), // 100ms
|
||||
BROKER_APPEND_LATENCY_AVG_MS, new AbnormalLatency(100), // 100ms
|
||||
BROKER_MAX_PENDING_APPEND_LATENCY_MS, new AbnormalLatency(10000), // 10s
|
||||
BROKER_MAX_PENDING_FETCH_LATENCY_MS, new AbnormalLatency(10000) // 10s
|
||||
);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
package kafka.automq;
|
||||
|
||||
import kafka.log.stream.s3.telemetry.exporter.ExporterConstants;
|
||||
import kafka.server.KafkaConfig;
|
||||
|
||||
import org.apache.kafka.common.config.ConfigDef;
|
||||
|
|
@ -79,7 +80,7 @@ public class AutoMQConfig {
|
|||
|
||||
public static final String S3_WAL_UPLOAD_INTERVAL_MS_CONFIG = "s3.wal.upload.interval.ms";
|
||||
public static final String S3_WAL_UPLOAD_INTERVAL_MS_DOC = "The interval at which WAL triggers upload to object storage. -1 means only upload by size trigger";
|
||||
public static final long S3_WAL_UPLOAD_INTERVAL_MS_DEFAULT = 60000L;
|
||||
public static final long S3_WAL_UPLOAD_INTERVAL_MS_DEFAULT = -1L;
|
||||
|
||||
public static final String S3_STREAM_SPLIT_SIZE_CONFIG = "s3.stream.object.split.size";
|
||||
public static final String S3_STREAM_SPLIT_SIZE_DOC = "The S3 stream object split size threshold when upload delta WAL or compact stream set object.";
|
||||
|
|
@ -113,7 +114,7 @@ public class AutoMQConfig {
|
|||
public static final String S3_STREAM_SET_OBJECT_COMPACTION_INTERVAL_CONFIG = "s3.stream.set.object.compaction.interval.minutes";
|
||||
public static final String S3_STREAM_SET_OBJECT_COMPACTION_INTERVAL_DOC = "Set the interpublic static final String for Stream object compaction. The smaller this value, the smaller the scale of metadata storage, and the earlier the data can become compact. " +
|
||||
"However, the number of compactions that the final generated stream object goes through will increase.";
|
||||
public static final int S3_STREAM_SET_OBJECT_COMPACTION_INTERVAL = 5; // 5min
|
||||
public static final int S3_STREAM_SET_OBJECT_COMPACTION_INTERVAL = 10; // 10min
|
||||
|
||||
public static final String S3_STREAM_SET_OBJECT_COMPACTION_CACHE_SIZE_CONFIG = "s3.stream.set.object.compaction.cache.size";
|
||||
public static final String S3_STREAM_SET_OBJECT_COMPACTION_CACHE_SIZE_DOC = "The size of memory is available during the Stream object compaction process. The larger this value, the lower the cost of API calls.";
|
||||
|
|
@ -153,7 +154,7 @@ public class AutoMQConfig {
|
|||
public static final String S3_NETWORK_BASELINE_BANDWIDTH_CONFIG = "s3.network.baseline.bandwidth";
|
||||
public static final String S3_NETWORK_BASELINE_BANDWIDTH_DOC = "The total available bandwidth for object storage requests. This is used to prevent stream set object compaction and catch-up read from monopolizing normal read and write traffic. Produce and Consume will also separately consume traffic in and traffic out. " +
|
||||
"For example, suppose this value is set to 100MB/s, and the normal read and write traffic is 80MB/s, then the available traffic for stream set object compaction is 20MB/s.";
|
||||
public static final long S3_NETWORK_BASELINE_BANDWIDTH = 1024 * 1024 * 1024; // 1GBps
|
||||
public static final long S3_NETWORK_BASELINE_BANDWIDTH = 100 * 1024 * 1024; // 100MB/s
|
||||
|
||||
public static final String S3_NETWORK_REFILL_PERIOD_MS_CONFIG = "s3.network.refill.period.ms";
|
||||
public static final String S3_NETWORK_REFILL_PERIOD_MS_DOC = "The network bandwidth token refill period in milliseconds.";
|
||||
|
|
@ -250,11 +251,6 @@ public class AutoMQConfig {
|
|||
public static final String S3_TELEMETRY_OPS_ENABLED_CONFIG = "s3.telemetry.ops.enabled";
|
||||
public static final String S3_TELEMETRY_OPS_ENABLED_DOC = "[DEPRECATED] use s3.telemetry.metrics.uri instead.";
|
||||
|
||||
private static final String TELEMETRY_EXPORTER_TYPE_OTLP = "otlp";
|
||||
private static final String TELEMETRY_EXPORTER_TYPE_PROMETHEUS = "prometheus";
|
||||
private static final String TELEMETRY_EXPORTER_TYPE_OPS = "ops";
|
||||
public static final String URI_DELIMITER = "://?";
|
||||
|
||||
// Deprecated config end
|
||||
|
||||
public static void define(ConfigDef configDef) {
|
||||
|
|
@ -313,14 +309,12 @@ public class AutoMQConfig {
|
|||
.define(AutoMQConfig.TABLE_TOPIC_SCHEMA_REGISTRY_URL_CONFIG, STRING, null, MEDIUM, AutoMQConfig.TABLE_TOPIC_SCHEMA_REGISTRY_URL_DOC);
|
||||
}
|
||||
|
||||
private final long nodeEpoch = System.currentTimeMillis();
|
||||
private List<BucketURI> dataBuckets;
|
||||
private List<BucketURI> opsBuckets;
|
||||
private String walConfig;
|
||||
private String metricsExporterURI;
|
||||
private List<Pair<String, String>> baseLabels;
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
private Optional<List<BucketURI>> zoneRouterChannels;
|
||||
private Optional<BucketURI> zoneRouterChannels;
|
||||
|
||||
public AutoMQConfig setup(KafkaConfig config) {
|
||||
dataBuckets = genDataBuckets(config);
|
||||
|
|
@ -332,10 +326,6 @@ public class AutoMQConfig {
|
|||
return this;
|
||||
}
|
||||
|
||||
public long nodeEpoch() {
|
||||
return nodeEpoch;
|
||||
}
|
||||
|
||||
public List<BucketURI> dataBuckets() {
|
||||
return dataBuckets;
|
||||
}
|
||||
|
|
@ -356,7 +346,7 @@ public class AutoMQConfig {
|
|||
return baseLabels;
|
||||
}
|
||||
|
||||
public Optional<List<BucketURI>> zoneRouterChannels() {
|
||||
public Optional<BucketURI> zoneRouterChannels() {
|
||||
return zoneRouterChannels;
|
||||
}
|
||||
|
||||
|
|
@ -407,7 +397,7 @@ public class AutoMQConfig {
|
|||
if (uri == null) {
|
||||
uri = buildMetrixExporterURIWithOldConfigs(config);
|
||||
}
|
||||
if (!uri.contains(TELEMETRY_EXPORTER_TYPE_OPS)) {
|
||||
if (!uri.contains(ExporterConstants.OPS_TYPE)) {
|
||||
uri += "," + buildOpsExporterURI();
|
||||
}
|
||||
return uri;
|
||||
|
|
@ -424,10 +414,10 @@ public class AutoMQConfig {
|
|||
for (String exporterType : exporterTypeArray) {
|
||||
exporterType = exporterType.trim();
|
||||
switch (exporterType) {
|
||||
case TELEMETRY_EXPORTER_TYPE_OTLP:
|
||||
case ExporterConstants.OTLP_TYPE:
|
||||
exportedUris.add(buildOTLPExporterURI(kafkaConfig));
|
||||
break;
|
||||
case TELEMETRY_EXPORTER_TYPE_PROMETHEUS:
|
||||
case ExporterConstants.PROMETHEUS_TYPE:
|
||||
exportedUris.add(buildPrometheusExporterURI(kafkaConfig));
|
||||
break;
|
||||
default:
|
||||
|
|
@ -445,31 +435,26 @@ public class AutoMQConfig {
|
|||
}
|
||||
|
||||
private static String buildOTLPExporterURI(KafkaConfig kafkaConfig) {
|
||||
String endpoint = kafkaConfig.getString(S3_TELEMETRY_EXPORTER_OTLP_ENDPOINT_CONFIG);
|
||||
if (StringUtils.isBlank(endpoint)) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder uriBuilder = new StringBuilder()
|
||||
.append(TELEMETRY_EXPORTER_TYPE_OTLP)
|
||||
.append("://?endpoint=").append(endpoint);
|
||||
String protocol = kafkaConfig.getString(S3_TELEMETRY_EXPORTER_OTLP_PROTOCOL_CONFIG);
|
||||
if (StringUtils.isNotBlank(protocol)) {
|
||||
uriBuilder.append("&protocol=").append(protocol);
|
||||
}
|
||||
.append(ExporterConstants.OTLP_TYPE)
|
||||
.append(ExporterConstants.URI_DELIMITER)
|
||||
.append(ExporterConstants.ENDPOINT).append("=").append(kafkaConfig.getString(S3_TELEMETRY_EXPORTER_OTLP_ENDPOINT_CONFIG))
|
||||
.append("&")
|
||||
.append(ExporterConstants.PROTOCOL).append("=").append(kafkaConfig.getString(S3_TELEMETRY_EXPORTER_OTLP_PROTOCOL_CONFIG));
|
||||
if (kafkaConfig.getBoolean(S3_TELEMETRY_EXPORTER_OTLP_COMPRESSION_ENABLE_CONFIG)) {
|
||||
uriBuilder.append("&compression=gzip");
|
||||
uriBuilder.append("&").append(ExporterConstants.COMPRESSION).append("=").append("gzip");
|
||||
}
|
||||
return uriBuilder.toString();
|
||||
}
|
||||
|
||||
private static String buildPrometheusExporterURI(KafkaConfig kafkaConfig) {
|
||||
return TELEMETRY_EXPORTER_TYPE_PROMETHEUS + URI_DELIMITER +
|
||||
"host" + "=" + kafkaConfig.getString(S3_METRICS_EXPORTER_PROM_HOST_CONFIG) + "&" +
|
||||
"port" + "=" + kafkaConfig.getInt(S3_METRICS_EXPORTER_PROM_PORT_CONFIG);
|
||||
return ExporterConstants.PROMETHEUS_TYPE + ExporterConstants.URI_DELIMITER +
|
||||
ExporterConstants.HOST + "=" + kafkaConfig.getString(S3_METRICS_EXPORTER_PROM_HOST_CONFIG) + "&" +
|
||||
ExporterConstants.PORT + "=" + kafkaConfig.getInt(S3_METRICS_EXPORTER_PROM_PORT_CONFIG);
|
||||
}
|
||||
|
||||
private static String buildOpsExporterURI() {
|
||||
return TELEMETRY_EXPORTER_TYPE_OPS + URI_DELIMITER;
|
||||
return ExporterConstants.OPS_TYPE + ExporterConstants.URI_DELIMITER;
|
||||
}
|
||||
|
||||
private static List<Pair<String, String>> parseBaseLabels(KafkaConfig config) {
|
||||
|
|
@ -490,7 +475,7 @@ public class AutoMQConfig {
|
|||
}
|
||||
|
||||
|
||||
private static Optional<List<BucketURI>> genZoneRouterChannels(KafkaConfig config) {
|
||||
private static Optional<BucketURI> genZoneRouterChannels(KafkaConfig config) {
|
||||
String str = config.getString(ZONE_ROUTER_CHANNELS_CONFIG);
|
||||
if (StringUtils.isBlank(str)) {
|
||||
return Optional.empty();
|
||||
|
|
@ -498,8 +483,10 @@ public class AutoMQConfig {
|
|||
List<BucketURI> buckets = BucketURI.parseBuckets(str);
|
||||
if (buckets.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else if (buckets.size() > 1) {
|
||||
throw new IllegalArgumentException(ZONE_ROUTER_CHANNELS_CONFIG + " only supports one object storage, but it's config with " + str);
|
||||
} else {
|
||||
return Optional.of(buckets);
|
||||
return Optional.of(buckets.get(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,10 +27,10 @@ public interface FailedNode {
|
|||
int id();
|
||||
|
||||
static FailedNode from(NodeRuntimeMetadata node) {
|
||||
return new DefaultFailedNode(node.id(), node.epoch());
|
||||
return new K8sFailedNode(node.id());
|
||||
}
|
||||
|
||||
static FailedNode from(FailoverContext context) {
|
||||
return new DefaultFailedNode(context.getNodeId(), context.getNodeEpoch());
|
||||
return new K8sFailedNode(context.getNodeId());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ public class FailoverControlManager implements AutoCloseable {
|
|||
node.getNodeId(),
|
||||
// There are node epochs in both streamControlManager and nodeControlManager, and they are the same in most cases.
|
||||
// However, in some rare cases, the node epoch in streamControlManager may be updated earlier than the node epoch in nodeControlManager.
|
||||
// So we use the node epoch in streamControlManager as the source of truth.
|
||||
// So we use the node epoch in nodeControlManager as the source of truth.
|
||||
nodeEpochMap.get(node.getNodeId()),
|
||||
node.getWalConfig(),
|
||||
node.getTags(),
|
||||
|
|
@ -265,6 +265,22 @@ public class FailoverControlManager implements AutoCloseable {
|
|||
.filter(NodeRuntimeMetadata::shouldFailover)
|
||||
.map(DefaultFailedWal::from)
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
maybeRemoveControllerNode(allNodes, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void maybeRemoveControllerNode(List<NodeRuntimeMetadata> allNodes, List<FailedWal> failedWALList) {
|
||||
long inactiveControllerCount = allNodes.stream()
|
||||
.filter(NodeRuntimeMetadata::isController)
|
||||
.filter(node -> !node.isActive())
|
||||
.count();
|
||||
if (inactiveControllerCount > 1) {
|
||||
LOGGER.warn("{} controller nodes is inactive, will not failover any controller node", inactiveControllerCount);
|
||||
Set<Integer> controllerNodeIds = allNodes.stream()
|
||||
.filter(NodeRuntimeMetadata::isController)
|
||||
.map(NodeRuntimeMetadata::id)
|
||||
.collect(Collectors.toSet());
|
||||
failedWALList.removeIf(wal -> controllerNodeIds.contains(wal.nodeId()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue