From 659dd2e49f6c68fcdbe4d051a7b7cb0b1487b772 Mon Sep 17 00:00:00 2001 From: David Jacot Date: Thu, 9 Feb 2023 09:10:28 +0100 Subject: [PATCH] KAFKA-14048: Add new `__consumer_offsets` records from KIP-848 (#13203) This patch adds the new (only the new ones) `__consumer_offsets` records as described in [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol#KIP848:TheNextGenerationoftheConsumerRebalanceProtocol-Records). Reviewers: Christo Lolov , Mickael Maison --- ...nsumerGroupCurrentMemberAssignmentKey.json | 28 ++++++++++ ...umerGroupCurrentMemberAssignmentValue.json | 37 +++++++++++++ .../ConsumerGroupMemberMetadataKey.json | 28 ++++++++++ .../ConsumerGroupMemberMetadataValue.json | 53 +++++++++++++++++++ .../message/ConsumerGroupMetadataKey.json | 26 +++++++++ .../message/ConsumerGroupMetadataValue.json | 26 +++++++++ .../ConsumerGroupPartitionMetadataKey.json | 26 +++++++++ .../ConsumerGroupPartitionMetadataValue.json | 33 ++++++++++++ ...onsumerGroupTargetAssignmentMemberKey.json | 28 ++++++++++ ...sumerGroupTargetAssignmentMemberValue.json | 35 ++++++++++++ ...sumerGroupTargetAssignmentMetadataKey.json | 26 +++++++++ ...merGroupTargetAssignmentMetadataValue.json | 26 +++++++++ 12 files changed, 372 insertions(+) create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentValue.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataValue.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataValue.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataValue.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberValue.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataKey.json create mode 100644 group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataValue.json diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentKey.json new file mode 100644 index 00000000000..c89ba9ab30c --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentKey.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupCurrentMemberAssignmentKey", + "validVersions": "8", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "8", + "about": "The group id." }, + { "name": "MemberId", "type": "string", "versions": "8", + "about": "The member id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentValue.json new file mode 100644 index 00000000000..fc2e801adaa --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupCurrentMemberAssignmentValue.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupCurrentMemberAssignmentValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "MemberEpoch", "versions": "0+", "type": "int32", + "about": "The member epoch." }, + { "name": "Error", "versions": "0+", "type": "int8", + "about": "The error reported by the assignor." }, + { "name": "TopicPartitions", "versions": "0+", "type": "[]TopicPartition", + "about": "The partitions assigned to this member.", "fields": [ + { "name": "TopicId", "versions": "0+", "type": "uuid" }, + { "name": "Partitions", "versions": "0+", "type": "[]int32" } + ]}, + { "name": "MetadataVersion", "versions": "0+", "type": "int16", + "about": "The version of the metadata bytes." }, + { "name": "MetadataBytes", "versions": "0+", "type": "bytes", + "about": "The metadata bytes." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataKey.json new file mode 100644 index 00000000000..2c27192af2e --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataKey.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupMemberMetadataKey", + "validVersions": "5", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "5", + "about": "The group id." }, + { "name": "MemberId", "type": "string", "versions": "5", + "about": "The member id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataValue.json new file mode 100644 index 00000000000..77aef727858 --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupMemberMetadataValue.json @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupMemberMetadataValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "GroupEpoch", "versions": "0+", "type": "int32", + "about": "The group epoch." }, + { "name": "InstanceId", "versions": "0+", "nullableVersions": "0+", "type": "string", + "about": "The (optional) instance id." }, + { "name": "RackId", "versions": "0+", "nullableVersions": "0+", "type": "string", + "about": "The (optional) rack id." }, + { "name": "ClientId", "versions": "0+", "type": "string", + "about": "The client id." }, + { "name": "ClientHost", "versions": "0+", "type": "string", + "about": "The client host." }, + { "name": "SubscribedTopicNames", "versions": "0+", "type": "[]string", + "about": "The list of subscribed topic names." }, + { "name": "SubscribedTopicRegex", "versions": "0+", "nullableVersions": "0+", "type": "string", + "about": "The subscribed topic regular expression." }, + { "name": "Assignors", "versions": "0+", "type": "[]Assignor", + "about": "The list of assignors.", "fields": [ + { "name": "Name", "versions": "0+", "type": "string", + "about": "The assignor name." }, + { "name": "MinimumVersion", "versions": "0+", "type": "int16", + "about": "The minimum version supported by the assignor." }, + { "name": "MaximumVersion", "versions": "0+", "type": "int16", + "about": "The maximum version supported by the assignor." }, + { "name": "Reason", "versions": "0+", "type": "int8", + "about": "The reason reported by the assignor." }, + { "name": "Version", "versions": "0+", "type": "int16", + "about": "The version used to serialize the metadata." }, + { "name": "Metadata", "versions": "0+", "type": "bytes", + "about": "The metadata." } + ]} + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataKey.json new file mode 100644 index 00000000000..a1bfb0d0502 --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataKey.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupMetadataKey", + "validVersions": "3", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "3", + "about": "The group id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataValue.json new file mode 100644 index 00000000000..035919eeffa --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupMetadataValue.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupMetadataValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "Epoch", "versions": "0+", "type": "int32", + "about": "The group epoch." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataKey.json new file mode 100644 index 00000000000..c6866bd34b7 --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataKey.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupPartitionMetadataKey", + "validVersions": "4", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "4", + "about": "The group id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataValue.json new file mode 100644 index 00000000000..b81ff93bdba --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupPartitionMetadataValue.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupPartitionMetadataValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "Epoch", "versions": "0+", "type": "int32", + "about": "The group epoch." }, + { "name": "Topics", "versions": "0+", "type": "[]TopicMetadata", + "about": "The list of topic metadata.", "fields": [ + { "name": "TopicId", "versions": "0+", "type": "uuid", + "about": "The topic id." }, + { "name": "NumPartitions", "versions": "0+", "type": "int32", + "about": "The number of partitions of the topic." } + ]} + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberKey.json new file mode 100644 index 00000000000..10b94f26e6d --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberKey.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupTargetAssignmentMemberKey", + "validVersions": "7", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "7", + "about": "The group id." }, + { "name": "MemberId", "type": "string", "versions": "7", + "about": "The member id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberValue.json new file mode 100644 index 00000000000..d056eab6d85 --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMemberValue.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupTargetAssignmentMemberValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "Error", "versions": "0+", "type": "int8", + "about": "The assigned error."}, + { "name": "TopicPartitions", "versions": "0+", "type": "[]TopicPartition", + "about": "The assigned partitions.", "fields": [ + { "name": "TopicId", "versions": "0+", "type": "uuid" }, + { "name": "Partitions", "versions": "0+", "type": "[]int32" } + ]}, + { "name": "MetadataVersion", "versions": "0+", "type": "int16", + "about": "The version of the assigned metadata." }, + { "name": "MetadataBytes", "versions": "0+", "type": "bytes", + "about": "The assigned metadata." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataKey.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataKey.json new file mode 100644 index 00000000000..e78d557fb61 --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataKey.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupTargetAssignmentMetadataKey", + "validVersions": "6", + "flexibleVersions": "none", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "6", + "about": "The group id." } + ] +} diff --git a/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataValue.json b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataValue.json new file mode 100644 index 00000000000..6c73971b3ea --- /dev/null +++ b/group-coordinator/src/main/resources/common/message/ConsumerGroupTargetAssignmentMetadataValue.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// KIP-848 is in development. This schema is subject to non-backwards-compatible changes. +{ + "type": "data", + "name": "ConsumerGroupTargetAssignmentMetadataValue", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "AssignmentEpoch", "versions": "0+", "type": "int32", + "about": "The assignment epoch." } + ] +}