mirror of https://github.com/apache/kafka.git
KAFKA-3365; Add documentation method for protocol types and update doc generation (#4735)
Reviewers: Sandor Murakozi <smurakozi@gmail.com>, Magnus Edenhill <magnus@edenhill.se>, Jason Gustafson <jason@confluent.io>
This commit is contained in:
Andras Beni
Jason Gustafson
parent
98bb75a58f
commit
462342210f
|
|
@ -633,6 +633,13 @@ project(':core') {
|
|||
standardOutput = new File(generatedDocsDir, "protocol_errors.html").newOutputStream()
|
||||
}
|
||||
|
||||
task genProtocolTypesDocs(type: JavaExec) {
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
main = 'org.apache.kafka.common.protocol.types.Type'
|
||||
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
||||
standardOutput = new File(generatedDocsDir, "protocol_types.html").newOutputStream()
|
||||
}
|
||||
|
||||
task genProtocolApiKeyDocs(type: JavaExec) {
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
main = 'org.apache.kafka.common.protocol.ApiKeys'
|
||||
|
|
@ -696,7 +703,7 @@ project(':core') {
|
|||
standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream()
|
||||
}
|
||||
|
||||
task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
|
||||
task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
|
||||
'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
|
||||
'genKafkaConfigDocs', 'genTopicConfigDocs',
|
||||
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
|
||||
|
|
|
|||
|
|
@ -16,12 +16,16 @@
|
|||
*/
|
||||
package org.apache.kafka.common.protocol.types;
|
||||
|
||||
import org.apache.kafka.common.protocol.types.Type.DocumentedType;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Represents a type for an array of a particular type
|
||||
*/
|
||||
public class ArrayOf extends Type {
|
||||
public class ArrayOf extends DocumentedType {
|
||||
|
||||
private static final String ARRAY_TYPE_NAME = "ARRAY";
|
||||
|
||||
private final Type type;
|
||||
private final boolean nullable;
|
||||
|
|
@ -93,7 +97,7 @@ public class ArrayOf extends Type {
|
|||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ARRAY(" + type + ")";
|
||||
return ARRAY_TYPE_NAME + "(" + type + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -110,4 +114,18 @@ public class ArrayOf extends Type {
|
|||
throw new SchemaException("Not an Object[].");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String typeName() {
|
||||
return ARRAY_TYPE_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a sequence of objects of a given type T. " +
|
||||
"Type T can be either a primitive type (e.g. " + STRING + ") or a structure. " +
|
||||
"First, the length N is given as an " + INT32 + ". Then N instances of type T follow. " +
|
||||
"A null array is represented with a length of -1. " +
|
||||
"In protocol documentation an array of T instances is referred to as [T].";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,6 +193,4 @@ public class Schema extends Type {
|
|||
public void visit(ArrayOf array) {}
|
||||
public void visit(Type field) {}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,6 +63,29 @@ public abstract class Type {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* A Type that can return its description for documentation purposes.
|
||||
*/
|
||||
public static abstract class DocumentedType extends Type {
|
||||
|
||||
/**
|
||||
* Short name of the type to identify it in documentation;
|
||||
* @return the name of the type
|
||||
*/
|
||||
public abstract String typeName();
|
||||
|
||||
/**
|
||||
* Documentation of the Type.
|
||||
*
|
||||
* @return details about valid values, representation
|
||||
*/
|
||||
public abstract String documentation();
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return typeName();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* The Boolean type represents a boolean value in a byte by using
|
||||
* the value of 0 to represent false, and 1 to represent true.
|
||||
|
|
@ -70,7 +93,7 @@ public abstract class Type {
|
|||
* If for some reason a value that is not 0 or 1 is read,
|
||||
* then any non-zero value will return true.
|
||||
*/
|
||||
public static final Type BOOLEAN = new Type() {
|
||||
public static final DocumentedType BOOLEAN = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
if ((Boolean) o)
|
||||
|
|
@ -91,7 +114,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "BOOLEAN";
|
||||
}
|
||||
|
||||
|
|
@ -102,9 +125,16 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a Boolean.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a boolean value in a byte. " +
|
||||
"Values 0 and 1 are used to represent false and true respectively. " +
|
||||
"When reading a boolean value, any non-zero value is considered true.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type INT8 = new Type() {
|
||||
public static final DocumentedType INT8 = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
buffer.put((Byte) o);
|
||||
|
|
@ -121,7 +151,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "INT8";
|
||||
}
|
||||
|
||||
|
|
@ -132,9 +162,14 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a Byte.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>7</sup> and 2<sup>7</sup>-1 inclusive.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type INT16 = new Type() {
|
||||
public static final DocumentedType INT16 = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
buffer.putShort((Short) o);
|
||||
|
|
@ -151,7 +186,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "INT16";
|
||||
}
|
||||
|
||||
|
|
@ -162,9 +197,15 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a Short.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>15</sup> and 2<sup>15</sup>-1 inclusive. " +
|
||||
"The values are encoded using two bytes in network byte order (big-endian).";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type INT32 = new Type() {
|
||||
public static final DocumentedType INT32 = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
buffer.putInt((Integer) o);
|
||||
|
|
@ -181,7 +222,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "INT32";
|
||||
}
|
||||
|
||||
|
|
@ -192,9 +233,15 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not an Integer.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>31</sup> and 2<sup>31</sup>-1 inclusive. " +
|
||||
"The values are encoded using four bytes in network byte order (big-endian).";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type UNSIGNED_INT32 = new Type() {
|
||||
public static final DocumentedType UNSIGNED_INT32 = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
ByteUtils.writeUnsignedInt(buffer, (long) o);
|
||||
|
|
@ -211,7 +258,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "UINT32";
|
||||
}
|
||||
|
||||
|
|
@ -222,9 +269,15 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a Long.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between 0 and 2<sup>32</sup>-1 inclusive. " +
|
||||
"The values are encoded using four bytes in network byte order (big-endian).";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type INT64 = new Type() {
|
||||
public static final DocumentedType INT64 = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
buffer.putLong((Long) o);
|
||||
|
|
@ -241,7 +294,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "INT64";
|
||||
}
|
||||
|
||||
|
|
@ -252,9 +305,15 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a Long.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>63</sup> and 2<sup>63</sup>-1 inclusive. " +
|
||||
"The values are encoded using eight bytes in network byte order (big-endian).";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type STRING = new Type() {
|
||||
public static final DocumentedType STRING = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
byte[] bytes = Utils.utf8((String) o);
|
||||
|
|
@ -282,7 +341,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "STRING";
|
||||
}
|
||||
|
||||
|
|
@ -293,9 +352,16 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a String.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a sequence of characters. First the length N is given as an " + INT16 +
|
||||
". Then N bytes follow which are the UTF-8 encoding of the character sequence. " +
|
||||
"Length must not be negative.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type NULLABLE_STRING = new Type() {
|
||||
public static final DocumentedType NULLABLE_STRING = new DocumentedType() {
|
||||
@Override
|
||||
public boolean isNullable() {
|
||||
return true;
|
||||
|
|
@ -336,7 +402,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "NULLABLE_STRING";
|
||||
}
|
||||
|
||||
|
|
@ -350,9 +416,16 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a String.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a sequence of characters or null. For non-null strings, first the length N is given as an " + INT16 +
|
||||
". Then N bytes follow which are the UTF-8 encoding of the character sequence. " +
|
||||
"A null value is encoded with length of -1 and there are no following bytes.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type BYTES = new Type() {
|
||||
public static final DocumentedType BYTES = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
ByteBuffer arg = (ByteBuffer) o;
|
||||
|
|
@ -383,7 +456,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "BYTES";
|
||||
}
|
||||
|
||||
|
|
@ -394,9 +467,15 @@ public abstract class Type {
|
|||
else
|
||||
throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a raw sequence of bytes. First the length N is given as an " + INT32 +
|
||||
". Then N bytes follow.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type NULLABLE_BYTES = new Type() {
|
||||
public static final DocumentedType NULLABLE_BYTES = new DocumentedType() {
|
||||
@Override
|
||||
public boolean isNullable() {
|
||||
return true;
|
||||
|
|
@ -440,7 +519,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "NULLABLE_BYTES";
|
||||
}
|
||||
|
||||
|
|
@ -454,9 +533,15 @@ public abstract class Type {
|
|||
|
||||
throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a raw sequence of bytes or null. For non-null values, first the length N is given as an " + INT32 +
|
||||
". Then N bytes follow. A null value is encoded with length of -1 and there are no following bytes.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type RECORDS = new Type() {
|
||||
public static final DocumentedType RECORDS = new DocumentedType() {
|
||||
@Override
|
||||
public boolean isNullable() {
|
||||
return true;
|
||||
|
|
@ -486,7 +571,7 @@ public abstract class Type {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "RECORDS";
|
||||
}
|
||||
|
||||
|
|
@ -500,9 +585,16 @@ public abstract class Type {
|
|||
|
||||
throw new SchemaException(item + " is not an instance of " + Records.class.getName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents a sequence of Kafka records as " + NULLABLE_BYTES + ". " +
|
||||
"For a detailed description of records see " +
|
||||
"<a href=\"/documentation/#messageformat\">Message Sets</a>.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type VARINT = new Type() {
|
||||
public static final DocumentedType VARINT = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
ByteUtils.writeVarint((Integer) o, buffer);
|
||||
|
|
@ -520,7 +612,7 @@ public abstract class Type {
|
|||
throw new SchemaException(item + " is not an integer");
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "VARINT";
|
||||
}
|
||||
|
||||
|
|
@ -528,9 +620,16 @@ public abstract class Type {
|
|||
public int sizeOf(Object o) {
|
||||
return ByteUtils.sizeOfVarint((Integer) o);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>31</sup> and 2<sup>31</sup>-1 inclusive. " +
|
||||
"Encoding follows the variable-length zig-zag encoding from " +
|
||||
" <a href=\"http://code.google.com/apis/protocolbuffers/docs/encoding.html\"> Google Protocol Buffers</a>.";
|
||||
}
|
||||
};
|
||||
|
||||
public static final Type VARLONG = new Type() {
|
||||
public static final DocumentedType VARLONG = new DocumentedType() {
|
||||
@Override
|
||||
public void write(ByteBuffer buffer, Object o) {
|
||||
ByteUtils.writeVarlong((Long) o, buffer);
|
||||
|
|
@ -548,7 +647,7 @@ public abstract class Type {
|
|||
throw new SchemaException(item + " is not a long");
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
public String typeName() {
|
||||
return "VARLONG";
|
||||
}
|
||||
|
||||
|
|
@ -556,6 +655,43 @@ public abstract class Type {
|
|||
public int sizeOf(Object o) {
|
||||
return ByteUtils.sizeOfVarlong((Long) o);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String documentation() {
|
||||
return "Represents an integer between -2<sup>63</sup> and 2<sup>63</sup>-1 inclusive. " +
|
||||
"Encoding follows the variable-length zig-zag encoding from " +
|
||||
" <a href=\"http://code.google.com/apis/protocolbuffers/docs/encoding.html\"> Google Protocol Buffers</a>.";
|
||||
}
|
||||
};
|
||||
|
||||
private static String toHtml() {
|
||||
|
||||
DocumentedType[] types = {
|
||||
BOOLEAN, INT8, INT16, INT32, INT64,
|
||||
UNSIGNED_INT32, VARINT, VARLONG,
|
||||
STRING, NULLABLE_STRING, BYTES, NULLABLE_BYTES,
|
||||
RECORDS, new ArrayOf(STRING)};
|
||||
final StringBuilder b = new StringBuilder();
|
||||
b.append("<table class=\"data-table\"><tbody>\n");
|
||||
b.append("<tr>");
|
||||
b.append("<th>Type</th>\n");
|
||||
b.append("<th>Description</th>\n");
|
||||
b.append("</tr>\n");
|
||||
for (DocumentedType type : types) {
|
||||
b.append("<tr>");
|
||||
b.append("<td>");
|
||||
b.append(type.typeName());
|
||||
b.append("</td>");
|
||||
b.append("<td>");
|
||||
b.append(type.documentation());
|
||||
b.append("</td>");
|
||||
b.append("</tr>\n");
|
||||
}
|
||||
b.append("</table>\n");
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(toHtml());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -163,18 +163,7 @@ Kafka request. SASL/GSSAPI authentication is performed starting with this packet
|
|||
<h5><a id="protocol_types" href="#protocol_types">Protocol Primitive Types</a></h5>
|
||||
|
||||
<p>The protocol is built out of the following primitive types.</p>
|
||||
|
||||
<p><b>Fixed Width Primitives</b><p>
|
||||
|
||||
<p>int8, int16, int32, int64 - Signed integers with the given precision (in bits) stored in big endian order.</p>
|
||||
|
||||
<p><b>Variable Length Primitives</b><p>
|
||||
|
||||
<p>bytes, string - These types consist of a signed integer giving a length N followed by N bytes of content. A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32.</p>
|
||||
|
||||
<p><b>Arrays</b><p>
|
||||
|
||||
<p>This is a notation for handling repeated structures. These will always be encoded as an int32 size containing the length N followed by N repetitions of the structure which can itself be made up of other primitive types. In the BNF grammars below we will show an array of a structure foo as [foo].</p>
|
||||
<!--#include virtual="generated/protocol_types.html" -->
|
||||
|
||||
<h5><a id="protocol_grammar" href="#protocol_grammar">Notes on reading the request format grammars</a></h5>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue