root
/
kafka
mirror of https://github.com/apache/kafka.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

MINOR: additional kip-182 doc updates 

Author: Damian Guy <damian.guy@gmail.com>

Reviewers: Michael G. Noll <michael@confluent.io>, Bill Bejeck <bill@confluent.io>, Matthias J. Sax <matthias@confluent.io>, Ismael Juma <ismael@juma.me.uk>

Closes #3971 from dguy/kip-182-docs
This commit is contained in:
Damian Guy 2017-10-02 13:20:49 -07:00
parent cdbf806e2d
commit cc84686a4a
1 changed files with 128 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

244
docs/streams/developer-guide.html
View File

@ -1383,65 +1383,72 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
// Java 8+ examples, using lambda expressions // Java 8+ examples, using lambda expressions
// Aggregating with time-based windowing (here: with 5-minute tumbling windows) // Aggregating with time-based windowing (here: with 5-minute tumbling windows)
KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream.aggregate( KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream
() -> 0L, /* initializer */ .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(5))) /* time-based window */
(aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */ .aggregate(
TimeWindows.of(TimeUnit.MINUTES.toMillis(5)), /* time-based window */ () -> 0L, /* initializer */
Serdes.Long(), /* serde for aggregate value */ (aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */
"time-windowed-aggregated-stream-store" /* state store name */); Materialized.&lt;String, Long, WindowStore&lt;Bytes, byte[]&gt;&gt;as("time-windowed-aggregated-stream-store") /* state store name */
.withValueSerde(Serdes.Long())); /* serde for aggregate value */
// Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes) // Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes)
KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionizedAggregatedStream = groupedStream.aggregate( KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionizedAggregatedStream = groupedStream
() -> 0L, /* initializer */ .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(5))) /* session window */
(aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */ .aggregate(
(aggKey, leftAggValue, rightAggValue) -> leftAggValue + rightAggValue, /* session merger */ () -> 0L, /* initializer */
SessionWindows.with(TimeUnit.MINUTES.toMillis(5)), /* session window */ (aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */
Serdes.Long(), /* serde for aggregate value */ (aggKey, leftAggValue, rightAggValue) -> leftAggValue + rightAggValue, /* session merger */
"sessionized-aggregated-stream-store" /* state store name */); Materialized.&lt;String, Long, SessionStore&lt;Bytes, byte[]&gt;&gt;as("sessionized-aggregated-stream-store") /* state store name */
.withValueSerde(Serdes.Long())); /* serde for aggregate value */
// Java 7 examples // Java 7 examples
// Aggregating with time-based windowing (here: with 5-minute tumbling windows) // Aggregating with time-based windowing (here: with 5-minute tumbling windows)
KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream.aggregate( KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream
new Initializer&lt;Long&gt;() { /* initializer */ .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(5))) /* time-based window */
@Override .aggregate(
public Long apply() { new Initializer&lt;Long&gt;() { /* initializer */
return 0L; @Override
} public Long apply() {
}, return 0L;
new Aggregator&lt;String, Long, Long&gt;() { /* adder */ }
@Override },
public Long apply(String aggKey, Long newValue, Long aggValue) { new Aggregator&lt;String, Long, Long&gt;() { /* adder */
return aggValue + newValue; @Override
} public Long apply(String aggKey, Long newValue, Long aggValue) {
}, return aggValue + newValue;
TimeWindows.of(TimeUnit.MINUTES.toMillis(5)), /* time-based window */ }
Serdes.Long(), /* serde for aggregate value */ },
"time-windowed-aggregated-stream-store" /* state store name */); Materialized.&lt;String, Long, WindowStore&lt;Bytes, byte[]&gt;&gt;as("time-windowed-aggregated-stream-store") /* state store name */
.withValueSerde(Serdes.Long()) /* serde for aggregate value */
);
// Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes) // Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes)
KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionizedAggregatedStream = groupedStream.aggregate( KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionizedAggregatedStream = groupedStream
new Initializer&lt;Long&gt;() { /* initializer */ .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(5))) /* session window */
@Override .aggregate(
public Long apply() { new Initializer&lt;Long&gt;() { /* initializer */
return 0L; @Override
} public Long apply() {
}, return 0L;
new Aggregator&lt;String, Long, Long&gt;() { /* adder */ }
@Override },
public Long apply(String aggKey, Long newValue, Long aggValue) { new Aggregator&lt;String, Long, Long&gt;() { /* adder */
return aggValue + newValue; @Override
} public Long apply(String aggKey, Long newValue, Long aggValue) {
}, return aggValue + newValue;
new Merger&lt;String, Long&gt;() { /* session merger */ }
@Override },
public Long apply(String aggKey, Long leftAggValue, Long rightAggValue) { new Merger&lt;String, Long&gt;() { /* session merger */
return rightAggValue + leftAggValue; @Override
} public Long apply(String aggKey, Long leftAggValue, Long rightAggValue) {
}, return rightAggValue + leftAggValue;
SessionWindows.with(TimeUnit.MINUTES.toMillis(5)), /* session window */ }
Serdes.Long(), /* serde for aggregate value */ },
"sessionized-aggregated-stream-store" /* state store name */); Materialized.&lt;String, Long, SessionStore&lt;Bytes, byte[]&gt;&gt;as("sessionized-aggregated-stream-store") /* state store name */
.withValueSerde(Serdes.Long()) /* serde for aggregate value */
);
</pre> </pre>
<p> <p>
@ -1478,12 +1485,10 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
KGroupedTable&lt;String, Long&gt; groupedTable = ...; KGroupedTable&lt;String, Long&gt; groupedTable = ...;
// Counting a KGroupedStream // Counting a KGroupedStream
KTable&lt;String, Long&gt; aggregatedStream = groupedStream.count( KTable&lt;String, Long&gt; aggregatedStream = groupedStream.count();
"counted-stream-store" /* state store name */);
// Counting a KGroupedTable // Counting a KGroupedTable
KTable&lt;String, Long&gt; aggregatedTable = groupedTable.count( KTable&lt;String, Long&gt; aggregatedTable = groupedTable.count();
"counted-table-store" /* state store name */);
</pre> </pre>
<p> <p>
Detailed behavior for <code>KGroupedStream</code>: Detailed behavior for <code>KGroupedStream</code>:
@ -1518,14 +1523,14 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
KGroupedStream&lt;String, Long&gt; groupedStream = ...; KGroupedStream&lt;String, Long&gt; groupedStream = ...;
// Counting a KGroupedStream with time-based windowing (here: with 5-minute tumbling windows) // Counting a KGroupedStream with time-based windowing (here: with 5-minute tumbling windows)
KTable&lt;Windowed&lt;String&gt;, Long&gt; aggregatedStream = groupedStream.count( KTable&lt;Windowed&lt;String&gt;, Long&gt; aggregatedStream = groupedStream
TimeWindows.of(TimeUnit.MINUTES.toMillis(5)), /* time-based window */ .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(5))) /* time-based window */
"time-windowed-counted-stream-store" /* state store name */); .count();
// Counting a KGroupedStream with session-based windowing (here: with 5-minute inactivity gaps) // Counting a KGroupedStream with session-based windowing (here: with 5-minute inactivity gaps)
KTable&lt;Windowed&lt;String&gt;, Long&gt; aggregatedStream = groupedStream.count( KTable&lt;Windowed&lt;String&gt;, Long&gt; aggregatedStream = groupedStream
SessionWindows.with(TimeUnit.MINUTES.toMillis(5)), /* session window */ .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(5))) /* session window */
"sessionized-counted-stream-store" /* state store name */); .count();
</pre> </pre>
<p> <p>
Detailed behavior: Detailed behavior:
@ -1561,14 +1566,14 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
// Reducing a KGroupedStream // Reducing a KGroupedStream
KTable&lt;String, Long&gt; aggregatedStream = groupedStream.reduce( KTable&lt;String, Long&gt; aggregatedStream = groupedStream.reduce(
(aggValue, newValue) -> aggValue + newValue, /* adder */ (aggValue, newValue) -> aggValue + newValue /* adder */
"reduced-stream-store" /* state store name */); );
// Reducing a KGroupedTable // Reducing a KGroupedTable
KTable&lt;String, Long&gt; aggregatedTable = groupedTable.reduce( KTable&lt;String, Long&gt; aggregatedTable = groupedTable.reduce(
(aggValue, newValue) -> aggValue + newValue, /* adder */ (aggValue, newValue) -> aggValue + newValue, /* adder */
(aggValue, oldValue) -> aggValue - oldValue, /* subtractor */ (aggValue, oldValue) -> aggValue - oldValue /* subtractor */
"reduced-table-store" /* state store name */); );
// Java 7 examples // Java 7 examples
@ -1580,8 +1585,8 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
public Long apply(Long aggValue, Long newValue) { public Long apply(Long aggValue, Long newValue) {
return aggValue + newValue; return aggValue + newValue;
} }
}, }
"reduced-stream-store" /* state store name */); );
// Reducing a KGroupedTable // Reducing a KGroupedTable
KTable&lt;String, Long&gt; aggregatedTable = groupedTable.reduce( KTable&lt;String, Long&gt; aggregatedTable = groupedTable.reduce(
@ -1596,8 +1601,8 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
public Long apply(Long aggValue, Long oldValue) { public Long apply(Long aggValue, Long oldValue) {
return aggValue - oldValue; return aggValue - oldValue;
} }
}, }
"reduced-table-store" /* state store name */); );
</pre> </pre>
<p> <p>
Detailed behavior for <code>KGroupedStream</code>: Detailed behavior for <code>KGroupedStream</code>:
@ -1659,41 +1664,39 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
// Java 8+ examples, using lambda expressions // Java 8+ examples, using lambda expressions
// Aggregating with time-based windowing (here: with 5-minute tumbling windows) // Aggregating with time-based windowing (here: with 5-minute tumbling windows)
KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream.reduce( KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream
(aggValue, newValue) -> aggValue + newValue, /* adder */ .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(5))) /* time-based window */
TimeWindows.of(TimeUnit.MINUTES.toMillis(5)), /* time-based window */ .reduce((aggValue, newValue) -> aggValue + newValue /* adder */);
"time-windowed-reduced-stream-store" /* state store name */);
// Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes) // Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes)
KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionzedAggregatedStream = groupedStream.reduce( KTable&lt;Windowed&lt;String&gt;, Long&gt; sessionzedAggregatedStream = groupedStream
(aggValue, newValue) -> aggValue + newValue, /* adder */ .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(5))) /* session window */
SessionWindows.with(TimeUnit.MINUTES.toMillis(5)), /* session window */ .reduce((aggValue, newValue) -> aggValue + newValue); /* adder */
"sessionized-reduced-stream-store" /* state store name */);
// Java 7 examples // Java 7 examples
// Aggregating with time-based windowing (here: with 5-minute tumbling windows) // Aggregating with time-based windowing (here: with 5-minute tumbling windows)
KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream.reduce( KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream
new Reducer&lt;Long&gt;() { /* adder */ .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(5))) /* time-based window */
@Override .reduce(
public Long apply(Long aggValue, Long newValue) { new Reducer&lt;Long&gt;() { /* adder */
return aggValue + newValue; @Override
} public Long apply(Long aggValue, Long newValue) {
}, return aggValue + newValue;
TimeWindows.of(TimeUnit.MINUTES.toMillis(5)), /* time-based window */ }
"time-windowed-reduced-stream-store" /* state store name */); });
// Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes) // Aggregating with session-based windowing (here: with an inactivity gap of 5 minutes)
KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream.reduce( KTable&lt;Windowed&lt;String&gt;, Long&gt; timeWindowedAggregatedStream = groupedStream
new Reducer&lt;Long&gt;() { /* adder */ .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(5))) /* session window */
@Override .reduce(
public Long apply(Long aggValue, Long newValue) { new Reducer&lt;Long&gt;() { /* adder */
return aggValue + newValue; @Override
} public Long apply(Long aggValue, Long newValue) {
}, return aggValue + newValue;
SessionWindows.with(TimeUnit.MINUTES.toMillis(5)), /* session window */ }
"sessionized-reduced-stream-store" /* state store name */); });
</pre> </pre>
<p> <p>
@ -1723,16 +1726,22 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
</p> </p>
<pre class="brush: java;"> <pre class="brush: java;">
// Key: word, value: count // Key: word, value: count
Properties streamsProperties == ...;
// specify the default serdes so we don't need to elsewhere.
streamsProperties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
streamsProperties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
StreamsConfig config = new StreamsConfig(streamsProperties);
KStream&lt;String, Integer&gt; wordCounts = ...; KStream&lt;String, Integer&gt; wordCounts = ...;
KGroupedStream&lt;String, Integer&gt; groupedStream = wordCounts KGroupedStream&lt;String, Integer&gt; groupedStream = wordCounts
.groupByKey(Serialized.with(Serdes.String(), Serdes.Integer())); .groupByKey();
KTable&lt;String, Integer&gt; aggregated = groupedStream.aggregate( KTable&lt;String, Integer&gt; aggregated = groupedStream.aggregate(
() -> 0, /* initializer */ () -> 0, /* initializer */
(aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */ (aggKey, newValue, aggValue) -> aggValue + newValue /* adder */
Serdes.Integer(), /* serde for aggregate value */ );
"aggregated-stream-store" /* state store name */);
</pre> </pre>
<p> <p>
@ -1836,8 +1845,10 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
() -> 0, /* initializer */ () -> 0, /* initializer */
(aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */ (aggKey, newValue, aggValue) -> aggValue + newValue, /* adder */
(aggKey, oldValue, aggValue) -> aggValue - oldValue, /* subtractor */ (aggKey, oldValue, aggValue) -> aggValue - oldValue, /* subtractor */
Serdes.Integer(), /* serde for aggregate value */ Materialized.&lt;String, Integer, KeyValueStore&lt;Bytes, byte[]&gt;&gt;as("aggregated-table-store")
"aggregated-table-store" /* state store name */); .withKeySerde(Serdes.String() /* serde for aggregate key */)
.withValueSerde(Serdes.Long() /* serde for aggregate value */)
);
</pre> </pre>
<p> <p>
<b>Impact of <a href=#streams_developer-guide_memory-management_record-cache>record caches</a></b>: <b>Impact of <a href=#streams_developer-guide_memory-management_record-cache>record caches</a></b>:
@ -2253,7 +2264,10 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
.groupBy((key, word) -> word, Serialized.with(stringSerde, stringSerde)); .groupBy((key, word) -> word, Serialized.with(stringSerde, stringSerde));
// Create a window state store named "CountsWindowStore" that contains the word counts for every minute // Create a window state store named "CountsWindowStore" that contains the word counts for every minute
groupedByWord.count(TimeWindows.of(60000), "CountsWindowStore"); groupedByWord.windowedBy(TimeWindows.of(60000))
.count(Materialized.&lt;String, Long, WindowStore&lt;Bytes, byte[]&gt;&gt;as("CountsWindowStore")
withKeySerde(Serdes.String()); // count() sets value serde to Serdes.Long() automatically
);
</pre> </pre>
<p> <p>
@ -2396,14 +2410,14 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
Topology topology = ...; Topology topology = ...;
ProcessorSupplier processorSuppler = ...; ProcessorSupplier processorSuppler = ...;
// Create CustomStoreSupplier for store name the-custom-store // Create CustomStoreBuilder for store name the-custom-store
MyCustomStoreSuppler customStoreSupplier = new MyCustomStoreSupplier("the-custom-store"); MyCustomStoreBuilder customStoreBuilder = new MyCustomStoreBuilder("the-custom-store");
// Add the source topic // Add the source topic
topology.addSource("input", "inputTopic"); topology.addSource("input", "inputTopic");
// Add a custom processor that reads from the source topic // Add a custom processor that reads from the source topic
topology.addProcessor("the-processor", processorSupplier, "input"); topology.addProcessor("the-processor", processorSupplier, "input");
// Connect your custom state store to the custom processor above // Connect your custom state store to the custom processor above
topology.addStateStore(customStoreSupplier, "the-processor"); topology.addStateStore(customStoreBuilder, "the-processor");
KafkaStreams streams = new KafkaStreams(topology, config); KafkaStreams streams = new KafkaStreams(topology, config);
streams.start(); streams.start();
@ -2478,7 +2492,7 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
// This call to `count()` creates a state store named "word-count". // This call to `count()` creates a state store named "word-count".
// The state store is discoverable and can be queried interactively. // The state store is discoverable and can be queried interactively.
groupedByWord.count("word-count"); groupedByWord.count(Materialized.&ltString, Long, KeyValueStore&lt;Bytes, byte[]&gt;&gt;as("word-count"));
// Start an instance of the topology // Start an instance of the topology
KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfiguration); KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfiguration);
@ -2835,22 +2849,20 @@ Note that in the <code>WordCountProcessor</code> implementation, users need to r
<p> <p>
For changelog topics you can also override the default configs on a per store basis. For changelog topics you can also override the default configs on a per store basis.
This can be done by using any method overload that has a <code>StateStoreSupplier</code> as a parameter: This can be done by using any method overload that has a <code>Materialized</code> as a parameter:
</p> </p>
<pre class="brush: java;"> <pre class="brush: java;">
// a map to add topic config // a map to add topic config
Map&lt;String, String&gt; topicConfig = new HashMap<>(); Map&lt;String, String&gt; topicConfig = new HashMap&lt;&gt;();
topicConfig.put(TopicConfig.SEGMENT_MS_CONFIG, "10000"); topicConfig.put(TopicConfig.SEGMENT_MS_CONFIG, "10000");
StateStoreSupplier supplier = Stores.create("store") final Materialized&lt;String, Long, KeyValueStore&lt;Bytes, byte[]&gt;&gt; materialized = Materialized.as("store")
.withKeys(Serdes.String()) .withKeySerde(Serdes.String())
.withValues(Serdes.String()) .withValueSerde(Serdes.String())
.persistent() .withLoggingEnabled(topicConfig); // pass in the config overrides
.enableLogging(topicConfig) // pass in the config overrides
.build(); groupedStream.count(materialized)
groupedStream.count(supplier)
</pre> </pre>
<h4><a id="streams_execute" href="#streams_execute">Executing Your Kafka Streams Application</a></h4> <h4><a id="streams_execute" href="#streams_execute">Executing Your Kafka Streams Application</a></h4>