root
/
kafka
mirror of https://github.com/apache/kafka.git
1
0
Fork 0
Code Issues Actions 2 Packages Projects Releases Wiki Activity

KAFKA-2421: Upgrade LZ4 to version 1.3 

A few notes on the added test:
 * I verified this test fails when changing between snappy 1.1.1.2 and 1.1.1.7 (per KAFKA-2189)
 * The hard coded numbers are passing before and after lzo change

Author: Grant Henke <granthenke@gmail.com>

Reviewers: Ismael Juma, Guozhang Wang

Closes #552 from granthenke/lz4
This commit is contained in:
Grant Henke 2015-12-01 11:57:34 -08:00 committed by Guozhang Wang
parent 0a52ddfd03
commit 69269e76a4
4 changed files with 45 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -408,7 +408,7 @@ project(':clients') {
dependencies {
compile "$slf4japi"
compile 'org.xerial.snappy:snappy-java:1.1.2'
compile 'net.jpountz.lz4:lz4:1.2.0'
compile 'net.jpountz.lz4:lz4:1.3'
testCompile 'org.bouncycastle:bcpkix-jdk15on:1.52'
testCompile "$junit"

2
clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockInputStream.java
View File

@ -174,7 +174,7 @@ public final class KafkaLZ4BlockInputStream extends FilterInputStream {
@Override
public int read(byte[] b, int off, int len) throws IOException {
net.jpountz.util.Utils.checkRange(b, off, len);
net.jpountz.util.SafeUtils.checkRange(b, off, len);
if (finished) {
return -1;
}

2
clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockOutputStream.java
View File

@ -180,7 +180,7 @@ public final class KafkaLZ4BlockOutputStream extends FilterOutputStream {
@Override
public void write(byte[] b, int off, int len) throws IOException {
net.jpountz.util.Utils.checkRange(b, off, len);
net.jpountz.util.SafeUtils.checkRange(b, off, len);
ensureNotFinished();
int bufferRemainingLength = maxBlockSize - bufferOffset;

31
core/src/test/scala/unit/kafka/message/MessageCompressionTest.scala
View File

@ -36,18 +36,39 @@ class MessageCompressionTest extends JUnitSuite {
testSimpleCompressDecompress(codec)
}
// A quick test to ensure any growth or increase in compression size is known when upgrading libraries
@Test
def testCompressSize() {
val bytes1k: Array[Byte] = (0 until 1000).map(_.toByte).toArray
val bytes2k: Array[Byte] = (1000 until 2000).map(_.toByte).toArray
val bytes3k: Array[Byte] = (3000 until 4000).map(_.toByte).toArray
val messages: List[Message] = List(new Message(bytes1k), new Message(bytes2k), new Message(bytes3k))
testCompressSize(GZIPCompressionCodec, messages, 388)
if(isSnappyAvailable)
testCompressSize(SnappyCompressionCodec, messages, 491)
if(isLZ4Available)
testCompressSize(LZ4CompressionCodec, messages, 380)
}
def testSimpleCompressDecompress(compressionCodec: CompressionCodec) {
val messages = List[Message](new Message("hi there".getBytes), new Message("I am fine".getBytes), new Message("I am not so well today".getBytes))
val messageSet = new ByteBufferMessageSet(compressionCodec = compressionCodec, messages = messages:_*)
assertEquals(compressionCodec, messageSet.shallowIterator.next.message.compressionCodec)
assertEquals(compressionCodec, messageSet.shallowIterator.next().message.compressionCodec)
val decompressed = messageSet.iterator.map(_.message).toList
assertEquals(messages, decompressed)
}
def testCompressSize(compressionCodec: CompressionCodec, messages: List[Message], expectedSize: Int) {
val messageSet = new ByteBufferMessageSet(compressionCodec = compressionCodec, messages = messages:_*)
assertEquals(s"$compressionCodec size has changed.", expectedSize, messageSet.sizeInBytes)
}
def isSnappyAvailable(): Boolean = {
def isSnappyAvailable: Boolean = {
try {
val snappy = new org.xerial.snappy.SnappyOutputStream(new ByteArrayOutputStream())
new org.xerial.snappy.SnappyOutputStream(new ByteArrayOutputStream())
true
} catch {
case e: UnsatisfiedLinkError => false
@ -55,9 +76,9 @@ class MessageCompressionTest extends JUnitSuite {
}
}
def isLZ4Available(): Boolean = {
def isLZ4Available: Boolean = {
try {
val lz4 = new net.jpountz.lz4.LZ4BlockOutputStream(new ByteArrayOutputStream())
new net.jpountz.lz4.LZ4BlockOutputStream(new ByteArrayOutputStream())
true
} catch {
case e: UnsatisfiedLinkError => false