mirror of https://github.com/apache/kafka.git
KAFKA-17569 Rewrite TestLinearWriteSpeed by Java (#17736)
Reviewers: Chia-Ping Tsai <chia7712@gmail.com>
This commit is contained in:
TengYao Chi
GitHub
parent
056a76e2b9
commit
0e4d8b3e86
|
|
@ -3318,6 +3318,7 @@ project(':jmh-benchmarks') {
|
|||
implementation project(':metadata')
|
||||
implementation project(':storage')
|
||||
implementation project(':streams')
|
||||
implementation project(':transaction-coordinator')
|
||||
implementation project(':core')
|
||||
implementation project(':connect:api')
|
||||
implementation project(':connect:transforms')
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@
|
|||
<allow pkg="org.apache.kafka.image"/>
|
||||
<allow pkg="org.apache.kafka.metadata"/>
|
||||
<allow pkg="org.apache.kafka.timeline" />
|
||||
<allow pkg="org.apache.kafka.coordinator.transaction"/>
|
||||
<allow pkg="org.apache.kafka.connect" />
|
||||
<allow pkg="org.apache.kafka.network" />
|
||||
<allow class="org.apache.kafka.raft.QuorumConfig"/>
|
||||
|
|
|
|||
|
|
@ -1,254 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package kafka
|
||||
|
||||
import java.io._
|
||||
import java.nio._
|
||||
import java.nio.channels._
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.util.{Properties, Random}
|
||||
import joptsimple._
|
||||
import kafka.log._
|
||||
import org.apache.kafka.common.compress.{Compression, GzipCompression, Lz4Compression, ZstdCompression}
|
||||
import org.apache.kafka.common.config.TopicConfig
|
||||
import org.apache.kafka.common.record._
|
||||
import org.apache.kafka.common.utils.{Exit, Time, Utils}
|
||||
import org.apache.kafka.coordinator.transaction.TransactionLogConfig
|
||||
import org.apache.kafka.server.util.{KafkaScheduler, Scheduler}
|
||||
import org.apache.kafka.server.util.CommandLineUtils
|
||||
import org.apache.kafka.storage.internals.log.{LogConfig, LogDirFailureChannel, ProducerStateManagerConfig}
|
||||
import org.apache.kafka.storage.log.metrics.BrokerTopicStats
|
||||
|
||||
import scala.math.max
|
||||
|
||||
/**
|
||||
* This test does linear writes using either a kafka log or a file and measures throughput and latency.
|
||||
*/
|
||||
object TestLinearWriteSpeed {
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val parser = new OptionParser(false)
|
||||
val dirOpt = parser.accepts("dir", "The directory to write to.")
|
||||
.withRequiredArg
|
||||
.describedAs("path")
|
||||
.ofType(classOf[java.lang.String])
|
||||
.defaultsTo(System.getProperty("java.io.tmpdir"))
|
||||
val bytesOpt = parser.accepts("bytes", "REQUIRED: The total number of bytes to write.")
|
||||
.withRequiredArg
|
||||
.describedAs("num_bytes")
|
||||
.ofType(classOf[java.lang.Long])
|
||||
val sizeOpt = parser.accepts("size", "REQUIRED: The size of each write.")
|
||||
.withRequiredArg
|
||||
.describedAs("num_bytes")
|
||||
.ofType(classOf[java.lang.Integer])
|
||||
val messageSizeOpt = parser.accepts("message-size", "REQUIRED: The size of each message in the message set.")
|
||||
.withRequiredArg
|
||||
.describedAs("num_bytes")
|
||||
.ofType(classOf[java.lang.Integer])
|
||||
.defaultsTo(1024)
|
||||
val filesOpt = parser.accepts("files", "REQUIRED: The number of logs or files.")
|
||||
.withRequiredArg
|
||||
.describedAs("num_files")
|
||||
.ofType(classOf[java.lang.Integer])
|
||||
.defaultsTo(1)
|
||||
val reportingIntervalOpt = parser.accepts("reporting-interval", "The number of ms between updates.")
|
||||
.withRequiredArg
|
||||
.describedAs("ms")
|
||||
.ofType(classOf[java.lang.Long])
|
||||
.defaultsTo(1000L)
|
||||
val maxThroughputOpt = parser.accepts("max-throughput-mb", "The maximum throughput.")
|
||||
.withRequiredArg
|
||||
.describedAs("mb")
|
||||
.ofType(classOf[java.lang.Integer])
|
||||
.defaultsTo(Integer.MAX_VALUE)
|
||||
val flushIntervalOpt = parser.accepts("flush-interval", "The number of messages between flushes")
|
||||
.withRequiredArg()
|
||||
.describedAs("message_count")
|
||||
.ofType(classOf[java.lang.Long])
|
||||
.defaultsTo(Long.MaxValue)
|
||||
val compressionCodecOpt = parser.accepts("compression", "The compression codec to use")
|
||||
.withRequiredArg
|
||||
.describedAs("codec")
|
||||
.ofType(classOf[java.lang.String])
|
||||
.defaultsTo(CompressionType.NONE.name)
|
||||
val compressionLevelOpt = parser.accepts("level", "The compression level to use")
|
||||
.withRequiredArg
|
||||
.describedAs("level")
|
||||
.ofType(classOf[java.lang.Integer])
|
||||
.defaultsTo(0)
|
||||
val mmapOpt = parser.accepts("mmap", "Do writes to memory-mapped files.")
|
||||
val channelOpt = parser.accepts("channel", "Do writes to file channels.")
|
||||
val logOpt = parser.accepts("log", "Do writes to kafka logs.")
|
||||
|
||||
val options = parser.parse(args : _*)
|
||||
|
||||
CommandLineUtils.checkRequiredArgs(parser, options, bytesOpt, sizeOpt, filesOpt)
|
||||
|
||||
var bytesToWrite = options.valueOf(bytesOpt).longValue
|
||||
val bufferSize = options.valueOf(sizeOpt).intValue
|
||||
val numFiles = options.valueOf(filesOpt).intValue
|
||||
val reportingInterval = options.valueOf(reportingIntervalOpt).longValue
|
||||
val dir = options.valueOf(dirOpt)
|
||||
val maxThroughputBytes = options.valueOf(maxThroughputOpt).intValue * 1024L * 1024L
|
||||
val buffer = ByteBuffer.allocate(bufferSize)
|
||||
val messageSize = options.valueOf(messageSizeOpt).intValue
|
||||
val flushInterval = options.valueOf(flushIntervalOpt).longValue
|
||||
val compressionType = CompressionType.forName(options.valueOf(compressionCodecOpt))
|
||||
val compressionBuilder = Compression.of(compressionType)
|
||||
val compressionLevel = options.valueOf(compressionLevelOpt)
|
||||
compressionType match {
|
||||
case CompressionType.GZIP => compressionBuilder.asInstanceOf[GzipCompression.Builder].level(compressionLevel)
|
||||
case CompressionType.LZ4 => compressionBuilder.asInstanceOf[Lz4Compression.Builder].level(compressionLevel)
|
||||
case CompressionType.ZSTD => compressionBuilder.asInstanceOf[ZstdCompression.Builder].level(compressionLevel)
|
||||
case _ => //Noop
|
||||
}
|
||||
val compression = compressionBuilder.build()
|
||||
val rand = new Random
|
||||
rand.nextBytes(buffer.array)
|
||||
val numMessages = bufferSize / (messageSize + Records.LOG_OVERHEAD)
|
||||
val createTime = System.currentTimeMillis
|
||||
val messageSet = {
|
||||
val records = (0 until numMessages).map(_ => new SimpleRecord(createTime, null, new Array[Byte](messageSize)))
|
||||
MemoryRecords.withRecords(compression, records: _*)
|
||||
}
|
||||
|
||||
val writables = new Array[Writable](numFiles)
|
||||
val scheduler = new KafkaScheduler(1)
|
||||
scheduler.startup()
|
||||
for (i <- 0 until numFiles) {
|
||||
if (options.has(mmapOpt)) {
|
||||
writables(i) = new MmapWritable(new File(dir, "kafka-test-" + i + ".dat"), bytesToWrite / numFiles, buffer)
|
||||
} else if (options.has(channelOpt)) {
|
||||
writables(i) = new ChannelWritable(new File(dir, "kafka-test-" + i + ".dat"), buffer)
|
||||
} else if (options.has(logOpt)) {
|
||||
val segmentSize = rand.nextInt(512)*1024*1024 + 64*1024*1024 // vary size to avoid herd effect
|
||||
val logProperties = new Properties()
|
||||
logProperties.put(TopicConfig.SEGMENT_BYTES_CONFIG, segmentSize: java.lang.Integer)
|
||||
logProperties.put(TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG, flushInterval: java.lang.Long)
|
||||
writables(i) = new LogWritable(new File(dir, "kafka-test-" + i), new LogConfig(logProperties), scheduler, messageSet)
|
||||
} else {
|
||||
System.err.println("Must specify what to write to with one of --log, --channel, or --mmap")
|
||||
Exit.exit(1)
|
||||
}
|
||||
}
|
||||
bytesToWrite = (bytesToWrite / numFiles) * numFiles
|
||||
|
||||
println("%10s\t%10s\t%10s".format("mb_sec", "avg_latency", "max_latency"))
|
||||
|
||||
val beginTest = System.nanoTime
|
||||
var maxLatency = 0L
|
||||
var totalLatency = 0L
|
||||
var count = 0L
|
||||
var written = 0L
|
||||
var totalWritten = 0L
|
||||
var lastReport = beginTest
|
||||
while (totalWritten + bufferSize < bytesToWrite) {
|
||||
val start = System.nanoTime
|
||||
val writeSize = writables((count % numFiles).toInt.abs).write()
|
||||
val elapsed = System.nanoTime - start
|
||||
maxLatency = max(elapsed, maxLatency)
|
||||
totalLatency += elapsed
|
||||
written += writeSize
|
||||
count += 1
|
||||
totalWritten += writeSize
|
||||
if ((start - lastReport)/(1000.0*1000.0) > reportingInterval.doubleValue) {
|
||||
val elapsedSecs = (start - lastReport) / (1000.0*1000.0*1000.0)
|
||||
val mb = written / (1024.0*1024.0)
|
||||
println("%10.3f\t%10.3f\t%10.3f".format(mb / elapsedSecs, totalLatency / count.toDouble / (1000.0*1000.0), maxLatency / (1000.0 * 1000.0)))
|
||||
lastReport = start
|
||||
written = 0
|
||||
maxLatency = 0L
|
||||
totalLatency = 0L
|
||||
} else if (written > maxThroughputBytes * (reportingInterval / 1000.0)) {
|
||||
// if we have written enough, just sit out this reporting interval
|
||||
val lastReportMs = lastReport / (1000*1000)
|
||||
val now = System.nanoTime / (1000*1000)
|
||||
val sleepMs = lastReportMs + reportingInterval - now
|
||||
if (sleepMs > 0)
|
||||
Thread.sleep(sleepMs)
|
||||
}
|
||||
}
|
||||
val elapsedSecs = (System.nanoTime - beginTest) / (1000.0*1000.0*1000.0)
|
||||
println((bytesToWrite / (1024.0 * 1024.0 * elapsedSecs)).toString + " MB per sec")
|
||||
scheduler.shutdown()
|
||||
}
|
||||
|
||||
trait Writable {
|
||||
def write(): Int
|
||||
def close(): Unit
|
||||
}
|
||||
|
||||
class MmapWritable(val file: File, size: Long, val content: ByteBuffer) extends Writable {
|
||||
file.deleteOnExit()
|
||||
val raf = new RandomAccessFile(file, "rw")
|
||||
raf.setLength(size)
|
||||
val buffer = raf.getChannel.map(FileChannel.MapMode.READ_WRITE, 0, raf.length())
|
||||
def write(): Int = {
|
||||
buffer.put(content)
|
||||
content.rewind()
|
||||
content.limit()
|
||||
}
|
||||
def close(): Unit = {
|
||||
raf.close()
|
||||
Utils.delete(file)
|
||||
}
|
||||
}
|
||||
|
||||
class ChannelWritable(val file: File, val content: ByteBuffer) extends Writable {
|
||||
file.deleteOnExit()
|
||||
val channel: FileChannel = FileChannel.open(file.toPath, StandardOpenOption.CREATE, StandardOpenOption.READ,
|
||||
StandardOpenOption.WRITE)
|
||||
def write(): Int = {
|
||||
channel.write(content)
|
||||
content.rewind()
|
||||
content.limit()
|
||||
}
|
||||
def close(): Unit = {
|
||||
channel.close()
|
||||
Utils.delete(file)
|
||||
}
|
||||
}
|
||||
|
||||
class LogWritable(val dir: File, config: LogConfig, scheduler: Scheduler, val messages: MemoryRecords) extends Writable {
|
||||
Utils.delete(dir)
|
||||
val log: UnifiedLog = UnifiedLog(
|
||||
dir = dir,
|
||||
config = config,
|
||||
logStartOffset = 0L,
|
||||
recoveryPoint = 0L,
|
||||
scheduler = scheduler,
|
||||
brokerTopicStats = new BrokerTopicStats,
|
||||
time = Time.SYSTEM,
|
||||
maxTransactionTimeoutMs = 5 * 60 * 1000,
|
||||
producerStateManagerConfig = new ProducerStateManagerConfig(TransactionLogConfig.PRODUCER_ID_EXPIRATION_MS_DEFAULT, false),
|
||||
producerIdExpirationCheckIntervalMs = TransactionLogConfig.PRODUCER_ID_EXPIRATION_CHECK_INTERVAL_MS_DEFAULT,
|
||||
logDirFailureChannel = new LogDirFailureChannel(10),
|
||||
topicId = None,
|
||||
keepPartitionMetadataFile = true
|
||||
)
|
||||
def write(): Int = {
|
||||
log.appendAsLeader(messages, leaderEpoch = 0)
|
||||
messages.sizeInBytes
|
||||
}
|
||||
def close(): Unit = {
|
||||
log.close()
|
||||
Utils.delete(log.dir)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,342 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.kafka.jmh.log;
|
||||
|
||||
import kafka.log.UnifiedLog;
|
||||
|
||||
import org.apache.kafka.common.compress.Compression;
|
||||
import org.apache.kafka.common.compress.GzipCompression;
|
||||
import org.apache.kafka.common.compress.Lz4Compression;
|
||||
import org.apache.kafka.common.compress.ZstdCompression;
|
||||
import org.apache.kafka.common.config.TopicConfig;
|
||||
import org.apache.kafka.common.record.CompressionType;
|
||||
import org.apache.kafka.common.record.MemoryRecords;
|
||||
import org.apache.kafka.common.record.Records;
|
||||
import org.apache.kafka.common.record.SimpleRecord;
|
||||
import org.apache.kafka.common.utils.CopyOnWriteMap;
|
||||
import org.apache.kafka.common.utils.Exit;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.apache.kafka.coordinator.transaction.TransactionLogConfig;
|
||||
import org.apache.kafka.server.common.MetadataVersion;
|
||||
import org.apache.kafka.server.common.RequestLocal;
|
||||
import org.apache.kafka.server.util.CommandLineUtils;
|
||||
import org.apache.kafka.server.util.KafkaScheduler;
|
||||
import org.apache.kafka.server.util.Scheduler;
|
||||
import org.apache.kafka.storage.internals.log.AppendOrigin;
|
||||
import org.apache.kafka.storage.internals.log.LogConfig;
|
||||
import org.apache.kafka.storage.internals.log.LogDirFailureChannel;
|
||||
import org.apache.kafka.storage.internals.log.LogOffsetsListener;
|
||||
import org.apache.kafka.storage.internals.log.ProducerStateManagerConfig;
|
||||
import org.apache.kafka.storage.internals.log.VerificationGuard;
|
||||
import org.apache.kafka.storage.log.metrics.BrokerTopicStats;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.MappedByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
|
||||
import joptsimple.OptionParser;
|
||||
import joptsimple.OptionSet;
|
||||
import joptsimple.OptionSpec;
|
||||
import scala.Option;
|
||||
|
||||
public class TestLinearWriteSpeed {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
OptionParser parser = new OptionParser();
|
||||
|
||||
OptionSpec<String> dirOpt = parser.accepts("dir", "The directory to write to.")
|
||||
.withRequiredArg()
|
||||
.describedAs("path")
|
||||
.ofType(String.class)
|
||||
.defaultsTo(System.getProperty("java.io.tmpdir"));
|
||||
|
||||
OptionSpec<Long> bytesOpt = parser.accepts("bytes", "REQUIRED: The total number of bytes to write.")
|
||||
.withRequiredArg()
|
||||
.describedAs("num_bytes")
|
||||
.ofType(Long.class);
|
||||
|
||||
OptionSpec<Integer> sizeOpt = parser.accepts("size", "REQUIRED: The size of each write.")
|
||||
.withRequiredArg()
|
||||
.describedAs("num_bytes")
|
||||
.ofType(Integer.class);
|
||||
|
||||
OptionSpec<Integer> messageSizeOpt = parser.accepts("message-size", "REQUIRED: The size of each message in the message set.")
|
||||
.withRequiredArg()
|
||||
.describedAs("num_bytes")
|
||||
.ofType(Integer.class)
|
||||
.defaultsTo(1024);
|
||||
|
||||
OptionSpec<Integer> filesOpt = parser.accepts("files", "REQUIRED: The number of logs or files.")
|
||||
.withRequiredArg()
|
||||
.describedAs("num_files")
|
||||
.ofType(Integer.class)
|
||||
.defaultsTo(1);
|
||||
|
||||
OptionSpec<Long> reportingIntervalOpt = parser.accepts("reporting-interval", "The number of ms between updates.")
|
||||
.withRequiredArg()
|
||||
.describedAs("ms")
|
||||
.ofType(Long.class)
|
||||
.defaultsTo(1000L);
|
||||
|
||||
OptionSpec<Integer> maxThroughputOpt = parser.accepts("max-throughput-mb", "The maximum throughput.")
|
||||
.withRequiredArg()
|
||||
.describedAs("mb")
|
||||
.ofType(Integer.class)
|
||||
.defaultsTo(Integer.MAX_VALUE);
|
||||
|
||||
OptionSpec<Long> flushIntervalOpt = parser.accepts("flush-interval", "The number of messages between flushes")
|
||||
.withRequiredArg()
|
||||
.describedAs("message_count")
|
||||
.ofType(Long.class)
|
||||
.defaultsTo(Long.MAX_VALUE);
|
||||
|
||||
OptionSpec<String> compressionCodecOpt = parser.accepts("compression", "The compression codec to use")
|
||||
.withRequiredArg()
|
||||
.describedAs("codec")
|
||||
.ofType(String.class)
|
||||
.defaultsTo(CompressionType.NONE.name);
|
||||
|
||||
OptionSpec<Integer> compressionLevelOpt = parser.accepts("level", "The compression level to use")
|
||||
.withRequiredArg()
|
||||
.describedAs("level")
|
||||
.ofType(Integer.class)
|
||||
.defaultsTo(0);
|
||||
|
||||
OptionSpec<Void> mmapOpt = parser.accepts("mmap", "Do writes to memory-mapped files.");
|
||||
OptionSpec<Void> channelOpt = parser.accepts("channel", "Do writes to file channels.");
|
||||
OptionSpec<Void> logOpt = parser.accepts("log", "Do writes to kafka logs.");
|
||||
OptionSet options = parser.parse(args);
|
||||
CommandLineUtils.checkRequiredArgs(parser, options, bytesOpt, sizeOpt, filesOpt);
|
||||
|
||||
long bytesToWrite = options.valueOf(bytesOpt);
|
||||
int bufferSize = options.valueOf(sizeOpt);
|
||||
int numFiles = options.valueOf(filesOpt);
|
||||
long reportingInterval = options.valueOf(reportingIntervalOpt);
|
||||
String dir = options.valueOf(dirOpt);
|
||||
long maxThroughputBytes = options.valueOf(maxThroughputOpt) * 1024L * 1024L;
|
||||
ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
|
||||
int messageSize = options.valueOf(messageSizeOpt);
|
||||
long flushInterval = options.valueOf(flushIntervalOpt);
|
||||
CompressionType compressionType = CompressionType.forName(options.valueOf(compressionCodecOpt));
|
||||
Compression.Builder<? extends Compression> compressionBuilder = Compression.of(compressionType);
|
||||
int compressionLevel = options.valueOf(compressionLevelOpt);
|
||||
|
||||
setupCompression(compressionType, compressionBuilder, compressionLevel);
|
||||
|
||||
ThreadLocalRandom.current().nextBytes(buffer.array());
|
||||
int numMessages = bufferSize / (messageSize + Records.LOG_OVERHEAD);
|
||||
long createTime = System.currentTimeMillis();
|
||||
|
||||
List<SimpleRecord> recordsList = new ArrayList<>();
|
||||
for (int i = 0; i < numMessages; i++) {
|
||||
recordsList.add(new SimpleRecord(createTime, null, new byte[messageSize]));
|
||||
}
|
||||
|
||||
MemoryRecords messageSet = MemoryRecords.withRecords(Compression.NONE, recordsList.toArray(new SimpleRecord[0]));
|
||||
Writable[] writables = new Writable[numFiles];
|
||||
KafkaScheduler scheduler = new KafkaScheduler(1);
|
||||
scheduler.startup();
|
||||
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
if (options.has(mmapOpt)) {
|
||||
writables[i] = new MmapWritable(new File(dir, "kafka-test-" + i + ".dat"), bytesToWrite / numFiles, buffer);
|
||||
} else if (options.has(channelOpt)) {
|
||||
writables[i] = new ChannelWritable(new File(dir, "kafka-test-" + i + ".dat"), buffer);
|
||||
} else if (options.has(logOpt)) {
|
||||
int segmentSize = ThreadLocalRandom.current().nextInt(512) * 1024 * 1024 + 64 * 1024 * 1024;
|
||||
Properties logProperties = new Properties();
|
||||
logProperties.put(TopicConfig.SEGMENT_BYTES_CONFIG, Integer.toString(segmentSize));
|
||||
logProperties.put(TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG, Long.toString(flushInterval));
|
||||
LogConfig logConfig = new LogConfig(logProperties);
|
||||
writables[i] = new LogWritable(new File(dir, "kafka-test-" + i), logConfig, scheduler, messageSet);
|
||||
} else {
|
||||
System.err.println("Must specify what to write to with one of --log, --channel, or --mmap");
|
||||
Exit.exit(1);
|
||||
}
|
||||
}
|
||||
bytesToWrite = (bytesToWrite / numFiles) * numFiles;
|
||||
|
||||
System.out.printf("%10s\t%10s\t%10s%n", "mb_sec", "avg_latency", "max_latency");
|
||||
|
||||
long beginTest = System.nanoTime();
|
||||
long maxLatency = 0L;
|
||||
long totalLatency = 0L;
|
||||
long count = 0L;
|
||||
long written = 0L;
|
||||
long totalWritten = 0L;
|
||||
long lastReport = beginTest;
|
||||
|
||||
while (totalWritten + bufferSize < bytesToWrite) {
|
||||
long start = System.nanoTime();
|
||||
int writeSize = writables[(int) (count % numFiles)].write();
|
||||
long elapsed = System.nanoTime() - start;
|
||||
maxLatency = Math.max(elapsed, maxLatency);
|
||||
totalLatency += elapsed;
|
||||
written += writeSize;
|
||||
count += 1;
|
||||
totalWritten += writeSize;
|
||||
if ((start - lastReport) / (1000.0 * 1000.0) > reportingInterval) {
|
||||
double elapsedSecs = (start - lastReport) / (1000.0 * 1000.0 * 1000.0);
|
||||
double mb = written / (1024.0 * 1024.0);
|
||||
System.out.printf("%10.3f\t%10.3f\t%10.3f%n", mb / elapsedSecs, (totalLatency / (double) count) / (1000.0 * 1000.0), maxLatency / (1000.0 * 1000.0));
|
||||
lastReport = start;
|
||||
written = 0;
|
||||
maxLatency = 0L;
|
||||
totalLatency = 0L;
|
||||
} else if (written > maxThroughputBytes * (reportingInterval / 1000.0)) {
|
||||
long lastReportMs = lastReport / (1000 * 1000);
|
||||
long now = System.nanoTime() / (1000 * 1000);
|
||||
long sleepMs = lastReportMs + reportingInterval - now;
|
||||
if (sleepMs > 0)
|
||||
Thread.sleep(sleepMs);
|
||||
}
|
||||
}
|
||||
double elapsedSecs = (System.nanoTime() - beginTest) / (1000.0 * 1000.0 * 1000.0);
|
||||
System.out.println((bytesToWrite / (1024.0 * 1024.0 * elapsedSecs)) + " MB per sec");
|
||||
scheduler.shutdown();
|
||||
}
|
||||
|
||||
private static void setupCompression(CompressionType compressionType,
|
||||
Compression.Builder<? extends Compression> compressionBuilder,
|
||||
int compressionLevel) {
|
||||
switch (compressionType) {
|
||||
case GZIP:
|
||||
((GzipCompression.Builder) compressionBuilder).level(compressionLevel);
|
||||
break;
|
||||
case LZ4:
|
||||
((Lz4Compression.Builder) compressionBuilder).level(compressionLevel);
|
||||
break;
|
||||
case ZSTD:
|
||||
((ZstdCompression.Builder) compressionBuilder).level(compressionLevel);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
interface Writable {
|
||||
int write() throws IOException;
|
||||
|
||||
void close() throws IOException;
|
||||
}
|
||||
|
||||
static class MmapWritable implements Writable {
|
||||
File file;
|
||||
ByteBuffer content;
|
||||
RandomAccessFile raf;
|
||||
MappedByteBuffer buffer;
|
||||
|
||||
public MmapWritable(File file, long size, ByteBuffer content) throws IOException {
|
||||
this.file = file;
|
||||
this.content = content;
|
||||
file.deleteOnExit();
|
||||
raf = new RandomAccessFile(file, "rw");
|
||||
raf.setLength(size);
|
||||
buffer = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, raf.length());
|
||||
}
|
||||
|
||||
public int write() {
|
||||
buffer.put(content);
|
||||
content.rewind();
|
||||
return content.limit();
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
raf.close();
|
||||
Utils.delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
static class ChannelWritable implements Writable {
|
||||
File file;
|
||||
ByteBuffer content;
|
||||
FileChannel channel;
|
||||
|
||||
public ChannelWritable(File file, ByteBuffer content) throws IOException {
|
||||
this.file = file;
|
||||
this.content = content;
|
||||
file.deleteOnExit();
|
||||
channel = FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
|
||||
}
|
||||
|
||||
public int write() throws IOException {
|
||||
channel.write(content);
|
||||
content.rewind();
|
||||
return content.limit();
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
channel.close();
|
||||
Utils.delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
static class LogWritable implements Writable {
|
||||
MemoryRecords messages;
|
||||
UnifiedLog log;
|
||||
|
||||
public LogWritable(File dir, LogConfig config, Scheduler scheduler, MemoryRecords messages) throws IOException {
|
||||
this.messages = messages;
|
||||
Utils.delete(dir);
|
||||
this.log = UnifiedLog.apply(
|
||||
dir,
|
||||
config,
|
||||
0L,
|
||||
0L,
|
||||
scheduler,
|
||||
new BrokerTopicStats(),
|
||||
Time.SYSTEM,
|
||||
5 * 60 * 1000,
|
||||
new ProducerStateManagerConfig(TransactionLogConfig.PRODUCER_ID_EXPIRATION_MS_DEFAULT, false),
|
||||
TransactionLogConfig.PRODUCER_ID_EXPIRATION_CHECK_INTERVAL_MS_DEFAULT,
|
||||
new LogDirFailureChannel(10),
|
||||
true,
|
||||
Option.empty(),
|
||||
true,
|
||||
new CopyOnWriteMap<>(),
|
||||
false,
|
||||
LogOffsetsListener.NO_OP_OFFSETS_LISTENER
|
||||
);
|
||||
}
|
||||
|
||||
public int write() {
|
||||
log.appendAsLeader(
|
||||
messages,
|
||||
0,
|
||||
AppendOrigin.CLIENT,
|
||||
MetadataVersion.latestProduction(),
|
||||
RequestLocal.noCaching(),
|
||||
VerificationGuard.SENTINEL
|
||||
);
|
||||
return messages.sizeInBytes();
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
log.close();
|
||||
Utils.delete(log.dir());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue