root
/
kafka
mirror of https://github.com/apache/kafka.git
1
0
Fork 0
Code Issues Actions 2 Packages Projects Releases Wiki Activity

KAFKA-3548: Use root locale for case transformation of constant strings 

For enums and other constant strings, use locale independent case conversions to enable comparisons to work regardless of the default locale.

Author: Rajini Sivaram <rajinisivaram@googlemail.com>

Reviewers: Manikumar Reddy, Ismael Juma, Guozhang Wang, Gwen Shapira

Closes #1220 from rajinisivaram/KAFKA-3548
This commit is contained in:
Rajini Sivaram 2016-04-20 18:54:30 -07:00 committed by Gwen Shapira
parent f213625fc5
commit 9d71489ff0
23 changed files with 57 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
checkstyle/checkstyle.xml
View File

@ -84,5 +84,12 @@
<module name="MethodParamPad"/>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<!-- locale-sensitive methods should specify locale -->
<module name="Regexp">
<property name="format" value="\.to(Lower|Upper)Case\(\)"/>
<property name="illegalPattern" value="true"/>
<property name="ignoreComments" value="true"/>
</module>
</module>
</module>

3
clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
View File

@ -51,6 +51,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@ -586,7 +587,7 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), time);
this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs,
config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG));
OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase());
OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT));
this.subscriptions = new SubscriptionState(offsetResetStrategy);
List<PartitionAssignor> assignors = config.getConfiguredInstances(
ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,

3
clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
View File

@ -61,6 +61,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@ -291,7 +292,7 @@ public class Fetcher<K, V> {
else
throw new NoOffsetForPartitionException(partition);
log.debug("Resetting offset for partition {} to {} offset.", partition, strategy.name().toLowerCase());
log.debug("Resetting offset for partition {} to {} offset.", partition, strategy.name().toLowerCase(Locale.ROOT));
long offset = listOffset(partition, timestamp);
// we might lose the assignment while fetching the offset, so check it is still active

2
clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
View File

@ -338,7 +338,7 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
private static int parseAcks(String acksString) {
try {
return acksString.trim().toLowerCase().equals("all") ? -1 : Integer.parseInt(acksString.trim());
return acksString.trim().equalsIgnoreCase("all") ? -1 : Integer.parseInt(acksString.trim());
} catch (NumberFormatException e) {
throw new ConfigException("Invalid configuration value for 'acks': " + acksString);
}

9
clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
View File

@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@ -891,7 +892,7 @@ public class ConfigDef {
b.append(def.documentation);
b.append("</td>");
b.append("<td>");
b.append(def.type.toString().toLowerCase());
b.append(def.type.toString().toLowerCase(Locale.ROOT));
b.append("</td>");
b.append("<td>");
if (def.hasDefault()) {
@ -908,7 +909,7 @@ public class ConfigDef {
b.append(def.validator != null ? def.validator.toString() : "");
b.append("</td>");
b.append("<td>");
b.append(def.importance.toString().toLowerCase());
b.append(def.importance.toString().toLowerCase(Locale.ROOT));
b.append("</td>");
b.append("</tr>\n");
}
@ -937,7 +938,7 @@ public class ConfigDef {
b.append("\n\n");
}
b.append(" * Type: ");
b.append(def.type.toString().toLowerCase());
b.append(def.type.toString().toLowerCase(Locale.ROOT));
b.append("\n");
if (def.defaultValue != null) {
b.append(" * Default: ");
@ -951,7 +952,7 @@ public class ConfigDef {
b.append("\n");
}
b.append(" * Importance: ");
b.append(def.importance.toString().toLowerCase());
b.append(def.importance.toString().toLowerCase(Locale.ROOT));
b.append("\n\n");
}
return b.toString();

3
clients/src/main/java/org/apache/kafka/common/metrics/stats/Rate.java
View File

@ -13,6 +13,7 @@
package org.apache.kafka.common.metrics.stats;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.metrics.MeasurableStat;
@ -48,7 +49,7 @@ public class Rate implements MeasurableStat {
}
public String unitName() {
return unit.name().substring(0, unit.name().length() - 2).toLowerCase();
return unit.name().substring(0, unit.name().length() - 2).toLowerCase(Locale.ROOT);
}
@Override

3
clients/src/main/java/org/apache/kafka/common/protocol/SecurityProtocol.java
View File

@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@ -85,7 +86,7 @@ public enum SecurityProtocol {
/** Case insensitive lookup by protocol name */
public static SecurityProtocol forName(String name) {
return SecurityProtocol.valueOf(name.toUpperCase());
return SecurityProtocol.valueOf(name.toUpperCase(Locale.ROOT));
}
/**

3
clients/src/test/java/org/apache/kafka/test/MockConsumerInterceptor.java
View File

@ -29,6 +29,7 @@ import org.apache.kafka.common.config.ConfigException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@ -59,7 +60,7 @@ public class MockConsumerInterceptor implements ConsumerInterceptor<String, Stri
record.timestamp(), record.timestampType(),
record.checksum(), record.serializedKeySize(),
record.serializedValueSize(),
record.key(), record.value().toUpperCase()));
record.key(), record.value().toUpperCase(Locale.ROOT)));
}
recordMap.put(tp, lst);
}

3
connect/api/src/main/java/org/apache/kafka/connect/data/Schema.java
View File

@ -18,6 +18,7 @@
package org.apache.kafka.connect.data;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
@ -48,7 +49,7 @@ public interface Schema {
private String name;
Type() {
this.name = this.name().toLowerCase();
this.name = this.name().toLowerCase(Locale.ROOT);
}
public String getName() {

3
core/src/main/scala/kafka/consumer/ConsumerConfig.scala
View File

@ -21,6 +21,7 @@ import java.util.Properties
import kafka.api.OffsetRequest
import kafka.utils._
import kafka.common.{InvalidConfigException, Config}
import java.util.Locale
object ConsumerConfig extends Config {
val RefreshMetadataBackoffMs = 200
@ -163,7 +164,7 @@ class ConsumerConfig private (val props: VerifiableProperties) extends ZKConfig(
val offsetsCommitMaxRetries = props.getInt("offsets.commit.max.retries", OffsetsCommitMaxRetries)
/** Specify whether offsets should be committed to "zookeeper" (default) or "kafka" */
val offsetsStorage = props.getString("offsets.storage", OffsetsStorage).toLowerCase
val offsetsStorage = props.getString("offsets.storage", OffsetsStorage).toLowerCase(Locale.ROOT)
/** If you are using "kafka" as offsets.storage, you can dual commit offsets to ZooKeeper (in addition to Kafka). This
* is required during migration from zookeeper-based offset storage to kafka-based offset storage. With respect to any

5
core/src/main/scala/kafka/log/LogConfig.scala
View File

@ -27,6 +27,7 @@ import kafka.server.KafkaConfig
import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}
import org.apache.kafka.common.record.TimestampType
import org.apache.kafka.common.utils.Utils
import java.util.Locale
object Defaults {
val SegmentSize = kafka.server.Defaults.LogSegmentBytes
@ -70,10 +71,10 @@ case class LogConfig(props: java.util.Map[_, _]) extends AbstractConfig(LogConfi
val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)
val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)
val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)
val compact = getString(LogConfig.CleanupPolicyProp).toLowerCase != LogConfig.Delete
val compact = getString(LogConfig.CleanupPolicyProp).toLowerCase(Locale.ROOT) != LogConfig.Delete
val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)
val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)
val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase
val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)
val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)
val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))
val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))

8
core/src/main/scala/kafka/message/CompressionCodec.scala
View File

@ -17,6 +17,8 @@
package kafka.message
import java.util.Locale
object CompressionCodec {
def getCompressionCodec(codec: Int): CompressionCodec = {
codec match {
@ -28,7 +30,7 @@ object CompressionCodec {
}
}
def getCompressionCodec(name: String): CompressionCodec = {
name.toLowerCase match {
name.toLowerCase(Locale.ROOT) match {
case NoCompressionCodec.name => NoCompressionCodec
case GZIPCompressionCodec.name => GZIPCompressionCodec
case SnappyCompressionCodec.name => SnappyCompressionCodec
@ -43,10 +45,10 @@ object BrokerCompressionCodec {
val brokerCompressionCodecs = List(UncompressedCodec, SnappyCompressionCodec, LZ4CompressionCodec, GZIPCompressionCodec, ProducerCompressionCodec)
val brokerCompressionOptions = brokerCompressionCodecs.map(codec => codec.name)
def isValid(compressionType: String): Boolean = brokerCompressionOptions.contains(compressionType.toLowerCase())
def isValid(compressionType: String): Boolean = brokerCompressionOptions.contains(compressionType.toLowerCase(Locale.ROOT))
def getCompressionCodec(compressionType: String): CompressionCodec = {
compressionType.toLowerCase match {
compressionType.toLowerCase(Locale.ROOT) match {
case UncompressedCodec.name => NoCompressionCodec
case _ => CompressionCodec.getCompressionCodec(compressionType)
}

4
core/src/main/scala/kafka/tools/ConsoleConsumer.scala
View File

@ -349,9 +349,9 @@ class DefaultMessageFormatter extends MessageFormatter {
override def init(props: Properties) {
if (props.containsKey("print.timestamp"))
printTimestamp = props.getProperty("print.timestamp").trim.toLowerCase.equals("true")
printTimestamp = props.getProperty("print.timestamp").trim.equalsIgnoreCase("true")
if (props.containsKey("print.key"))
printKey = props.getProperty("print.key").trim.toLowerCase.equals("true")
printKey = props.getProperty("print.key").trim.equalsIgnoreCase("true")
if (props.containsKey("key.separator"))
keySeparator = props.getProperty("key.separator").getBytes
if (props.containsKey("line.separator"))

4
core/src/main/scala/kafka/tools/ConsoleProducer.scala
View File

@ -295,11 +295,11 @@ object ConsoleProducer {
override def init(inputStream: InputStream, props: Properties) {
topic = props.getProperty("topic")
if (props.containsKey("parse.key"))
parseKey = props.getProperty("parse.key").trim.toLowerCase.equals("true")
parseKey = props.getProperty("parse.key").trim.equalsIgnoreCase("true")
if (props.containsKey("key.separator"))
keySeparator = props.getProperty("key.separator")
if (props.containsKey("ignore.error"))
ignoreError = props.getProperty("ignore.error").trim.toLowerCase.equals("true")
ignoreError = props.getProperty("ignore.error").trim.equalsIgnoreCase("true")
reader = new BufferedReader(new InputStreamReader(inputStream))
}

3
core/src/main/scala/kafka/utils/Log4jController.scala
View File

@ -20,6 +20,7 @@ package kafka.utils
import org.apache.log4j.{Logger, Level, LogManager}
import java.util
import java.util.Locale
object Log4jController {
@ -81,7 +82,7 @@ private class Log4jController extends Log4jControllerMBean {
def setLogLevel(loggerName: String, level: String) = {
val log = newLogger(loggerName)
if (!loggerName.trim.isEmpty && !level.trim.isEmpty && log != null) {
log.setLevel(Level.toLevel(level.toUpperCase))
log.setLevel(Level.toLevel(level.toUpperCase(Locale.ROOT)))
true
}
else false

4
core/src/main/scala/kafka/utils/Os.scala
View File

@ -17,7 +17,9 @@
package kafka.utils
import java.util.Locale
object Os {
val name = System.getProperty("os.name").toLowerCase
val name = System.getProperty("os.name").toLowerCase(Locale.ROOT)
val isWindows = name.startsWith("windows")
}

3
core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
View File

@ -35,6 +35,7 @@ import org.junit.Test
import scala.collection.JavaConverters._
import scala.collection.mutable.Buffer
import java.util.Locale
/* We have some tests in this class instead of `BaseConsumerTest` in order to keep the build time under control. */
class PlaintextConsumerTest extends BaseConsumerTest {
@ -606,7 +607,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
for (i <- 0 until numRecords) {
val record = records.get(i)
assertEquals(s"key $i", new String(record.key()))
assertEquals(s"value $i$appendStr".toUpperCase, new String(record.value()))
assertEquals(s"value $i$appendStr".toUpperCase(Locale.ROOT), new String(record.value()))
}
// commit sync and verify onCommit is called

2
examples/src/main/java/kafka/examples/KafkaConsumerProducerDemo.java
View File

@ -18,7 +18,7 @@ package kafka.examples;
public class KafkaConsumerProducerDemo {
public static void main(String[] args) {
boolean isAsync = args.length == 0 || !args[0].trim().toLowerCase().equals("sync");
boolean isAsync = args.length == 0 || !args[0].trim().equalsIgnoreCase("sync");
Producer producerThread = new Producer(KafkaProperties.TOPIC, isAsync);
producerThread.start();

3
streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
View File

@ -29,6 +29,7 @@ import org.apache.kafka.streams.kstream.KeyValueMapper;
import org.apache.kafka.streams.kstream.ValueMapper;
import java.util.Arrays;
import java.util.Locale;
import java.util.Properties;
/**
@ -63,7 +64,7 @@ public class WordCountDemo {
.flatMapValues(new ValueMapper<String, Iterable<String>>() {
@Override
public Iterable<String> apply(String value) {
return Arrays.asList(value.toLowerCase().split(" "));
return Arrays.asList(value.toLowerCase(Locale.getDefault()).split(" "));
}
}).map(new KeyValueMapper<String, String, KeyValue<String, String>>() {
@Override

3
streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java
View File

@ -30,6 +30,7 @@ import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.Stores;
import java.util.Locale;
import java.util.Properties;
/**
@ -63,7 +64,7 @@ public class WordCountProcessorDemo {
@Override
public void process(String dummy, String line) {
String[] words = line.toLowerCase().split(" ");
String[] words = line.toLowerCase(Locale.getDefault()).split(" ");
for (String word : words) {
Integer oldValue = this.kvStore.get(word);

3
streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
View File

@ -26,6 +26,7 @@ import org.apache.kafka.test.MockProcessorSupplier;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Locale;
import static org.junit.Assert.assertEquals;
@ -42,7 +43,7 @@ public class KStreamFlatMapValuesTest {
@Override
public Iterable<String> apply(String value) {
ArrayList<String> result = new ArrayList<String>();
result.add(value.toLowerCase());
result.add(value.toLowerCase(Locale.ROOT));
result.add(value);
return result;
}

3
streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
View File

@ -26,6 +26,7 @@ import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.test.KStreamTestDriver;
import org.junit.Test;
import java.util.List;
import java.util.Locale;
import java.util.ArrayList;
import java.util.Arrays;
@ -60,7 +61,7 @@ public class KStreamForeachTest {
new ForeachAction<Integer, String>() {
@Override
public void apply(Integer key, String value) {
actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase()));
actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase(Locale.ROOT)));
}
};

3
streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableForeachTest.java
View File

@ -26,6 +26,7 @@ import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.test.KStreamTestDriver;
import org.junit.Test;
import java.util.List;
import java.util.Locale;
import java.util.ArrayList;
import java.util.Arrays;
@ -60,7 +61,7 @@ public class KTableForeachTest {
new ForeachAction<Integer, String>() {
@Override
public void apply(Integer key, String value) {
actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase()));
actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase(Locale.ROOT)));
}
};