mirror of https://github.com/apache/kafka.git
KAFKA-14669: Use the generated docs for MirrorMaker configs in the doc (#13658)
Reviewers: Chris Egerton <chrise@aiven.io>
This commit is contained in:
Gantigmaa Selenge
GitHub
parent
fd3b1137d2
commit
5cb2504c49
|
|
@ -1070,7 +1070,7 @@ project(':core') {
|
|||
':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs',
|
||||
':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs',
|
||||
':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs',
|
||||
':connect:mirror:genMirrorHeartbeatConfigDocs'], type: Tar) {
|
||||
':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs'], type: Tar) {
|
||||
archiveClassifier = 'site-docs'
|
||||
compression = Compression.GZIP
|
||||
from project.file("$rootDir/docs")
|
||||
|
|
@ -2998,6 +2998,13 @@ project(':connect:mirror') {
|
|||
duplicatesStrategy 'exclude'
|
||||
}
|
||||
|
||||
task genMirrorConnectorConfigDocs(type: JavaExec) {
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
mainClass = 'org.apache.kafka.connect.mirror.MirrorConnectorConfig'
|
||||
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
||||
standardOutput = new File(generatedDocsDir, "mirror_connector_config.html").newOutputStream()
|
||||
}
|
||||
|
||||
task genMirrorSourceConfigDocs(type: JavaExec) {
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
mainClass = 'org.apache.kafka.connect.mirror.MirrorSourceConfig'
|
||||
|
|
|
|||
|
|
@ -165,94 +165,98 @@ public class MirrorCheckpointConfig extends MirrorConnectorConfig {
|
|||
return Duration.ofMillis(getLong(CONSUMER_POLL_TIMEOUT_MILLIS));
|
||||
}
|
||||
|
||||
protected static final ConfigDef CONNECTOR_CONFIG_DEF = new ConfigDef(BASE_CONNECTOR_CONFIG_DEF)
|
||||
.define(
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS,
|
||||
ConfigDef.Type.LONG,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DOC)
|
||||
.define(
|
||||
GROUPS,
|
||||
ConfigDef.Type.LIST,
|
||||
GROUPS_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
GROUPS_DOC)
|
||||
.define(
|
||||
GROUPS_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
GROUPS_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
GROUPS_EXCLUDE_DOC)
|
||||
.define(
|
||||
GROUPS_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + GROUPS_EXCLUDE + " instead.")
|
||||
.define(
|
||||
GROUP_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
GROUP_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
GROUP_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
REFRESH_GROUPS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
REFRESH_GROUPS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_GROUPS_ENABLED_DOC)
|
||||
.define(
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
EMIT_CHECKPOINTS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
EMIT_CHECKPOINTS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_CHECKPOINTS_ENABLED_DOC)
|
||||
.define(
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
SYNC_GROUP_OFFSETS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_GROUP_OFFSETS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_GROUP_OFFSETS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR,
|
||||
ConfigDef.Type.SHORT,
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_LOCATION,
|
||||
ConfigDef.Type.STRING,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DEFAULT,
|
||||
ConfigDef.ValidString.in(SOURCE_CLUSTER_ALIAS_DEFAULT, TARGET_CLUSTER_ALIAS_DEFAULT),
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DOC)
|
||||
.define(
|
||||
TOPIC_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
TOPIC_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
TOPIC_FILTER_CLASS_DOC);
|
||||
private static ConfigDef defineCheckpointConfig(ConfigDef baseConfig) {
|
||||
return baseConfig
|
||||
.define(
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS,
|
||||
ConfigDef.Type.LONG,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DOC)
|
||||
.define(
|
||||
GROUPS,
|
||||
ConfigDef.Type.LIST,
|
||||
GROUPS_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
GROUPS_DOC)
|
||||
.define(
|
||||
GROUPS_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
GROUPS_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
GROUPS_EXCLUDE_DOC)
|
||||
.define(
|
||||
GROUPS_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + GROUPS_EXCLUDE + " instead.")
|
||||
.define(
|
||||
GROUP_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
GROUP_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
GROUP_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
REFRESH_GROUPS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
REFRESH_GROUPS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_GROUPS_ENABLED_DOC)
|
||||
.define(
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_GROUPS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
EMIT_CHECKPOINTS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
EMIT_CHECKPOINTS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_CHECKPOINTS_ENABLED_DOC)
|
||||
.define(
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_CHECKPOINTS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
SYNC_GROUP_OFFSETS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_GROUP_OFFSETS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_GROUP_OFFSETS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_GROUP_OFFSETS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR,
|
||||
ConfigDef.Type.SHORT,
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CHECKPOINTS_TOPIC_REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_LOCATION,
|
||||
ConfigDef.Type.STRING,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DEFAULT,
|
||||
ConfigDef.ValidString.in(SOURCE_CLUSTER_ALIAS_DEFAULT, TARGET_CLUSTER_ALIAS_DEFAULT),
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DOC)
|
||||
.define(
|
||||
TOPIC_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
TOPIC_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
TOPIC_FILTER_CLASS_DOC);
|
||||
}
|
||||
|
||||
protected final static ConfigDef CONNECTOR_CONFIG_DEF = defineCheckpointConfig(new ConfigDef(BASE_CONNECTOR_CONFIG_DEF));
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(CONNECTOR_CONFIG_DEF.toHtml(4, config -> "mirror_checkpoint_" + config));
|
||||
System.out.println(defineCheckpointConfig(new ConfigDef()).toHtml(4, config -> "mirror_checkpoint_" + config));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -306,4 +306,8 @@ public abstract class MirrorConnectorConfig extends AbstractConfig {
|
|||
)
|
||||
.withClientSslSupport()
|
||||
.withClientSaslSupport();
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(BASE_CONNECTOR_CONFIG_DEF.toHtml(4, config -> "mirror_connector_" + config));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,27 +57,31 @@ public class MirrorHeartbeatConfig extends MirrorConnectorConfig {
|
|||
return getShort(HEARTBEATS_TOPIC_REPLICATION_FACTOR);
|
||||
}
|
||||
|
||||
protected static final ConfigDef CONNECTOR_CONFIG_DEF = new ConfigDef(BASE_CONNECTOR_CONFIG_DEF)
|
||||
.define(
|
||||
private static ConfigDef defineHeartbeatConfig(ConfigDef baseConfig) {
|
||||
return baseConfig
|
||||
.define(
|
||||
EMIT_HEARTBEATS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
EMIT_HEARTBEATS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_HEARTBEATS_ENABLED_DOC)
|
||||
.define(
|
||||
.define(
|
||||
EMIT_HEARTBEATS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
EMIT_HEARTBEATS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
EMIT_HEARTBEATS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
.define(
|
||||
HEARTBEATS_TOPIC_REPLICATION_FACTOR,
|
||||
ConfigDef.Type.SHORT,
|
||||
HEARTBEATS_TOPIC_REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
HEARTBEATS_TOPIC_REPLICATION_FACTOR_DOC);
|
||||
}
|
||||
|
||||
protected final static ConfigDef CONNECTOR_CONFIG_DEF = defineHeartbeatConfig(new ConfigDef(BASE_CONNECTOR_CONFIG_DEF));
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(CONNECTOR_CONFIG_DEF.toHtml(4, config -> "mirror_heartbeat_" + config));
|
||||
System.out.println(defineHeartbeatConfig(new ConfigDef()).toHtml(4, config -> "mirror_heartbeat_" + config));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -220,131 +220,135 @@ public class MirrorSourceConfig extends MirrorConnectorConfig {
|
|||
return getBoolean(ADD_SOURCE_ALIAS_TO_METRICS);
|
||||
}
|
||||
|
||||
protected static final ConfigDef CONNECTOR_CONFIG_DEF = new ConfigDef(BASE_CONNECTOR_CONFIG_DEF)
|
||||
.define(
|
||||
TOPICS,
|
||||
ConfigDef.Type.LIST,
|
||||
TOPICS_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
TOPICS_DOC)
|
||||
.define(
|
||||
TOPICS_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
TOPICS_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
TOPICS_EXCLUDE_DOC)
|
||||
.define(
|
||||
TOPICS_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + TOPICS_EXCLUDE + " instead.")
|
||||
.define(
|
||||
CONFIG_PROPERTIES_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
CONFIG_PROPERTIES_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
CONFIG_PROPERTIES_EXCLUDE_DOC)
|
||||
.define(
|
||||
CONFIG_PROPERTIES_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE + " instead.")
|
||||
.define(
|
||||
TOPIC_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
TOPIC_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
TOPIC_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
CONFIG_PROPERTY_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
CONFIG_PROPERTY_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONFIG_PROPERTY_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS,
|
||||
ConfigDef.Type.LONG,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DOC)
|
||||
.define(
|
||||
REFRESH_TOPICS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
REFRESH_TOPICS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_TOPICS_ENABLED_DOC)
|
||||
.define(
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_CONFIGS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_TOPIC_CONFIGS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_CONFIGS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
USE_INCREMENTAL_ALTER_CONFIGS,
|
||||
ConfigDef.Type.STRING,
|
||||
REQUEST_INCREMENTAL_ALTER_CONFIGS,
|
||||
in(REQUEST_INCREMENTAL_ALTER_CONFIGS, REQUIRE_INCREMENTAL_ALTER_CONFIGS, NEVER_USE_INCREMENTAL_ALTER_CONFIGS),
|
||||
ConfigDef.Importance.LOW,
|
||||
USE_INCREMENTAL_ALTER_CONFIG_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_ACLS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_TOPIC_ACLS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_ACLS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
REPLICATION_FACTOR,
|
||||
ConfigDef.Type.INT,
|
||||
REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR,
|
||||
ConfigDef.Type.SHORT,
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_LAG_MAX,
|
||||
ConfigDef.Type.LONG,
|
||||
OFFSET_LAG_MAX_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_LAG_MAX_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_LOCATION,
|
||||
ConfigDef.Type.STRING,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DEFAULT,
|
||||
in(SOURCE_CLUSTER_ALIAS_DEFAULT, TARGET_CLUSTER_ALIAS_DEFAULT),
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DOC)
|
||||
.define(
|
||||
ADD_SOURCE_ALIAS_TO_METRICS,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
ADD_SOURCE_ALIAS_TO_METRICS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
ADD_SOURCE_ALIAS_TO_METRICS_DOC);
|
||||
private static ConfigDef defineSourceConfig(ConfigDef baseConfig) {
|
||||
return baseConfig
|
||||
.define(
|
||||
TOPICS,
|
||||
ConfigDef.Type.LIST,
|
||||
TOPICS_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
TOPICS_DOC)
|
||||
.define(
|
||||
TOPICS_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
TOPICS_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
TOPICS_EXCLUDE_DOC)
|
||||
.define(
|
||||
TOPICS_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + TOPICS_EXCLUDE + " instead.")
|
||||
.define(
|
||||
CONFIG_PROPERTIES_EXCLUDE,
|
||||
ConfigDef.Type.LIST,
|
||||
CONFIG_PROPERTIES_EXCLUDE_DEFAULT,
|
||||
ConfigDef.Importance.HIGH,
|
||||
CONFIG_PROPERTIES_EXCLUDE_DOC)
|
||||
.define(
|
||||
CONFIG_PROPERTIES_EXCLUDE_ALIAS,
|
||||
ConfigDef.Type.LIST,
|
||||
null,
|
||||
ConfigDef.Importance.HIGH,
|
||||
"Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE + " instead.")
|
||||
.define(
|
||||
TOPIC_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
TOPIC_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
TOPIC_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
CONFIG_PROPERTY_FILTER_CLASS,
|
||||
ConfigDef.Type.CLASS,
|
||||
CONFIG_PROPERTY_FILTER_CLASS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONFIG_PROPERTY_FILTER_CLASS_DOC)
|
||||
.define(
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS,
|
||||
ConfigDef.Type.LONG,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
CONSUMER_POLL_TIMEOUT_MILLIS_DOC)
|
||||
.define(
|
||||
REFRESH_TOPICS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
REFRESH_TOPICS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_TOPICS_ENABLED_DOC)
|
||||
.define(
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REFRESH_TOPICS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_CONFIGS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_TOPIC_CONFIGS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_CONFIGS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_CONFIGS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
USE_INCREMENTAL_ALTER_CONFIGS,
|
||||
ConfigDef.Type.STRING,
|
||||
REQUEST_INCREMENTAL_ALTER_CONFIGS,
|
||||
in(REQUEST_INCREMENTAL_ALTER_CONFIGS, REQUIRE_INCREMENTAL_ALTER_CONFIGS, NEVER_USE_INCREMENTAL_ALTER_CONFIGS),
|
||||
ConfigDef.Importance.LOW,
|
||||
USE_INCREMENTAL_ALTER_CONFIG_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_ACLS_ENABLED,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
SYNC_TOPIC_ACLS_ENABLED_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_ACLS_ENABLED_DOC)
|
||||
.define(
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS,
|
||||
ConfigDef.Type.LONG,
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
SYNC_TOPIC_ACLS_INTERVAL_SECONDS_DOC)
|
||||
.define(
|
||||
REPLICATION_FACTOR,
|
||||
ConfigDef.Type.INT,
|
||||
REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR,
|
||||
ConfigDef.Type.SHORT,
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_REPLICATION_FACTOR_DOC)
|
||||
.define(
|
||||
OFFSET_LAG_MAX,
|
||||
ConfigDef.Type.LONG,
|
||||
OFFSET_LAG_MAX_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_LAG_MAX_DOC)
|
||||
.define(
|
||||
OFFSET_SYNCS_TOPIC_LOCATION,
|
||||
ConfigDef.Type.STRING,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DEFAULT,
|
||||
in(SOURCE_CLUSTER_ALIAS_DEFAULT, TARGET_CLUSTER_ALIAS_DEFAULT),
|
||||
ConfigDef.Importance.LOW,
|
||||
OFFSET_SYNCS_TOPIC_LOCATION_DOC)
|
||||
.define(
|
||||
ADD_SOURCE_ALIAS_TO_METRICS,
|
||||
ConfigDef.Type.BOOLEAN,
|
||||
ADD_SOURCE_ALIAS_TO_METRICS_DEFAULT,
|
||||
ConfigDef.Importance.LOW,
|
||||
ADD_SOURCE_ALIAS_TO_METRICS_DOC);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(CONNECTOR_CONFIG_DEF.toHtml(4, config -> "mirror_source_" + config));
|
||||
protected final static ConfigDef CONNECTOR_CONFIG_DEF = defineSourceConfig(new ConfigDef(BASE_CONNECTOR_CONFIG_DEF));
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(defineSourceConfig(new ConfigDef()).toHtml(4, config -> "mirror_source_" + config));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -267,23 +267,42 @@
|
|||
Below is the configuration of the Kafka Admin client library.
|
||||
<!--#include virtual="generated/admin_client_config.html" -->
|
||||
|
||||
<h3 class="anchor-heading"><a id="systemproperties" class="anchor-link"></a><a href="#systemproperties">3.8 System Properties</a></h3>
|
||||
<h3 class="anchor-heading"><a id="mirrormakerconfigs" class="anchor-link"></a><a href="#mirrormakerconfigs">3.8 MirrorMaker Configs</a></h3>
|
||||
Below is the configuration of the connectors that make up MirrorMaker 2.
|
||||
|
||||
<h4 class="anchor-heading"><a id="mirrormakercommonconfigs" class="anchor-link"></a><a href="#mirrormakercommonconfigs">3.8.1 MirrorMaker Common Configs</a></h4>
|
||||
Below are the common configuration properties that apply to all three connectors.
|
||||
<!--#include virtual="generated/mirror_connector_config.html" -->
|
||||
|
||||
<h4 class="anchor-heading"><a id="mirrormakersourceconfigs" class="anchor-link"></a><a href="#mirrormakersourceconfigs">3.8.2 MirrorMaker Source Configs</a></h4>
|
||||
Below is the configuration of MirrorMaker 2 source connector for replicating topics.
|
||||
<!--#include virtual="generated/mirror_source_config.html" -->
|
||||
|
||||
<h4 class="anchor-heading"><a id="mirrormakercheckpointconfigs" class="anchor-link"></a><a href="#mirrormakercheckpointconfigs">3.8.3 MirrorMaker Checkpoint Configs</a></h4>
|
||||
Below is the configuration of MirrorMaker 2 checkpoint connector for emitting consumer offset checkpoints.
|
||||
<!--#include virtual="generated/mirror_checkpoint_config.html" -->
|
||||
|
||||
<h4 class="anchor-heading"><a id="mirrormakerheartbeatconfigs" class="anchor-link"></a><a href="#mirrormakerheartbeatconfigs">3.8.4 MirrorMaker HeartBeat Configs</a></h4>
|
||||
Below is the configuration of MirrorMaker 2 heartbeat connector for checking connectivity between connectors and clusters.
|
||||
<!--#include virtual="generated/mirror_heartbeat_config.html" -->
|
||||
|
||||
<h3 class="anchor-heading"><a id="systemproperties" class="anchor-link"></a><a href="#systemproperties">3.9 System Properties</a></h3>
|
||||
Kafka supports some configuration that can be enabled through Java system properties. System properties are usually set by passing the -D flag to the Java virtual machine in which Kafka components are running.
|
||||
Below are the supported system properties.
|
||||
<ul class="config-list">
|
||||
<li>
|
||||
<h4><a id="org.apache.kafka.disallowed.login.modules"></a><a id="systemproperties_org.apache.kafka.disallowed.login.modules" href="#systemproperties_org.apache.kafka.disallowed.login.modules">org.apache.kafka.disallowed.login.modules</a></h4>
|
||||
<p>This system property is used to disable the problematic login modules usage in SASL JAAS configuration. This property accepts comma-separated list of loginModule names. By default <b>com.sun.security.auth.module.JndiLoginModule</b> loginModule is disabled.
|
||||
<p>If users want to enable JndiLoginModule, users need to explicitly reset the system property like below. We advise the users to validate configurations and only allow trusted JNDI configurations. For more details <a href="https://nvd.nist.gov/vuln/detail/CVE-2023-25194">CVE-2023-25194</a>.
|
||||
<p><pre class="brush: bash;"> -Dorg.apache.kafka.disallowed.login.modules=</pre>
|
||||
<p>To disable more loginModules, update the system property with comma-separated loginModule names. Make sure to explicitly add <b>JndiLoginModule</b> module name to the comma-separated list like below.
|
||||
<p><pre class="brush: bash;"> -Dorg.apache.kafka.disallowed.login.modules=com.sun.security.auth.module.JndiLoginModule,com.ibm.security.auth.module.LdapLoginModule,com.ibm.security.auth.module.Krb5LoginModule</pre>
|
||||
<table><tbody>
|
||||
<tr><th>Since:</th><td>3.4.0</td></tr>
|
||||
<tr><th>Default Value:</th><td>com.sun.security.auth.module.JndiLoginModule</td></tr>
|
||||
</tbody></table>
|
||||
</li>
|
||||
</ul>
|
||||
<li>
|
||||
<h4><a id="org.apache.kafka.disallowed.login.modules"></a><a id="systemproperties_org.apache.kafka.disallowed.login.modules" href="#systemproperties_org.apache.kafka.disallowed.login.modules">org.apache.kafka.disallowed.login.modules</a></h4>
|
||||
<p>This system property is used to disable the problematic login modules usage in SASL JAAS configuration. This property accepts comma-separated list of loginModule names. By default <b>com.sun.security.auth.module.JndiLoginModule</b> loginModule is disabled.
|
||||
<p>If users want to enable JndiLoginModule, users need to explicitly reset the system property like below. We advise the users to validate configurations and only allow trusted JNDI configurations. For more details <a href="https://nvd.nist.gov/vuln/detail/CVE-2023-25194">CVE-2023-25194</a>.
|
||||
<p><pre class="brush: bash;"> -Dorg.apache.kafka.disallowed.login.modules=</pre>
|
||||
<p>To disable more loginModules, update the system property with comma-separated loginModule names. Make sure to explicitly add <b>JndiLoginModule</b> module name to the comma-separated list like below.
|
||||
<p><pre class="brush: bash;"> -Dorg.apache.kafka.disallowed.login.modules=com.sun.security.auth.module.JndiLoginModule,com.ibm.security.auth.module.LdapLoginModule,com.ibm.security.auth.module.Krb5LoginModule</pre>
|
||||
<table><tbody>
|
||||
<tr><th>Since:</th><td>3.4.0</td></tr>
|
||||
<tr><th>Default Value:</th><td>com.sun.security.auth.module.JndiLoginModule</td></tr>
|
||||
</tbody></table>
|
||||
</li>
|
||||
</ul>
|
||||
</script>
|
||||
|
||||
<div class="p-configuration"></div>
|
||||
|
|
|
|||
|
|
@ -805,27 +805,9 @@ us-west->us-east.emit.heartbeats = false
|
|||
</code></pre>
|
||||
|
||||
<p>
|
||||
Additional configuration settings are supported, some of which are listed below. In most cases, you can leave these settings at their default values. See <a href="https://github.com/apache/kafka/blob/trunk/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java">MirrorMakerConfig</a> and <a href="https://github.com/apache/kafka/blob/trunk/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorConnectorConfig.java">MirrorConnectorConfig</a> for further details.
|
||||
Additional configuration settings are supported which can be left with their default values in most cases. See <a href="/documentation/#mirrormakerconfigs">MirrorMaker Configs</a>.
|
||||
</p>
|
||||
|
||||
<ul>
|
||||
<li><code>refresh.topics.enabled</code>: whether to check for new topics in the source cluster periodically (default: true)
|
||||
<li><code>refresh.topics.interval.seconds</code>: frequency of checking for new topics in the source cluster; lower values than the default may lead to performance degradation (default: 600, every ten minutes)
|
||||
<li><code>refresh.groups.enabled</code>: whether to check for new consumer groups in the source cluster periodically (default: true)
|
||||
<li><code>refresh.groups.interval.seconds</code>: frequency of checking for new consumer groups in the source cluster; lower values than the default may lead to performance degradation (default: 600, every ten minutes)
|
||||
<li><code>sync.topic.configs.enabled</code>: whether to replicate topic configurations from the source cluster (default: true)
|
||||
<li><code>sync.topic.acls.enabled</code>: whether to sync ACLs from the source cluster (default: true)
|
||||
<li><code>emit.heartbeats.enabled</code>: whether to emit heartbeats periodically (default: true)
|
||||
<li><code>emit.heartbeats.interval.seconds</code>: frequency at which heartbeats are emitted (default: 1, every one seconds)
|
||||
<li><code>heartbeats.topic.replication.factor</code>: replication factor of MirrorMaker's internal heartbeat topics (default: 3)
|
||||
<li><code>emit.checkpoints.enabled</code>: whether to emit MirrorMaker's consumer offsets periodically (default: true)
|
||||
<li><code>emit.checkpoints.interval.seconds</code>: frequency at which checkpoints are emitted (default: 60, every minute)
|
||||
<li><code>checkpoints.topic.replication.factor</code>: replication factor of MirrorMaker's internal checkpoints topics (default: 3)
|
||||
<li><code>sync.group.offsets.enabled</code>: whether to periodically write the translated offsets of replicated consumer groups (in the source cluster) to <code>__consumer_offsets</code> topic in target cluster, as long as no active consumers in that group are connected to the target cluster (default: false)
|
||||
<li><code>sync.group.offsets.interval.seconds</code>: frequency at which consumer group offsets are synced (default: 60, every minute)
|
||||
<li><code>offset-syncs.topic.replication.factor</code>: replication factor of MirrorMaker's internal offset-sync topics (default: 3)
|
||||
</ul>
|
||||
|
||||
<h5 class="anchor-heading"><a id="georeplication-flow-secure" class="anchor-link"></a><a href="#georeplication-flow-secure">Securing Replication Flows</a></h5>
|
||||
|
||||
<p>
|
||||
|
|
|
|||
Loading…
Reference in New Issue