root
/
kafka
mirror of https://github.com/apache/kafka.git
1
0
Fork 0
Code Issues Actions 2 Packages Projects Releases Wiki Activity

KAFKA-14645: Use plugin classloader when retrieving connector plugin config definitions (#13148) 

Reviewers: Mickael Maison <mickael.maison@gmail.com>, Greg Harris <gharris1727@gmail.com>
This commit is contained in:
Chris Egerton 2023-01-30 12:06:02 -05:00 committed by GitHub
parent 72cfc994f5
commit 17559d581e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
View File

@ -797,12 +797,18 @@ public abstract class AbstractHerder implements Herder, TaskStatus.Listener, Con
@Override
public List<ConfigKeyInfo> connectorPluginConfig(String pluginName) {
List<ConfigKeyInfo> results = new ArrayList<>();
ConfigDef configDefs;
try {
Plugins p = plugins();
Class<?> pluginClass;
try {
pluginClass = p.pluginClass(pluginName);
} catch (ClassNotFoundException cnfe) {
throw new NotFoundException("Unknown plugin " + pluginName + ".");
}
try (LoaderSwap loaderSwap = p.withClassLoader(pluginClass.getClassLoader())) {
Object plugin = p.newPlugin(pluginName);
PluginType pluginType = PluginType.from(plugin.getClass());
ConfigDef configDefs;
switch (pluginType) {
case SINK:
case SOURCE:
@ -823,13 +829,14 @@ public abstract class AbstractHerder implements Herder, TaskStatus.Listener, Con
default:
throw new BadRequestException("Invalid plugin type " + pluginType + ". Valid types are sink, source, converter, header_converter, transformation, predicate.");
}
} catch (ClassNotFoundException cnfe) {
throw new NotFoundException("Unknown plugin " + pluginName + ".");
}
List<ConfigKeyInfo> results = new ArrayList<>();
for (ConfigDef.ConfigKey configKey : configDefs.configKeys().values()) {
results.add(AbstractHerder.convertConfigKey(configKey));
}
return results;
} catch (ClassNotFoundException e) {
throw new ConnectException("Failed to load plugin class or one of its dependencies", e);
}
}
}

4
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java
View File

@ -123,6 +123,10 @@ public class Plugins {
);
}
public Class<?> pluginClass(String classOrAlias) throws ClassNotFoundException {
return pluginClass(delegatingLoader, classOrAlias, Object.class);
}
public static ClassLoader compareAndSwapLoaders(ClassLoader loader) {
ClassLoader current = Thread.currentThread().getContextClassLoader();
if (!current.equals(loader)) {

76
connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
View File

@ -67,6 +67,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues;
@ -899,48 +900,51 @@ public class AbstractHerderTest {
}
@Test
public void testConnectorPluginConfig() throws Exception {
public void testSinkConnectorPluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("sink", SampleSinkConnector::new, SampleSinkConnector::config);
}
@Test
public void testSourceConnectorPluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("source", SampleSourceConnector::new, SampleSourceConnector::config);
}
@Test
public void testConverterPluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("converter", SampleConverterWithHeaders::new, SampleConverterWithHeaders::config);
}
@Test
public void testHeaderConverterPluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("header-converter", SampleHeaderConverter::new, SampleHeaderConverter::config);
}
@Test
public void testPredicatePluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("predicate", SamplePredicate::new, SamplePredicate::config);
}
@Test
public void testTransformationPluginConfig() throws ClassNotFoundException {
testConnectorPluginConfig("transformation", SampleTransformation::new, SampleTransformation::config);
}
private <T> void testConnectorPluginConfig(String pluginName, Supplier<T> newPluginInstance, Function<T, ConfigDef> pluginConfig) throws ClassNotFoundException {
AbstractHerder herder = mock(AbstractHerder.class, withSettings()
.useConstructor(worker, workerId, kafkaClusterId, statusStore, configStore, noneConnectorClientConfigOverridePolicy)
.defaultAnswer(CALLS_REAL_METHODS));
when(plugins.newPlugin(anyString())).then(invocation -> {
String name = invocation.getArgument(0);
switch (name) {
case "sink": return new SampleSinkConnector();
case "source": return new SampleSourceConnector();
case "converter": return new SampleConverterWithHeaders();
case "header-converter": return new SampleHeaderConverter();
case "predicate": return new SamplePredicate();
default: return new SampleTransformation<>();
}
});
when(plugins.pluginClass(pluginName)).then(invocation -> newPluginInstance.get().getClass());
when(plugins.newPlugin(anyString())).then(invocation -> newPluginInstance.get());
when(herder.plugins()).thenReturn(plugins);
List<ConfigKeyInfo> sinkConnectorConfigs = herder.connectorPluginConfig("sink");
assertNotNull(sinkConnectorConfigs);
assertEquals(new SampleSinkConnector().config().names().size(), sinkConnectorConfigs.size());
List<ConfigKeyInfo> configs = herder.connectorPluginConfig(pluginName);
assertNotNull(configs);
List<ConfigKeyInfo> sourceConnectorConfigs = herder.connectorPluginConfig("source");
assertNotNull(sourceConnectorConfigs);
assertEquals(new SampleSourceConnector().config().names().size(), sourceConnectorConfigs.size());
List<ConfigKeyInfo> converterConfigs = herder.connectorPluginConfig("converter");
assertNotNull(converterConfigs);
assertEquals(new SampleConverterWithHeaders().config().names().size(), converterConfigs.size());
List<ConfigKeyInfo> headerConverterConfigs = herder.connectorPluginConfig("header-converter");
assertNotNull(headerConverterConfigs);
assertEquals(new SampleHeaderConverter().config().names().size(), headerConverterConfigs.size());
List<ConfigKeyInfo> predicateConfigs = herder.connectorPluginConfig("predicate");
assertNotNull(predicateConfigs);
assertEquals(new SamplePredicate().config().names().size(), predicateConfigs.size());
List<ConfigKeyInfo> transformationConfigs = herder.connectorPluginConfig("transformation");
assertNotNull(transformationConfigs);
assertEquals(new SampleTransformation<>().config().names().size(), transformationConfigs.size());
ConfigDef expectedConfig = pluginConfig.apply(newPluginInstance.get());
assertEquals(expectedConfig.names().size(), configs.size());
// Make sure that we used the correct class loader when interacting with the plugin
verify(plugins).withClassLoader(newPluginInstance.get().getClass().getClassLoader());
}
@Test(expected = NotFoundException.class)
@ -950,17 +954,19 @@ public class AbstractHerderTest {
.useConstructor(worker, workerId, kafkaClusterId, statusStore, configStore, noneConnectorClientConfigOverridePolicy)
.defaultAnswer(CALLS_REAL_METHODS));
when(worker.getPlugins()).thenReturn(plugins);
when(plugins.newPlugin(anyString())).thenThrow(new ClassNotFoundException());
when(plugins.pluginClass(anyString())).thenThrow(new ClassNotFoundException());
herder.connectorPluginConfig(connName);
}
@Test(expected = BadRequestException.class)
@SuppressWarnings({"rawtypes", "unchecked"})
public void testGetConnectorConfigDefWithInvalidPluginType() throws Exception {
String connName = "AnotherPlugin";
AbstractHerder herder = mock(AbstractHerder.class, withSettings()
.useConstructor(worker, workerId, kafkaClusterId, statusStore, configStore, noneConnectorClientConfigOverridePolicy)
.defaultAnswer(CALLS_REAL_METHODS));
when(worker.getPlugins()).thenReturn(plugins);
when(plugins.pluginClass(anyString())).thenReturn((Class) Object.class);
when(plugins.newPlugin(anyString())).thenReturn(new DirectoryConfigProvider());
herder.connectorPluginConfig(connName);
}