diff --git a/spring-boot-cli/src/test/java/cli/command/CustomCompilerAutoConfiguration.java b/spring-boot-cli/src/test/java/cli/command/CustomCompilerAutoConfiguration.java
deleted file mode 100644
index 7da36c57c09..00000000000
--- a/spring-boot-cli/src/test/java/cli/command/CustomCompilerAutoConfiguration.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2012-2013 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cli.command;
-
-import groovy.lang.GroovyClassLoader;
-
-import org.codehaus.groovy.ast.ClassNode;
-import org.codehaus.groovy.classgen.GeneratorContext;
-import org.codehaus.groovy.control.CompilationFailedException;
-import org.codehaus.groovy.control.SourceUnit;
-import org.springframework.boot.cli.compiler.CompilerAutoConfiguration;
-import org.springframework.boot.cli.compiler.GroovyCompilerConfiguration;
-
-/**
- * @author Andy Wilkinson
- */
-public class CustomCompilerAutoConfiguration extends CompilerAutoConfiguration {
-
-	@Override
-	public void apply(GroovyClassLoader loader,
-			GroovyCompilerConfiguration configuration, GeneratorContext generatorContext,
-			SourceUnit source, ClassNode classNode) throws CompilationFailedException {
-		System.out.println("Customising compiler configuration");
-	}
-
-}
diff --git a/spring-boot-cli/src/test/java/org/springframework/boot/cli/command/InitCommandTests.java b/spring-boot-cli/src/test/java/org/springframework/boot/cli/command/InitCommandTests.java
index b913bf0adef..f6020ef3b95 100644
--- a/spring-boot-cli/src/test/java/org/springframework/boot/cli/command/InitCommandTests.java
+++ b/spring-boot-cli/src/test/java/org/springframework/boot/cli/command/InitCommandTests.java
@@ -18,8 +18,6 @@ package org.springframework.boot.cli.command;
 
 import groovy.lang.GroovyClassLoader;
 
-import java.util.ServiceLoader;
-
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -27,9 +25,6 @@ import org.junit.Test;
 import org.springframework.boot.OutputCapture;
 import org.springframework.boot.cli.Command;
 import org.springframework.boot.cli.SpringCli;
-import org.springframework.boot.cli.compiler.CompilerAutoConfiguration;
-
-import cli.command.CustomCompilerAutoConfiguration;
 
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
@@ -72,22 +67,6 @@ public class InitCommandTests {
 		this.command.run("src/test/resources/grab.groovy");
 		verify(this.cli, times(this.defaultCount + 1)).register(any(Command.class));
 		assertTrue(this.output.toString().contains("Hello Grab"));
-
-		Iterable<CompilerAutoConfiguration> autoConfigurations = ServiceLoader.load(
-				CompilerAutoConfiguration.class, Thread.currentThread()
-						.getContextClassLoader());
-
-		boolean foundCustomConfiguration = false;
-
-		for (CompilerAutoConfiguration autoConfiguration : autoConfigurations) {
-			if (CustomCompilerAutoConfiguration.class.getName().equals(
-					autoConfiguration.getClass().getName())) {
-				foundCustomConfiguration = true;
-				break;
-			}
-		}
-
-		assertTrue(foundCustomConfiguration);
 	}
 
 	@Test
diff --git a/spring-boot-cli/src/test/plugins/custom/META-INF/services/org.springframework.boot.cli.compiler.CompilerAutoConfiguration b/spring-boot-cli/src/test/plugins/custom/META-INF/services/org.springframework.boot.cli.compiler.CompilerAutoConfiguration
deleted file mode 100644
index c59f47ea9ec..00000000000
--- a/spring-boot-cli/src/test/plugins/custom/META-INF/services/org.springframework.boot.cli.compiler.CompilerAutoConfiguration
+++ /dev/null
@@ -1 +0,0 @@
-cli.command.CustomCompilerAutoConfiguration
\ No newline at end of file
diff --git a/spring-boot-cli/src/test/plugins/custom/custom/0.0.1/custom-0.0.1.jar b/spring-boot-cli/src/test/plugins/custom/custom/0.0.1/custom-0.0.1.jar
index 062524811f1..6623163c931 100644
Binary files a/spring-boot-cli/src/test/plugins/custom/custom/0.0.1/custom-0.0.1.jar and b/spring-boot-cli/src/test/plugins/custom/custom/0.0.1/custom-0.0.1.jar differ