Add WebTestClient HtmlUnit Support

Fixes gh-4534

config/spring-security-config.gradle

@@ -36,6 +36,9 @@ dependencies {
 	testCompile 'ch.qos.logback:logback-classic'
 	testCompile 'javax.annotation:jsr250-api:1.0'
 	testCompile 'ldapsdk:ldapsdk:4.1'
+	testCompile('net.sourceforge.htmlunit:htmlunit') {
+		exclude group: 'commons-logging', module: 'commons-logging'
+	}
 	testCompile 'org.codehaus.groovy:groovy-all'
 	testCompile 'org.eclipse.persistence:javax.persistence'
 	testCompile 'org.hibernate:hibernate-entitymanager'
@@ -43,6 +46,13 @@ dependencies {
 	testCompile ('org.openid4java:openid4java-nodeps') {
 		exclude group: 'com.google.code.guice', module: 'guice'
 	}
+	testCompile('org.seleniumhq.selenium:htmlunit-driver') {
+		exclude group: 'commons-logging', module: 'commons-logging'
+	}
+	testCompile('org.seleniumhq.selenium:selenium-java') {
+		exclude group: 'commons-logging', module: 'commons-logging'
+		exclude group: 'io.netty', module: 'netty'
+	}
 	testCompile 'org.slf4j:jcl-over-slf4j'
 	testCompile 'org.springframework.ldap:spring-ldap-core'
 	testCompile 'org.springframework:spring-expression'

config/src/test/java/org/springframework/security/htmlunit/server/HtmlUnitWebTestClient.java

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2002-2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.security.htmlunit.server;
+
+import java.net.URI;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import com.gargoylesoftware.htmlunit.FormEncodingType;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.WebRequest;
+
+import com.gargoylesoftware.htmlunit.util.NameValuePair;
+import org.springframework.http.HttpCookie;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseCookie;
+import org.springframework.lang.Nullable;
+import org.springframework.test.web.reactive.server.FluxExchangeResult;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.util.Assert;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.reactive.function.BodyInserters;
+import org.springframework.web.reactive.function.client.ClientRequest;
+import org.springframework.web.reactive.function.client.ClientResponse;
+import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
+import org.springframework.web.reactive.function.client.ExchangeFunction;
+import reactor.core.publisher.Mono;
+
+final class HtmlUnitWebTestClient {
+
+	private final WebClient webClient;
+
+	private final WebTestClient webTestClient;
+
+	public HtmlUnitWebTestClient(WebClient webClient, WebTestClient webTestClient) {
+		Assert.notNull(webClient, "WebClient must not be null");
+		Assert.notNull(webTestClient, "WebTestClient must not be null");
+		this.webClient = webClient;
+		this.webTestClient = webTestClient.mutate()
+			.filter(new FollowRedirects())
+			.filter(new CookieManager())
+			.build();
+	}
+
+	public FluxExchangeResult<String> getResponse(WebRequest webRequest) {
+		WebTestClient.RequestBodySpec request = this.webTestClient
+				.method(httpMethod(webRequest))
+				.uri(uri(webRequest));
+		contentType(request, webRequest);
+		cookies(request, webRequest);
+		headers(request, webRequest);
+
+
+		return content(request, webRequest).exchange().returnResult(String.class);
+	}
+
+	private WebTestClient.RequestHeadersSpec<?> content(WebTestClient.RequestBodySpec request, WebRequest webRequest) {
+		String requestBody = webRequest.getRequestBody();
+		if (requestBody == null) {
+			List<NameValuePair> params = webRequest.getRequestParameters();
+			if(params != null && !params.isEmpty()) {
+				return request.body(BodyInserters.fromFormData(formData(params)));
+			}
+			return request;
+		}
+		return request.body(BodyInserters.fromObject(requestBody));
+	}
+
+	private MultiValueMap<String,String> formData(List<NameValuePair> params) {
+		MultiValueMap<String,String> result = new LinkedMultiValueMap<>(params.size());
+		params.forEach( pair -> result.add(pair.getName(), pair.getValue()));
+		return result;
+	}
+
+	private void contentType(WebTestClient.RequestBodySpec request, WebRequest webRequest) {
+		String contentType = header("Content-Type", webRequest);
+		if (contentType == null) {
+			FormEncodingType encodingType = webRequest.getEncodingType();
+			if (encodingType != null) {
+				contentType = encodingType.getName();
+			}
+		}
+		MediaType mediaType = contentType == null ? MediaType.ALL : MediaType.parseMediaType(contentType);
+		request.contentType(mediaType);
+	}
+
+	private void cookies(WebTestClient.RequestBodySpec request, WebRequest webRequest) {
+		String cookieHeaderValue = header("Cookie", webRequest);
+		if (cookieHeaderValue != null) {
+			StringTokenizer tokens = new StringTokenizer(cookieHeaderValue, "=;");
+			while (tokens.hasMoreTokens()) {
+				String cookieName = tokens.nextToken().trim();
+				Assert.isTrue(tokens.hasMoreTokens(),
+						() -> "Expected value for cookie name '" + cookieName +
+								"': full cookie header was [" + cookieHeaderValue + "]");
+				String cookieValue = tokens.nextToken().trim();
+				request.cookie(cookieName, cookieValue);
+			}
+		}
+
+		Set<com.gargoylesoftware.htmlunit.util.Cookie> managedCookies = this.webClient.getCookies(webRequest.getUrl());
+		for (com.gargoylesoftware.htmlunit.util.Cookie cookie : managedCookies) {
+			request.cookie(cookie.getName(), cookie.getValue());
+		}
+	}
+
+	@Nullable
+	private String header(String headerName, WebRequest webRequest) {
+		return webRequest.getAdditionalHeaders().get(headerName);
+	}
+
+	private void headers(WebTestClient.RequestBodySpec request, WebRequest webRequest) {
+		webRequest.getAdditionalHeaders().forEach( (name,value) -> request.header(name, value));
+	}
+
+	private HttpMethod httpMethod(WebRequest webRequest) {
+		String httpMethod = webRequest.getHttpMethod().name();
+		return HttpMethod.valueOf(httpMethod);
+	}
+
+	private URI uri(WebRequest webRequest) {
+		URL url = webRequest.getUrl();
+		return URI.create(url.toExternalForm());
+	}
+
+	static class FollowRedirects implements ExchangeFilterFunction {
+		@Override
+		public Mono<ClientResponse> filter(ClientRequest request, ExchangeFunction next) {
+			return next.exchange(request)
+				.flatMap( response -> redirectIfNecessary(request, next, response));
+		}
+
+		private Mono<ClientResponse> redirectIfNecessary(ClientRequest request, ExchangeFunction next, ClientResponse response) {
+			URI location = response.headers().asHttpHeaders().getLocation();
+			if(location != null) {
+				ClientRequest redirect = ClientRequest.method(HttpMethod.GET, URI.create("http://localhost" + location.toASCIIString()))
+					.headers(headers -> headers.addAll(request.headers()))
+					.cookies(cookies -> cookies.addAll(request.cookies()))
+					.attributes(attributes -> attributes.putAll(request.attributes()))
+					.build();
+
+				return next.exchange(redirect).flatMap( r -> redirectIfNecessary(request, next, r));
+			}
+
+			return Mono.just(response);
+		}
+	}
+
+	static class CookieManager implements ExchangeFilterFunction {
+		private Map<String, ResponseCookie> cookies = new HashMap<>();
+
+		@Override
+		public Mono<ClientResponse> filter(ClientRequest request, ExchangeFunction next) {
+			return next.exchange(withClientCookies(request))
+				.doOnSuccess( response -> {
+					response.cookies().values().forEach( cookies -> {
+						cookies.forEach( cookie -> {
+							if(cookie.getMaxAge().isZero()) {
+								this.cookies.remove(cookie.getName());
+							} else {
+								this.cookies.put(cookie.getName(), cookie);
+							}
+						});
+					});
+				});
+		}
+
+		private ClientRequest withClientCookies(ClientRequest request) {
+			return ClientRequest.from(request)
+				.cookies( c -> {
+					c.addAll(clientCookies());
+				}).build();
+		}
+
+		private MultiValueMap<String,String> clientCookies() {
+			MultiValueMap<String,String> result = new LinkedMultiValueMap<>(this.cookies.size());
+			this.cookies.values().forEach( cookie ->
+				result.add(cookie.getName(), cookie.getValue())
+			);
+			return result;
+		}
+	}
+}

config/src/test/java/org/springframework/security/htmlunit/server/MockWebResponseBuilder.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2002-2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.security.htmlunit.server;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.gargoylesoftware.htmlunit.WebRequest;
+import com.gargoylesoftware.htmlunit.WebResponse;
+import com.gargoylesoftware.htmlunit.WebResponseData;
+import com.gargoylesoftware.htmlunit.util.NameValuePair;
+
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.web.reactive.server.FluxExchangeResult;
+import org.springframework.util.Assert;
+
+/**
+ * @author Rob Winch
+ * @since 5.0
+ */
+final class MockWebResponseBuilder {
+	private final long startTime;
+
+	private final WebRequest webRequest;
+
+	private final FluxExchangeResult<String> exchangeResult;
+
+
+	public MockWebResponseBuilder(long startTime, WebRequest webRequest, FluxExchangeResult<String> exchangeResult) {
+		Assert.notNull(webRequest, "WebRequest must not be null");
+		Assert.notNull(exchangeResult, "FluxExchangeResult must not be null");
+		this.startTime = startTime;
+		this.webRequest = webRequest;
+		this.exchangeResult = exchangeResult;
+	}
+
+
+	public WebResponse build() throws IOException {
+		WebResponseData webResponseData = webResponseData();
+		long endTime = System.currentTimeMillis();
+		return new WebResponse(webResponseData, this.webRequest, endTime - this.startTime);
+	}
+
+	private WebResponseData webResponseData() throws IOException {
+		List<NameValuePair> responseHeaders = responseHeaders();
+		HttpStatus status = this.exchangeResult.getStatus();
+		return new WebResponseData(this.exchangeResult.getResponseBodyContent(), status.value(), status.getReasonPhrase(), responseHeaders);
+	}
+
+	private List<NameValuePair> responseHeaders() {
+		HttpHeaders responseHeaders = this.exchangeResult.getResponseHeaders();
+		List<NameValuePair> result = new ArrayList<>(responseHeaders.size());
+		responseHeaders.forEach( (headerName, headerValues) ->
+			headerValues.forEach( headerValue ->
+				result.add(new NameValuePair(headerName, headerValue))
+			)
+		);
+		return result;
+	}
+
+}

config/src/test/java/org/springframework/security/htmlunit/server/WebTestClientHtmlUnitDriverBuilder.java

@@ -0,0 +1,48 @@
+/*
+ *
+ *  * Copyright 2002-2017 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.security.htmlunit.server;
+
+import com.gargoylesoftware.htmlunit.WebClient;
+import org.openqa.selenium.WebDriver;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.test.web.servlet.htmlunit.webdriver.WebConnectionHtmlUnitDriver;
+
+/**
+ * @author Rob Winch
+ * @since 5.0
+ */
+public class WebTestClientHtmlUnitDriverBuilder {
+	private final WebTestClient webTestClient;
+
+	private WebTestClientHtmlUnitDriverBuilder(WebTestClient webTestClient) {
+		this.webTestClient = webTestClient;
+	}
+
+	public WebDriver build() {
+		WebConnectionHtmlUnitDriver driver = new WebConnectionHtmlUnitDriver();
+		WebClient webClient = driver.getWebClient();
+		WebTestClientWebConnection connection = new WebTestClientWebConnection(this.webTestClient, webClient);
+		driver.setWebConnection(connection);
+		return driver;
+	}
+
+	public static WebTestClientHtmlUnitDriverBuilder webTestClientSetup(WebTestClient webTestClient) {
+		return new WebTestClientHtmlUnitDriverBuilder(webTestClient);
+	}
+}

config/src/test/java/org/springframework/security/htmlunit/server/WebTestClientHtmlUnitDriverBuilderTests.java

@@ -0,0 +1,131 @@
+/*
+ *
+ *  * Copyright 2002-2017 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.security.htmlunit.server;
+
+import org.junit.Test;
+import org.openqa.selenium.WebDriver;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseCookie;
+import org.springframework.http.server.reactive.ServerHttpResponse;
+import org.springframework.security.web.util.TextEscapeUtils;
+import org.springframework.stereotype.Controller;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.web.bind.annotation.CookieValue;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+import reactor.core.publisher.Mono;
+
+import java.net.URI;
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Rob Winch
+ * @since 5.0
+ */
+public class WebTestClientHtmlUnitDriverBuilderTests {
+
+	@Test
+	public void helloWorld() {
+		WebTestClient webTestClient = WebTestClient
+			.bindToController(new HelloWorldController())
+			.build();
+		WebDriver driver = WebTestClientHtmlUnitDriverBuilder
+			.webTestClientSetup(webTestClient).build();
+
+		driver.get("http://localhost/");
+
+		assertThat(driver.getPageSource()).contains("Hello World");
+	}
+
+	/**
+	 * @author Rob Winch
+	 * @since 5.0
+	 */
+	@Controller
+	class HelloWorldController {
+		@ResponseBody
+		@GetMapping(produces = MediaType.TEXT_HTML_VALUE)
+		public String index() {
+			return "<html>\n"
+				+ "<head>\n"
+				+ "<title>Hello World</title>\n"
+				+ "</head>\n"
+				+ "<body>\n"
+				+ "<h1>Hello World</h1>\n"
+				+ "</body>\n"
+				+ "</html>";
+		}
+	}
+
+	@Test
+	public void cookies() {
+		WebTestClient webTestClient = WebTestClient
+			.bindToController(new CookieController())
+			.build();
+		WebDriver driver = WebTestClientHtmlUnitDriverBuilder
+			.webTestClientSetup(webTestClient).build();
+
+		driver.get("http://localhost/cookie");
+
+		assertThat(driver.getPageSource()).contains("theCookie");
+
+		driver.get("http://localhost/cookie/delete");
+
+		assertThat(driver.getPageSource()).contains("null");
+	}
+
+	@Controller
+	@ResponseBody
+	class CookieController {
+		@GetMapping(path = "/", produces = MediaType.TEXT_HTML_VALUE)
+		public String view(@CookieValue(required = false) String cookieName) {
+			return "<html>\n"
+				+ "<head>\n"
+				+ "<title>Hello World</title>\n"
+				+ "</head>\n"
+				+ "<body>\n"
+				+ "<h1>" + TextEscapeUtils.escapeEntities(cookieName) + "</h1>\n"
+				+ "</body>\n"
+				+ "</html>";
+		}
+
+		@GetMapping("/cookie")
+		public Mono<Void> setCookie(ServerHttpResponse response) {
+			response.addCookie(ResponseCookie.from("cookieName", "theCookie").build());
+			return redirect(response);
+		}
+
+		private Mono<Void> redirect(ServerHttpResponse response) {
+			response.setStatusCode(HttpStatus.MOVED_PERMANENTLY);
+			response.getHeaders().setLocation(URI.create("/"));
+			return response.setComplete();
+		}
+
+		@GetMapping("/cookie/delete")
+		public Mono<Void> deleteCookie(ServerHttpResponse response) {
+			response.addCookie(
+				ResponseCookie.from("cookieName", "").maxAge(Duration.ofSeconds(0)).build());
+			return redirect(response);
+		}
+	}
+}

config/src/test/java/org/springframework/security/htmlunit/server/WebTestClientWebConnection.java

@@ -0,0 +1,95 @@
+/*
+ *
+ *  * Copyright 2002-2017 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.security.htmlunit.server;
+
+import java.io.IOException;
+
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.WebConnection;
+import com.gargoylesoftware.htmlunit.WebRequest;
+import com.gargoylesoftware.htmlunit.WebResponse;
+
+import org.springframework.lang.Nullable;
+import org.springframework.test.web.reactive.server.FluxExchangeResult;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.util.Assert;
+
+/**
+ * @author Rob Winch
+ * @since 5.0
+ */
+public class WebTestClientWebConnection implements WebConnection {
+
+	private final WebTestClient webTestClient;
+
+	private final String contextPath;
+	private final HtmlUnitWebTestClient requestBuilder;
+
+	private WebClient webClient;
+
+
+	public WebTestClientWebConnection(WebTestClient webTestClient, WebClient webClient) {
+		this(webTestClient, webClient, "");
+	}
+
+	public WebTestClientWebConnection(WebTestClient webTestClient, WebClient webClient, String contextPath) {
+		Assert.notNull(webTestClient, "MockMvc must not be null");
+		Assert.notNull(webClient, "WebClient must not be null");
+		validateContextPath(contextPath);
+
+		this.webClient = webClient;
+		this.webTestClient = webTestClient;
+		this.contextPath = contextPath;
+		this.requestBuilder = new HtmlUnitWebTestClient(this.webClient, this.webTestClient);
+	}
+
+	/**
+	 * Validate the supplied {@code contextPath}.
+	 * <p>If the value is not {@code null}, it must conform to
+	 * {@link javax.servlet.http.HttpServletRequest#getContextPath()} which
+	 * states that it can be an empty string and otherwise must start with
+	 * a "/" character and not end with a "/" character.
+	 * @param contextPath the path to validate
+	 */
+	static void validateContextPath(@Nullable String contextPath) {
+		if (contextPath == null || "".equals(contextPath)) {
+			return;
+		}
+		Assert.isTrue(contextPath.startsWith("/"), () -> "contextPath '" + contextPath + "' must start with '/'.");
+		Assert.isTrue(!contextPath.endsWith("/"), () -> "contextPath '" + contextPath + "' must not end with '/'.");
+	}
+
+
+	public void setWebClient(WebClient webClient) {
+		Assert.notNull(webClient, "WebClient must not be null");
+		this.webClient = webClient;
+	}
+
+	@Override
+	public WebResponse getResponse(WebRequest webRequest) throws IOException {
+		long startTime = System.currentTimeMillis();
+
+		FluxExchangeResult<String> exchangeResult = this.requestBuilder.getResponse(webRequest);
+
+		return new MockWebResponseBuilder(startTime, webRequest, exchangeResult).build();
+	}
+
+	@Override
+	public void close() {}
+}

gradle/dependency-management.gradle

@@ -20,6 +20,7 @@ dependencyManagement {
 		dependency 'org.powermock:powermock-module-junit4:1.6.2'
 		dependency 'org.powermock:powermock-reflect:1.6.2'
 		dependency 'org.python:jython:2.5.0'
+		dependency 'org.seleniumhq.selenium:selenium-java:3.4.0'
 		dependency 'org.spockframework:spock-core:1.0-groovy-2.4'
 		dependency 'org.spockframework:spock-spring:1.0-groovy-2.4'
 	}

webflux/src/test/java/org/springframework/security/test/web/reactive/server/WebTestClientBuilder.java

@@ -18,12 +18,15 @@
 package org.springframework.security.test.web.reactive.server;
 
 import org.springframework.http.HttpStatus;
+import org.springframework.security.web.server.SecurityWebFilterChain;
+import org.springframework.security.web.server.WebFilterChainFilter;
 import org.springframework.test.web.reactive.server.WebTestClient;
 import org.springframework.test.web.reactive.server.WebTestClient.Builder;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.WebFilter;
+import reactor.core.publisher.Flux;
 
 /**
  * Provides a convenient mechanism for running {@link WebTestClient} against
@@ -39,6 +42,10 @@ public class WebTestClientBuilder {
 		return WebTestClient.bindToController(new Http200RestController()).webFilter(webFilters).configureClient();
 	}
 
+	public static Builder bindToWebFilters(SecurityWebFilterChain securityWebFilterChain) {
+		return bindToWebFilters(WebFilterChainFilter.fromSecurityWebFilterChains(securityWebFilterChain));
+	}
+
 	@RestController
 	static class Http200RestController {
 		@RequestMapping("/**")