Auto-configure Elasticsearch REST clients

This commit adds auto-configuration support for both `RestClient` and
`RestHighLevelClient` which are provided by `elasticsearch-rest-client`
and `elasticsearch-rest-high-level-client` dependencies respectively.

`RestClient` is associated with configuration properties in the
`spring.elasticsearch.rest.*` namespace, since this is the component
taking care of HTTP communication with the actual Elasticsearch node.

`RestHighLevelClient` wraps the first one and naturally inherits that
configuration.

Closes gh-12600

spring-boot-project/spring-boot-autoconfigure/pom.xml

@@ -305,6 +305,16 @@
 			<artifactId>ehcache</artifactId>
 			<optional>true</optional>
 		</dependency>
+		<dependency>
+			<groupId>org.elasticsearch.client</groupId>
+			<artifactId>elasticsearch-rest-client</artifactId>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>org.elasticsearch.client</groupId>
+			<artifactId>elasticsearch-rest-high-level-client</artifactId>
+			<optional>true</optional>
+		</dependency>
 		<dependency>
 			<groupId>org.freemarker</groupId>
 			<artifactId>freemarker</artifactId>

spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/elasticsearch/rest/RestClientAutoConfiguration.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2012-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.autoconfigure.elasticsearch.rest;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.boot.context.properties.PropertyMapper;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * {@link EnableAutoConfiguration Auto-Configuration}
+ * for Elasticseach REST clients.
+ *
+ * @author Brian Clozel
+ * @since 2.1.0
+ */
+@Configuration
+@ConditionalOnClass(RestClient.class)
+@EnableConfigurationProperties(RestClientProperties.class)
+public class RestClientAutoConfiguration {
+
+
+	private final RestClientProperties properties;
+
+	private final List<RestClientBuilderCustomizer> builderCustomizers;
+
+	public RestClientAutoConfiguration(RestClientProperties properties,
+			ObjectProvider<List<RestClientBuilderCustomizer>> builderCustomizers) {
+		this.properties = properties;
+		this.builderCustomizers = builderCustomizers.getIfAvailable(Collections::emptyList);
+	}
+
+	@Bean(destroyMethod = "close")
+	@ConditionalOnMissingBean
+	public RestClient restClient() {
+		RestClientBuilder builder = configureBuilder();
+		return builder.build();
+	}
+
+	protected RestClientBuilder configureBuilder() {
+		HttpHost[] hosts = this.properties.getUris().stream()
+				.map(HttpHost::create).toArray(HttpHost[]::new);
+		RestClientBuilder builder = RestClient.builder(hosts);
+		PropertyMapper map = PropertyMapper.get();
+		map.from(this.properties::getUsername).whenHasText().to((username) -> {
+			CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+			Credentials credentials = new UsernamePasswordCredentials(
+					this.properties.getUsername(), this.properties.getPassword());
+			credentialsProvider.setCredentials(AuthScope.ANY, credentials);
+			builder.setHttpClientConfigCallback(httpClientBuilder ->
+					httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+		});
+		this.builderCustomizers.forEach((customizer) -> customizer.customize(builder));
+		return builder;
+	}
+
+	@Configuration
+	@ConditionalOnClass(RestHighLevelClient.class)
+	public static class RestHighLevelClientConfiguration {
+
+		@Bean
+		@ConditionalOnMissingBean
+		public RestHighLevelClient restHighLevelClient(RestClient restClient) {
+			return new RestHighLevelClient(restClient);
+		}
+
+	}
+
+}

spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/elasticsearch/rest/RestClientBuilderCustomizer.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2012-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.autoconfigure.elasticsearch.rest;
+
+import org.elasticsearch.client.RestClientBuilder;
+
+/**
+ * Callback interface that can be implemented by beans wishing to further customize the
+ * {@link org.elasticsearch.client.RestClient} via a {@link RestClientBuilder} whilst
+ * retaining default auto-configuration.
+ *
+ * @author Brian Clozel
+ * @since 2.1.0
+ */
+@FunctionalInterface
+public interface RestClientBuilderCustomizer {
+
+	/**
+	 * Customize the {@link RestClientBuilder}.
+	 * @param builder the builder to customize
+	 */
+	void customize(RestClientBuilder builder);
+}

spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/elasticsearch/rest/RestClientProperties.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2012-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.autoconfigure.elasticsearch.rest;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Configuration properties for Elasticsearch REST clients.
+ *
+ * @author Brian Clozel
+ * @since 2.1.0
+ */
+@ConfigurationProperties(prefix = "spring.elasticsearch.rest")
+public class RestClientProperties {
+
+	/**
+	 * Comma-separated list of the Elasticsearch instances to use.
+	 */
+	private List<String> uris = new ArrayList<>(
+			Collections.singletonList("http://localhost:9200"));
+
+	/**
+	 * Credentials username.
+	 */
+	private String username;
+
+	/**
+	 * Credentials password.
+	 */
+	private String password;
+
+
+
+	public List<String> getUris() {
+		return this.uris;
+	}
+
+	public void setUris(List<String> uris) {
+		this.uris = uris;
+	}
+
+	public String getUsername() {
+		return this.username;
+	}
+
+	public void setUsername(String username) {
+		this.username = username;
+	}
+
+	public String getPassword() {
+		return this.password;
+	}
+
+	public void setPassword(String password) {
+		this.password = password;
+	}
+}

spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/elasticsearch/rest/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2012-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Auto-configuration for Elasticsearch REST clients.
+ */
+package org.springframework.boot.autoconfigure.elasticsearch.rest;

spring-boot-project/spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json

@@ -212,6 +212,12 @@
         "http://localhost:9200"
       ]
     },
+    {
+      "name": "spring.elasticsearch.rest.uris",
+      "defaultValue": [
+        "http://localhost:9200"
+      ]
+    },
     {
       "name": "spring.info.build.location",
       "defaultValue": "classpath:META-INF/build-info.properties"

spring-boot-project/spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories

@@ -56,6 +56,7 @@ org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfigura
 org.springframework.boot.autoconfigure.data.rest.RepositoryRestMvcAutoConfiguration,\
 org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration,\
 org.springframework.boot.autoconfigure.elasticsearch.jest.JestAutoConfiguration,\
+org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration,\
 org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration,\
 org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration,\
 org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration,\

spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/elasticsearch/jest/JestAutoConfigurationTests.java

@@ -25,10 +25,8 @@ import io.searchbox.action.Action;
 import io.searchbox.client.JestClient;
 import io.searchbox.client.JestResult;
 import io.searchbox.client.http.JestHttpClient;
+import io.searchbox.core.Get;
 import io.searchbox.core.Index;
-import io.searchbox.core.Search;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -74,9 +72,8 @@ public class JestAutoConfigurationTests {
 
 	@Test
 	public void jestClientOnLocalhostByDefault() {
-		this.contextRunner
-				.run((context) -> assertThat(context.getBeansOfType(JestClient.class))
-						.hasSize(1));
+		this.contextRunner.run((context) ->
+				assertThat(context).hasSingleBean(JestClient.class));
 	}
 
 	@Test
@@ -84,8 +81,7 @@ public class JestAutoConfigurationTests {
 		this.contextRunner.withUserConfiguration(CustomJestClient.class)
 				.withPropertyValues(
 						"spring.elasticsearch.jest.uris[0]=http://localhost:9200")
-				.run((context) -> assertThat(context.getBeansOfType(JestClient.class))
-						.hasSize(1));
+				.run((context) -> assertThat(context).hasSingleBean(JestClient.class));
 	}
 
 	@Test
@@ -134,15 +130,11 @@ public class JestAutoConfigurationTests {
 					Map<String, String> source = new HashMap<>();
 					source.put("a", "alpha");
 					source.put("b", "bravo");
-					Index index = new Index.Builder(source).index("foo").type("bar")
-							.build();
+					Index index = new Index.Builder(source).index("foo")
+							.type("bar").id("1").build();
 					execute(client, index);
-					SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-					searchSourceBuilder.query(QueryBuilders.matchQuery("a", "alpha"));
-					assertThat(execute(client,
-							new Search.Builder(searchSourceBuilder.toString())
-									.addIndex("foo").build()).getResponseCode())
-											.isEqualTo(200);
+					Get getRequest = new Get.Builder("foo", "1").build();
+					assertThat(execute(client, getRequest).getResponseCode()).isEqualTo(200);
 				}));
 	}
 

spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/elasticsearch/rest/RestClientAutoConfigurationTests.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2012-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.autoconfigure.elasticsearch.rest;
+
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Test;
+
+import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchNodeTemplate;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.ReflectionUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for {@link RestClientAutoConfiguration}
+ *
+ * @author Brian Clozel
+ */
+public class RestClientAutoConfigurationTests {
+
+	private ApplicationContextRunner contextRunner = new ApplicationContextRunner()
+			.withConfiguration(AutoConfigurations.of(RestClientAutoConfiguration.class));
+
+	@Test
+	public void configureShouldCreateBothRestClientVariants() {
+		this.contextRunner.run((context) -> {
+			assertThat(context).hasSingleBean(RestClient.class)
+					.hasSingleBean(RestHighLevelClient.class);
+		});
+	}
+
+	@Test
+	public void configureWhenCustomClientShouldBackOff() {
+		this.contextRunner
+				.withUserConfiguration(CustomRestClientConfiguration.class)
+				.run((context) -> {
+					assertThat(context).hasSingleBean(RestClient.class)
+							.hasBean("customRestClient");
+				});
+	}
+
+	@Test
+	public void configureWhenBuilderCustomizerShouldApply() {
+		this.contextRunner
+				.withUserConfiguration(BuilderCustomizerConfiguration.class)
+				.run((context) -> {
+					assertThat(context).hasSingleBean(RestClient.class);
+					RestClient restClient = context.getBean(RestClient.class);
+					Field field = ReflectionUtils.findField(RestClient.class,
+							"maxRetryTimeoutMillis");
+					ReflectionUtils.makeAccessible(field);
+					assertThat(ReflectionUtils.getField(field, restClient))
+							.isEqualTo(42L);
+				});
+	}
+
+	@Test
+	public void restClientCanQueryElasticsearchNode() {
+		new ElasticsearchNodeTemplate().doWithNode((node) -> this.contextRunner
+				.withPropertyValues("spring.elasticsearch.rest.uris=http://localhost:"
+						+ node.getHttpPort())
+				.run((context) -> {
+					RestHighLevelClient client = context.getBean(RestHighLevelClient.class);
+					Map<String, String> source = new HashMap<>();
+					source.put("a", "alpha");
+					source.put("b", "bravo");
+					IndexRequest index = new IndexRequest("foo", "bar", "1")
+							.source(source);
+					client.index(index);
+					GetRequest getRequest = new GetRequest("foo", "bar", "1");
+					assertThat(client.get(getRequest).isExists()).isTrue();
+				}));
+	}
+
+	@Configuration
+	static class CustomRestClientConfiguration {
+
+		@Bean
+		public RestClient customRestClient() {
+			return mock(RestClient.class);
+		}
+	}
+
+	@Configuration
+	static class BuilderCustomizerConfiguration {
+
+		@Bean
+		public RestClientBuilderCustomizer myCustomizer() {
+			return builder -> builder.setMaxRetryTimeoutMillis(42);
+		}
+	}
+
+}

spring-boot-project/spring-boot-dependencies/pom.xml

@@ -1757,6 +1757,16 @@
 				<artifactId>transport-netty4-client</artifactId>
 				<version>${elasticsearch.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>org.elasticsearch.client</groupId>
+				<artifactId>elasticsearch-rest-client</artifactId>
+				<version>${elasticsearch.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.elasticsearch.client</groupId>
+				<artifactId>elasticsearch-rest-high-level-client</artifactId>
+				<version>${elasticsearch.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>org.firebirdsql.jdbc</groupId>
 				<artifactId>jaybird-jdk17</artifactId>

spring-boot-project/spring-boot-docs/src/main/asciidoc/appendix-application-properties.adoc

@@ -717,6 +717,11 @@ content into your application. Rather, pick only the properties that you need.
 	spring.elasticsearch.jest.uris=http://localhost:9200 # Comma-separated list of the Elasticsearch instances to use.
 	spring.elasticsearch.jest.username= # Login username.
 
+	# Elasticsearch REST clients ({sc-spring-boot-autoconfigure}/elasticsearch/rest/RestClientProperties.{sc-ext}[RestClientProperties])
+	spring.elasticsearch.rest.password= # Credentials username.
+    spring.elasticsearch.rest.uris=http://localhost:9200 # Comma-separated list of the Elasticsearch instances to use.
+    spring.elasticsearch.rest.username= # Credentials password.
+
 	# H2 Web Console ({sc-spring-boot-autoconfigure}/h2/H2ConsoleProperties.{sc-ext}[H2ConsoleProperties])
 	spring.h2.console.enabled=false # Whether to enable the console.
 	spring.h2.console.path=/h2-console # Path at which the console is available.

spring-boot-project/spring-boot-docs/src/main/asciidoc/spring-boot-features.adoc

@@ -4221,14 +4221,45 @@ https://projects.spring.io/spring-data-solr/[reference documentation].
 
 [[boot-features-elasticsearch]]
 === Elasticsearch
-http://www.elasticsearch.org/[Elasticsearch] is an open source, distributed, real-time
-search and analytics engine. Spring Boot offers basic auto-configuration for
-Elasticsearch and the abstractions on top of it provided by
-https://github.com/spring-projects/spring-data-elasticsearch[Spring Data Elasticsearch].
-There is a `spring-boot-starter-data-elasticsearch` "`Starter`" for collecting the
-dependencies in a convenient way. Spring Boot also supports
-https://github.com/searchbox-io/Jest[Jest].
+https://www.elastic.co/products/elasticsearch[Elasticsearch] is an open source,
+distributed, RESTful search and analytics engine. Spring Boot offers basic
+auto-configuration for Elasticsearch.
 
+Spring Boot supports several HTTP clients:
+
+* The official Java "Low Level" and "High Level" REST clients
+* https://github.com/searchbox-io/Jest[Jest]
+
+The transport client is still being used by
+https://github.com/spring-projects/spring-data-elasticsearch[Spring Data Elasticsearch],
+which you can start using with the `spring-boot-starter-data-elasticsearch` "`Starter`".
+
+
+[[boot-features-connecting-to-elasticsearch-rest]]
+==== Connecting to Elasticsearch by REST clients
+Elasticsearch ships
+https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/index.html[two different REST clients]
+that you can use to query a cluster: the "Low Level" client and the "High Level" client.
+
+If you have the `org.elasticsearch.client:elasticsearch-rest-client` dependency on the
+classpath, Spring Boot will auto-configure and register a `RestClient` bean that
+by default targets `http://localhost:9200`.
+You can further tune how `RestClient` is configured, as shown in the following example:
+
+[source,properties,indent=0]
+----
+	spring.elasticsearch.rest.uris=http://search.example.com:9200
+	spring.elasticsearch.rest.username=user
+	spring.elasticsearch.rest.password=secret
+----
+
+You can also register an arbitrary number of beans that implement
+`RestClientBuilderCustomizer` for more advanced customizations.
+To take full control over the registration, define a `RestClient` bean.
+
+If you have the `org.elasticsearch.client:elasticsearch-rest-high-level-client` dependency
+on the classpath, Spring Boot will auto-configure a `RestHighLevelClient`, which wraps
+any existing `RestClient` bean, reusing its HTTP configuration.
 
 
 [[boot-features-connecting-to-elasticsearch-jest]]