From 3f76eb2097f2aa951922a3891eaea91992648999 Mon Sep 17 00:00:00 2001 From: Madhura Bhave Date: Tue, 23 Jun 2020 15:39:12 -0700 Subject: [PATCH] Add volume mount property source support Add support for volume mounted directories where the filename becomes the property key and the file contents becomes the value. Support is provided via a dedicated `VolumeMountDirectoryPropertySource` class which can either be used directly, or via a "volumemount:/..." `spring.config.import` location. Closes gh-19990 Co-authored-by: Phillip Webb --- .../docs/asciidoc/spring-boot-features.adoc | 44 ++- .../config/VolumeMountConfigDataLoader.java | 41 +++ .../config/VolumeMountConfigDataLocation.java | 66 ++++ ...VolumeMountConfigDataLocationResolver.java | 44 +++ .../VolumeMountDirectoryPropertySource.java | 313 ++++++++++++++++++ .../main/resources/META-INF/spring.factories | 6 +- .../VolumeMountConfigDataLoaderTests.java | 58 ++++ ...eMountConfigDataLocationResolverTests.java | 56 ++++ .../VolumeMountConfigDataLocationTests.java | 58 ++++ ...lumeMountDirectoryPropertySourceTests.java | 217 ++++++++++++ 10 files changed, 900 insertions(+), 3 deletions(-) create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLoader.java create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocation.java create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationResolver.java create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/env/VolumeMountDirectoryPropertySource.java create mode 100644 spring-boot-project/spring-boot/src/test/java/org/springframework/boot/context/config/VolumeMountConfigDataLoaderTests.java create mode 100644 spring-boot-project/spring-boot/src/test/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationResolverTests.java create mode 100644 spring-boot-project/spring-boot/src/test/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationTests.java create mode 100644 spring-boot-project/spring-boot/src/test/java/org/springframework/boot/env/VolumeMountDirectoryPropertySourceTests.java diff --git a/spring-boot-project/spring-boot-docs/src/docs/asciidoc/spring-boot-features.adoc b/spring-boot-project/spring-boot-docs/src/docs/asciidoc/spring-boot-features.adoc index 6d37c67a1b2..d3fe2dff5c3 100644 --- a/spring-boot-project/spring-boot-docs/src/docs/asciidoc/spring-boot-features.adoc +++ b/spring-boot-project/spring-boot-docs/src/docs/asciidoc/spring-boot-features.adoc @@ -670,7 +670,7 @@ Locations will be processed in the order that they are defined, with later impor [TIP] ==== Spring Boot includes pluggable API that allows various different location addresses to be supported. -By default you can import Java Properties and YAML. +By default you can import Java Properties, YAML and volume mounts. Third-party jars can offer support for additional technologies (there's no requirement for files to be local). For example, you can imagine config data being from external stores such as Consul, Apache ZooKeeper or Netflix Archaius. @@ -680,6 +680,48 @@ If you want to support your own locations, see the `ConfigDataLocationResolver` +[[boot-features-external-config-files-voumemounts]] +==== Using Volume Mount Properties +When running applications on a cloud platform (such as Kubernetes) you often need to read config values that the platform supplies. +It's not uncommon to use environment variables for such purposes, but this can have drawbacks, especially if the value is supposed to be kept secret. + +As an alternative to environment variables, many cloud platforms now allow you to map configuration into mounted data volumes. +For example, Kubernetes can volume mount both https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#populate-a-volume-with-data-stored-in-a-configmap[`ConfigMaps`] and https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-files-from-a-pod[`Secrets`]. + +There are two common volume mount patterns that can be use: + +. A single file contains a complete set of properties (usually written as YAML). +. Multiple files are written to a directory with the filename becoming the '`key`' and the contents becoming the '`value`'. + +For the first case, you can import the YAML or Properties file directly using `spring.config.import` as described <>. +For the second case, you need to use the `volumemount:` prefix so that Spring Boot knows it needs to expose all the files as properties. + +As an example, let's imagine that Kubernetes has mounted the following volume: + +[source,indent=0] +---- + etc/ + config/ + myapp/ + username + password +---- + +The contents of the `username` file would be a config value, and the contents of `password` would be a secret. + +To import these properties, you can add the following to your `application.properties` file: + +[source,properties,indent=0] +---- +spring.config.import=volumemount:/etc/config +---- + +You can then access or inject `myapp.username` and `myapp.password` properties from the `Environment` in the usual way. + +TIP: Volume mounted values can be bound to both string `String` and `byte[]` types depending on the contents expected. + + + [[boot-features-external-config-placeholders-in-properties]] ==== Property Placeholders The values in `application.properties` and `application.yml` are filtered through the existing `Environment` when they are used, so you can refer back to previously defined values (for example, from System properties). diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLoader.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLoader.java new file mode 100644 index 00000000000..2fc409c178b --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLoader.java @@ -0,0 +1,41 @@ +/* + * Copyright 2012-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.context.config; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; + +import org.springframework.boot.env.VolumeMountDirectoryPropertySource; + +/** + * {@link ConfigDataLoader} for directory locations mounted as volumes. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +class VolumeMountConfigDataLoader implements ConfigDataLoader { + + @Override + public ConfigData load(VolumeMountConfigDataLocation location) throws IOException { + Path path = location.getPath(); + String name = "Volume mount config '" + path + "'"; + VolumeMountDirectoryPropertySource source = new VolumeMountDirectoryPropertySource(name, path); + return new ConfigData(Collections.singletonList(source)); + } + +} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocation.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocation.java new file mode 100644 index 00000000000..358d7c229d5 --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocation.java @@ -0,0 +1,66 @@ +/* + * Copyright 2012-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.context.config; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +import org.springframework.util.Assert; + +/** + * {@link ConfigDataLocation} backed by a directory mounted as a volume. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +class VolumeMountConfigDataLocation extends ConfigDataLocation { + + private final Path path; + + VolumeMountConfigDataLocation(String path) { + Assert.notNull(path, "Path must not be null"); + this.path = Paths.get(path).toAbsolutePath(); + } + + Path getPath() { + return this.path; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + VolumeMountConfigDataLocation other = (VolumeMountConfigDataLocation) obj; + return Objects.equals(this.path, other.path); + } + + @Override + public int hashCode() { + return this.path.hashCode(); + } + + @Override + public String toString() { + return "volume mount [" + this.path + "]"; + } + +} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationResolver.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationResolver.java new file mode 100644 index 00000000000..9f3fd9e0fc1 --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/context/config/VolumeMountConfigDataLocationResolver.java @@ -0,0 +1,44 @@ +/* + * Copyright 2012-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.context.config; + +import java.util.Collections; +import java.util.List; + +/** + * {@link ConfigDataLocationResolver} for volume mounted locations such as Kubernetes + * ConfigMaps and Secrets. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +class VolumeMountConfigDataLocationResolver implements ConfigDataLocationResolver { + + private static final String PREFIX = "volumemount:"; + + @Override + public boolean isResolvable(ConfigDataLocationResolverContext context, String location) { + return location.startsWith(PREFIX); + } + + @Override + public List resolve(ConfigDataLocationResolverContext context, String location) { + VolumeMountConfigDataLocation resolved = new VolumeMountConfigDataLocation(location.substring(PREFIX.length())); + return Collections.singletonList(resolved); + } + +} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/env/VolumeMountDirectoryPropertySource.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/env/VolumeMountDirectoryPropertySource.java new file mode 100644 index 00000000000..b478c328b90 --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/env/VolumeMountDirectoryPropertySource.java @@ -0,0 +1,313 @@ +/* + * Copyright 2012-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.env; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.springframework.boot.convert.ApplicationConversionService; +import org.springframework.boot.origin.Origin; +import org.springframework.boot.origin.OriginLookup; +import org.springframework.boot.origin.OriginProvider; +import org.springframework.boot.origin.TextResourceOrigin; +import org.springframework.boot.origin.TextResourceOrigin.Location; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.InputStreamSource; +import org.springframework.core.io.PathResource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; +import org.springframework.util.FileCopyUtils; +import org.springframework.util.StringUtils; + +/** + * {@link PropertySource} backed by a directory that contains files for each value. The + * {@link PropertySource} will recursively scan a given source directory and expose a + * property for each file found. The property name will be the filename, and the property + * value will be the contents of the file. + *

+ * Directories are only scanned when the source is first created. The directory is not + * monitored for updates, so files should not be added or removed. However, the contents + * of a file can be updated as long as the property source was created with a + * {@link Option#ALWAYS_READ} option. Nested folders are included in the source, but with + * a {@code '.'} rather than {@code '/'} used as the path separator. + *

+ * Property values are returned as {@link Value} instances which allows them to be treated + * either as an {@link InputStreamSource} or as a {@link CharSequence}. In addition, if + * used with an {@link Environment} configured with an + * {@link ApplicationConversionService}, property values can be converted to a + * {@code String} or {@code byte[]}. + *

+ * This property source is typically used to read Kubernetes {@code configMap} volume + * mounts. + * + * @author Phillip Webb + * @since 2.4.0 + */ +public class VolumeMountDirectoryPropertySource extends EnumerablePropertySource implements OriginLookup { + + private static final int MAX_DEPTH = 100; + + private final Map propertyFiles; + + private final String[] names; + + private final Set