From 3fda7445228f90015d3cbf13b300f6a96961c709 Mon Sep 17 00:00:00 2001 From: Dave Syer Date: Tue, 5 May 2015 11:50:37 +0100 Subject: [PATCH] Add samples and tweak metrics reader/writers till they work --- .../MetricRepositoryAutoConfiguration.java | 12 +++- .../autoconfigure/MetricsProperties.java | 24 +++++++- .../aggregate/AggregateMetricReader.java | 2 +- .../metrics/export/MetricCopyExporter.java | 24 +++++--- .../DefaultOpenTsdbNamingStrategy.java | 22 +++++--- .../redis/RedisMetricRepository.java | 3 + .../asciidoc/production-ready-features.adoc | 47 ++++++++++------ .../src/main/resources/application.properties | 3 +- .../README.adoc | 22 ++++++++ .../redis/AggregateMetricsConfiguration.java | 56 +++++++++++++++++++ .../metrics/redis/ExportProperties.java | 12 ++++ .../src/main/resources/application.properties | 3 +- 12 files changed, 191 insertions(+), 39 deletions(-) create mode 100644 spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/AggregateMetricsConfiguration.java diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricRepositoryAutoConfiguration.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricRepositoryAutoConfiguration.java index 4992c6c76a9..c2308e7a9d6 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricRepositoryAutoConfiguration.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricRepositoryAutoConfiguration.java @@ -175,6 +175,9 @@ public class MetricRepositoryAutoConfiguration { @Autowired(required = false) private List writers; + @Autowired + private MetricsProperties metrics; + @Autowired(required = false) @Qualifier("actuatorMetricRepository") private MetricWriter actuatorMetricRepository; @@ -188,13 +191,20 @@ public class MetricRepositoryAutoConfiguration { && writers.contains(this.actuatorMetricRepository)) { writers.remove(this.actuatorMetricRepository); } - return new MetricCopyExporter(reader, new CompositeMetricWriter(writers)) { + MetricCopyExporter exporter = new MetricCopyExporter(reader, + new CompositeMetricWriter(writers)) { @Scheduled(fixedDelayString = "${spring.metrics.export.delayMillis:5000}") @Override public void export() { super.export(); } }; + if (this.metrics.getExport().getIncludes() != null + || this.metrics.getExport().getExcludes() != null) { + exporter.setIncludes(this.metrics.getExport().getIncludes()); + exporter.setExcludes(this.metrics.getExport().getExcludes()); + } + return exporter; } } diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricsProperties.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricsProperties.java index 2070fc18aef..20f2be9200a 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricsProperties.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/autoconfigure/MetricsProperties.java @@ -43,11 +43,31 @@ public class MetricsProperties { private boolean enabled = true; /** - * Flag to switch off an optimization based on not exporting unchanged metric - * values. + * Flag to switch off any available optimizations based on not exporting unchanged + * metric values. */ private boolean ignoreTimestamps = false; + private String[] includes; + + private String[] excludes; + + public String[] getIncludes() { + return this.includes; + } + + public void setIncludes(String[] includes) { + this.includes = includes; + } + + public String[] getExcludes() { + return this.excludes; + } + + public void setExcludes(String[] excludes) { + this.excludes = excludes; + } + public boolean isEnabled() { return this.enabled; } diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/aggregate/AggregateMetricReader.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/aggregate/AggregateMetricReader.java index b3cb11f7305..2c8513503ee 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/aggregate/AggregateMetricReader.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/aggregate/AggregateMetricReader.java @@ -115,7 +115,7 @@ public class AggregateMetricReader implements MetricReader { if (aggregate == null) { aggregate = new Metric(name, metric.getValue(), metric.getTimestamp()); } - else if (key.startsWith("counter")) { + else if (key.contains("counter.")) { // accumulate all values aggregate = new Metric(name, metric.increment( aggregate.getValue().intValue()).getValue(), metric.getTimestamp()); diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/export/MetricCopyExporter.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/export/MetricCopyExporter.java index bbb3aec2077..5bbccc8021b 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/export/MetricCopyExporter.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/export/MetricCopyExporter.java @@ -46,11 +46,15 @@ public class MetricCopyExporter extends AbstractMetricExporter { } public void setIncludes(String... includes) { - this.includes = includes; + if (includes != null) { + this.includes = includes; + } } public void setExcludes(String... excludes) { - this.excludes = excludes; + if (excludes != null) { + this.excludes = excludes; + } } public MetricCopyExporter(MetricReader reader, MetricWriter writer, String prefix) { @@ -61,7 +65,8 @@ public class MetricCopyExporter extends AbstractMetricExporter { @Override protected Iterable> next(String group) { - if (this.includes.length == 0 && this.excludes.length == 0) { + if ((this.includes != null || this.includes.length == 0) + && (this.excludes != null || this.excludes.length == 0)) { return this.reader.findAll(); } return new Iterable>() { @@ -108,7 +113,8 @@ public class MetricCopyExporter extends AbstractMetricExporter { boolean matched = false; while (this.iterator.hasNext() && !matched) { metric = this.iterator.next(); - if (MetricCopyExporter.this.includes.length == 0) { + if (MetricCopyExporter.this.includes == null + || MetricCopyExporter.this.includes.length == 0) { matched = true; } else { @@ -119,10 +125,12 @@ public class MetricCopyExporter extends AbstractMetricExporter { } } } - for (String pattern : MetricCopyExporter.this.excludes) { - if (PatternMatchUtils.simpleMatch(pattern, metric.getName())) { - matched = false; - break; + if (MetricCopyExporter.this.excludes != null) { + for (String pattern : MetricCopyExporter.this.excludes) { + if (PatternMatchUtils.simpleMatch(pattern, metric.getName())) { + matched = false; + break; + } } } } diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/opentsdb/DefaultOpenTsdbNamingStrategy.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/opentsdb/DefaultOpenTsdbNamingStrategy.java index 9b52ad54a1c..50ec9d0831f 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/opentsdb/DefaultOpenTsdbNamingStrategy.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/opentsdb/DefaultOpenTsdbNamingStrategy.java @@ -24,18 +24,22 @@ import org.springframework.util.ObjectUtils; /** * A naming strategy that just passes through the metric name, together with tags from a - * set of static values. Open TSDB requires at least one tag, so one is always added for - * you: the {@value #PREFIX_KEY} key is added with a unique value "spring.X" where X is an - * object hash code ID for this (the naming stategy). In most cases this will be unique - * enough to allow aggregation of the underlying metrics in Open TSDB, but normally it is - * best to provide your own tags, including a prefix if you know one (overwriting the - * default). + * set of static values. Open TSDB requires at least one tag, so tags are always added for + * you: the {@value #DOMAIN_KEY} key is added with a value "spring", and the + * {@value #PROCESS_KEY} key is added with a value equal to the object hash of "this" (the + * naming strategy). The "domain" value is a system identifier - it would be common to all + * processes in the same distributed system. In most cases this will be unique enough to + * allow aggregation of the underlying metrics in Open TSDB, but normally it is best to + * provide your own tags, including a prefix and process identifier if you know one + * (overwriting the default). * * @author Dave Syer */ public class DefaultOpenTsdbNamingStrategy implements OpenTsdbNamingStrategy { - public static final String PREFIX_KEY = "prefix"; + public static final String DOMAIN_KEY = "domain"; + + public static final String PROCESS_KEY = "process"; /** * Tags to apply to every metric. Open TSDB requires at least one tag, so a "prefix" @@ -46,8 +50,8 @@ public class DefaultOpenTsdbNamingStrategy implements OpenTsdbNamingStrategy { private Map cache = new HashMap(); public DefaultOpenTsdbNamingStrategy() { - this.tags.put(PREFIX_KEY, - "spring." + ObjectUtils.getIdentityHexString(this)); + this.tags.put(DOMAIN_KEY, "org.springframework.metrics"); + this.tags.put(PROCESS_KEY, ObjectUtils.getIdentityHexString(this)); } public void setTags(Map staticTags) { diff --git a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/repository/redis/RedisMetricRepository.java b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/repository/redis/RedisMetricRepository.java index fb9dca99df1..5155798778d 100644 --- a/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/repository/redis/RedisMetricRepository.java +++ b/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/metrics/repository/redis/RedisMetricRepository.java @@ -29,6 +29,7 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.core.BoundZSetOperations; import org.springframework.data.redis.core.RedisOperations; import org.springframework.util.Assert; +import org.springframework.util.StringUtils; /** * A {@link MetricRepository} implementation for a redis backend. Metric values are stored @@ -92,6 +93,8 @@ public class RedisMetricRepository implements MetricRepository { public RedisMetricRepository(RedisConnectionFactory redisConnectionFactory, String prefix, String key) { Assert.notNull(redisConnectionFactory, "RedisConnectionFactory must not be null"); + Assert.state(StringUtils.hasText(prefix), "Prefix must be non-empty"); + Assert.state(StringUtils.hasText(key), "Key must be non-empty"); this.redisOperations = RedisUtils.stringTemplate(redisConnectionFactory); if (!prefix.endsWith(".")) { prefix = prefix + "."; diff --git a/spring-boot-docs/src/main/asciidoc/production-ready-features.adoc b/spring-boot-docs/src/main/asciidoc/production-ready-features.adoc index 69d1dc09e59..e699f61abc1 100644 --- a/spring-boot-docs/src/main/asciidoc/production-ready-features.adoc +++ b/spring-boot-docs/src/main/asciidoc/production-ready-features.adoc @@ -943,10 +943,10 @@ Example: [source,java,indent=0] ---- -@Value("${spring.application.name:application}.${random.value:0000}") -private String prefix = "metrics"; +@Value("metrics.mysystem.${random.value:0000}.${spring.application.name:application}") +private String prefix = "metrics.mysystem"; -@Value("${metrics.key:METRICSKEY}") +@Value("${metrics.key:keys.mysystem}") private String key = "METRICSKEY"; @Bean @@ -955,35 +955,50 @@ MetricWriter metricWriter() { } ---- +The prefix is constructed with the application name at the end, so it can easily be used +to identify a group of processes with the same logical name later. + +NOTE: it's important to set both the key and the prefix. The key is used for all +repository operations, and can be shared by multiple repositories. If multiple +repositories share a key (like in the case where you need to aggregate across them), then +you normally have a read-only "master" repository that has a short, but identifiable, +prefix (like "metrics.mysystem"), and many write-only repositories with prefixes that +start with the master prefix (like `metrics.mysystem.*` in the example above). It is +efficient to read all the keys from a "master" repository like that, but inefficient to +read a subset with a longer prefix (e.g. using one of the writing repositories). + [[production-ready-metric-writers-export-to-open-tdsb]] ==== Example: Export to Open TSDB -If you provide a `@Bean` of type `OpenTsdbHttpMetricWriter` the metrics are exported to -http://opentsdb.net/[Open TSDB] for aggregation. The `OpenTsdbHttpMetricWriter` has a -`url` property that you need to set to the Open TSDB "/put" endpoint, e.g. -`http://localhost:4242/api/put`). It also has a `namingStrategy` that you can customize -or configure to make the metrics match the data structure you need on the server. By -default it just passes through the metric name as an Open TSDB metric name and adds a tag -"prefix" with value "spring.X" where "X" is the object hash of the default naming -strategy. Thus, after running the application and generating some metrics (e.g. by pinging -the home page) you can inspect the metrics in the TDB UI (http://localhost:4242 by -default). Example: +If you provide a `@Bean` of type `OpenTsdbHttpMetricWriter` the metrics are exported to +http://opentsdb.net/[Open TSDB] for aggregation. The `OpenTsdbHttpMetricWriter` has a +`url` property that you need to set to the Open TSDB "/put" endpoint, e.g. +`http://localhost:4242/api/put`). It also has a `namingStrategy` that you can customize or +configure to make the metrics match the data structure you need on the server. By default +it just passes through the metric name as an Open TSDB metric name and adds a tag "domain" +with value "org.springframework.metrics" and another tag "process" with value equals to +the object hash of the naming strategy. Thus, after running the application and generating +some metrics (e.g. by pinging the home page) you can inspect the metrics in the TDB UI +(http://localhost:4242 by default). Example: + +[source,indent=0] ---- curl localhost:4242/api/query?start=1h-ago&m=max:counter.status.200.root [ { "metric": "counter.status.200.root", "tags": { - "prefix": "spring.b968a76" + "domain": "org.springframework.metrics", + "process": "b968a76" }, "aggregateTags": [], "dps": { "1430492872": 2, "1430492875": 6 } - } + } ] ---- @@ -1057,7 +1072,7 @@ results to the "/metrics" endpoint. Example: @Bean protected MetricReader repository(RedisConnectionFactory connectionFactory) { RedisMetricRepository repository = new RedisMetricRepository(connectionFactory, - "metrics", "METRICSKEY"); + "mysystem", "myorg.keys"); return repository; } diff --git a/spring-boot-samples/spring-boot-sample-metrics-opentsdb/src/main/resources/application.properties b/spring-boot-samples/spring-boot-sample-metrics-opentsdb/src/main/resources/application.properties index 80dfb9df521..22fc8604904 100644 --- a/spring-boot-samples/spring-boot-sample-metrics-opentsdb/src/main/resources/application.properties +++ b/spring-boot-samples/spring-boot-sample-metrics-opentsdb/src/main/resources/application.properties @@ -1 +1,2 @@ -service.name: Phil \ No newline at end of file +service.name: Phil +metrics.names.tags.process: ${spring.application.name:application}:${random.value:0000} \ No newline at end of file diff --git a/spring-boot-samples/spring-boot-sample-metrics-redis/README.adoc b/spring-boot-samples/spring-boot-sample-metrics-redis/README.adoc index a50efbf7520..3038d548170 100644 --- a/spring-boot-samples/spring-boot-sample-metrics-redis/README.adoc +++ b/spring-boot-samples/spring-boot-sample-metrics-redis/README.adoc @@ -22,3 +22,25 @@ the result in Redis, e.g. 2) "4" ---- +There is also an `AggregateMetricReader` with public metrics in the application context, +and you can see the result in the "/metrics" (metrics with names in "aggregate.*"). +The way the Redis repository was set up (with a random key in the metric names) makes the +aggregates work across restarts of the same application, or across a scaled up application +running in multiple processes. E.g. + +[source,indent=0] +---- + $ curl localhost:8080/metrics + { + ... + "aggregate.application.counter.status.200.metrics": 12, + "aggregate.application.counter.status.200.root": 29, + "aggregate.application.gauge.response.metrics": 43, + "aggregate.application.gauge.response.root": 5, + "counter.status.200.root": 2, + "counter.status.200.metrics": 1, + "gauge.response.metrics": 43, + "gauge.response.root": 5 + } +---- + diff --git a/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/AggregateMetricsConfiguration.java b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/AggregateMetricsConfiguration.java new file mode 100644 index 00000000000..a8b3fdddfae --- /dev/null +++ b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/AggregateMetricsConfiguration.java @@ -0,0 +1,56 @@ +/* + * Copyright 2012-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.metrics.redis; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.endpoint.MetricReaderPublicMetrics; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.aggregate.AggregateMetricReader; +import org.springframework.boot.actuate.metrics.reader.MetricReader; +import org.springframework.boot.actuate.metrics.repository.redis.RedisMetricRepository; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; + +/** + * @author Dave Syer + */ +@Configuration +public class AggregateMetricsConfiguration { + + @Autowired + private ExportProperties export; + + @Autowired + private RedisConnectionFactory connectionFactory; + + @Bean + public PublicMetrics metricsAggregate() { + return new MetricReaderPublicMetrics(aggregatesMetricReader()); + } + + private MetricReader globalMetricsForAggregation() { + return new RedisMetricRepository(this.connectionFactory, + this.export.getAggregatePrefix(), this.export.getKey()); + } + + private MetricReader aggregatesMetricReader() { + AggregateMetricReader repository = new AggregateMetricReader( + globalMetricsForAggregation()); + return repository; + } +} diff --git a/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/ExportProperties.java b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/ExportProperties.java index 3e27955f8d5..769325d3f11 100644 --- a/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/ExportProperties.java +++ b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/java/sample/metrics/redis/ExportProperties.java @@ -17,6 +17,7 @@ package sample.metrics.redis; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; @ConfigurationProperties("metrics.export") class ExportProperties { @@ -40,4 +41,15 @@ class ExportProperties { this.key = key; } + public String getAggregatePrefix() { + String[] tokens = StringUtils.delimitedListToStringArray(this.prefix, "."); + if (tokens.length > 1) { + if (StringUtils.hasText(tokens[1])) { + // If the prefix has 2 or more non-trivial parts, use the first 1 + return tokens[0]; + } + } + return this.prefix; + } + } \ No newline at end of file diff --git a/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/resources/application.properties b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/resources/application.properties index 01ba46e43df..8f126a3c808 100644 --- a/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/resources/application.properties +++ b/spring-boot-samples/spring-boot-sample-metrics-redis/src/main/resources/application.properties @@ -1,2 +1,3 @@ service.name: Phil -metrics.export.prefix: ${spring.application.name:application}:${random.value:0000} \ No newline at end of file +metrics.export.prefix: metrics.sample.${random.value:0000}.${spring.application.name:application} +metrics.export.key: keys.metrics.sample