elasticsearch/docs/reference/rest-api/usage.asciidoc

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

553 lines
13 KiB
Plaintext
Raw Normal View History

2020-02-21 01:25:57 +08:00
[role="xpack"]
[[usage-api]]
== Usage API
.New API reference
[sidebar]
--
For the most up-to-date API details, refer to {api-es}/group/endpoint-xpack[Usage APIs].
--
2020-02-21 01:25:57 +08:00
Provides usage information about the installed {xpack} features.
[discrete]
[[usage-api-request]]
=== {api-request-title}
`GET /_xpack/usage`
[discrete]
[[usage-api-prereqs]]
=== {api-prereq-title}
* If the {es} {security-features} are enabled, you must have the `monitor` or
`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
2020-02-21 01:25:57 +08:00
[discrete]
[[usage-api-desc]]
=== {api-description-title}
This API provides information about which features are currently enabled and
available under the current license and some usage statistics.
2020-02-21 01:25:57 +08:00
[discrete]
[[usage-api-query-parms]]
=== {api-query-parms-title}
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
2020-02-21 01:25:57 +08:00
[discrete]
[[usage-api-example]]
=== {api-examples-title}
[source,console]
------------------------------------------------------------
GET /_xpack/usage
------------------------------------------------------------
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/]
// This response filter removes watcher logging results if they are included
// to avoid errors in the CI builds.
// Same for ES|QL functions, that is a long list and quickly evolving.
2020-02-21 01:25:57 +08:00
[source,console-result]
------------------------------------------------------------
{
"security" : {
"available" : true,
"enabled" : true,
...
2020-02-21 01:25:57 +08:00
},
"monitoring" : {
"available" : true,
"enabled" : true,
"collection_enabled" : false,
"enabled_exporters" : {
"local" : 1
}
},
"watcher" : {
"available" : true,
"enabled" : true,
"execution" : {
"actions" : {
"_all" : {
"total" : 0,
"total_time_in_ms" : 0
}
2020-02-21 01:25:57 +08:00
}
},
2020-02-21 01:25:57 +08:00
"watch" : {
"input" : {
"_all" : {
"total" : 0,
"active" : 0
}
},
"trigger" : {
"_all" : {
"total" : 0,
"active" : 0
}
}
},
"count" : {
"total" : 0,
"active" : 0
}
},
"graph" : {
"available" : true,
"enabled" : true
},
"ml" : {
"available" : true,
"enabled" : true,
"jobs" : {
"_all" : {
"count" : 0,
"detectors" : {
...
},
"created_by" : { },
"model_size" : {
...
},
"forecasts" : {
"total" : 0,
"forecasted_jobs" : 0
}
}
},
"datafeeds" : {
"_all" : {
"count" : 0
}
},
"data_frame_analytics_jobs" : {
"_all" : {
"count" : 0
},
"analysis_counts": { },
"memory_usage": {
"peak_usage_bytes": {
"min": 0.0,
"max": 0.0,
"avg": 0.0,
"total": 0.0
}
2020-02-21 01:25:57 +08:00
}
},
"inference" : {
"ingest_processors" : {
"_all" : {
"num_docs_processed" : {
"max" : 0,
"sum" : 0,
"min" : 0
},
"pipelines" : {
"count" : 0
},
"num_failures" : {
"max" : 0,
"sum" : 0,
"min" : 0
},
"time_ms" : {
"max" : 0,
"sum" : 0,
"min" : 0
}
}
},
"trained_models" : {
"_all" : {
"count": 1
},
"count": {
"total": 1,
"prepackaged": 1,
"other": 0
},
"model_size_bytes": {
"min": 0.0,
"max": 0.0,
"avg": 0.0,
"total": 0.0
},
"estimated_operations": {
"min": 0.0,
"max": 0.0,
"avg": 0.0,
"total": 0.0
2020-02-21 01:25:57 +08:00
}
},
"deployments": {
"count": 0,
"inference_counts": {
"total": 0.0,
"min": 0.0,
"avg": 0.0,
"max": 0.0
},
"stats_by_model": [],
"model_sizes_bytes": {
"total": 0.0,
"min": 0.0,
"avg": 0.0,
"max": 0.0
},
"time_ms": {
"avg": 0.0
}
2020-02-21 01:25:57 +08:00
}
},
"node_count" : 1,
"memory": {
anomaly_detectors_memory_bytes: 0,
data_frame_analytics_memory_bytes: 0,
pytorch_inference_memory_bytes: 0,
total_used_memory_bytes: 0
}
2020-02-21 01:25:57 +08:00
},
"inference": {
"available" : true,
"enabled" : true,
"models" : [
...
]
},
2020-02-21 01:25:57 +08:00
"logstash" : {
"available" : true,
"enabled" : true
},
"eql" : {
"available" : true,
"enabled" : true
},
2023-05-25 07:17:02 +08:00
"esql" : {
"available" : true,
"enabled" : true,
"features" : {
"eval" : 0,
"stats" : 0,
2023-08-14 21:52:43 +08:00
"dissect": 0,
2023-05-25 07:17:02 +08:00
"grok" : 0,
"limit" : 0,
"where" : 0,
"sort" : 0,
"drop" : 0,
"show" : 0,
"rename" : 0,
"mv_expand" : 0,
"keep" : 0,
"enrich" : 0,
"from" : 0,
"row" : 0
2023-05-25 07:17:02 +08:00
},
"queries" : {
"rest" : {
"total" : 0,
"failed" : 0
},
"kibana" : {
"total" : 0,
"failed" : 0
},
"_all" : {
"total" : 0,
"failed" : 0
}
}
},
2020-02-21 01:25:57 +08:00
"sql" : {
"available" : true,
"enabled" : true,
"features" : {
"having" : 0,
"subselect" : 0,
"limit" : 0,
"orderby" : 0,
"where" : 0,
"join" : 0,
"groupby" : 0,
"command" : 0,
"local" : 0
},
"queries" : {
"rest" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"cli" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"canvas" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"odbc" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"jdbc" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"odbc32" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"odbc64" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"_all" : {
"total" : 0,
"paging" : 0,
"failed" : 0
},
"translate" : {
"count" : 0
}
}
},
"rollup" : {
"available" : true,
"enabled" : true,
...
2020-02-21 01:25:57 +08:00
},
"ilm" : {
"policy_count" : 3,
"policy_stats" : [
...
]
},
"slm" : {
"available" : true,
"enabled" : true
},
"ccr" : {
"available" : true,
"enabled" : true,
"follower_indices_count" : 0,
"auto_follow_patterns_count" : 0
},
"transform" : {
"available" : true,
"enabled" : true
},
"voting_only" : {
"available" : true,
"enabled" : true
},
"searchable_snapshots" : {
"available" : true,
"enabled" : true,
"indices_count" : 0,
"full_copy_indices_count" : 0,
"shared_cache_indices_count" : 0
},
2020-02-21 01:25:57 +08:00
"spatial" : {
"available" : true,
"enabled" : true
},
"analytics" : {
"available" : true,
"enabled" : true,
"stats": {
"boxplot_usage" : 0,
"top_metrics_usage" : 0,
"normalize_usage" : 0,
"cumulative_cardinality_usage" : 0,
"t_test_usage" : 0,
"rate_usage" : 0,
"string_stats_usage" : 0,
"moving_percentiles_usage" : 0,
"multi_terms_usage" : 0
}
},
"data_streams" : {
"available" : true,
"enabled" : true,
"data_streams" : 0,
"indices_count" : 0
Add telemetry for data tiers (#63031) This commit adds telemetry for our data tier formalization. This telemetry helps determine the topology of the cluster with regard to the content, hot, warm, & cold tiers/roles. An example of the telemetry looks like: ``` GET /_xpack/usage?human { ... "data_tiers" : { "available" : true, "enabled" : true, "data_warm" : { ... }, "data_cold" : { ... }, "data_content" : { "node_count" : 1, "index_count" : 6, "total_shard_count" : 6, "primary_shard_count" : 6, "doc_count" : 71, "total_size" : "59.6kb", "total_size_bytes" : 61110, "primary_size" : "59.6kb", "primary_size_bytes" : 61110, "primary_shard_size_avg" : "9.9kb", "primary_shard_size_avg_bytes" : 10185, "primary_shard_size_median" : "8kb", "primary_shard_size_median_bytes" : 8254, "primary_shard_size_mad" : "7.2kb", "primary_shard_size_mad_bytes" : 7391 }, "data_hot" : { ... } } } ``` The fields are as follows: - node_count :: number of nodes with this tier/role - index_count :: number of indices on this tier - total_shard_count :: total number of shards for all nodes in this tier - primary_shard_count :: number of primary shards for all nodes in this tier - doc_count :: number of documents for all nodes in this tier - total_size_bytes :: total number of bytes for all shards for all nodes in this tier - primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier - primary_shard_size_avg_bytes :: average shard size for primary shard in this tier - primary_shard_size_median_bytes :: median shard size for primary shard in this tier - primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier Relates to #60848
2020-10-01 22:35:10 +08:00
},
"data_lifecycle" : {
"available": true,
"enabled": true,
"count": 0,
"default_rollover_used": true,
Update data stream lifecycle telemetry to track global retention (#112451) Currently, the data stream lifecycle telemetry has the following structure: ``` { .... "data_lifecycle" : { "available": true, "enabled": true, "count": 0, "default_rollover_used": true, "retention": { "minimum_millis": 0, "maximum_millis": 0, "average_millis": 0.0 } }.... ``` In the snippet above you can see that we track: - The amount of data streams managed by the data stream lifecycle by `count` - If the default rollover has been overwritten by `default_rollover_used` - The min, max and average of the `data_retention` configured on a data stream level. In this PR we propose the following extention: ``` .... "data_lifecycle" : { "available": true, "enabled": true, "count": 0, "default_rollover_used": true, "effective_retention": { #https://github.com/elastic/dev/issues/2537 "retained_data_streams": 5, "minimum_millis": 0, # Only if retained data streams > 1 "maximum_millis": 0, "average_millis": 0.0 }, "data_retention": { "configured_data_streams": 5, "minimum_millis": 0, # Only if retained data streams > 1 "maximum_millis": 0, "average_millis": 0.0 }, "global_retention": { "default": { "defined": true/false, "affected_data_streams": 0, "millis": 0 }, "max": { "defined": true/false, "affected_data_streams": 0, "millis": 0 } } ``` With this extension we are tracking: - The amount of data streams managed by the data stream lifecycle by `count` - If the default rollover has been overwritten by `default_rollover_used` - The min, max and average of the `data_retention` configured on a data stream level and the number of data streams that have it configured. We add the min, max and avg only if there are data streams with data retention configuration to avoid messing with the stats in a dashboard. - The min, max and average of the `effective_retention` and the number of data streams that are retained. We add the min, max and avg only if there are retained data streams to avoid messing with the stats in a dashboard. - Global retention stats, if they are defined, if the number of the affected data streams and the actual value. The above metrics allow us to answer questions like: - How many data streams are affected by global retention. - How big is the difference between the longest data retention compared to max global retention. - How much does the effective retention diverging from the data retention, this will show the impact of the global retention.
2024-09-11 16:31:04 +08:00
"data_retention": {
"configured_data_streams": 0
},
"effective_retention": {
"retained_data_streams": 0
},
"global_retention": {
"default": {
"defined": false
},
"max": {
"defined": false
}
}
},
Add telemetry for data tiers (#63031) This commit adds telemetry for our data tier formalization. This telemetry helps determine the topology of the cluster with regard to the content, hot, warm, & cold tiers/roles. An example of the telemetry looks like: ``` GET /_xpack/usage?human { ... "data_tiers" : { "available" : true, "enabled" : true, "data_warm" : { ... }, "data_cold" : { ... }, "data_content" : { "node_count" : 1, "index_count" : 6, "total_shard_count" : 6, "primary_shard_count" : 6, "doc_count" : 71, "total_size" : "59.6kb", "total_size_bytes" : 61110, "primary_size" : "59.6kb", "primary_size_bytes" : 61110, "primary_shard_size_avg" : "9.9kb", "primary_shard_size_avg_bytes" : 10185, "primary_shard_size_median" : "8kb", "primary_shard_size_median_bytes" : 8254, "primary_shard_size_mad" : "7.2kb", "primary_shard_size_mad_bytes" : 7391 }, "data_hot" : { ... } } } ``` The fields are as follows: - node_count :: number of nodes with this tier/role - index_count :: number of indices on this tier - total_shard_count :: total number of shards for all nodes in this tier - primary_shard_count :: number of primary shards for all nodes in this tier - doc_count :: number of documents for all nodes in this tier - total_size_bytes :: total number of bytes for all shards for all nodes in this tier - primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier - primary_shard_size_avg_bytes :: average shard size for primary shard in this tier - primary_shard_size_median_bytes :: median shard size for primary shard in this tier - primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier Relates to #60848
2020-10-01 22:35:10 +08:00
"data_tiers" : {
"available" : true,
"enabled" : true,
"data_warm" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_frozen" : {
"node_count" : 1,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
Add telemetry for data tiers (#63031) This commit adds telemetry for our data tier formalization. This telemetry helps determine the topology of the cluster with regard to the content, hot, warm, & cold tiers/roles. An example of the telemetry looks like: ``` GET /_xpack/usage?human { ... "data_tiers" : { "available" : true, "enabled" : true, "data_warm" : { ... }, "data_cold" : { ... }, "data_content" : { "node_count" : 1, "index_count" : 6, "total_shard_count" : 6, "primary_shard_count" : 6, "doc_count" : 71, "total_size" : "59.6kb", "total_size_bytes" : 61110, "primary_size" : "59.6kb", "primary_size_bytes" : 61110, "primary_shard_size_avg" : "9.9kb", "primary_shard_size_avg_bytes" : 10185, "primary_shard_size_median" : "8kb", "primary_shard_size_median_bytes" : 8254, "primary_shard_size_mad" : "7.2kb", "primary_shard_size_mad_bytes" : 7391 }, "data_hot" : { ... } } } ``` The fields are as follows: - node_count :: number of nodes with this tier/role - index_count :: number of indices on this tier - total_shard_count :: total number of shards for all nodes in this tier - primary_shard_count :: number of primary shards for all nodes in this tier - doc_count :: number of documents for all nodes in this tier - total_size_bytes :: total number of bytes for all shards for all nodes in this tier - primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier - primary_shard_size_avg_bytes :: average shard size for primary shard in this tier - primary_shard_size_median_bytes :: median shard size for primary shard in this tier - primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier Relates to #60848
2020-10-01 22:35:10 +08:00
"data_cold" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_content" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_hot" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
}
},
"aggregate_metric" : {
"available" : true,
"enabled" : true
},
"archive" : {
"available" : true,
"enabled" : true,
"indices_count" : 0
},
"health_api" : {
"available" : true,
"enabled" : true,
"invocations": {
"total": 0
}
},
"remote_clusters": {
"size": 0,
"mode": {
"proxy": 0,
"sniff": 0
},
"security": {
"cert": 0,
"api_key": 0
}
},
"enterprise_search" : {
"available": true,
"enabled": true,
"search_applications" : {
"count": 0
},
"analytics_collections": {
"count": 0
},
"query_rulesets": {
"total_rule_count": 0,
"total_count": 0,
"min_rule_count": 0,
"max_rule_count": 0
}
},
"universal_profiling" : {
"available" : true,
"enabled" : true
},
"logsdb": {
"available": true,
"enabled": true,
"indices_count": 0,
"indices_with_synthetic_source": 0,
"num_docs": 0,
"size_in_bytes": 0,
"has_custom_cutoff_date": false
2020-02-21 01:25:57 +08:00
}
}
------------------------------------------------------------
// TESTRESPONSE[s/"security" : \{[^\}]*\},/"security" : $body.$_path,/]
// TESTRESPONSE[s/"rollup" : \{[^\}]*\},/"rollup" : $body.$_path,/]
2020-02-21 01:25:57 +08:00
// TESTRESPONSE[s/"detectors" : \{[^\}]*\},/"detectors" : $body.$_path,/]
// TESTRESPONSE[s/"model_size" : \{[^\}]*\},/"model_size" : $body.$_path,/]
// TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql" : $body.$_path,/]
// TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
// TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
// TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
// TESTRESPONSE[s/"models" : \[[^\]]*\]/"models" : $body.$_path/]
// TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/]
2020-02-21 01:25:57 +08:00
// TESTRESPONSE[s/ : true/ : $body.$_path/]
// TESTRESPONSE[s/ : false/ : $body.$_path/]
// TESTRESPONSE[s/ : (\-)?[0-9]+/ : $body.$_path/]
2020-02-21 01:25:57 +08:00
// These replacements do a few things:
// 1. Ignore the contents of the `policy_stats` object because we don't know all
// of the policies that will be in it. And because we figure folks don't need
// to see an exhaustive list anyway.
// 2. Handling eql, which is disabled by default on release builds and enabled
// everywhere else during the initial implementation phase until its release
// 3. Ignore the contents of the `slm` object because it might contain policies
// 4. Ignore the contents of the `health_api` object because we do not know if the api has been called or not
// 5. All of the numbers and strings on the right hand side of *every* field in
2020-02-21 01:25:57 +08:00
// the response are ignored. So we're really only asserting things about the
// the shape of this response, not the values in it.
// 6. Ignore the contents of the `inference.models` array because the models might not yet have been initialized
// 7. Ignore the contents of data streams until the failure store is tech preview.