2020-02-21 01:25:57 +08:00
|
|
|
[role="xpack"]
|
|
|
|
|
[[usage-api]]
|
|
|
|
|
== Usage API
|
|
|
|
|
|
2025-01-01 04:02:59 +08:00
|
|
|
.New API reference
|
|
|
|
|
[sidebar]
|
|
|
|
|
--
|
|
|
|
|
For the most up-to-date API details, refer to {api-es}/group/endpoint-xpack[Usage APIs].
|
|
|
|
|
--
|
|
|
|
|
|
2020-02-21 01:25:57 +08:00
|
|
|
Provides usage information about the installed {xpack} features.
|
|
|
|
|
|
|
|
|
|
[discrete]
|
|
|
|
|
[[usage-api-request]]
|
|
|
|
|
=== {api-request-title}
|
|
|
|
|
|
|
|
|
|
`GET /_xpack/usage`
|
|
|
|
|
|
2021-01-27 22:06:06 +08:00
|
|
|
[discrete]
|
|
|
|
|
[[usage-api-prereqs]]
|
|
|
|
|
=== {api-prereq-title}
|
|
|
|
|
|
|
|
|
|
* If the {es} {security-features} are enabled, you must have the `monitor` or
|
|
|
|
|
`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
|
|
|
|
|
|
2020-02-21 01:25:57 +08:00
|
|
|
[discrete]
|
|
|
|
|
[[usage-api-desc]]
|
|
|
|
|
=== {api-description-title}
|
|
|
|
|
|
|
|
|
|
This API provides information about which features are currently enabled and
|
2020-04-14 01:16:12 +08:00
|
|
|
available under the current license and some usage statistics.
|
2020-02-21 01:25:57 +08:00
|
|
|
|
|
|
|
|
[discrete]
|
|
|
|
|
[[usage-api-query-parms]]
|
|
|
|
|
=== {api-query-parms-title}
|
|
|
|
|
|
2024-04-17 20:37:07 +08:00
|
|
|
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
|
2020-02-21 01:25:57 +08:00
|
|
|
|
|
|
|
|
[discrete]
|
|
|
|
|
[[usage-api-example]]
|
|
|
|
|
=== {api-examples-title}
|
|
|
|
|
|
|
|
|
|
[source,console]
|
|
|
|
|
------------------------------------------------------------
|
|
|
|
|
GET /_xpack/usage
|
|
|
|
|
------------------------------------------------------------
|
2024-10-21 15:36:14 +08:00
|
|
|
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/]
|
2021-02-10 00:24:14 +08:00
|
|
|
// This response filter removes watcher logging results if they are included
|
|
|
|
|
// to avoid errors in the CI builds.
|
2024-10-21 15:36:14 +08:00
|
|
|
// Same for ES|QL functions, that is a long list and quickly evolving.
|
2020-02-21 01:25:57 +08:00
|
|
|
|
|
|
|
|
[source,console-result]
|
|
|
|
|
------------------------------------------------------------
|
|
|
|
|
{
|
|
|
|
|
"security" : {
|
|
|
|
|
"available" : true,
|
2023-09-13 02:53:41 +08:00
|
|
|
"enabled" : true,
|
|
|
|
|
...
|
2020-02-21 01:25:57 +08:00
|
|
|
},
|
|
|
|
|
"monitoring" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"collection_enabled" : false,
|
|
|
|
|
"enabled_exporters" : {
|
|
|
|
|
"local" : 1
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"watcher" : {
|
2021-01-28 05:42:22 +08:00
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"execution" : {
|
|
|
|
|
"actions" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"total_time_in_ms" : 0
|
|
|
|
|
}
|
2020-02-21 01:25:57 +08:00
|
|
|
}
|
2021-01-28 05:42:22 +08:00
|
|
|
},
|
2020-02-21 01:25:57 +08:00
|
|
|
"watch" : {
|
|
|
|
|
"input" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"active" : 0
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"trigger" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"active" : 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"count" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"active" : 0
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"graph" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
|
|
|
|
"ml" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"jobs" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"count" : 0,
|
|
|
|
|
"detectors" : {
|
|
|
|
|
...
|
|
|
|
|
},
|
|
|
|
|
"created_by" : { },
|
|
|
|
|
"model_size" : {
|
|
|
|
|
...
|
|
|
|
|
},
|
|
|
|
|
"forecasts" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"forecasted_jobs" : 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"datafeeds" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"count" : 0
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"data_frame_analytics_jobs" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"count" : 0
|
2021-02-24 21:45:26 +08:00
|
|
|
},
|
|
|
|
|
"analysis_counts": { },
|
|
|
|
|
"memory_usage": {
|
|
|
|
|
"peak_usage_bytes": {
|
|
|
|
|
"min": 0.0,
|
|
|
|
|
"max": 0.0,
|
|
|
|
|
"avg": 0.0,
|
|
|
|
|
"total": 0.0
|
|
|
|
|
}
|
2020-02-21 01:25:57 +08:00
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"inference" : {
|
|
|
|
|
"ingest_processors" : {
|
|
|
|
|
"_all" : {
|
|
|
|
|
"num_docs_processed" : {
|
|
|
|
|
"max" : 0,
|
|
|
|
|
"sum" : 0,
|
|
|
|
|
"min" : 0
|
|
|
|
|
},
|
|
|
|
|
"pipelines" : {
|
|
|
|
|
"count" : 0
|
|
|
|
|
},
|
|
|
|
|
"num_failures" : {
|
|
|
|
|
"max" : 0,
|
|
|
|
|
"sum" : 0,
|
|
|
|
|
"min" : 0
|
|
|
|
|
},
|
|
|
|
|
"time_ms" : {
|
|
|
|
|
"max" : 0,
|
|
|
|
|
"sum" : 0,
|
|
|
|
|
"min" : 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"trained_models" : {
|
|
|
|
|
"_all" : {
|
2021-11-04 23:26:37 +08:00
|
|
|
"count": 1
|
2021-02-24 21:45:26 +08:00
|
|
|
},
|
|
|
|
|
"count": {
|
|
|
|
|
"total": 1,
|
|
|
|
|
"prepackaged": 1,
|
|
|
|
|
"other": 0
|
|
|
|
|
},
|
2021-11-10 23:23:25 +08:00
|
|
|
"model_size_bytes": {
|
2021-02-24 21:45:26 +08:00
|
|
|
"min": 0.0,
|
|
|
|
|
"max": 0.0,
|
|
|
|
|
"avg": 0.0,
|
|
|
|
|
"total": 0.0
|
|
|
|
|
},
|
|
|
|
|
"estimated_operations": {
|
|
|
|
|
"min": 0.0,
|
|
|
|
|
"max": 0.0,
|
|
|
|
|
"avg": 0.0,
|
|
|
|
|
"total": 0.0
|
2020-02-21 01:25:57 +08:00
|
|
|
}
|
2021-11-04 23:26:37 +08:00
|
|
|
},
|
|
|
|
|
"deployments": {
|
|
|
|
|
"count": 0,
|
|
|
|
|
"inference_counts": {
|
|
|
|
|
"total": 0.0,
|
|
|
|
|
"min": 0.0,
|
|
|
|
|
"avg": 0.0,
|
|
|
|
|
"max": 0.0
|
|
|
|
|
},
|
2023-11-13 15:50:15 +08:00
|
|
|
"stats_by_model": [],
|
2021-11-04 23:26:37 +08:00
|
|
|
"model_sizes_bytes": {
|
|
|
|
|
"total": 0.0,
|
|
|
|
|
"min": 0.0,
|
|
|
|
|
"avg": 0.0,
|
|
|
|
|
"max": 0.0
|
|
|
|
|
},
|
|
|
|
|
"time_ms": {
|
|
|
|
|
"avg": 0.0
|
|
|
|
|
}
|
2020-02-21 01:25:57 +08:00
|
|
|
}
|
|
|
|
|
},
|
2024-09-23 23:08:46 +08:00
|
|
|
"node_count" : 1,
|
|
|
|
|
"memory": {
|
|
|
|
|
anomaly_detectors_memory_bytes: 0,
|
|
|
|
|
data_frame_analytics_memory_bytes: 0,
|
|
|
|
|
pytorch_inference_memory_bytes: 0,
|
|
|
|
|
total_used_memory_bytes: 0
|
|
|
|
|
}
|
2020-02-21 01:25:57 +08:00
|
|
|
},
|
2023-12-04 21:27:54 +08:00
|
|
|
"inference": {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
2024-12-23 23:03:47 +08:00
|
|
|
"models" : [
|
|
|
|
|
...
|
[ML] Default inference endpoint for ELSER (#113873)
Adds a default configuration for the ELSER model. The config uses
adaptive allocations to automatically scale. Min number of allocations
is set to 1 for this PR, a follow up with change that to 0 and enable
scale from 0.
This end point is always visible in the GET API.
```
GET _inference
{
"endpoints": [
{
"inference_id": ".elser-2",
"task_type": "sparse_embedding",
"service": "elser",
"service_settings": {
"num_threads": 1,
"model_id": ".elser_model_2",
"adaptive_allocations": {
"enabled": true,
"min_number_of_allocations": 1,
"max_number_of_allocations": 8
}
},
"task_settings": {}
}
]
}
```
The default configuration can be used against without any prior setup.
If the model is not downloaded it is automatically downloaded. If it is
not deployed it is deployed
```
POST _inference/.elser-2
{
"input": "Automagically deploy and infer"
}
...
{
"sparse_embedding": [
{
"is_truncated": false,
"embedding": {
"##fer": 2.2107008,
"deployment": 2.1624098,
"deploy": 2.144009,
"auto": 1.9384763,
```
### Follow up tasks - [ ] Add default config for the E5 text embedding
model - [ ] Select platform specific version - [ ] Scale from 0 - [ ]
Chunking settings - What happens when the end point is deleted, can it
be deleted? - Can the default config be modified - chunking settings for
example? Probably not
2024-10-04 17:27:10 +08:00
|
|
|
]
|
2023-12-04 21:27:54 +08:00
|
|
|
},
|
2020-02-21 01:25:57 +08:00
|
|
|
"logstash" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
|
|
|
|
"eql" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
2023-05-25 07:17:02 +08:00
|
|
|
"esql" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"features" : {
|
|
|
|
|
"eval" : 0,
|
|
|
|
|
"stats" : 0,
|
2023-08-14 21:52:43 +08:00
|
|
|
"dissect": 0,
|
2023-05-25 07:17:02 +08:00
|
|
|
"grok" : 0,
|
|
|
|
|
"limit" : 0,
|
|
|
|
|
"where" : 0,
|
2023-12-06 08:24:00 +08:00
|
|
|
"sort" : 0,
|
|
|
|
|
"drop" : 0,
|
|
|
|
|
"show" : 0,
|
|
|
|
|
"rename" : 0,
|
|
|
|
|
"mv_expand" : 0,
|
|
|
|
|
"keep" : 0,
|
|
|
|
|
"enrich" : 0,
|
|
|
|
|
"from" : 0,
|
2024-10-09 00:37:55 +08:00
|
|
|
"row" : 0
|
2023-05-25 07:17:02 +08:00
|
|
|
},
|
|
|
|
|
"queries" : {
|
|
|
|
|
"rest" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"kibana" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"_all" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
2020-02-21 01:25:57 +08:00
|
|
|
"sql" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"features" : {
|
|
|
|
|
"having" : 0,
|
|
|
|
|
"subselect" : 0,
|
|
|
|
|
"limit" : 0,
|
|
|
|
|
"orderby" : 0,
|
|
|
|
|
"where" : 0,
|
|
|
|
|
"join" : 0,
|
|
|
|
|
"groupby" : 0,
|
|
|
|
|
"command" : 0,
|
|
|
|
|
"local" : 0
|
|
|
|
|
},
|
|
|
|
|
"queries" : {
|
|
|
|
|
"rest" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"cli" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"canvas" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"odbc" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"jdbc" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"odbc32" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"odbc64" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"_all" : {
|
|
|
|
|
"total" : 0,
|
|
|
|
|
"paging" : 0,
|
|
|
|
|
"failed" : 0
|
|
|
|
|
},
|
|
|
|
|
"translate" : {
|
|
|
|
|
"count" : 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"rollup" : {
|
|
|
|
|
"available" : true,
|
2024-05-08 01:02:54 +08:00
|
|
|
"enabled" : true,
|
|
|
|
|
...
|
2020-02-21 01:25:57 +08:00
|
|
|
},
|
|
|
|
|
"ilm" : {
|
|
|
|
|
"policy_count" : 3,
|
|
|
|
|
"policy_stats" : [
|
|
|
|
|
...
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"slm" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
|
|
|
|
"ccr" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"follower_indices_count" : 0,
|
|
|
|
|
"auto_follow_patterns_count" : 0
|
|
|
|
|
},
|
|
|
|
|
"transform" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
|
|
|
|
"voting_only" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
2020-07-07 16:41:42 +08:00
|
|
|
"searchable_snapshots" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
2021-03-17 01:02:34 +08:00
|
|
|
"indices_count" : 0,
|
|
|
|
|
"full_copy_indices_count" : 0,
|
|
|
|
|
"shared_cache_indices_count" : 0
|
2020-07-07 16:41:42 +08:00
|
|
|
},
|
2020-02-21 01:25:57 +08:00
|
|
|
"spatial" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
|
|
|
|
},
|
|
|
|
|
"analytics" : {
|
|
|
|
|
"available" : true,
|
2020-04-14 01:16:12 +08:00
|
|
|
"enabled" : true,
|
|
|
|
|
"stats": {
|
|
|
|
|
"boxplot_usage" : 0,
|
|
|
|
|
"top_metrics_usage" : 0,
|
2020-05-15 04:32:42 +08:00
|
|
|
"normalize_usage" : 0,
|
2020-04-14 01:16:12 +08:00
|
|
|
"cumulative_cardinality_usage" : 0,
|
|
|
|
|
"t_test_usage" : 0,
|
2020-08-25 23:32:20 +08:00
|
|
|
"rate_usage" : 0,
|
2020-05-12 16:30:52 +08:00
|
|
|
"string_stats_usage" : 0,
|
2021-02-04 02:13:33 +08:00
|
|
|
"moving_percentiles_usage" : 0,
|
|
|
|
|
"multi_terms_usage" : 0
|
2020-04-14 01:16:12 +08:00
|
|
|
}
|
2020-07-14 02:12:22 +08:00
|
|
|
},
|
|
|
|
|
"data_streams" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"data_streams" : 0,
|
|
|
|
|
"indices_count" : 0
|
Add telemetry for data tiers (#63031)
This commit adds telemetry for our data tier formalization. This telemetry helps determine the
topology of the cluster with regard to the content, hot, warm, & cold tiers/roles.
An example of the telemetry looks like:
```
GET /_xpack/usage?human
{
...
"data_tiers" : {
"available" : true,
"enabled" : true,
"data_warm" : {
...
},
"data_cold" : {
...
},
"data_content" : {
"node_count" : 1,
"index_count" : 6,
"total_shard_count" : 6,
"primary_shard_count" : 6,
"doc_count" : 71,
"total_size" : "59.6kb",
"total_size_bytes" : 61110,
"primary_size" : "59.6kb",
"primary_size_bytes" : 61110,
"primary_shard_size_avg" : "9.9kb",
"primary_shard_size_avg_bytes" : 10185,
"primary_shard_size_median" : "8kb",
"primary_shard_size_median_bytes" : 8254,
"primary_shard_size_mad" : "7.2kb",
"primary_shard_size_mad_bytes" : 7391
},
"data_hot" : {
...
}
}
}
```
The fields are as follows:
- node_count :: number of nodes with this tier/role
- index_count :: number of indices on this tier
- total_shard_count :: total number of shards for all nodes in this tier
- primary_shard_count :: number of primary shards for all nodes in this tier
- doc_count :: number of documents for all nodes in this tier
- total_size_bytes :: total number of bytes for all shards for all nodes in this tier
- primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier
- primary_shard_size_avg_bytes :: average shard size for primary shard in this tier
- primary_shard_size_median_bytes :: median shard size for primary shard in this tier
- primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier
Relates to #60848
2020-10-01 22:35:10 +08:00
|
|
|
},
|
2023-05-30 18:48:03 +08:00
|
|
|
"data_lifecycle" : {
|
|
|
|
|
"available": true,
|
|
|
|
|
"enabled": true,
|
2023-06-01 20:33:26 +08:00
|
|
|
"count": 0,
|
|
|
|
|
"default_rollover_used": true,
|
Update data stream lifecycle telemetry to track global retention (#112451)
Currently, the data stream lifecycle telemetry has the following
structure:
```
{
....
"data_lifecycle" : {
"available": true,
"enabled": true,
"count": 0,
"default_rollover_used": true,
"retention": {
"minimum_millis": 0,
"maximum_millis": 0,
"average_millis": 0.0
}
}....
```
In the snippet above you can see that we track:
- The amount of data streams managed by the data stream lifecycle by `count`
- If the default rollover has been overwritten by `default_rollover_used`
- The min, max and average of the `data_retention` configured on a data stream level.
In this PR we propose the following extention:
```
....
"data_lifecycle" : {
"available": true,
"enabled": true,
"count": 0,
"default_rollover_used": true,
"effective_retention": { #https://github.com/elastic/dev/issues/2537
"retained_data_streams": 5,
"minimum_millis": 0, # Only if retained data streams > 1
"maximum_millis": 0,
"average_millis": 0.0
},
"data_retention": {
"configured_data_streams": 5,
"minimum_millis": 0, # Only if retained data streams > 1
"maximum_millis": 0,
"average_millis": 0.0
},
"global_retention": {
"default": {
"defined": true/false,
"affected_data_streams": 0,
"millis": 0
},
"max": {
"defined": true/false,
"affected_data_streams": 0,
"millis": 0
}
}
```
With this extension we are tracking:
- The amount of data streams managed by the data stream lifecycle by `count`
- If the default rollover has been overwritten by `default_rollover_used`
- The min, max and average of the `data_retention` configured on a data stream level and the number of data streams that have it configured. We add the min, max and avg only if there are data streams with data retention configuration to avoid messing with the stats in a dashboard.
- The min, max and average of the `effective_retention` and the number of data streams that are retained. We add the min, max and avg only if there are retained data streams to avoid messing with the stats in a dashboard.
- Global retention stats, if they are defined, if the number of the affected data streams and the actual value.
The above metrics allow us to answer questions like:
- How many data streams are affected by global retention.
- How big is the difference between the longest data retention compared to max global retention.
- How much does the effective retention diverging from the data retention, this will show the impact of the global retention.
2024-09-11 16:31:04 +08:00
|
|
|
"data_retention": {
|
|
|
|
|
"configured_data_streams": 0
|
|
|
|
|
},
|
|
|
|
|
"effective_retention": {
|
|
|
|
|
"retained_data_streams": 0
|
|
|
|
|
},
|
|
|
|
|
"global_retention": {
|
|
|
|
|
"default": {
|
|
|
|
|
"defined": false
|
|
|
|
|
},
|
|
|
|
|
"max": {
|
|
|
|
|
"defined": false
|
|
|
|
|
}
|
2023-05-30 18:48:03 +08:00
|
|
|
}
|
|
|
|
|
},
|
Add telemetry for data tiers (#63031)
This commit adds telemetry for our data tier formalization. This telemetry helps determine the
topology of the cluster with regard to the content, hot, warm, & cold tiers/roles.
An example of the telemetry looks like:
```
GET /_xpack/usage?human
{
...
"data_tiers" : {
"available" : true,
"enabled" : true,
"data_warm" : {
...
},
"data_cold" : {
...
},
"data_content" : {
"node_count" : 1,
"index_count" : 6,
"total_shard_count" : 6,
"primary_shard_count" : 6,
"doc_count" : 71,
"total_size" : "59.6kb",
"total_size_bytes" : 61110,
"primary_size" : "59.6kb",
"primary_size_bytes" : 61110,
"primary_shard_size_avg" : "9.9kb",
"primary_shard_size_avg_bytes" : 10185,
"primary_shard_size_median" : "8kb",
"primary_shard_size_median_bytes" : 8254,
"primary_shard_size_mad" : "7.2kb",
"primary_shard_size_mad_bytes" : 7391
},
"data_hot" : {
...
}
}
}
```
The fields are as follows:
- node_count :: number of nodes with this tier/role
- index_count :: number of indices on this tier
- total_shard_count :: total number of shards for all nodes in this tier
- primary_shard_count :: number of primary shards for all nodes in this tier
- doc_count :: number of documents for all nodes in this tier
- total_size_bytes :: total number of bytes for all shards for all nodes in this tier
- primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier
- primary_shard_size_avg_bytes :: average shard size for primary shard in this tier
- primary_shard_size_median_bytes :: median shard size for primary shard in this tier
- primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier
Relates to #60848
2020-10-01 22:35:10 +08:00
|
|
|
"data_tiers" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"data_warm" : {
|
|
|
|
|
"node_count" : 0,
|
|
|
|
|
"index_count" : 0,
|
|
|
|
|
"total_shard_count" : 0,
|
|
|
|
|
"primary_shard_count" : 0,
|
|
|
|
|
"doc_count" : 0,
|
|
|
|
|
"total_size_bytes" : 0,
|
|
|
|
|
"primary_size_bytes" : 0,
|
|
|
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
|
|
|
"primary_shard_size_median_bytes" : 0,
|
|
|
|
|
"primary_shard_size_mad_bytes" : 0
|
|
|
|
|
},
|
2021-02-06 05:38:13 +08:00
|
|
|
"data_frozen" : {
|
|
|
|
|
"node_count" : 1,
|
|
|
|
|
"index_count" : 0,
|
|
|
|
|
"total_shard_count" : 0,
|
|
|
|
|
"primary_shard_count" : 0,
|
|
|
|
|
"doc_count" : 0,
|
|
|
|
|
"total_size_bytes" : 0,
|
|
|
|
|
"primary_size_bytes" : 0,
|
|
|
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
|
|
|
"primary_shard_size_median_bytes" : 0,
|
|
|
|
|
"primary_shard_size_mad_bytes" : 0
|
|
|
|
|
},
|
Add telemetry for data tiers (#63031)
This commit adds telemetry for our data tier formalization. This telemetry helps determine the
topology of the cluster with regard to the content, hot, warm, & cold tiers/roles.
An example of the telemetry looks like:
```
GET /_xpack/usage?human
{
...
"data_tiers" : {
"available" : true,
"enabled" : true,
"data_warm" : {
...
},
"data_cold" : {
...
},
"data_content" : {
"node_count" : 1,
"index_count" : 6,
"total_shard_count" : 6,
"primary_shard_count" : 6,
"doc_count" : 71,
"total_size" : "59.6kb",
"total_size_bytes" : 61110,
"primary_size" : "59.6kb",
"primary_size_bytes" : 61110,
"primary_shard_size_avg" : "9.9kb",
"primary_shard_size_avg_bytes" : 10185,
"primary_shard_size_median" : "8kb",
"primary_shard_size_median_bytes" : 8254,
"primary_shard_size_mad" : "7.2kb",
"primary_shard_size_mad_bytes" : 7391
},
"data_hot" : {
...
}
}
}
```
The fields are as follows:
- node_count :: number of nodes with this tier/role
- index_count :: number of indices on this tier
- total_shard_count :: total number of shards for all nodes in this tier
- primary_shard_count :: number of primary shards for all nodes in this tier
- doc_count :: number of documents for all nodes in this tier
- total_size_bytes :: total number of bytes for all shards for all nodes in this tier
- primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier
- primary_shard_size_avg_bytes :: average shard size for primary shard in this tier
- primary_shard_size_median_bytes :: median shard size for primary shard in this tier
- primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier
Relates to #60848
2020-10-01 22:35:10 +08:00
|
|
|
"data_cold" : {
|
|
|
|
|
"node_count" : 0,
|
|
|
|
|
"index_count" : 0,
|
|
|
|
|
"total_shard_count" : 0,
|
|
|
|
|
"primary_shard_count" : 0,
|
|
|
|
|
"doc_count" : 0,
|
|
|
|
|
"total_size_bytes" : 0,
|
|
|
|
|
"primary_size_bytes" : 0,
|
|
|
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
|
|
|
"primary_shard_size_median_bytes" : 0,
|
|
|
|
|
"primary_shard_size_mad_bytes" : 0
|
|
|
|
|
},
|
|
|
|
|
"data_content" : {
|
|
|
|
|
"node_count" : 0,
|
|
|
|
|
"index_count" : 0,
|
|
|
|
|
"total_shard_count" : 0,
|
|
|
|
|
"primary_shard_count" : 0,
|
|
|
|
|
"doc_count" : 0,
|
|
|
|
|
"total_size_bytes" : 0,
|
|
|
|
|
"primary_size_bytes" : 0,
|
|
|
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
|
|
|
"primary_shard_size_median_bytes" : 0,
|
|
|
|
|
"primary_shard_size_mad_bytes" : 0
|
|
|
|
|
},
|
|
|
|
|
"data_hot" : {
|
|
|
|
|
"node_count" : 0,
|
|
|
|
|
"index_count" : 0,
|
|
|
|
|
"total_shard_count" : 0,
|
|
|
|
|
"primary_shard_count" : 0,
|
|
|
|
|
"doc_count" : 0,
|
|
|
|
|
"total_size_bytes" : 0,
|
|
|
|
|
"primary_size_bytes" : 0,
|
|
|
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
|
|
|
"primary_shard_size_median_bytes" : 0,
|
|
|
|
|
"primary_shard_size_mad_bytes" : 0
|
|
|
|
|
}
|
2020-11-11 01:31:59 +08:00
|
|
|
},
|
|
|
|
|
"aggregate_metric" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
2022-02-16 15:31:41 +08:00
|
|
|
},
|
|
|
|
|
"archive" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"indices_count" : 0
|
2023-01-04 21:34:29 +08:00
|
|
|
},
|
|
|
|
|
"health_api" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true,
|
|
|
|
|
"invocations": {
|
|
|
|
|
"total": 0
|
|
|
|
|
}
|
2023-05-05 17:11:13 +08:00
|
|
|
},
|
2023-08-14 06:33:37 +08:00
|
|
|
"remote_clusters": {
|
|
|
|
|
"size": 0,
|
|
|
|
|
"mode": {
|
|
|
|
|
"proxy": 0,
|
|
|
|
|
"sniff": 0
|
|
|
|
|
},
|
|
|
|
|
"security": {
|
|
|
|
|
"cert": 0,
|
|
|
|
|
"api_key": 0
|
|
|
|
|
}
|
|
|
|
|
},
|
2023-05-05 17:11:13 +08:00
|
|
|
"enterprise_search" : {
|
|
|
|
|
"available": true,
|
|
|
|
|
"enabled": true,
|
|
|
|
|
"search_applications" : {
|
|
|
|
|
"count": 0
|
2023-05-27 00:10:54 +08:00
|
|
|
},
|
|
|
|
|
"analytics_collections": {
|
|
|
|
|
"count": 0
|
2023-08-07 22:38:12 +08:00
|
|
|
},
|
|
|
|
|
"query_rulesets": {
|
|
|
|
|
"total_rule_count": 0,
|
|
|
|
|
"total_count": 0,
|
|
|
|
|
"min_rule_count": 0,
|
|
|
|
|
"max_rule_count": 0
|
2023-05-05 17:11:13 +08:00
|
|
|
}
|
2023-10-09 13:58:04 +08:00
|
|
|
},
|
|
|
|
|
"universal_profiling" : {
|
|
|
|
|
"available" : true,
|
|
|
|
|
"enabled" : true
|
2024-11-01 23:25:40 +08:00
|
|
|
},
|
|
|
|
|
"logsdb": {
|
|
|
|
|
"available": true,
|
2025-01-29 22:03:28 +08:00
|
|
|
"enabled": true,
|
2024-11-01 23:25:40 +08:00
|
|
|
"indices_count": 0,
|
2024-12-23 23:03:47 +08:00
|
|
|
"indices_with_synthetic_source": 0,
|
|
|
|
|
"num_docs": 0,
|
|
|
|
|
"size_in_bytes": 0,
|
|
|
|
|
"has_custom_cutoff_date": false
|
2020-02-21 01:25:57 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
------------------------------------------------------------
|
2023-09-13 02:53:41 +08:00
|
|
|
// TESTRESPONSE[s/"security" : \{[^\}]*\},/"security" : $body.$_path,/]
|
2024-05-08 01:02:54 +08:00
|
|
|
// TESTRESPONSE[s/"rollup" : \{[^\}]*\},/"rollup" : $body.$_path,/]
|
2020-02-21 01:25:57 +08:00
|
|
|
// TESTRESPONSE[s/"detectors" : \{[^\}]*\},/"detectors" : $body.$_path,/]
|
|
|
|
|
// TESTRESPONSE[s/"model_size" : \{[^\}]*\},/"model_size" : $body.$_path,/]
|
|
|
|
|
// TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql" : $body.$_path,/]
|
|
|
|
|
// TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
|
|
|
|
|
// TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
|
2023-01-04 21:34:29 +08:00
|
|
|
// TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
|
2024-12-23 23:03:47 +08:00
|
|
|
// TESTRESPONSE[s/"models" : \[[^\]]*\]/"models" : $body.$_path/]
|
2024-05-23 21:45:11 +08:00
|
|
|
// TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/]
|
2020-02-21 01:25:57 +08:00
|
|
|
// TESTRESPONSE[s/ : true/ : $body.$_path/]
|
|
|
|
|
// TESTRESPONSE[s/ : false/ : $body.$_path/]
|
|
|
|
|
// TESTRESPONSE[s/ : (\-)?[0-9]+/ : $body.$_path/]
|
2021-02-10 00:24:14 +08:00
|
|
|
|
2020-02-21 01:25:57 +08:00
|
|
|
// These replacements do a few things:
|
|
|
|
|
// 1. Ignore the contents of the `policy_stats` object because we don't know all
|
|
|
|
|
// of the policies that will be in it. And because we figure folks don't need
|
|
|
|
|
// to see an exhaustive list anyway.
|
|
|
|
|
// 2. Handling eql, which is disabled by default on release builds and enabled
|
|
|
|
|
// everywhere else during the initial implementation phase until its release
|
|
|
|
|
// 3. Ignore the contents of the `slm` object because it might contain policies
|
2023-01-04 21:34:29 +08:00
|
|
|
// 4. Ignore the contents of the `health_api` object because we do not know if the api has been called or not
|
|
|
|
|
// 5. All of the numbers and strings on the right hand side of *every* field in
|
2020-02-21 01:25:57 +08:00
|
|
|
// the response are ignored. So we're really only asserting things about the
|
2020-03-05 00:35:03 +08:00
|
|
|
// the shape of this response, not the values in it.
|
2024-12-23 23:03:47 +08:00
|
|
|
// 6. Ignore the contents of the `inference.models` array because the models might not yet have been initialized
|
|
|
|
|
// 7. Ignore the contents of data streams until the failure store is tech preview.
|
