From 276755a0e4d0f80cffbf5cb02689967a12c0f6ec Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 11 Apr 2019 10:53:23 +0200 Subject: [PATCH] devenv: add elasticsearch v6 filebeat integration (#16493) Adds filebeat (log collector) to elasticsearch v6 docker block that collects Grafana logs. Also adds a new elasticsearch v6 datasource and test dashboard thru provisioning. --- devenv/datasources.yaml | 10 + ...ource_tests_elasticsearch_v6_filebeat.json | 255 ++++++++++++++++++ .../blocks/elastic6/docker-compose.yaml | 8 + devenv/docker/blocks/elastic6/filebeat.yml | 183 +++++++++++++ 4 files changed, 456 insertions(+) create mode 100644 devenv/dev-dashboards/datasource_tests_elasticsearch_v6_filebeat.json create mode 100644 devenv/docker/blocks/elastic6/filebeat.yml diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml index 1922743fca8..f50fdeeb5c4 100644 --- a/devenv/datasources.yaml +++ b/devenv/datasources.yaml @@ -95,6 +95,16 @@ datasources: timeField: "@timestamp" esVersion: 60 + - name: gdev-elasticsearch-v6-filebeat + type: elasticsearch + access: proxy + database: "[filebeat-6.2.4-]YYYY.MM.DD" + url: http://localhost:11200 + jsonData: + interval: Daily + timeField: "@timestamp" + esVersion: 60 + - name: gdev-mysql type: mysql url: localhost:3306 diff --git a/devenv/dev-dashboards/datasource_tests_elasticsearch_v6_filebeat.json b/devenv/dev-dashboards/datasource_tests_elasticsearch_v6_filebeat.json new file mode 100644 index 00000000000..6682f09d8e1 --- /dev/null +++ b/devenv/dev-dashboards/datasource_tests_elasticsearch_v6_filebeat.json @@ -0,0 +1,255 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1554902936982, + "links": [], + "panels": [ + { + "aliasColors": { + "error": "red" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-elasticsearch-v6-filebeat", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "fake": true, + "field": "fields.level", + "id": "3", + "settings": { + "min_doc_count": 1, + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "5m", + "min_doc_count": 1, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "select field", + "id": "1", + "type": "count" + } + ], + "query": "fields.app:grafana", + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [ + { + "text": "@timestamp", + "value": "@timestamp" + }, + { + "text": "fields.level", + "value": "fields.level" + }, + { + "text": "message", + "value": "message" + } + ], + "datasource": "gdev-elasticsearch-v6-filebeat", + "fontSize": "100%", + "gridPos": { + "h": 22, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "@timestamp", + "type": "date" + }, + { + "alias": "Level", + "colorMode": null, + "colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "fields.level", + "thresholds": [""], + "type": "string", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "Message", + "colorMode": null, + "colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"], + "decimals": 2, + "pattern": "message", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "bucketAggs": [], + "metrics": [ + { + "field": "select field", + "id": "1", + "meta": {}, + "settings": { + "size": 500 + }, + "type": "raw_document" + } + ], + "query": "fields.app:grafana", + "refId": "A", + "timeField": "@timestamp" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "transform": "json", + "type": "table" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": ["gdev", "elasticsearch"], + "templating": { + "list": [ + { + "datasource": "gdev-elasticsearch-v6-filebeat", + "filters": [], + "hide": 0, + "label": "", + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "Datasource tests - Elasticsearch v6 Filebeat", + "uid": "06tPt4gZz", + "version": 8 +} diff --git a/devenv/docker/blocks/elastic6/docker-compose.yaml b/devenv/docker/blocks/elastic6/docker-compose.yaml index dd2439f88e4..d7f92e2b6a5 100644 --- a/devenv/docker/blocks/elastic6/docker-compose.yaml +++ b/devenv/docker/blocks/elastic6/docker-compose.yaml @@ -13,3 +13,11 @@ environment: FD_DATASOURCE: elasticsearch6 FD_PORT: 11200 + + filebeat6: + image: docker.elastic.co/beats/filebeat:6.2.4 + command: filebeat -e -strict.perms=false + volumes: + - ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/log:/var/log:ro + - ../data/log:/var/log/grafana:ro diff --git a/devenv/docker/blocks/elastic6/filebeat.yml b/devenv/docker/blocks/elastic6/filebeat.yml new file mode 100644 index 00000000000..4b80ffe6196 --- /dev/null +++ b/devenv/docker/blocks/elastic6/filebeat.yml @@ -0,0 +1,183 @@ +###################### Filebeat Configuration Example ######################### + +# This file is an example configuration file highlighting only the most common +# options. The filebeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/filebeat/index.html + +# For more available modules and options, please see the filebeat.reference.yml sample +# configuration file. + +#=========================== Filebeat prospectors ============================= + +filebeat.prospectors: + +# Each - is a prospector. Most options can be set at the prospector level, so +# you can use different prospectors for various configurations. +# Below are the prospector specific configurations. + +- type: log + enabled: false + paths: + - /var/log/*.log +- type: log + enabled: true + paths: + - /var/log/grafana/grafana.log + include_lines: ['lvl=info'] + fields: + app: grafana + level: info +- type: log + enabled: true + paths: + - /var/log/grafana/grafana.log + include_lines: ['lvl=eror'] + fields: + app: grafana + level: error +- type: log + enabled: true + paths: + - /var/log/grafana/grafana.log + include_lines: ['lvl=warn'] + fields: + app: grafana + level: warning +- type: log + enabled: true + paths: + - /var/log/grafana/grafana.log + include_lines: ['lvl=dbug'] + fields: + app: grafana + level: debug + +#============================= Filebeat modules =============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +#==================== Elasticsearch template setting ========================== + +setup.template.settings: + index.number_of_shards: 3 + #index.codec: best_compression + #_source.enabled: false + +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + + +#============================== Dashboards ===================================== +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +#============================== Kibana ===================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + +#============================= Elastic Cloud ================================== + +# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Outputs ===================================== + +# Configure what output to use when sending the data collected by the beat. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["elasticsearch6:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + +#----------------------------- Logstash output -------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +#================================ Logging ===================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +#============================== Xpack Monitoring =============================== +# filebeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#xpack.monitoring.enabled: false + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. Any setting that is not set is +# automatically inherited from the Elasticsearch output configuration, so if you +# have the Elasticsearch output configured, you can simply uncomment the +# following line. +#xpack.monitoring.elasticsearch: \ No newline at end of file