diff --git a/devenv/frontend-service/Tiltfile b/devenv/frontend-service/Tiltfile index a753e475e80..efb5a9822e4 100644 --- a/devenv/frontend-service/Tiltfile +++ b/devenv/frontend-service/Tiltfile @@ -68,10 +68,13 @@ dc_resource("frontend-service", labels=["services"], ) dc_resource("alloy", labels=["observability"]) +dc_resource("prometheus", labels=["observability"]) dc_resource("loki", labels=["observability"]) -dc_resource("tempo-init", labels=["observability"]) dc_resource("tempo", labels=["observability"]) +dc_resource("postgres", labels=["misc"]) +dc_resource("tempo-init", labels=["misc"]) + # paths in tilt files are confusing.... # - if tilt is dealing the the path, it is relative to the Tiltfile # - if docker is dealing with the path, it is relative to the context diff --git a/devenv/frontend-service/configs/alloy/_databases.alloy b/devenv/frontend-service/configs/alloy/_databases.alloy index 9e050d70fd9..41a2c15b359 100644 --- a/devenv/frontend-service/configs/alloy/_databases.alloy +++ b/devenv/frontend-service/configs/alloy/_databases.alloy @@ -1,3 +1,22 @@ +// +// Metrics +prometheus.relabel "publish_metrics" { + rule { + action = "replace" + target_label = "instance" + replacement = constants.hostname + } + + forward_to = [prometheus.remote_write.tilt_prom.receiver] +} + +prometheus.remote_write "tilt_prom" { + endpoint { + url = "http://prometheus:9090/api/v1/write" + send_native_histograms = true + } +} + // // Logs loki.relabel "publish_logs" { diff --git a/devenv/frontend-service/configs/alloy/alloy.alloy b/devenv/frontend-service/configs/alloy/alloy.alloy index bd09daa1e87..1ed6dfe2a45 100644 --- a/devenv/frontend-service/configs/alloy/alloy.alloy +++ b/devenv/frontend-service/configs/alloy/alloy.alloy @@ -1,7 +1,6 @@ // // Self-config // - livedebugging { enabled = true } @@ -21,3 +20,13 @@ loki.relabel "alloy" { forward_to = [loki.relabel.publish_logs.receiver] } + +// Metrics +prometheus.exporter.self "alloy" { } + +prometheus.scrape "alloy" { + targets = prometheus.exporter.self.alloy.targets + scrape_interval = "60s" + forward_to = [prometheus.relabel.publish_metrics.receiver] + scrape_protocols = ["PrometheusProto", "OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"] +} diff --git a/devenv/frontend-service/configs/alloy/grafana.alloy b/devenv/frontend-service/configs/alloy/grafana.alloy new file mode 100644 index 00000000000..48e6ccc6eef --- /dev/null +++ b/devenv/frontend-service/configs/alloy/grafana.alloy @@ -0,0 +1,11 @@ +// +// Metrics +prometheus.scrape "grafana_dev" { + targets = [ + {__scheme__ = "http", __address__ = "grafana-api:3000", job = "grafana_dev", service_name = "grafana-api"}, + {__scheme__ = "http", __address__ = "frontend-service:3000", job = "grafana_dev", service_name = "frontend-service"}, + ] + scrape_interval = "15s" + forward_to = [prometheus.relabel.publish_metrics.receiver] + scrape_protocols = ["PrometheusProto", "OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"] +} diff --git a/devenv/frontend-service/docker-compose.yaml b/devenv/frontend-service/docker-compose.yaml index 3a238ba890d..ec599c2b67b 100644 --- a/devenv/frontend-service/docker-compose.yaml +++ b/devenv/frontend-service/docker-compose.yaml @@ -34,7 +34,7 @@ services: GF_DEFAULT_APP_MODE: development GF_PANELS_ENABLE_ALPHA: true GF_SERVER_CDN_URL: http://localhost:3010 - GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend + GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend enableNativeHTTPHistogram GF_DATABASE_URL: postgres://grafana:grafana@postgres:5432/grafana GF_SERVER_ROUTER_LOGGING: true GF_LOG_LEVEL: info @@ -62,6 +62,7 @@ services: GF_DEFAULT_APP_MODE: development GF_DEFAULT_TARGET: frontend-server GF_SECURITY_CONTENT_SECURITY_POLICY: false + GF_FEATURE_TOGGLES_ENABLE: enableNativeHTTPHistogram GF_SERVER_CDN_URL: http://localhost:3010 GF_SERVER_ROUTER_LOGGING: true GF_LOG_LEVEL: info @@ -95,6 +96,20 @@ services: - /alloy-config depends_on: - loki + labels: + - 'alloy.logs=true' + + prometheus: + image: prom/prometheus + volumes: + - prometheus-data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.enable-remote-write-receiver' + - '--enable-feature=native-histograms' + labels: + - 'alloy.logs=true' loki: image: grafana/loki @@ -120,6 +135,8 @@ services: - tempo-data:/var/lib/tempo - ./configs/tempo.yaml:/etc/tempo/tempo.yaml command: ['-config.file=/etc/tempo/tempo.yaml'] + labels: + - 'alloy.logs=true' depends_on: tempo-init: @@ -131,3 +148,4 @@ volumes: alloy-data: loki-data: tempo-data: + prometheus-data: diff --git a/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json b/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json index 1669392b6b6..e5b18aa04bc 100644 --- a/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json +++ b/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json @@ -18,9 +18,952 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 272, + "id": 138, "links": [], "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(grafana_build_info) by (job, revision, version, goversion, branch)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - process_start_time_seconds{service_name=\"grafana-api\"}", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Backend version", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value": 6, + "branch": 2, + "goversion": 3, + "job": 1, + "revision": 4, + "version": 5 + }, + "renameByName": { + "job": "service" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - process_start_time_seconds{service_name=\"grafana-api\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "grafana-api process age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "grafana_stat_totals_dashboard{service_name=\"grafana-api\"}", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Total dashboards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "grafana_stat_total_users{service_name=\"grafana-api\"}", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Total users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "grafana_stat_active_users{service_name=\"grafana-api\"}", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Active users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(grafana_stat_totals_datasource{service_name=\"grafana-api\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Total data sources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 3 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(process_start_time_seconds{service_name=\"frontend-service\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "frontend-service process age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 4, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "builder", + "expr": "grafana_http_request_duration_seconds{service_name=\"frontend-service\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "frontend-service requests", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "builder", + "expr": "grafana_http_request_duration_seconds{service_name=\"grafana-api\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "grafana-api requests", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "expr": "histogram_count(sum(\r\n rate(grafana_http_request_duration_seconds{service_name=\"frontend-service\", handler!=\"/api/health\"} [$__rate_interval])\r\n) by (handler, status_code)) > 0", + "instant": false, + "legendFormat": "{{handler}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "frontend-service HTTP traffic (requests per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fs-prometheus" + }, + "editorMode": "code", + "expr": "histogram_count(sum(\r\n rate(grafana_http_request_duration_seconds{service_name=\"grafana-api\", handler!=\"/metrics\", handler!=\"/api/health\"} [$__rate_interval])\r\n) by (handler, status_code)) > 0", + "instant": false, + "legendFormat": "{{handler}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "grafana-api HTTP traffic (requests per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 15, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"frontend-service\"} | logfmt | msg = `Request Completed` | path != `/metrics`\r\n| line_format `{{.method}} {{.status}} {{.path}} ({{.time_ms}}ms)`\r\n| label_format level=`{{ if hasPrefix \"4\" .status }}WARN{{else}}{{.level}}{{end}}`\r\n| label_format level=`{{ if hasPrefix \"5\" .status }}ERROR{{else}}{{.level}}{{end}}`", + "queryType": "range", + "refId": "A" + } + ], + "title": "Requests", + "type": "logs-new" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 16, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"grafana-api\"} | logfmt | msg=\"Request Completed\" | path != `/metrics`\r\n| label_format method=`{{ if hasPrefix \"GET\" .method }}GET {{else}}{{.method}}{{end}}`\r\n| line_format `{{.method}} {{.status}} {{.path}} ({{.time_ms}}ms)`\r\n| label_format level=`{{ if hasPrefix \"4\" .status }}WARN{{else}}{{.level}}{{end}}`\r\n| label_format level=`{{ if hasPrefix \"5\" .status }}ERROR{{else}}{{.level}}{{end}}`", + "queryType": "range", + "refId": "A" + } + ], + "title": "grafana-api requests", + "type": "logs-new" + }, { "datasource": { "type": "loki", @@ -139,10 +1082,166 @@ ] }, "gridPos": { - "h": 6, - "w": 24, + "h": 7, + "w": 12, "x": 0, - "y": 0 + "y": 30 + }, + "id": 18, + "interval": "60000ms", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "sum by (level, detected_level) (count_over_time({service_name=\"frontend-service\"}[$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "frontend-service logs volume", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 }, "id": 3, "interval": "60000ms", @@ -174,9 +1273,95 @@ "refId": "A" } ], - "title": "Logs volume", + "title": "grafana-api logs volume", "type": "timeseries" }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 19, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"frontend-service\"}\r\n| logfmt\r\n| level!=\"debug\"\r\n| level!=\"info\"\r\n# | line_format \"{{.logger}}: {{.msg}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "frontend-service errors", + "type": "logs-new" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"grafana-api\"}\r\n| logfmt\r\n| logger!=\"context\"\r\n| msg!=\"\"\r\n| level!=\"debug\"\r\n| level!=\"info\"\r\n| line_format \"{{.logger}}: {{.msg}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "grafana-api errors", + "type": "logs-new" + }, { "datasource": { "type": "loki", @@ -190,9 +1375,9 @@ "h": 11, "w": 24, "x": 0, - "y": 6 + "y": 45 }, - "id": 2, + "id": 1, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, @@ -212,12 +1397,12 @@ }, "direction": "backward", "editorMode": "code", - "expr": "{service_name=\"grafana-api\"}\r\n| logfmt\r\n| logger!=\"context\"\r\n| msg!=\"Request Completed\"\r\n| level!=\"debug\"\r\n| level!=\"info\"", + "expr": "{service_name=\"frontend-service\"}\r\n| logfmt\r\n| logger!=\"context\"\r\n| msg!=\"Request Completed\"", "queryType": "range", "refId": "A" } ], - "title": "Error logs", + "title": "All frontend-service logs", "type": "logs-new" }, { @@ -233,9 +1418,9 @@ "h": 11, "w": 24, "x": 0, - "y": 17 + "y": 56 }, - "id": 1, + "id": 20, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, @@ -260,7 +1445,7 @@ "refId": "A" } ], - "title": "All logs", + "title": "All grafana-api logs", "type": "logs-new" } ], @@ -278,5 +1463,5 @@ "timezone": "browser", "title": "Grafana Instance", "uid": "fd70c2ef-3841-4bcc-89af-b639e7f31800", - "version": 4 + "version": 6 } diff --git a/devenv/frontend-service/provisioning/datasources/default.yaml b/devenv/frontend-service/provisioning/datasources/default.yaml index 005bd631498..a43364d52c4 100644 --- a/devenv/frontend-service/provisioning/datasources/default.yaml +++ b/devenv/frontend-service/provisioning/datasources/default.yaml @@ -8,6 +8,11 @@ datasources: uid: PD8C576611E62080A type: grafana-testdata-datasource + - name: Prometheus Pmetrics + uid: fs-prometheus + type: prometheus + url: http://prometheus:9090 + - name: Loki Logs uid: fs-loki type: loki