From d8360c4e12c1e1e8b1b908821a56167e3c2d6219 Mon Sep 17 00:00:00 2001 From: Josh Hunt Date: Wed, 6 Aug 2025 09:40:48 +0100 Subject: [PATCH] FS: Add Loki and Alloy for local observability (#109209) --- devenv/frontend-service/Tiltfile | 6 +- .../configs/alloy/_databases.alloy | 15 + .../configs/alloy/alloy.alloy | 23 ++ .../configs/alloy/docker.alloy | 44 +++ devenv/frontend-service/docker-compose.yaml | 41 ++- devenv/frontend-service/nginx.conf | 2 +- .../dashboards/dashboards/instance.json | 282 ++++++++++++++++++ .../provisioning/dashboards/devenv.yaml | 8 + .../provisioning/datasources/default.yaml | 5 + 9 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 devenv/frontend-service/configs/alloy/_databases.alloy create mode 100644 devenv/frontend-service/configs/alloy/alloy.alloy create mode 100644 devenv/frontend-service/configs/alloy/docker.alloy create mode 100644 devenv/frontend-service/provisioning/dashboards/dashboards/instance.json diff --git a/devenv/frontend-service/Tiltfile b/devenv/frontend-service/Tiltfile index b7e6f81eca3..885d8eddaf3 100644 --- a/devenv/frontend-service/Tiltfile +++ b/devenv/frontend-service/Tiltfile @@ -56,10 +56,10 @@ local_resource( # --- Docker Compose docker_compose("./docker-compose.yaml") dc_resource("proxy", - resource_deps=["backend", "frontend-service"], + resource_deps=["grafana-api", "frontend-service"], labels=["services"] ) -dc_resource("backend", +dc_resource("grafana-api", resource_deps=["yarn start", "backend-build"], labels=["services"] ) @@ -67,6 +67,8 @@ dc_resource("frontend-service", resource_deps=["yarn start", "backend-build"], labels=["services"], ) +dc_resource("alloy", labels=["observability"]) +dc_resource("loki", labels=["observability"]) # paths in tilt files are confusing.... # - if tilt is dealing the the path, it is relative to the Tiltfile diff --git a/devenv/frontend-service/configs/alloy/_databases.alloy b/devenv/frontend-service/configs/alloy/_databases.alloy new file mode 100644 index 00000000000..eb1766c1346 --- /dev/null +++ b/devenv/frontend-service/configs/alloy/_databases.alloy @@ -0,0 +1,15 @@ +loki.relabel "publish_logs" { + rule { + action = "replace" + target_label = "instance" + replacement = constants.hostname + } + + forward_to = [loki.write.tilt_loki.receiver] +} + +loki.write "tilt_loki" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/devenv/frontend-service/configs/alloy/alloy.alloy b/devenv/frontend-service/configs/alloy/alloy.alloy new file mode 100644 index 00000000000..bd09daa1e87 --- /dev/null +++ b/devenv/frontend-service/configs/alloy/alloy.alloy @@ -0,0 +1,23 @@ +// +// Self-config +// + +livedebugging { + enabled = true +} + +// Alloy logs +logging { + level = "info" + write_to = [loki.relabel.alloy.receiver] +} + +loki.relabel "alloy" { + rule { + action = "replace" + target_label = "job" + replacement = "alloy" + } + + forward_to = [loki.relabel.publish_logs.receiver] +} diff --git a/devenv/frontend-service/configs/alloy/docker.alloy b/devenv/frontend-service/configs/alloy/docker.alloy new file mode 100644 index 00000000000..a081aa75d31 --- /dev/null +++ b/devenv/frontend-service/configs/alloy/docker.alloy @@ -0,0 +1,44 @@ +discovery.docker "tilt" { + host = "unix:///var/run/docker.sock" + + filter { + name = "label" + values = ["alloy.logs=true"] + } + + // Exposes output +} + +discovery.relabel "tilt" { + // Read from + targets = discovery.docker.tilt.targets + + rule { + action = "replace" + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + replacement = "$1" + target_label = "docker_container_name" + } + + rule { + action = "replace" + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "docker_compose_project" + } + + rule { + action = "replace" + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service_name" + } + + // Exposes output +} + +loki.source.docker "default" { + targets = discovery.relabel.tilt.output + host = "unix:///var/run/docker.sock" + labels = {"job" = "docker"} + forward_to = [loki.relabel.publish_logs.receiver] +} diff --git a/devenv/frontend-service/docker-compose.yaml b/devenv/frontend-service/docker-compose.yaml index 17a0ab3c7ef..9e6939d6e97 100644 --- a/devenv/frontend-service/docker-compose.yaml +++ b/devenv/frontend-service/docker-compose.yaml @@ -12,8 +12,13 @@ services: ports: - '3000:80' # Gateway - '3010:81' # CDN + depends_on: + - grafana-api + - frontend-service + labels: + - 'alloy.logs=true' - backend: + grafana-api: image: grafana-fs-dev build: context: ../.. @@ -25,6 +30,8 @@ services: - ./provisioning/dashboards:/grafana/conf/provisioning/dashboards - ../dev-dashboards:/grafana/conf/dev-dashboards environment: + GF_DEFAULT_APP_MODE: development + GF_PANELS_ENABLE_ALPHA: true GF_SERVER_CDN_URL: http://localhost:3010 GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend GF_DATABASE_TYPE: postgres @@ -34,6 +41,8 @@ services: GF_DATABASE_PASSWORD: grafana ports: - '3011:3000' + labels: + - 'alloy.logs=true' frontend-service: image: grafana-fs-dev @@ -43,6 +52,8 @@ services: entrypoint: ['bin/grafana', 'server', 'target'] ports: - '3012:3000' + labels: + - 'alloy.logs=true' environment: GF_DEFAULT_APP_MODE: development GF_DEFAULT_TARGET: frontend-server @@ -57,7 +68,35 @@ services: POSTGRES_DB: grafana volumes: - postgres-data:/var/lib/postgresql/data + labels: + - 'alloy.logs=true' + + alloy: + image: grafana/alloy:latest + volumes: + - ./configs/alloy:/alloy-config + - /var/run/docker.sock:/var/run/docker.sock # To scrape Docker container logs + - alloy-data:/var/lib/alloy/data + ports: + - '12346:12345' # Alloy UI + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /alloy-config + depends_on: + - loki + + loki: + image: grafana/loki + volumes: + - loki-data:/loki + command: -config.file=/etc/loki/local-config.yaml + labels: + - 'alloy.logs=true' volumes: backend-data: postgres-data: + alloy-data: + loki-data: diff --git a/devenv/frontend-service/nginx.conf b/devenv/frontend-service/nginx.conf index 1cf21295a36..5e2ad228eb3 100644 --- a/devenv/frontend-service/nginx.conf +++ b/devenv/frontend-service/nginx.conf @@ -2,7 +2,7 @@ # Instance ### upstream backend { - server backend:3000; + server grafana-api:3000; } upstream frontend { diff --git a/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json b/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json new file mode 100644 index 00000000000..1669392b6b6 --- /dev/null +++ b/devenv/frontend-service/provisioning/dashboards/dashboards/instance.json @@ -0,0 +1,282 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 272, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "interval": "60000ms", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "sum by (level, detected_level) (count_over_time({service_name=\"grafana-api\"}[$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs volume", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": false, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"grafana-api\"}\r\n| logfmt\r\n| logger!=\"context\"\r\n| msg!=\"Request Completed\"\r\n| level!=\"debug\"\r\n| level!=\"info\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error logs", + "type": "logs-new" + }, + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "showControls": false, + "showTime": false, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "pluginVersion": "12.2.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "fs-loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{service_name=\"grafana-api\"}\r\n| logfmt\r\n| logger!=\"context\"\r\n| msg!=\"Request Completed\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "All logs", + "type": "logs-new" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Grafana Instance", + "uid": "fd70c2ef-3841-4bcc-89af-b639e7f31800", + "version": 4 +} diff --git a/devenv/frontend-service/provisioning/dashboards/devenv.yaml b/devenv/frontend-service/provisioning/dashboards/devenv.yaml index c22f2d90141..bec2d040a21 100644 --- a/devenv/frontend-service/provisioning/dashboards/devenv.yaml +++ b/devenv/frontend-service/provisioning/dashboards/devenv.yaml @@ -11,3 +11,11 @@ providers: # This path is relative to the WORKDIR in the docker container. It is mounted # in the docker-compose.yml file. path: conf/dev-dashboards + - name: 'frontend-service dashboards' + folder: 'frontend-service dashboards' + folderUid: '' + type: file + allowUiUpdates: false + updateIntervalSeconds: 3600 + options: + path: conf/provisioning/dashboards/dashboards diff --git a/devenv/frontend-service/provisioning/datasources/default.yaml b/devenv/frontend-service/provisioning/datasources/default.yaml index a3e3c50f570..c99a93fd0b2 100644 --- a/devenv/frontend-service/provisioning/datasources/default.yaml +++ b/devenv/frontend-service/provisioning/datasources/default.yaml @@ -7,3 +7,8 @@ datasources: - name: Testdata uid: PD8C576611E62080A type: grafana-testdata-datasource + + - name: Loki Logs + uid: fs-loki + type: loki + url: http://loki:3100