FS: Add prometheus metrics to dev stack (#109952)

* FS: Add prometheus metrics to dev stack

* remove comments

devenv/frontend-service/Tiltfile

@@ -68,10 +68,13 @@ dc_resource("frontend-service",
   labels=["services"],
 )
 dc_resource("alloy", labels=["observability"])
+dc_resource("prometheus", labels=["observability"])
 dc_resource("loki", labels=["observability"])
-dc_resource("tempo-init", labels=["observability"])
 dc_resource("tempo", labels=["observability"])
 
+dc_resource("postgres", labels=["misc"])
+dc_resource("tempo-init", labels=["misc"])
+
 # paths in tilt files are confusing....
 # - if tilt is dealing the the path, it is relative to the Tiltfile
 # - if docker is dealing with the path, it is relative to the context

devenv/frontend-service/configs/alloy/_databases.alloy

@@ -1,3 +1,22 @@
+//
+// Metrics
+prometheus.relabel "publish_metrics" {
+	rule {
+		action       = "replace"
+		target_label = "instance"
+		replacement  = constants.hostname
+	}
+
+	forward_to = [prometheus.remote_write.tilt_prom.receiver]
+}
+
+prometheus.remote_write "tilt_prom" {
+	endpoint {
+		url                    = "http://prometheus:9090/api/v1/write"
+		send_native_histograms = true
+	}
+}
+
 //
 // Logs
 loki.relabel "publish_logs" {

devenv/frontend-service/configs/alloy/alloy.alloy

@@ -1,7 +1,6 @@
 // 
 // Self-config
 //
-
 livedebugging {
 	enabled = true
 }
@@ -21,3 +20,13 @@ loki.relabel "alloy" {
 
 	forward_to = [loki.relabel.publish_logs.receiver]
 }
+
+// Metrics
+prometheus.exporter.self "alloy" { }
+
+prometheus.scrape "alloy" {
+	targets          = prometheus.exporter.self.alloy.targets
+	scrape_interval  = "60s"
+	forward_to       = [prometheus.relabel.publish_metrics.receiver]
+	scrape_protocols = ["PrometheusProto", "OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"]
+}

devenv/frontend-service/configs/alloy/grafana.alloy

@@ -0,0 +1,11 @@
+//
+// Metrics
+prometheus.scrape "grafana_dev" {
+	targets = [
+		{__scheme__ = "http", __address__ = "grafana-api:3000", job = "grafana_dev", service_name = "grafana-api"},
+		{__scheme__ = "http", __address__ = "frontend-service:3000", job = "grafana_dev", service_name = "frontend-service"},
+	]
+	scrape_interval  = "15s"
+	forward_to       = [prometheus.relabel.publish_metrics.receiver]
+	scrape_protocols = ["PrometheusProto", "OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"]
+}

devenv/frontend-service/docker-compose.yaml

@@ -34,7 +34,7 @@ services:
       GF_DEFAULT_APP_MODE: development
       GF_PANELS_ENABLE_ALPHA: true
       GF_SERVER_CDN_URL: http://localhost:3010
-      GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend
+      GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend enableNativeHTTPHistogram
       GF_DATABASE_URL: postgres://grafana:grafana@postgres:5432/grafana
       GF_SERVER_ROUTER_LOGGING: true
       GF_LOG_LEVEL: info
@@ -62,6 +62,7 @@ services:
       GF_DEFAULT_APP_MODE: development
       GF_DEFAULT_TARGET: frontend-server
       GF_SECURITY_CONTENT_SECURITY_POLICY: false
+      GF_FEATURE_TOGGLES_ENABLE: enableNativeHTTPHistogram
       GF_SERVER_CDN_URL: http://localhost:3010
       GF_SERVER_ROUTER_LOGGING: true
       GF_LOG_LEVEL: info
@@ -95,6 +96,20 @@ services:
       - /alloy-config
     depends_on:
       - loki
+    labels:
+      - 'alloy.logs=true'
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=native-histograms'
+    labels:
+      - 'alloy.logs=true'
 
   loki:
     image: grafana/loki
@@ -120,6 +135,8 @@ services:
       - tempo-data:/var/lib/tempo
       - ./configs/tempo.yaml:/etc/tempo/tempo.yaml
     command: ['-config.file=/etc/tempo/tempo.yaml']
+    labels:
+      - 'alloy.logs=true'
 
     depends_on:
       tempo-init:
@@ -131,3 +148,4 @@ volumes:
   alloy-data:
   loki-data:
   tempo-data:
+  prometheus-data:

devenv/frontend-service/provisioning/datasources/default.yaml

@@ -8,6 +8,11 @@ datasources:
     uid: PD8C576611E62080A
     type: grafana-testdata-datasource
 
+  - name: Prometheus Pmetrics
+    uid: fs-prometheus
+    type: prometheus
+    url: http://prometheus:9090
+
   - name: Loki Logs
     uid: fs-loki
     type: loki