Tempo: Devenv update to support all features (#52728)

* Tempo devenv

* Add logging to tempo devenv

* Update readme

* Formatting

* Update loki image

* Update tempo image

.gitignore

@@ -74,6 +74,7 @@ public/css/*.min.css
 # devenv
 /devenv/docker-compose.yaml
 /devenv/.env
+/devenv/docker/blocks/tempo/tempo-data/
 
 conf/custom.ini
 /conf/provisioning/**/custom.yaml

devenv/docker/blocks/tempo/README.md

@@ -1,4 +1,7 @@
-Right now Tempo is before stable release so this uses :latest tag which means there can be changes depending on when
+This devenv docker-compose.yaml will allow you to;
-you pull the image.
+- search traces
-
+- view traces
-For adding some traces easily you can run Loki block and enable tracing (see ../loki-promtail/README.md)
+- upload/download trace JSON files
+- view service graphs
+- view the APM table
+- search traces via Loki

devenv/docker/blocks/tempo/docker-compose.yaml

@@ -1,8 +1,111 @@
+x-logging: &default-logging
+  driver: loki
+  options:
+    loki-url: 'http://localhost:3100/api/prom/push'
+    labels: namespace
+    loki-relabel-config: |
+      - action: replace
+        source_labels: ["namespace","compose_service"]
+        separator: "/"
+        target_label: job
+      - action: replace
+        source_labels: ["container_name"]
+        target_label: instance
+
+version: "3"
+services:
+
+  db:
+    image: grafana/tns-db:9c1ab38
+    command:
+      - '-log.level=debug'
+    ports:
+      - 0.0.0.0:8000:80
+    environment:
+      JAEGER_ENDPOINT: 'http://tempo:14268/api/traces'
+      JAEGER_TAGS: job=tns/db
+      JAEGER_SAMPLER_TYPE: const
+      JAEGER_SAMPLER_PARAM: 1
+    labels:
+      namespace: tns
+    logging: *default-logging
+
+  app:
+    image: grafana/tns-app:9c1ab38
+    command:
+      - '-log.level=debug'
+      - 'http://db'
+    links:
+      - db
+    ports:
+      - 0.0.0.0:8001:80
+    environment:
+      JAEGER_ENDPOINT: 'http://tempo:14268/api/traces'
+      JAEGER_TAGS: job=tns/app
+      JAEGER_SAMPLER_TYPE: const
+      JAEGER_SAMPLER_PARAM: 1
+    labels:
+      namespace: tns
+    logging: *default-logging
+
+  loadgen:
+    image: grafana/tns-loadgen:9c1ab38
+    command:
+      - '-log.level=debug'
+      - 'http://app'
+    links:
+      - app
+    ports:
+      - 0.0.0.0:8002:80
+    environment:
+      JAEGER_ENDPOINT: 'http://tempo:14268/api/traces'
+      JAEGER_TAGS: job=tns/loadgen
+      JAEGER_SAMPLER_TYPE: const
+      JAEGER_SAMPLER_PARAM: 1
+    labels:
+      namespace: tns
+    logging: *default-logging
+
   tempo:
     image: grafana/tempo:latest
-    command: ["-config.file=/etc/tempo.yaml"]
+    command: 
-    ports:
+      - --config.file=/etc/tempo.yaml
-      - "6831:6831"
+      - --search.enabled=true
-      - "3101:3101"
     volumes:
       - ./docker/blocks/tempo/tempo.yaml:/etc/tempo.yaml
+      - ./docker/blocks/tempo/tempo-data:/tmp/tempo
+    ports:
+      - "14268:14268"  # jaeger ingest
+      - "3200:3200"   # tempo
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http
+
+  prometheus:
+    image: prom/prometheus:main
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+    volumes:
+      - ./docker/blocks/tempo/prometheus.yaml:/etc/prometheus.yaml
+    links:
+      - app
+      - db
+      - loadgen
+    ports:
+      - "9090:9090"
+    labels:
+      namespace: monitoring
+    logging: *default-logging
+
+  loki:
+    image: grafana/loki:main
+    command:
+      - -config.file=/etc/loki/local-config.yaml
+      - -table-manager.retention-period=1d
+      - -table-manager.retention-deletes-enabled=true
+    ports:
+      - "3100:3100"
+    labels:
+      namespace: monitoring
+    logging: *default-logging

devenv/docker/blocks/tempo/prometheus.yaml

@@ -0,0 +1,11 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
+  - job_name: 'tempo'
+    static_configs:
+      - targets: [ 'tempo:3200' ]

devenv/docker/blocks/tempo/tempo.yaml

@@ -1,12 +1,12 @@
-auth_enabled: false
+metrics_generator_enabled: true
 
 server:
-  http_listen_port: 3101
+  http_listen_port: 3200
 
 distributor:
   receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
     jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
-      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/master/receiver
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
         thrift_http:                   #
         grpc:                          # for a production deployment you should only enable the receivers you need!
         thrift_binary:
@@ -20,22 +20,42 @@ distributor:
 
 ingester:
   trace_idle_period: 10s               # the length of time after a trace has not received spans to consider it complete and flush it
+  max_block_bytes: 1_000_000           # cut the head block when it hits this size or ...
   max_block_duration: 5m               #   this much time passes
 
 compactor:
   compaction:
     compaction_window: 1h              # blocks in this time window will be compacted together
-    max_compaction_objects: 1000000    # maximum size of compacted blocks
+    max_block_bytes: 100_000_000       # maximum size of compacted blocks
     block_retention: 1h
     compacted_block_retention: 10m
 
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
 storage:
   trace:
     backend: local                     # backend configuration to use
+    block:
+      bloom_filter_false_positive: .05 # bloom filter false positive rate.  lower values create larger filters but fewer false positives
+      index_downsample_bytes: 1000     # number of bytes per index record
+      encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
     local:
       path: /tmp/tempo/blocks
     pool:
-      max_workers: 100                 # the worker pool mainly drives querying, but is also used for polling the blocklist
+      max_workers: 100                 # worker pool determines the number of parallel requests to the object store backend
       queue_depth: 10000
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]