Add inet_tcp_metrics Grafana dashboard & cluster example

It uses the commercial edition of RabbitMQ, requires a valid Tanzu
Network account.  Learn more: https://rabbitmq.com/tanzu

Signed-off-by: Gerhard Lazu <gerhard@lazu.co.uk>

deps/rabbitmq_prometheus/Makefile

@@ -123,7 +123,7 @@ docker-image-bump: ## diu | Bump Docker image version across all docker-compose-
 	@sed -i '' \
 	  -e 's|$(DOCKER_IMAGE_NAME):.*|$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)|g' \
 	  -e 's|pivotalrabbitmq/perf-test:.*|pivotalrabbitmq/perf-test:2.11.0-ubuntu|g' \
-	  docker/docker-compose-{overview,dist-tls,qq}.yml
+	  docker/docker-compose-{overview,dist-tls,qq,dist-metrics}.yml
 .PHONY: diu
 diu: docker-image-bump
 
@@ -204,6 +204,12 @@ qq: ## q   | Make RabbitMQ-Quorum-Queues-Raft panels come alive - HIGH LOAD
 .PHONY: q
 q: qq
 
+.PHONY: dist-metrics
+dist-metrics: ## dm  | Make inet_tcp_metrics come alive
+	@$(DOCKER_COMPOSE_UP)
+.PHONY: dm
+dm: dist-metrics
+
 .PHONY: h
 h:
 	@awk -F"[:#]" '/^[^\.][a-zA-Z\._\-]+:+.+##.+$$/ { printf "\033[36m%-24s\033[0m %s\n", $$1, $$4 }' $(MAKEFILE_LIST) \

deps/rabbitmq_prometheus/docker/docker-compose-dist-metrics.yml

@@ -0,0 +1,80 @@
+# https://docs.docker.com/compose/compose-file/
+version: "3.6"
+
+# https://docs.docker.com/compose/compose-file/#networks
+networks:
+  rabbitmq-prometheus:
+
+# https://docs.docker.com/compose/compose-file/#volumes
+volumes:
+  rabbitmq-prometheus_prometheus:
+  rabbitmq-prometheus_grafana:
+
+services:
+  rmq0-dist-metrics: &rabbitmq
+    # https://network.pivotal.io/products/rabbitmq/
+    # This is a commercial edition of RabbitMQ that requires a valid Tanzu Network account
+    # Learn more: https://rabbitmq.com/tanzu
+    image: dev.registry.pivotal.io/rabbitmq/vmware-tanzu-rabbitmq:2021.06
+    networks:
+      - "rabbitmq-prometheus"
+    ports:
+      - "5680:5672"
+      - "15680:15672"
+      - "15700:15692"
+    # https://unix.stackexchange.com/questions/71940/killing-tcp-connection-in-linux
+    # https://en.wikipedia.org/wiki/Tcpkill
+    # https://www.digitalocean.com/community/tutorials/iptables-essentials-common-firewall-rules-and-commands#block-an-ip-address
+    cap_add:
+      - ALL
+    hostname: rmq0-dist-metrics
+    environment:
+      # RABBITMQ_ERLANG_COOKIE: rabbitmq-prometheus
+      RABBITMQ_SERVER_START_ARGS: -proto_dist inet_tcp_metrics
+      # Uncomment the following line if you want debug logs & colour
+      # RABBITMQ_LOG: debug,+color
+    volumes:
+      # This does not work that well on Windows
+      # https://github.com/rabbitmq/rabbitmq-prometheus/commit/c4b04ea9bae877ff7d22a7085475965016933d91#commitcomment-40660523
+      - ./erlang.cookie:/var/lib/rabbitmq/.erlang.cookie
+      - ./rabbitmq-dist-metrics.conf:/etc/rabbitmq/rabbitmq.conf:ro
+      - ./rabbitmq-dist-metrics-definitions.json:/etc/rabbitmq/rabbitmq-definitions.json:ro
+    # we want to simulate hitting thresholds
+    ulimits:
+      nofile:
+        soft: "2000"
+        hard: "2000"
+  rmq1-dist-metrics:
+    << : *rabbitmq
+    hostname: rmq1-dist-metrics
+    ports:
+      - "5681:5672"
+      - "15681:15672"
+      - "15701:15692"
+  rmq2-dist-metrics:
+    << : *rabbitmq
+    hostname: rmq2-dist-metrics
+    ports:
+      - "5682:5672"
+      - "15682:15672"
+      - "15702:15692"
+
+  qq:
+    image: &perf-test-image pivotalrabbitmq/perf-test:2.11.0-ubuntu
+    networks:
+      - "rabbitmq-prometheus"
+    environment:
+      URIS: "amqp://guest:guest@rmq0-dist-metrics:5672/%2f,amqp://guest:guest@rmq1-dist-metrics:5672/%2f,amqp://guest:guest@rmq2-dist-metrics:5672/%2f"
+      CONFIRM: 50
+      QUEUE_PATTERN: "qq%d"
+      QUEUE_PATTERN_FROM: 1
+      QUEUE_PATTERN_TO: 3
+      PRODUCERS: 3
+      CONSUMERS: 3
+      QUEUE_ARGS: x-queue-type=quorum,x-max-length=1000
+      FLAG: persistent
+      AUTO_DELETE: "false"
+      RATE: 10
+      AUTOACK: "false"
+      SERVERS_STARTUP_TIMEOUT: &startup_timeout 30
+      METRICS_PROMETHEUS: "true"

deps/rabbitmq_prometheus/docker/prometheus.yml

@@ -46,6 +46,9 @@ scrape_configs:
           - 'rmq0-qq:15692'
           - 'rmq1-qq:15692'
           - 'rmq2-qq:15692'
+          - 'rmq0-dist-metrics:15692'
+          - 'rmq1-dist-metrics:15692'
+          - 'rmq2-dist-metrics:15692'
   - job_name: 'rabbitmq-perf-test'
     static_configs:
       - targets:

deps/rabbitmq_prometheus/docker/rabbitmq-dist-metrics-definitions.json

@@ -0,0 +1,49 @@
+{
+  "global_parameters": [
+    {"name": "cluster_name", "value": "rabbitmq-dist-metrics"}
+  ],
+  "permissions": [
+    {
+      "configure": ".*",
+      "read": ".*",
+      "user": "guest",
+      "vhost": "/",
+      "write": ".*"
+    }
+  ],
+  "policies": [
+    {
+      "apply-to": "queues",
+      "definition": {"ha-mode": "exactly", "ha-params": 1},
+      "name": "ha1",
+      "pattern": "ha1.*",
+      "priority": 0,
+      "vhost": "/"
+    },
+    {
+      "apply-to": "queues",
+      "definition": {"ha-mode": "exactly", "ha-params": 2},
+      "name": "ha2",
+      "pattern": "ha2.*",
+      "priority": 0,
+      "vhost": "/"
+    },
+    {
+      "apply-to": "queues",
+      "definition": {"ha-mode": "exactly", "ha-params": 3},
+      "name": "ha3",
+      "pattern": "ha3.*",
+      "priority": 0,
+      "vhost": "/"
+    }
+  ],
+  "users": [
+    {
+      "hashing_algorithm": "rabbit_password_hashing_sha256",
+      "name": "guest",
+      "password_hash": "hENva+fxJ7gnmaBK/WhwNHOYbvB53/QjNcqhtF4KqF7p21+x",
+      "tags": "administrator"
+    }
+  ],
+  "vhosts": [{"name": "/"}]
+}

deps/rabbitmq_prometheus/docker/rabbitmq-dist-metrics.conf

@@ -0,0 +1,32 @@
+# https://github.com/rabbitmq/rabbitmq-server/blob/master/docs/rabbitmq.conf.example
+loopback_users.guest = false
+listeners.tcp.default = 5672
+management.listener.port = 15672
+management.listener.ssl = false
+
+vm_memory_high_watermark.absolute = 768MiB
+vm_memory_high_watermark_paging_ratio = 0.2
+
+cluster_name = rabbitmq-dist-metrics
+
+cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
+cluster_formation.classic_config.nodes.1 = rabbit@rmq0-dist-metrics
+cluster_formation.classic_config.nodes.2 = rabbit@rmq1-dist-metrics
+cluster_formation.classic_config.nodes.3 = rabbit@rmq2-dist-metrics
+
+load_definitions = /etc/rabbitmq/rabbitmq-definitions.json
+
+# background_gc_enabled = true
+
+# Increase the 5s default so that we are below Prometheus' scrape interval,
+# but still refresh in time for Prometheus scrape
+# This is linked to Prometheus scrape interval & range used with rate()
+collect_statistics_interval = 10000
+
+# Run RabbitMQ Management in Management-only mode, no stats
+# https://github.com/rabbitmq/rabbitmq-management/pull/707
+# management.disable_stats = true
+
+# Return per-object metrics (unaggregated)
+# https://github.com/rabbitmq/rabbitmq-prometheus/pull/28
+# prometheus.return_per_object_metrics = true