From b08ef4583e6ed434e0acedb698f6d6fb6de8f223 Mon Sep 17 00:00:00 2001 From: David Ansari Date: Mon, 4 Jul 2022 13:45:59 +0000 Subject: [PATCH] Add Bazel targets start-cluster and stop-cluster to have the equivalent of `make start-cluster` and `make stop-cluster`. To create a 3-node RabbitMQ cluster: ``` bazel run --config=local start-cluster ``` To define number of nodes or a custom directory: ``` bazel run --config=local start-cluster NODES=5 TEST_TMPDIR="$HOME/scratch/myrabbit" ``` To stop the cluster: ``` bazel run --config=local stop-cluster ``` or, if started by the 2nd command: ``` bazel run --config=local stop-cluster NODES=5 TEST_TMPDIR="$HOME/scratch/myrabbit" ``` --- BUILD.bazel | 16 ++ rabbitmq_run.bzl | 2 + scripts/bazel/rabbitmq-run.sh | 317 ++++++++++++++++++++-------------- 3 files changed, 210 insertions(+), 125 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index b4fe4899cc..ea9dfe4c07 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -58,6 +58,22 @@ rabbitmq_run_command( subcommand = "run-broker", ) +# Allow us to `bazel run start-cluster` +# for the equivalent of `make start-cluster` +rabbitmq_run_command( + name = "start-cluster", + rabbitmq_run = ":rabbitmq-run", + subcommand = "start-cluster", +) + +# Allow us to `bazel run stop-cluster` +# for the equivalent of `make stop-cluster` +rabbitmq_run_command( + name = "stop-cluster", + rabbitmq_run = ":rabbitmq-run", + subcommand = "stop-cluster", +) + # `bazel run rabbitmqctl` rabbitmqctl( name = "rabbitmqctl", diff --git a/rabbitmq_run.bzl b/rabbitmq_run.bzl index 828f6b71ca..b2e5debae1 100644 --- a/rabbitmq_run.bzl +++ b/rabbitmq_run.bzl @@ -125,6 +125,8 @@ rabbitmq_run_command_private = rule( "run-broker", "start-background-broker", "stop-node", + "start-cluster", + "stop-cluster", ]), }, executable = True, diff --git a/scripts/bazel/rabbitmq-run.sh b/scripts/bazel/rabbitmq-run.sh index 3911bfc528..1cba1df40a 100644 --- a/scripts/bazel/rabbitmq-run.sh +++ b/scripts/bazel/rabbitmq-run.sh @@ -1,121 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +GREEN='\033[0;32m' +NO_COLOR='\033[0m' + rmq_realpath() { local path=$1 if [ -d "$path" ]; then cd "$path" && pwd elif [ -f "$path" ]; then - cd "$(dirname "$path")" && echo $(pwd)/$(basename "$path") + cd "$(dirname "$path")" && echo "$(pwd)/$(basename "$path")" else echo "$path" fi } -if [ -z ${TEST_SRCDIR+x} ]; then -BASE_DIR=$PWD -else -BASE_DIR=$TEST_SRCDIR/$TEST_WORKSPACE -fi - -if [ $1 = "-C" ]; then - cd $2 - shift 2 -fi - -for arg in "$@"; do - case $arg in - run-broker) - CMD="$arg" - ;; - start-background-broker) - CMD="$arg" - ;; - stop-node) - CMD="$arg" - ;; - set-resource-alarm) - CMD="$arg" - ;; - clear-resource-alarm) - CMD="$arg" - ;; - *) - export "$arg" - ;; - esac -done - -DEFAULT_PLUGINS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/plugins -if [ ! -z ${EXTRA_PLUGINS_DIR+x} ]; then - DEFAULT_PLUGINS_DIR=${DEFAULT_PLUGINS_DIR}:${EXTRA_PLUGINS_DIR} -fi - -TEST_TMPDIR=${TEST_TMPDIR:=${TMPDIR}/rabbitmq-test-instances} -RABBITMQ_SCRIPTS_DIR="$(rmq_realpath ${BASE_DIR}/{RABBITMQ_HOME}/sbin)" -RABBITMQ_PLUGINS=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-plugins -RABBITMQ_SERVER=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server -RABBITMQCTL=${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl - -export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER - -HOSTNAME="$(hostname -s)" - -RABBITMQ_NODENAME=${RABBITMQ_NODENAME:=rabbit@${HOSTNAME}} -RABBITMQ_NODENAME_FOR_PATHS=${RABBITMQ_NODENAME_FOR_PATHS:=${RABBITMQ_NODENAME}} -NODE_TMPDIR=${TEST_TMPDIR}/${RABBITMQ_NODENAME_FOR_PATHS} - -RABBITMQ_BASE=${NODE_TMPDIR} -RABBITMQ_PID_FILE=${NODE_TMPDIR}/${RABBITMQ_NODENAME_FOR_PATHS}.pid -RABBITMQ_LOG_BASE=${NODE_TMPDIR}/log -RABBITMQ_MNESIA_BASE=${NODE_TMPDIR}/mnesia -RABBITMQ_MNESIA_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME_FOR_PATHS} -RABBITMQ_QUORUM_DIR=${RABBITMQ_MNESIA_DIR}/quorum -RABBITMQ_STREAM_DIR=${RABBITMQ_MNESIA_DIR}/stream -RABBITMQ_PLUGINS_DIR=${RABBITMQ_PLUGINS_DIR:=${DEFAULT_PLUGINS_DIR}} -RABBITMQ_PLUGINS_EXPAND_DIR=${NODE_TMPDIR}/plugins -RABBITMQ_FEATURE_FLAGS_FILE=${NODE_TMPDIR}/feature_flags -RABBITMQ_ENABLED_PLUGINS_FILE=${NODE_TMPDIR}/enabled_plugins - -RABBITMQ_SERVER_START_ARGS="${RABBITMQ_SERVER_START_ARGS:=-ra wal_sync_method sync}" - -# Enable colourful debug logging by default -# To change this, set RABBITMQ_LOG to info, notice, warning etc. -RABBITMQ_LOG=${RABBITMQ_LOG:='debug,+color'} -export RABBITMQ_LOG - -if [ -z ${LEAVE_PLUGINS_DISABLED+x} ]; then - RABBITMQ_ENABLED_PLUGINS=${RABBITMQ_ENABLED_PLUGINS:=ALL} -else - RABBITMQ_ENABLED_PLUGINS=${RABBITMQ_ENABLED_PLUGINS:=} -fi - -mkdir -p ${TEST_TMPDIR} - -mkdir -p ${RABBITMQ_LOG_BASE} -mkdir -p ${RABBITMQ_MNESIA_BASE} -mkdir -p ${RABBITMQ_PLUGINS_EXPAND_DIR} - -export \ - RABBITMQ_NODENAME \ - RABBITMQ_NODE_IP_ADDRESS \ - RABBITMQ_BASE \ - RABBITMQ_PID_FILE \ - RABBITMQ_LOG_BASE \ - RABBITMQ_MNESIA_BASE \ - RABBITMQ_MNESIA_DIR \ - RABBITMQ_QUORUM_DIR \ - RABBITMQ_STREAM_DIR \ - RABBITMQ_FEATURE_FLAGS_FILE \ - RABBITMQ_PLUGINS_DIR \ - RABBITMQ_PLUGINS_EXPAND_DIR \ - RABBITMQ_SERVER_START_ARGS \ - RABBITMQ_ENABLED_PLUGINS \ - RABBITMQ_ENABLED_PLUGINS_FILE - write_config_file() { -cat << EOF > ${RABBITMQ_CONFIG_FILE} + cat << EOF > "$RABBITMQ_CONFIG_FILE" %% vim:ft=erlang: [ @@ -139,50 +41,215 @@ cat << EOF > ${RABBITMQ_CONFIG_FILE} EOF } +setup_node_env() { + local node_index="" + if [ -n "${1-}" ]; then + node_index="-$1" + unset RABBITMQ_NODENAME RABBITMQ_NODENAME_FOR_PATHS + fi + + RABBITMQ_NODENAME=${RABBITMQ_NODENAME:=rabbit${node_index}@${HOSTNAME}} + RABBITMQ_NODENAME_FOR_PATHS=${RABBITMQ_NODENAME_FOR_PATHS:=${RABBITMQ_NODENAME}} + NODE_TMPDIR=${TEST_TMPDIR}/${RABBITMQ_NODENAME_FOR_PATHS} + + RABBITMQ_BASE=${NODE_TMPDIR} + RABBITMQ_PID_FILE=${NODE_TMPDIR}/${RABBITMQ_NODENAME_FOR_PATHS}.pid + RABBITMQ_LOG_BASE=${NODE_TMPDIR}/log + RABBITMQ_MNESIA_BASE=${NODE_TMPDIR}/mnesia + RABBITMQ_MNESIA_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME_FOR_PATHS} + RABBITMQ_QUORUM_DIR=${RABBITMQ_MNESIA_DIR}/quorum + RABBITMQ_STREAM_DIR=${RABBITMQ_MNESIA_DIR}/stream + RABBITMQ_PLUGINS_EXPAND_DIR=${NODE_TMPDIR}/plugins + RABBITMQ_FEATURE_FLAGS_FILE=${NODE_TMPDIR}/feature_flags + RABBITMQ_ENABLED_PLUGINS_FILE=${NODE_TMPDIR}/enabled_plugins + + export \ + RABBITMQ_NODENAME \ + RABBITMQ_BASE \ + RABBITMQ_PID_FILE \ + RABBITMQ_LOG_BASE \ + RABBITMQ_MNESIA_BASE \ + RABBITMQ_MNESIA_DIR \ + RABBITMQ_QUORUM_DIR \ + RABBITMQ_STREAM_DIR \ + RABBITMQ_PLUGINS_EXPAND_DIR \ + RABBITMQ_FEATURE_FLAGS_FILE \ + RABBITMQ_ENABLED_PLUGINS_FILE + + mkdir -p "$TEST_TMPDIR" + mkdir -p "$RABBITMQ_LOG_BASE" + mkdir -p "$RABBITMQ_MNESIA_BASE" + mkdir -p "$RABBITMQ_PLUGINS_EXPAND_DIR" +} + +await_startup() { + RMQCTL_WAIT_TIMEOUT=${RMQCTL_WAIT_TIMEOUT:=60} + + # rabbitmqctl wait shells out to 'ps', which is broken in the bazel macOS + # sandbox (https://github.com/bazelbuild/bazel/issues/7448) + # adding "--spawn_strategy=local" to the invocation is a workaround + "$RABBITMQCTL" \ + -n "$RABBITMQ_NODENAME" \ + wait \ + --timeout "$RMQCTL_WAIT_TIMEOUT" \ + "$RABBITMQ_PID_FILE" + + "$RABBITMQCTL" \ + -n "$RABBITMQ_NODENAME" \ + await_startup + } + +if [ -z ${TEST_SRCDIR+x} ]; then + BASE_DIR=$PWD +else + BASE_DIR=$TEST_SRCDIR/$TEST_WORKSPACE +fi + +if [ "$1" = "-C" ]; then + cd "$2" + shift 2 +fi + +for arg in "$@"; do + case $arg in + run-broker) + CMD="$arg" + ;; + start-background-broker) + CMD="$arg" + ;; + stop-node) + CMD="$arg" + ;; + start-cluster) + CMD="$arg" + ;; + stop-cluster) + CMD="$arg" + ;; + set-resource-alarm) + CMD="$arg" + ;; + clear-resource-alarm) + CMD="$arg" + ;; + *) + export "$arg" + ;; + esac +done + +# shellcheck disable=SC1083 +DEFAULT_PLUGINS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/plugins +if [[ -n ${EXTRA_PLUGINS_DIR+x} ]]; then + DEFAULT_PLUGINS_DIR=${DEFAULT_PLUGINS_DIR}:${EXTRA_PLUGINS_DIR} +fi + +RABBITMQ_PLUGINS_DIR=${RABBITMQ_PLUGINS_DIR:=${DEFAULT_PLUGINS_DIR}} +export RABBITMQ_PLUGINS_DIR +RABBITMQ_SERVER_START_ARGS="${RABBITMQ_SERVER_START_ARGS:=-ra wal_sync_method sync}" +export RABBITMQ_SERVER_START_ARGS + +# Enable colourful debug logging by default +# To change this, set RABBITMQ_LOG to info, notice, warning etc. +RABBITMQ_LOG=${RABBITMQ_LOG:='debug,+color'} +export RABBITMQ_LOG + +if [ -z ${LEAVE_PLUGINS_DISABLED+x} ]; then + RABBITMQ_ENABLED_PLUGINS=${RABBITMQ_ENABLED_PLUGINS:=ALL} +else + RABBITMQ_ENABLED_PLUGINS=${RABBITMQ_ENABLED_PLUGINS:=} +fi +export RABBITMQ_ENABLED_PLUGINS + + +TEST_TMPDIR=${TEST_TMPDIR:=$(dirname "$(mktemp -u)")/rabbitmq-test-instances} +printf "RabbitMQ node(s) in directory $GREEN$(realpath "$TEST_TMPDIR")$NO_COLOR\n" + +# shellcheck disable=SC1083 +RABBITMQ_SCRIPTS_DIR="$(rmq_realpath "$BASE_DIR"/{RABBITMQ_HOME}/sbin)" +RABBITMQ_SERVER=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server +RABBITMQCTL=${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl +export RABBITMQ_SCRIPTS_DIR \ + RABBITMQ_SERVER \ + RABBITMQCTL + +HOSTNAME="$(hostname -s)" + case $CMD in run-broker) + setup_node_env export RABBITMQ_ALLOW_INPUT=true if [ -z ${RABBITMQ_CONFIG_FILE+x} ]; then export RABBITMQ_CONFIG_FILE=${TEST_TMPDIR}/test.config write_config_file fi - ${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server + "$RABBITMQ_SERVER" ;; start-background-broker) - RMQCTL_WAIT_TIMEOUT=${RMQCTL_WAIT_TIMEOUT:=60} - - ${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server \ - > ${RABBITMQ_LOG_BASE}/startup_log \ - 2> ${RABBITMQ_LOG_BASE}/startup_err & - - # rabbitmqctl wait shells out to 'ps', which is broken in the bazel macOS - # sandbox (https://github.com/bazelbuild/bazel/issues/7448) - # adding "--spawn_strategy=local" to the invocation is a workaround - ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl \ - -n ${RABBITMQ_NODENAME} \ - wait \ - --timeout ${RMQCTL_WAIT_TIMEOUT} \ - ${RABBITMQ_PID_FILE} - - ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl \ - --node ${RABBITMQ_NODENAME} \ - await_startup + setup_node_env + "$RABBITMQ_SERVER" \ + > "$RABBITMQ_LOG_BASE"/startup_log \ + 2> "$RABBITMQ_LOG_BASE"/startup_err & + await_startup ;; stop-node) - pid=$(test -f $RABBITMQ_PID_FILE && cat $RABBITMQ_PID_FILE); \ + setup_node_env + pid=$(test -f "$RABBITMQ_PID_FILE" && cat "$RABBITMQ_PID_FILE"); \ test "$pid" && \ kill -TERM "$pid" && \ echo "waiting for process to exit" && \ while ps -p "$pid" >/dev/null 2>&1; do sleep 1; done ;; + start-cluster) + nodes=${NODES:=3} + for ((n=0; n < nodes; n++)) + do + setup_node_env "$n" + + RABBITMQ_NODE_PORT=$((5672 + n)) \ + RABBITMQ_SERVER_START_ARGS=" \ + -rabbit loopback_users [] \ + -rabbitmq_management listener [{port,$((15672 + n))}] \ + -rabbitmq_mqtt tcp_listeners [$((1883 + n))] \ + -rabbitmq_web_mqtt tcp_config [{port,$((1893 + n))}] \ + -rabbitmq_web_mqtt_examples listener [{port,$((1903 + n))}] \ + -rabbitmq_stomp tcp_listeners [$((61613 + n))] \ + -rabbitmq_web_stomp tcp_config [{port,$((61623 + n))}] \ + -rabbitmq_web_stomp_examples listener [{port,$((61633 + n))}] \ + -rabbitmq_prometheus tcp_config [{port,$((15692 + n))}] \ + -rabbitmq_stream tcp_listeners [$((5552 + n))]" \ + "$RABBITMQ_SERVER" \ + > "$RABBITMQ_LOG_BASE"/startup_log \ + 2> "$RABBITMQ_LOG_BASE"/startup_err & + + await_startup + if [ -n "${nodename0-}" ]; then + "$RABBITMQCTL" -n "$RABBITMQ_NODENAME" stop_app + "$RABBITMQCTL" -n "$RABBITMQ_NODENAME" join_cluster "$nodename0" + "$RABBITMQCTL" -n "$RABBITMQ_NODENAME" start_app + else + nodename0=$RABBITMQ_NODENAME + fi + done + ;; + stop-cluster) + nodes=${NODES:=3} + for ((n=nodes-1; n >= 0; n--)) + do + "$RABBITMQCTL" -n "rabbit-$n@$HOSTNAME" stop + done + ;; set-resource-alarm) + setup_node_env ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \ - ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \ + "$RABBITMQCTL" -n "$RABBITMQ_NODENAME" \ eval "rabbit_alarm:set_alarm({{resource_limit, ${SOURCE}, node()}, []})." ;; clear-resource-alarm) + setup_node_env ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \ - ${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \ + "$RABBITMQCTL" -n "$RABBITMQ_NODENAME" \ eval "rabbit_alarm:clear_alarm({resource_limit, ${SOURCE}, node()})." ;; *)