Mixed version testing in bazel (#3200)

Unlike with gnu make, mixed version testing with bazel uses a package-generic-unix for the secondary umbrella rather than the source. This brings the benefit of being able to mixed version test releases built with older erlang versions (even though all nodes will run under the single version given to bazel)

This introduces new test labels, adding a `-mixed` suffix for every existing test. They can be skipped if necessary with `--test_tag_filters` (see the github actions workflow for an example)

As part of the change, it is now possible to run an old release of rabbit with rabbitmq_run rule, such as:

`bazel run @rabbitmq-server-generic-unix-3.8.17//:rabbitmq-run run-broker`
This commit is contained in:
Philip Kuryloski 2021-07-19 14:33:25 +02:00 committed by GitHub
parent 0f4cf2755d
commit d6399bbb5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 316 additions and 77 deletions

View File

@ -0,0 +1,82 @@
name: Test Mixed Version Clusters
on: push
jobs:
test-mixed-versions:
name: Test (Mixed Version Cluster)
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
erlang_version:
- "23"
- "24"
timeout-minutes: 120
steps:
- name: CHECKOUT REPOSITORY
uses: actions/checkout@v2.3.4
- name: CONFIGURE BAZEL
run: |
echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
cat << EOF >> user.bazelrc
build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
build:buildbuddy --tls_client_key=buildbuddy-key.pem
build:buildbuddy --build_metadata=ROLE=CI
build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
EOF
#! - name: Setup tmate session
#! uses: mxschmitt/action-tmate@v3
- name: RUN TESTS
run: |
bazelisk test //... \
--config=rbe-${{ matrix.erlang_version }} \
--test_tag_filters=mixed-version-cluster,-exclusive,-aws \
--verbose_failures
test-exclusive-mixed-versions:
name: Test (Exclusive Tests with Mixed Version Cluster)
runs-on: ubuntu-18.04
strategy:
matrix:
erlang_version:
- "23"
- "24"
timeout-minutes: 60
steps:
- name: CHECKOUT REPOSITORY
uses: actions/checkout@v2.3.4
- name: CONFIGURE OTP & ELIXIR
uses: erlef/setup-beam@v1.8
with:
otp-version: ${{ matrix.erlang_version }}
elixir-version: 1.11.4
- name: CONFIGURE BAZEL
run: |
ERLANG_HOME="$(dirname $(dirname $(which erl)))"
ELIXIR_HOME="$(dirname $(dirname $(which iex)))"
echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
cat << EOF >> user.bazelrc
build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
build:buildbuddy --tls_client_key=buildbuddy-key.pem
build:buildbuddy --build_metadata=ROLE=CI
build:buildbuddy --build_metadata=VISIBILITY=PRIVATE
build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-exclusive-${{ matrix.erlang_version }}
build --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
build --@bazel-erlang//:erlang_home=${ERLANG_HOME}
build --//:elixir_home=${ELIXIR_HOME}
EOF
#! - name: Setup tmate session
#! uses: mxschmitt/action-tmate@v3
- name: RUN EXCLUSIVE TESTS
run: |
MIXED_EXCLUSIVE_TESTS=$(bazel query 'attr(tags, "mixed-version-cluster", attr(tags, "exclusive", tests(//...)))')
bazelisk test $MIXED_EXCLUSIVE_TESTS \
--config=buildbuddy \
--test_tag_filters=-aws \
--build_tests_only \
--test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
--verbose_failures

View File

@ -32,7 +32,7 @@ jobs:
run: |
bazelisk test //... \
--config=rbe-${{ matrix.erlang_version }} \
--test_tag_filters=-exclusive,-aws \
--test_tag_filters=-exclusive,-aws,-mixed-version-cluster \
--verbose_failures
test-exclusive:
name: Test (Exclusive Tests)
@ -75,7 +75,7 @@ jobs:
run: |
bazelisk test //... \
--config=buildbuddy \
--test_tag_filters=exclusive,-aws \
--test_tag_filters=exclusive,-aws,-mixed-version-cluster \
--build_tests_only \
--test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
--verbose_failures

View File

@ -0,0 +1,18 @@
load("@//:rabbitmq_package_generic_unix.bzl", "rabbitmq_package_generic_unix")
load("@//:rabbitmq_run.bzl", "rabbitmq_run")
rabbitmq_package_generic_unix(
name = "broker-home",
sbin = glob(["sbin/*"]),
escript = glob(["escript/*"]),
plugins = [
"//plugins:standard_plugins",
"//plugins:inet_tcp_proxy_ez",
],
)
rabbitmq_run(
name = "rabbitmq-run",
home = ":broker-home",
visibility = ["//visibility:public"],
)

View File

@ -16,6 +16,19 @@ load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
buildbuddy(name = "buildbuddy_toolchain")
http_archive(
name = "rules_pkg",
sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
"https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
],
)
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
rules_pkg_dependencies()
http_archive(
name = "bazel-erlang",
strip_prefix = "bazel-erlang-main",
@ -51,3 +64,34 @@ git_repository(
load("//deps/amqp10_client:activemq.bzl", "activemq_archive")
activemq_archive()
ADD_PLUGINS_DIR_BUILD_FILE = """set -euo pipefail
cat << EOF > plugins/BUILD.bazel
load("@rules_pkg//:pkg.bzl", "pkg_zip")
pkg_zip(
name = "inet_tcp_proxy_ez",
package_dir = "inet_tcp_proxy/ebin",
srcs = [
"@inet_tcp_proxy//:bazel_erlang_lib",
],
package_file_name = "inet_tcp_proxy.ez",
visibility = ["//visibility:public"],
)
filegroup(
name = "standard_plugins",
srcs = glob(["*.ez"]),
visibility = ["//visibility:public"],
)
EOF
"""
http_archive(
name = "rabbitmq-server-generic-unix-3.8.18",
build_file = "@//:BUILD.package_generic_unix",
patch_cmds = [ADD_PLUGINS_DIR_BUILD_FILE],
strip_prefix = "rabbitmq_server-3.8.18",
urls = ["https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.8.18/rabbitmq-server-generic-unix-3.8.18.tar.xz"],
)

View File

@ -10,6 +10,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("amqp_client/include/amqp_client.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
-compile(export_all).
@ -247,15 +248,14 @@ quorum_queue_leadership_transfer(Config) ->
Config, Nodenames),
case AllTheSame of
true ->
rabbit_ct_helpers:await_condition(
fun () ->
LocalLeaders = rabbit_ct_broker_helpers:rpc(
Config, A,
rabbit_amqqueue,
list_local_leaders,
[]),
length(LocalLeaders) =:= 0
end, 20000);
?awaitMatch(
LocalLeaders when length(LocalLeaders) == 0,
rabbit_ct_broker_helpers:rpc(
Config, A,
rabbit_amqqueue,
list_local_leaders,
[]),
20000);
false ->
ct:pal(
?LOW_IMPORTANCE,

View File

@ -118,7 +118,17 @@ init_per_suite(Config0) ->
end_per_suite(Config) ->
rabbit_ct_helpers:run_teardown_steps(Config).
init_per_group(cluster_size_3_parallel = Group, Config) ->
case rabbit_ct_helpers:is_mixed_versions() of
true ->
{skip, "not mixed versions compatible"};
_ ->
init_per_group1(Group, Config)
end;
init_per_group(Group, Config) ->
init_per_group1(Group, Config).
init_per_group1(Group, Config) ->
ClusterSize = case Group of
single_node -> 1;
single_node_parallel -> 1;

View File

@ -117,10 +117,15 @@ init_per_group(cluster_size_1 = Group, Config) ->
]),
init_per_group1(Group, Config1);
init_per_group(cluster_size_2 = Group, Config) ->
Config1 = rabbit_ct_helpers:set_config(Config, [
{rmq_nodes_count, 2}
]),
init_per_group1(Group, Config1).
case rabbit_ct_helpers:is_mixed_versions() of
true ->
{skip, "not mixed versions compatible"};
_ ->
Config1 = rabbit_ct_helpers:set_config(Config, [
{rmq_nodes_count, 2}
]),
init_per_group1(Group, Config1)
end.
init_per_group1(Group, Config) ->
SetupFederation = case Group of

View File

@ -91,6 +91,10 @@ end_per_group(_, Config) ->
Steps = Teardown0 ++ Teardown1,
rabbit_ct_helpers:run_teardown_steps(Config, Steps).
init_per_testcase(Testcase, Config)
when Testcase == is_quorum_critical_test
orelse Testcase == is_mirror_sync_critical_test ->
{skip, "not mixed versions compatible"};
init_per_testcase(Testcase, Config) ->
rabbit_ct_helpers:testcase_started(Config, Testcase).

View File

@ -78,13 +78,20 @@ end_per_group(_, Config) ->
Config.
init_per_testcase(quorum_queue_stats = Testcase, Config) ->
case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
ok ->
rabbit_ct_helpers:testcase_started(Config, Testcase),
rabbit_ct_helpers:run_steps(
Config, rabbit_ct_client_helpers:setup_steps());
Skip ->
Skip
case rabbit_ct_helpers:is_mixed_versions() of
true ->
{skip, "not mixed versions compatible"};
_ ->
case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
ok ->
rabbit_ct_helpers:testcase_started(Config, Testcase),
rabbit_ct_helpers:run_steps(
Config, rabbit_ct_client_helpers:setup_steps());
{skip, _} = Skip ->
Skip;
Other ->
{skip, Other}
end
end;
init_per_testcase(Testcase, Config) ->
rabbit_ct_helpers:testcase_started(Config, Testcase),

View File

@ -42,30 +42,37 @@ groups() ->
].
init_per_suite(Config) ->
inets:start(),
rabbit_ct_helpers:log_environment(),
Config1 = rabbit_ct_helpers:set_config(
Config, [
{ecs_region, "eu-west-1"},
{ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
{ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
{ecs_instance_role, "ecs-peer-discovery-aws"},
{ecs_cluster_size, ?CLUSTER_SIZE},
{rabbitmq_default_user, "test"},
{rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
{rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
]),
Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
rabbit_ct_helpers:run_steps(
Config2, [
fun rabbit_ct_helpers:init_skip_as_error_flag/1,
fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
fun aws_ecs_util:ensure_aws_cli/1,
fun aws_ecs_util:ensure_ecs_cli/1,
fun aws_ecs_util:init_aws_credentials/1,
fun aws_ecs_util:ensure_rabbitmq_image/1,
fun aws_ecs_util:start_ecs_cluster/1
]).
case rabbit_ct_helpers:is_mixed_versions() of
true ->
%% These test would like passed in mixed versions, but they won't
%% actually honor mixed versions as currently specified via env var
{skip, "not mixed versions compatible"};
_ ->
inets:start(),
rabbit_ct_helpers:log_environment(),
Config1 = rabbit_ct_helpers:set_config(
Config, [
{ecs_region, "eu-west-1"},
{ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
{ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
{ecs_instance_role, "ecs-peer-discovery-aws"},
{ecs_cluster_size, ?CLUSTER_SIZE},
{rabbitmq_default_user, "test"},
{rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
{rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
]),
Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
rabbit_ct_helpers:run_steps(
Config2, [
fun rabbit_ct_helpers:init_skip_as_error_flag/1,
fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
fun aws_ecs_util:ensure_aws_cli/1,
fun aws_ecs_util:ensure_ecs_cli/1,
fun aws_ecs_util:init_aws_credentials/1,
fun aws_ecs_util:ensure_rabbitmq_image/1,
fun aws_ecs_util:start_ecs_cluster/1
])
end.
end_per_suite(Config) ->
rabbit_ct_helpers:run_teardown_steps(Config).

View File

@ -5,7 +5,7 @@ load(
"erlang_lib",
"test_erlang_lib",
)
load("@bazel-erlang//:ct_sharded.bzl", "ct_suite")
load("@bazel-erlang//:ct_sharded.bzl", "ct_suite", "ct_suite_variant")
load("//:rabbitmq_home.bzl", "rabbitmq_home")
load("//:rabbitmq_run.bzl", "rabbitmq_run")
@ -108,15 +108,24 @@ def broker_for_integration_suites():
def rabbitmq_integration_suite(
package,
name = None,
tags = [],
data = [],
erlc_opts = [],
additional_hdrs = [],
additional_srcs = [],
test_env = {},
tools = [],
deps = [],
runtime_deps = [],
**kwargs):
ct_suite(
name = name,
suite_name = name,
tags = tags,
erlc_opts = RABBITMQ_TEST_ERLC_OPTS + erlc_opts,
additional_hdrs = additional_hdrs,
additional_srcs = additional_srcs,
data = [
"@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
"@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
@ -144,7 +153,42 @@ def rabbitmq_integration_suite(
] + deps,
**kwargs
)
return kwargs["name"]
ct_suite_variant(
name = name + "-mixed",
suite_name = name,
tags = tags + ["mixed-version-cluster"],
data = [
"@rabbitmq_ct_helpers//tools/tls-certs:Makefile",
"@rabbitmq_ct_helpers//tools/tls-certs:openssl.cnf.in",
] + data,
test_env = dict({
"SKIP_MAKE_TEST_DIST": "true",
"RABBITMQ_FEATURE_FLAGS": "",
"RABBITMQ_RUN": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/rabbitmq-for-tests-run".format(package),
"RABBITMQCTL": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmqctl".format(package),
"RABBITMQ_PLUGINS": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-plugins".format(package),
"RABBITMQ_QUEUES": "$TEST_SRCDIR/$TEST_WORKSPACE/{}/broker-for-tests-home/sbin/rabbitmq-queues".format(package),
"RABBITMQ_RUN_SECONDARY": "$TEST_SRCDIR/rabbitmq-server-generic-unix-3.8.18/rabbitmq-run",
}.items() + test_env.items()),
tools = [
":rabbitmq-for-tests-run",
"@rabbitmq-server-generic-unix-3.8.18//:rabbitmq-run",
] + tools,
runtime_deps = [
"//deps/rabbitmq_cli:elixir_as_bazel_erlang_lib",
"//deps/rabbitmq_cli:rabbitmqctl",
"@rabbitmq_ct_client_helpers//:bazel_erlang_lib",
] + runtime_deps,
deps = [
"//deps/amqp_client:bazel_erlang_lib",
"//deps/rabbit_common:bazel_erlang_lib",
"@rabbitmq_ct_helpers//:bazel_erlang_lib",
] + deps,
**kwargs
)
return name
def assert_suites(suite_names, suite_files):
for f in suite_files:

View File

@ -6,7 +6,6 @@ RabbitmqHomeInfo = provider(
"sbin": "Files making up the sbin dir",
"escript": "Files making up the escript dir",
"plugins": "Files making up the plugins dir",
"erlang_version": "Version of the Erlang compiler used",
},
)
@ -118,7 +117,6 @@ def _impl(ctx):
sbin = scripts,
escript = escripts,
plugins = plugins,
erlang_version = erlang_versions[0],
),
DefaultInfo(
files = depset(scripts + escripts + plugins),
@ -141,7 +139,6 @@ rabbitmq_home = rule(
allow_files = True,
),
"_rabbitmqctl_escript": attr.label(default = "//deps/rabbitmq_cli:rabbitmqctl"),
"_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
"plugins": attr.label_list(),
},
)

View File

@ -0,0 +1,26 @@
load("@//:rabbitmq_home.bzl", "RabbitmqHomeInfo")
def _impl(ctx):
scripts = ctx.files.sbin
escripts = ctx.files.escript
plugins = ctx.files.plugins
return [
RabbitmqHomeInfo(
sbin = scripts,
escript = escripts,
plugins = plugins,
),
DefaultInfo(
files = depset(scripts + escripts + plugins),
),
]
rabbitmq_package_generic_unix = rule(
implementation = _impl,
attrs = {
"sbin": attr.label_list(allow_files = True),
"escript": attr.label_list(allow_files = True),
"plugins": attr.label_list(allow_files = True),
},
)

View File

@ -3,38 +3,34 @@ load("@bazel-erlang//:bazel_erlang_lib.bzl", "path_join")
load("@bazel-erlang//:ct.bzl", "sanitize_sname")
load(":rabbitmq_home.bzl", "RabbitmqHomeInfo")
# Note: Theses rules take advantage of the fact that when the files from
# the rabbitmq_home rule are used as runfiles, they are linked in
# at their declared relative paths. In other words, since
# rabbitmq_home declares "sbin/rabbitmq-server", is still at
# "sbin/rabbitmq-server" when our script runs.
def _dirname(p):
return p.rpartition("/")[0]
def _rabbitmq_home_info_root_short_path(rabbitmq_home):
return _dirname(_dirname(rabbitmq_home.sbin[0].short_path))
def _impl(ctx):
erlang_version = ctx.attr._erlang_version[ErlangVersionProvider].version
rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
if rabbitmq_home.erlang_version != erlang_version:
fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
root = _rabbitmq_home_info_root_short_path(rabbitmq_home)
erl_libs = ":".join(
[p.short_path for p in rabbitmq_home.plugins],
)
erl_libs = [path_join(root, "plugins")]
ctx.actions.expand_template(
template = ctx.file._template,
output = ctx.outputs.executable,
substitutions = {
"{RABBITMQ_HOME}": ctx.attr.home.label.name,
"{ERL_LIBS}": erl_libs,
"{RABBITMQ_HOME}": root,
"{ERL_LIBS}": ":".join(erl_libs),
"{ERLANG_HOME}": ctx.attr._erlang_home[ErlangHomeProvider].path,
"{SNAME}": sanitize_sname("sbb-" + ctx.attr.name),
},
is_executable = True,
)
return [DefaultInfo(
runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list()),
)]
runfiles = ctx.runfiles(ctx.attr.home[DefaultInfo].files.to_list())
return [DefaultInfo(runfiles = runfiles)]
rabbitmq_run = rule(
implementation = _impl,
@ -44,7 +40,6 @@ rabbitmq_run = rule(
allow_single_file = True,
),
"_erlang_home": attr.label(default = "@bazel-erlang//:erlang_home"),
"_erlang_version": attr.label(default = "@bazel-erlang//:erlang_version"),
"home": attr.label(providers = [RabbitmqHomeInfo]),
},
executable = True,

View File

@ -6,9 +6,6 @@ def _impl(ctx):
rabbitmq_home = ctx.attr.home[RabbitmqHomeInfo]
if rabbitmq_home.erlang_version != erlang_version:
fail("Mismatched erlang versions", erlang_version, rabbitmq_home.erlang_version)
script = """
exec ./{home}/sbin/{cmd} $@
""".format(

View File

@ -1,8 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
# https://stackoverflow.com/a/4774063
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
if [ -z ${TEST_SRCDIR+x} ]; then
BASE_DIR=$PWD
else
BASE_DIR=$TEST_SRCDIR/$TEST_WORKSPACE
fi
if [ $1 = "-C" ]; then
cd $2
@ -32,13 +35,13 @@ for arg in "$@"; do
esac
done
DEFAULT_PLUGINS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/plugins
DEFAULT_PLUGINS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/plugins
if [ ! -z ${EXTRA_PLUGINS_DIR+x} ]; then
DEFAULT_PLUGINS_DIR=${DEFAULT_PLUGINS_DIR}:${EXTRA_PLUGINS_DIR}
fi
TEST_TMPDIR=${TEST_TMPDIR:=${TMPDIR}/rabbitmq-test-instances}
RABBITMQ_SCRIPTS_DIR=${SCRIPTPATH}/{RABBITMQ_HOME}/sbin
RABBITMQ_SCRIPTS_DIR=${BASE_DIR}/{RABBITMQ_HOME}/sbin
RABBITMQ_PLUGINS=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-plugins
RABBITMQ_SERVER=${RABBITMQ_SCRIPTS_DIR}/rabbitmq-server
RABBITMQCTL=${RABBITMQ_SCRIPTS_DIR}/rabbitmqzctl
@ -161,12 +164,12 @@ case $CMD in
while ps -p "$pid" >/dev/null 2>&1; do sleep 1; done
;;
set-resource-alarm)
ERL_LIBS="{ERL_LIBS}" \
ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
eval "rabbit_alarm:set_alarm({{resource_limit, ${SOURCE}, node()}, []})."
;;
clear-resource-alarm)
ERL_LIBS="{ERL_LIBS}" \
ERL_LIBS="${BASE_DIR}/{ERL_LIBS}" \
${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl -n ${RABBITMQ_NODENAME} \
eval "rabbit_alarm:clear_alarm({resource_limit, ${SOURCE}, node()})."
;;