Introduce a way to disable virtual host reconciliation

Certain test suites need virtual hosts to be down
for an extended period of time, and this feature
gets in a way ;)

deps/rabbit/BUILD.bazel

@@ -144,7 +144,9 @@ _APP_ENV = """[
         %% EOL date for the current release series, if known/announced
 		{release_series_eol_date, none},
 
-		{vhost_process_reconciliation_run_interval, 30}
+		{vhost_process_reconciliation_run_interval, 30},
+        %% for testing
+        {vhost_process_reconciliation_enabled, true}
 	  ]
 """
 

deps/rabbit/Makefile

@@ -125,7 +125,9 @@ define PROJECT_ENV
       	{dead_letter_worker_publisher_confirm_timeout, 180000},
 		%% EOL date for the current release series, if known/announced
 		{release_series_eol_date, none},
-		{vhost_process_reconciliation_run_interval, 30}
+		{vhost_process_reconciliation_run_interval, 30},
+		%% for testing
+		{vhost_process_reconciliation_enabled, true}
 	  ]
 endef
 

deps/rabbit/src/rabbit_vhosts.erl

@@ -19,6 +19,9 @@
     boot/0,
     reconcile/0,
     reconcile_once/0,
+    is_reconciliation_enabled/0,
+    disable_reconciliation/0,
+    enable_reconciliation/0,
     start_processes_for_all/0,
     start_on_all_nodes/2,
     on_node_up/1
@@ -37,7 +40,10 @@ list_names() -> rabbit_db_vhost:list().
 boot() ->
     _ = start_processes_for_all(),
     _ = increment_run_counter(),
-    _ = maybe_start_timer(reconcile),
+    _ = case is_reconciliation_enabled() of
+        false -> ok;
+        true -> maybe_start_timer(reconcile)
+    end,
     ok.
 
 %% Performs a round of virtual host process reconciliation and sets up a timer to
@@ -45,9 +51,13 @@ boot() ->
 %% See start_processes_for_all/1.
 -spec reconcile() -> 'ok'.
 reconcile() ->
+    case is_reconciliation_enabled() of
+        false -> ok;
+        true  ->
             _ = reconcile_once(),
             _ = maybe_start_timer(?FUNCTION_NAME),
-    ok.
+            ok
+    end.
 
 %% Performs a round of virtual host process reconciliation but does not schedule any future runs.
 %% See start_processes_for_all/1.
@@ -62,11 +72,33 @@ reconcile_once() ->
 
 -spec on_node_up(Node :: node()) -> 'ok'.
 on_node_up(_Node) ->
+    case is_reconciliation_enabled() of
+        false -> ok;
+        true  ->
             DelayInSeconds = 10,
             Delay = DelayInSeconds * 1000,
             rabbit_log:debug("Will reschedule virtual host process reconciliation after ~b seconds", [DelayInSeconds]),
-    _ = timer:apply_after(Delay, ?MODULE, reconcile_once, []),
-    ok.
+            timer:apply_after(Delay, ?MODULE, reconcile_once, []),
+            ok
+    end.
+
+-spec is_reconciliation_enabled() -> boolean().
+is_reconciliation_enabled() ->
+    application:get_env(rabbit, vhost_process_reconciliation_enabled, true).
+
+-spec enable_reconciliation() -> 'ok'.
+enable_reconciliation() ->
+    %% reset the auto-stop counter
+    persistent_term:put(?PERSISTENT_TERM_COUNTER_KEY, 0),
+    application:set_env(rabbit, vhost_process_reconciliation_enabled, true).
+
+-spec disable_reconciliation() -> 'ok'.
+disable_reconciliation() ->
+    application:set_env(rabbit, vhost_process_reconciliation_enabled, false).
+
+-spec reconciliation_interval() -> non_neg_integer().
+reconciliation_interval() ->
+    application:get_env(rabbit, vhost_process_reconciliation_run_interval, 30).
 
 %% Starts a virtual host process on every specified nodes.
 %% Only exists to allow for "virtual host process repair"
@@ -117,14 +149,18 @@ increment_run_counter() ->
 -spec maybe_start_timer(atom()) -> ok | {ok, timer:tref()} | {error, any()}.
 maybe_start_timer(FunName) ->
     N = get_run_counter(),
-    DelayInSeconds = application:get_env(rabbit, vhost_process_reconciliation_run_interval, 30),
+    DelayInSeconds = reconciliation_interval(),
     case N >= 10 of
         true ->
             %% Stop after ten runs
             rabbit_log:debug("Will stop virtual host process reconciliation after ~tp runs", [N]),
             ok;
         false ->
+            case is_reconciliation_enabled() of
+                false -> ok;
+                true  ->
                     Delay = DelayInSeconds * 1000,
                     rabbit_log:debug("Will reschedule virtual host process reconciliation after ~b seconds", [DelayInSeconds]),
                     timer:apply_after(Delay, ?MODULE, FunName, [])
+            end
     end.

deps/rabbit/test/rabbitmqctl_integration_SUITE.erl

@@ -67,7 +67,11 @@ create_n_node_cluster(Config0, NumNodes) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config0, [{rmq_nodes_count, NumNodes},
                           {rmq_nodes_clustered, true}]),
-    rabbit_ct_helpers:run_steps(Config1,
+    Config2 = rabbit_ct_helpers:merge_app_env(
+        Config1, {rabbit, [
+            {vhost_process_reconciliation_enabled, false}
+        ]}),
+    rabbit_ct_helpers:run_steps(Config2,
                                 rabbit_ct_broker_helpers:setup_steps() ++
                                 rabbit_ct_client_helpers:setup_steps()).
 
@@ -100,9 +104,12 @@ end_per_group(_, Config) ->
     Config.
 
 init_per_testcase(list_queues_stopped, Config0) ->
-    %% Start node 3 to crash it's queues
+    %% Start node 3 to kill a few virtual hosts on it
     rabbit_ct_broker_helpers:start_node(Config0, 2),
-    %% Make vhost "down" on nodes 2 and 3
+    %% Disable virtual host reconciliation
+    rabbit_ct_broker_helpers:rpc(Config0, 1, rabbit_vhosts, disable_reconciliation, []),
+    rabbit_ct_broker_helpers:rpc(Config0, 2, rabbit_vhosts, disable_reconciliation, []),
+    %% Terminate default virtual host's processes on nodes 2 and 3
     ok = rabbit_ct_broker_helpers:force_vhost_failure(Config0, 1, <<"/">>),
     ok = rabbit_ct_broker_helpers:force_vhost_failure(Config0, 2, <<"/">>),
 
@@ -118,6 +125,7 @@ end_per_testcase(Testcase, Config0) ->
 %%----------------------------------------------------------------------------
 %% Test cases
 %%----------------------------------------------------------------------------
+
 list_queues_local(Config) ->
     Node1Queues = lists:nth(1, ?config(per_node_queues, Config)),
     Node2Queues = lists:nth(2, ?config(per_node_queues, Config)),
@@ -147,9 +155,9 @@ list_queues_stopped(Config) ->
 
     Expected =
         lists:sort([ {Q, <<"running">>} || Q <- Node1Queues ] ++
-                       %% Node is running. Vhost is down
+                       %% Node is running but virtual host is down with reconciliation disabled
                        [ {Q, <<"stopped">>} || Q <- Node2Queues ] ++
-                       %% Node is not running. Vhost is down
+                       %% Node is not running
                        [ {Q, <<"down">>} || Q <- Node3Queues ]),
 
     ?awaitMatch(