Merge pull request #14171 from rabbitmq/mixed-version-with-otp26-rmq-3.13

Test mixed version with 3.13.7/OTP26 (weekly)

.github/workflows/test-make-target.yaml

@@ -15,6 +15,10 @@ on:
         required: false
         default: false
         type: boolean
+      previous_version:
+        required: false
+        default: 'tags/v4.0.9'
+        type: string
       make_target:
         required: true
         type: string
@@ -62,7 +66,7 @@ jobs:
       uses: dsaltares/fetch-gh-release-asset@master
       if: inputs.mixed_clusters
       with:
-        version: 'tags/v4.0.9'
+        version: ${{ inputs.previous_version }}
         regex: true
         file: "rabbitmq-server-generic-unix-\\d.+\\.tar\\.xz"
         target: ./

.github/workflows/test-mixed.yaml

@@ -0,0 +1,25 @@
+name: Test Mixed with 3.13
+on:
+  schedule:
+    # at 2:20am on Sundays
+    - cron: 20 2 * * 0
+  workflow_dispatch:
+jobs:
+  test-mixed-clusters:
+    name: Test mixed clusters
+    strategy:
+      fail-fast: false
+      matrix:
+        erlang_version:
+        - '26'
+        elixir_version:
+        - '1.18'
+        metadata_store:
+          - mnesia
+    uses: ./.github/workflows/test-make-tests.yaml
+    with:
+      erlang_version: ${{ matrix.erlang_version }}
+      elixir_version: ${{ matrix.elixir_version }}
+      previous_version: 'tags/v3.13.7'
+      metadata_store: ${{ matrix.metadata_store }}
+      mixed_clusters: true

deps/rabbit/test/amqp_client_SUITE.erl

@@ -326,7 +326,15 @@ init_per_testcase(T, Config)
     end;
 init_per_testcase(T, Config)
   when T =:= leader_transfer_quorum_queue_credit_single orelse
-       T =:= leader_transfer_quorum_queue_credit_batches ->
+       T =:= leader_transfer_quorum_queue_credit_batches orelse
+       T =:= async_notify_unsettled_classic_queue orelse
+       T =:= leader_transfer_stream_credit_single orelse
+       T =:= dead_letter_into_stream orelse
+       T =:= classic_queue_on_new_node orelse
+       T =:= leader_transfer_quorum_queue_send orelse
+       T =:= last_queue_confirms orelse
+       T =:= leader_transfer_stream_credit_batches orelse
+       T =:= leader_transfer_stream_send ->
     %% These test cases flake with feature flag 'rabbitmq_4.0.0' disabled.
     case rabbit_ct_broker_helpers:enable_feature_flag(Config, 'rabbitmq_4.0.0') of
         ok ->
@@ -342,14 +350,6 @@ init_per_testcase(T = immutable_bare_message, Config) ->
             {skip, "RabbitMQ is known to wrongfully modify the bare message with feature "
              "flag rabbitmq_4.0.0 disabled"}
     end;
-init_per_testcase(T = dead_letter_into_stream, Config) ->
-    case rabbit_ct_broker_helpers:enable_feature_flag(Config, message_containers_deaths_v2) of
-        ok ->
-            rabbit_ct_helpers:testcase_started(Config, T);
-        _ ->
-            {skip, "This test is known to fail with feature flag message_containers_deaths_v2 disabled "
-             "due to missing feature https://github.com/rabbitmq/rabbitmq-server/issues/11173"}
-    end;
 init_per_testcase(T = dead_letter_reject, Config) ->
     case rabbit_ct_broker_helpers:enable_feature_flag(Config, message_containers_deaths_v2) of
         ok ->

deps/rabbit/test/clustering_management_SUITE.erl

@@ -146,12 +146,26 @@ init_per_group(unclustered_2_nodes, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config, [{rmq_nodes_clustered, false}]),
     rabbit_ct_helpers:merge_app_env(
-      Config1, {rabbit, [{forced_feature_flags_on_init, []}]});
+      Config1, {rabbit, [{forced_feature_flags_on_init, [
+                                                         restart_streams,
+                                                         stream_sac_coordinator_unblock_group,
+                                                         stream_update_config_command,
+                                                         stream_filtering,
+                                                         message_containers,
+                                                         quorum_queue_non_voters
+                                                        ]}]});
 init_per_group(unclustered_3_nodes, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config, [{rmq_nodes_clustered, false}]),
     rabbit_ct_helpers:merge_app_env(
-      Config1, {rabbit, [{forced_feature_flags_on_init, []}]});
+      Config1, {rabbit, [{forced_feature_flags_on_init, [
+                                                         restart_streams,
+                                                         stream_sac_coordinator_unblock_group,
+                                                         stream_update_config_command,
+                                                         stream_filtering,
+                                                         message_containers,
+                                                         quorum_queue_non_voters
+                                                        ]}]});
 init_per_group(clustered_2_nodes, Config) ->
     rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]);
 init_per_group(clustered_3_nodes, Config) ->

deps/rabbit/test/dead_lettering_SUITE.erl

@@ -184,6 +184,14 @@ init_per_testcase(T, Config)
     %% * stream is known to fail due to https://github.com/rabbitmq/rabbitmq-server/issues/11173
     ok = rabbit_ct_broker_helpers:enable_feature_flag(Config, message_containers_deaths_v2),
     init_per_testcase0(T, Config);
+init_per_testcase(T, Config)
+  when T =:= dead_letter_headers_should_not_be_appended_for_republish ->
+    case rabbit_ct_broker_helpers:enable_feature_flag(Config, 'rabbitmq_4.0.0') of
+        ok ->
+            init_per_testcase0(T, Config);
+        _ ->
+            {skip, "The expectations of this test don't match 3.13 behaviour"}
+    end;
 init_per_testcase(Testcase, Config) ->
     init_per_testcase0(Testcase, Config).
 

deps/rabbit/test/dynamic_qq_SUITE.erl

@@ -222,6 +222,8 @@ quorum_unaffected_after_vhost_failure(Config) ->
 forget_cluster_node(Config) ->
     %% Tests that quorum queues shrink when forget_cluster_node
     %% operations are issues.
+    quorum_queue_SUITE:check_quorum_queues_v4_compat(Config),
+
     [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
 

deps/rabbit/test/feature_flags_SUITE.erl

@@ -197,7 +197,14 @@ init_per_group(clustering, Config) ->
                  {rmq_nodes_clustered, false},
                  {start_rmq_with_plugins_disabled, true}]),
     Config2 = rabbit_ct_helpers:merge_app_env(
-                Config1, {rabbit, [{forced_feature_flags_on_init, []}]}),
+                Config1, {rabbit, [{forced_feature_flags_on_init, [
+                                                                   restart_streams,
+                                                                   stream_sac_coordinator_unblock_group,
+                                                                   stream_update_config_command,
+                                                                   stream_filtering,
+                                                                   message_containers,
+                                                                   quorum_queue_non_voters
+                                                                  ]}]}),
     rabbit_ct_helpers:run_setup_steps(Config2, [fun prepare_my_plugin/1]);
 init_per_group(activating_plugin, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
@@ -1277,6 +1284,13 @@ activating_plugin_with_new_ff_enabled(Config) ->
     ok.
 
 enable_plugin_feature_flag_after_deactivating_plugin(Config) ->
+    case rabbit_ct_broker_helpers:is_feature_flag_enabled(Config, 'rabbitmq_4.0.0') of
+        true ->
+            ok;
+        false ->
+            throw({skip, "this test triggers a bug present in 3.13"})
+    end,
+
     FFSubsysOk = is_feature_flag_subsystem_available(Config),
 
     log_feature_flags_of_all_nodes(Config),
@@ -1307,6 +1321,13 @@ enable_plugin_feature_flag_after_deactivating_plugin(Config) ->
     ok.
 
 restart_node_with_unknown_enabled_feature_flag(Config) ->
+    case rabbit_ct_broker_helpers:is_feature_flag_enabled(Config, 'rabbitmq_4.0.0') of
+        true ->
+            ok;
+        false ->
+            throw({skip, "this test triggers a bug present in 3.13"})
+    end,
+
     FFSubsysOk = is_feature_flag_subsystem_available(Config),
 
     log_feature_flags_of_all_nodes(Config),

deps/rabbit/test/queue_length_limits_SUITE.erl

@@ -82,8 +82,17 @@ init_per_group(mnesia_parallel_tests = Group, Config0) ->
     Config = rabbit_ct_helpers:set_config(Config0, [{metadata_store, mnesia}]),
     init_per_group0(Group, Config);
 init_per_group(khepri_parallel_tests = Group, Config0) ->
+    %% this is very hacky way of skipping the tests, but the khepri_db
+    %% flag exists in 3,13, but it's not compatible with 4.x. We can remove
+    %% this after 4.2
+    SecondaryDist = os:getenv("SECONDARY_DIST", ""),
+    case string:str(SecondaryDist, "3.13.") == 0 of
+        true ->
             Config = rabbit_ct_helpers:set_config(Config0, [{metadata_store, khepri}]),
-    init_per_group0(Group, Config).
+            init_per_group0(Group, Config);
+        _ ->
+            {skip, "Khepri was not supported in 3.13"}
+    end.
 
 init_per_group0(Group, Config) ->
     case lists:member({group, Group}, all()) of

deps/rabbit/test/quorum_queue_SUITE.erl

@@ -322,6 +322,10 @@ init_per_testcase(T, Config)
 init_per_testcase(Testcase, Config) ->
     ClusterSize = ?config(rmq_nodes_count, Config),
     IsMixed = rabbit_ct_helpers:is_mixed_versions(),
+    RabbitMQ3 = case rabbit_ct_broker_helpers:enable_feature_flag(Config, 'rabbitmq_4.0.0') of
+                    ok -> false;
+                    _ -> true
+                end,
     SameKhepriMacVers = (
       rabbit_ct_broker_helpers:do_nodes_run_same_ra_machine_version(
         Config, khepri_machine)),
@@ -359,6 +363,8 @@ init_per_testcase(Testcase, Config) ->
             {skip, "reclaim_memory_with_wrong_queue_type isn't mixed versions compatible"};
         peek_with_wrong_queue_type when IsMixed ->
             {skip, "peek_with_wrong_queue_type isn't mixed versions compatible"};
+        cancel_consumer_gh_3729 when IsMixed andalso RabbitMQ3 ->
+            {skip, "this test is not compatible with RabbitMQ 3.13.x"};
         _ ->
             Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase),
             rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []),
@@ -1324,6 +1330,10 @@ force_shrink_member_to_current_member(Config) ->
     end.
 
 force_all_queues_shrink_member_to_current_member(Config) ->
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "Should not run in mixed version environments"};
+        _ ->
             [Server0, Server1, Server2] =
             rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
@@ -1366,9 +1376,14 @@ force_all_queues_shrink_member_to_current_member(Config) ->
                  {ok, Q0} = rpc:call(Server0, rabbit_amqqueue, lookup, [Q, <<"/">>]),
                  #{nodes := Nodes0} = amqqueue:get_type_state(Q0),
                  ?assertEqual(3, length(Nodes0))
-    end || Q <- QQs].
+             end || Q <- QQs]
+    end.
 
 force_vhost_queues_shrink_member_to_current_member(Config) ->
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "Should not run in mixed version environments"};
+        _ ->
             [Server0, Server1, Server2] =
                 rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
@@ -1432,9 +1447,12 @@ force_vhost_queues_shrink_member_to_current_member(Config) ->
                 {ok, Q0} = rpc:call(Server0, rabbit_amqqueue, lookup, [Q, VHost]),
                 #{nodes := Nodes0} = amqqueue:get_type_state(Q0),
                 ?assertEqual(3, length(Nodes0))
-    end || Q <- QQs, VHost <- VHosts].
+            end || Q <- QQs, VHost <- VHosts]
+    end.
 
 force_checkpoint_on_queue(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     [Server0, Server1, Server2] =
         rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server0),
@@ -1501,6 +1519,8 @@ force_checkpoint_on_queue(Config) ->
       end).
 
 force_checkpoint(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     [Server0, _Server1, _Server2] =
         rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server0),
@@ -1722,6 +1742,7 @@ subscribe_from_each(Config) ->
     ok.
 
 dont_leak_file_handles(Config) ->
+    check_quorum_queues_v4_compat(Config),
 
     [Server0 | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
@@ -1770,6 +1791,8 @@ dont_leak_file_handles(Config) ->
     ok.
 
 gh_12635(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     % https://github.com/rabbitmq/rabbitmq-server/issues/12635
     [Server0, _Server1, Server2] =
         rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
@@ -2477,8 +2500,6 @@ metrics_cleanup_on_leader_crash(Config) ->
     publish(Ch, QQ),
     publish(Ch, QQ),
 
-    wait_for_messages_ready([Server], RaName, 3),
-    wait_for_messages_pending_ack([Server], RaName, 0),
     {ok, _, {Name, Leader}} = ra:members({RaName, Server}),
     QRes = rabbit_misc:r(<<"/">>, queue, QQ),
     rabbit_ct_helpers:await_condition(
@@ -3268,6 +3289,8 @@ subscribe_redelivery_limit(Config) ->
     end.
 
 subscribe_redelivery_limit_disable(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
@@ -3664,6 +3687,8 @@ queue_length_limit_reject_publish(Config) ->
     ok.
 
 queue_length_limit_policy_cleared(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
@@ -3965,7 +3990,7 @@ receive_and_ack(Ch) ->
     end.
 
 message_ttl_policy(Config) ->
-    %% Using ttl is very difficul to guarantee 100% test rate success, unless
+    %% Using ttl is very difficult to guarantee 100% test rate success, unless
     %% using really high ttl values. Previously, this test used 1s and 3s ttl,
     %% but expected to see first the messages in the queue and then the messages
     %% gone. A slow CI run, would fail the first assertion as the messages would
@@ -3985,9 +4010,8 @@ message_ttl_policy(Config) ->
     VHost = <<"%2F">>,
     RaName = binary_to_atom(<<VHost/binary, "_", QQ/binary>>, utf8),
 
-    QueryFun = fun rabbit_fifo:overview/1,
+    ?awaitMatch({ok, #{machine := #{config := #{msg_ttl := 1000}}}, _},
-    ?awaitMatch({ok, {_, #{config := #{msg_ttl := 1000}}}, _},
+                rpc:call(Server, ra, member_overview, [RaName]),
-                rpc:call(Server, ra, local_query, [RaName, QueryFun]),
                 ?DEFAULT_AWAIT),
     Msg1 = <<"msg1">>,
     Msg2 = <<"msg11">>,
@@ -3999,8 +4023,8 @@ message_ttl_policy(Config) ->
     ok = rabbit_ct_broker_helpers:set_policy(Config, 0, <<"msg-ttl">>,
                                              QQ, <<"queues">>,
                                              [{<<"message-ttl">>, 1000}]),
-    {ok, {_, Overview2}, _} = rpc:call(Server, ra, local_query, [RaName, QueryFun]),
+    {ok, Overview2, _} = rpc:call(Server, ra, member_overview, [RaName]),
-    ?assertMatch(#{config := #{msg_ttl := 1000}}, Overview2),
+    ?assertMatch(#{machine := #{config := #{msg_ttl := 1000}}}, Overview2),
     publish(Ch, QQ, Msg1),
     wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]),
     wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]),
@@ -4723,6 +4747,8 @@ select_nodes_with_least_replicas_node_down(Config) ->
      || Q <- Qs].
 
 requeue_multiple_true(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     Ch = rabbit_ct_client_helpers:open_channel(Config),
     QQ = ?config(queue_name, Config),
     ?assertEqual({'queue.declare_ok', QQ, 0, 0},
@@ -4761,6 +4787,8 @@ requeue_multiple_true(Config) ->
                  amqp_channel:call(Ch, #'queue.delete'{queue = QQ})).
 
 requeue_multiple_false(Config) ->
+    check_quorum_queues_v4_compat(Config),
+
     Ch = rabbit_ct_client_helpers:open_channel(Config),
     QQ = ?config(queue_name, Config),
     ?assertEqual({'queue.declare_ok', QQ, 0, 0},

deps/rabbit/test/rabbit_fifo_prop_SUITE.erl

@@ -1123,8 +1123,8 @@ two_nodes_different_otp_version(_Config) ->
                 pong ->
                     case is_same_otp_version(Node) of
                         true ->
-                            ct:fail("expected CT node and 'rabbit_fifo_prop@localhost' "
+                            {skip, "expected CT node and 'rabbit_fifo_prop@localhost' "
-                                    "to have different OTP versions");
+                                    "to have different OTP versions"};
                         false ->
                             Prefixes = ["rabbit_fifo", "rabbit_misc", "mc",
                                         "lqueue", "priority_queue", "ra_"],

deps/rabbit/test/rabbit_stream_queue_SUITE.erl

@@ -238,7 +238,14 @@ init_per_group1(Group, Config) ->
                        Config1
                end,
     Config1c = rabbit_ct_helpers:merge_app_env(
-                 Config1b, {rabbit, [{forced_feature_flags_on_init, []}]}),
+                 Config1b, {rabbit, [{forced_feature_flags_on_init, [
+                                                                     restart_streams,
+                                                                     stream_sac_coordinator_unblock_group,
+                                                                     stream_update_config_command,
+                                                                     stream_filtering,
+                                                                     message_containers,
+                                                                     quorum_queue_non_voters
+                                                                    ]}]}),
     Ret = rabbit_ct_helpers:run_steps(Config1c,
                                       [fun merge_app_env/1 ] ++
                                       rabbit_ct_broker_helpers:setup_steps()),

deps/rabbitmq_cli/test/ctl/set_permissions_command_test.exs

@@ -96,10 +96,10 @@ defmodule SetPermissionsCommandTest do
 
   @tag user: @user, vhost: @root
   test "run: invalid regex patterns returns an error", context do
-    assert @command.run(
+    assert match?({:error, {:invalid_regexp, ~c"*", _}}, @command.run(
              [context[:user], "^#{context[:user]}-.*", ".*", "*"],
              context[:opts]
-           ) == {:error, {:invalid_regexp, ~c"*", {~c"quantifier does not follow a repeatable item", 0}}}
+           ))
 
     # asserts that the failed command didn't change anything
     u = Enum.find(list_permissions(context[:vhost]), fn x -> x[:user] == context[:user] end)

deps/rabbitmq_cli/test/ctl/set_permissions_globally_command_test.exs

@@ -97,10 +97,10 @@ defmodule SetPermissionsGloballyCommandTest do
     p2 = Enum.find(list_permissions(@vhost2), fn x -> x[:user] == context[:user] end)
     p3 = Enum.find(list_permissions(@vhost3), fn x -> x[:user] == context[:user] end)
 
-    assert @command.run(
+  assert match?({:error, {:invalid_regexp, ~c"*", _}}, @command.run(
            [context[:user], "^#{context[:user]}-.*", ".*", "*"],
            context[:opts]
-           ) == {:error, {:invalid_regexp, ~c"*", {~c"quantifier does not follow a repeatable item", 0}}}
+         ))
 
     # asserts that the failed command didn't change anything
     p4 = Enum.find(list_permissions(@vhost1), fn x -> x[:user] == context[:user] end)

deps/rabbitmq_mqtt/test/mqtt_shared_SUITE.erl

@@ -164,7 +164,17 @@ init_per_suite(Config) ->
                 Config, {rabbit, [
                                   {quorum_tick_interval, 1000},
                                   {stream_tick_interval, 1000},
-                                  {forced_feature_flags_on_init, []},
+                                  {forced_feature_flags_on_init, [
+                                                                  delete_ra_cluster_mqtt_node,
+                                                                  mqtt_v5,
+                                                                  rabbit_mqtt_qos0_queue,
+                                                                  restart_streams,
+                                                                  stream_sac_coordinator_unblock_group,
+                                                                  stream_update_config_command,
+                                                                  stream_filtering,
+                                                                  message_containers,
+                                                                  quorum_queue_non_voters
+                                                                 ]},
                                   {start_rmq_with_plugins_disabled, true}
                                  ]}),
     rabbit_ct_helpers:run_setup_steps(Config1).