From 1eb80a11bd13dc79f0a27af4de6a14e31093b1b8 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 9 May 2018 10:06:20 -0500 Subject: [PATCH 01/35] Change channel_max default to 2048 Closes #1593. --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6068979317..5b8835e3cc 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ define PROJECT_ENV %% 0 ("no limit") would make a better default, but that %% breaks the QPid Java client {frame_max, 131072}, - {channel_max, 0}, + %% see rabbitmq-server#1593 + {channel_max, 2048}, {connection_max, infinity}, {heartbeat, 60}, {msg_store_file_size_limit, 16777216}, @@ -62,7 +63,7 @@ define PROJECT_ENV ]}, {halt_on_upgrade_failure, true}, {hipe_compile, false}, - %% see bug 24513 for how this list was created + %% see bug 24513 [in legacy Bugzilla] for how this list was created {hipe_modules, [rabbit_reader, rabbit_channel, gen_server2, rabbit_exchange, rabbit_command_assembler, rabbit_framing_amqp_0_9_1, rabbit_basic, From 7a3898110e9bd76fc42f51c3e7eba78135ac9e0f Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 9 May 2018 12:39:19 -0500 Subject: [PATCH 02/35] Use 2047 Since channel 0 exists on every connection for negotiation and error communication. 655365 = (1 << 16) - 1, so 2047 = (1 << 11) - 1. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5b8835e3cc..da56b2ad9c 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ define PROJECT_ENV %% breaks the QPid Java client {frame_max, 131072}, %% see rabbitmq-server#1593 - {channel_max, 2048}, + {channel_max, 2047}, {connection_max, infinity}, {heartbeat, 60}, {msg_store_file_size_limit, 16777216}, From 54c9b85836199597fb3b633fdeb2d725ab9d45ad Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Thu, 10 May 2018 13:43:39 -0500 Subject: [PATCH 03/35] Make policy validation aware of the max-priority argument References #1590. [#157380396] --- src/rabbit_policies.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rabbit_policies.erl b/src/rabbit_policies.erl index f48189b210..0faad7259d 100644 --- a/src/rabbit_policies.erl +++ b/src/rabbit_policies.erl @@ -41,6 +41,7 @@ register() -> {policy_validator, <<"expires">>}, {policy_validator, <<"max-length">>}, {policy_validator, <<"max-length-bytes">>}, + {policy_validator, <<"max-priority">>}, {policy_validator, <<"queue-mode">>}, {policy_validator, <<"overflow">>}, {operator_policy_validator, <<"expires">>}, @@ -100,6 +101,12 @@ validate_policy0(<<"max-length-bytes">>, Value) validate_policy0(<<"max-length-bytes">>, Value) -> {error, "~p is not a valid maximum length in bytes", [Value]}; +validate_policy0(<<"max-priority">>, Value) + when is_integer(Value), Value >= 0, Value =< 255 -> + ok; +validate_policy0(<<"max-priority">>, Value) -> + {error, "~p is not a valid max priority (must be an integer in the 1-255 range)", [Value]}; + validate_policy0(<<"queue-mode">>, <<"default">>) -> ok; validate_policy0(<<"queue-mode">>, <<"lazy">>) -> @@ -117,4 +124,3 @@ merge_policy_value(<<"message-ttl">>, Val, OpVal) -> min(Val, OpVal); merge_policy_value(<<"max-length">>, Val, OpVal) -> min(Val, OpVal); merge_policy_value(<<"max-length-bytes">>, Val, OpVal) -> min(Val, OpVal); merge_policy_value(<<"expires">>, Val, OpVal) -> min(Val, OpVal). - From 08636f98cef491c62bf67d6b865333d65f0d3b9f Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 11 May 2018 00:05:07 -0500 Subject: [PATCH 04/35] Reject max-priority arguments >= 256 This is the value we advertise in the docs and it should be enforced to avoid process explosion e.g. when an overflow value is provided. Part of #1590. [#157380396] --- src/rabbit_amqqueue.erl | 9 ++++++++- src/rabbit_priority_queue.erl | 13 ++++++++----- test/priority_queue_SUITE.erl | 29 +++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index ac79d56358..eb00729baa 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -575,7 +575,7 @@ declare_args() -> {<<"x-dead-letter-routing-key">>, fun check_dlxrk_arg/2}, {<<"x-max-length">>, fun check_non_neg_int_arg/2}, {<<"x-max-length-bytes">>, fun check_non_neg_int_arg/2}, - {<<"x-max-priority">>, fun check_non_neg_int_arg/2}, + {<<"x-max-priority">>, fun check_max_priority_arg/2}, {<<"x-overflow">>, fun check_overflow/2}, {<<"x-queue-mode">>, fun check_queue_mode/2}]. @@ -611,6 +611,13 @@ check_message_ttl_arg({Type, Val}, Args) -> Error -> Error end. +check_max_priority_arg({Type, Val}, Args) -> + case check_non_neg_int_arg({Type, Val}, Args) of + ok when Val =< 255 -> ok; + ok -> {error, {max_value_exceeded, Val}}; + Error -> Error + end. + %% Note that the validity of x-dead-letter-exchange is already verified %% by rabbit_channel's queue.declare handler. check_dlxname_arg({longstr, _}, _) -> ok; diff --git a/src/rabbit_priority_queue.erl b/src/rabbit_priority_queue.erl index 5786bed6ba..b1eb83dddc 100644 --- a/src/rabbit_priority_queue.erl +++ b/src/rabbit_priority_queue.erl @@ -128,11 +128,14 @@ collapse_recovery(QNames, DupNames, Recovery) -> priorities(#amqqueue{arguments = Args}) -> Ints = [long, short, signedint, byte, unsignedbyte, unsignedshort, unsignedint], case rabbit_misc:table_lookup(Args, <<"x-max-priority">>) of - {Type, Max} -> case lists:member(Type, Ints) of - false -> none; - true -> lists:reverse(lists:seq(0, Max)) - end; - _ -> none + {Type, RequestedMax} -> + case lists:member(Type, Ints) of + false -> none; + true -> + Max = min(RequestedMax, ?MAX_SUPPORTED_PRIORITY), + lists:reverse(lists:seq(0, Max)) + end; + _ -> none end. %%---------------------------------------------------------------------------- diff --git a/test/priority_queue_SUITE.erl b/test/priority_queue_SUITE.erl index a1ae66dbbb..cfa98b94c7 100644 --- a/test/priority_queue_SUITE.erl +++ b/test/priority_queue_SUITE.erl @@ -17,6 +17,7 @@ -module(priority_queue_SUITE). -include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). -include_lib("amqp_client/include/amqp_client.hrl"). -compile(export_all). @@ -46,7 +47,9 @@ groups() -> simple_order, straight_through, invoke, - gen_server2_stats + gen_server2_stats, + negative_max_priorities, + max_priorities_above_hard_limit ]}, {cluster_size_3, [], [ mirror_queue_auto_ack, @@ -192,6 +195,28 @@ straight_through(Config) -> rabbit_ct_client_helpers:close_connection(Conn), passed. +max_priorities_above_hard_limit(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"max_priorities_above_hard_limit">>, + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + %% Note that lower values (e.g. 300) will cause overflow the byte type here. + %% However, values >= 256 would still be rejected when used by + %% other clients + declare(Ch, Q, 3000)), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +negative_max_priorities(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"negative_max_priorities">>, + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + declare(Ch, Q, -10)), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + + invoke(Config) -> %% Synthetic test to check the invoke callback, as the bug tested here %% is only triggered with a race condition. @@ -669,7 +694,7 @@ get_ok(Ch, Q, Ack, PBin) -> {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = PBin2}} = amqp_channel:call(Ch, #'basic.get'{queue = Q, no_ack = Ack =:= no_ack}), - PBin = PBin2, + ?assertEqual(PBin, PBin2), maybe_ack(Ch, Ack, DTag). get_payload(Ch, Q, Ack, Ps) -> From f5aa1fbe043395806d9b9ed8780892924431466c Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sun, 13 May 2018 01:32:39 -0500 Subject: [PATCH 05/35] Take policy-configured max-priority into account Part of #1590. --- src/rabbit_amqqueue_process.erl | 4 ++++ src/rabbit_priority_queue.erl | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index a3c8f99519..746679168f 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -382,6 +382,7 @@ process_args_policy(State = #q{q = Q, {<<"message-ttl">>, fun res_min/2, fun init_ttl/2}, {<<"max-length">>, fun res_min/2, fun init_max_length/2}, {<<"max-length-bytes">>, fun res_min/2, fun init_max_bytes/2}, + {<<"max-priority">>, fun res_arg/2, fun init_max_priority/2}, {<<"overflow">>, fun res_arg/2, fun init_overflow/2}, {<<"queue-mode">>, fun res_arg/2, fun init_queue_mode/2}], drop_expired_msgs( @@ -426,6 +427,9 @@ init_max_bytes(MaxBytes, State) -> {_Dropped, State1} = maybe_drop_head(State#q{max_bytes = MaxBytes}), State1. +init_max_priority(_MaxPriority, State) -> + State. + init_overflow(undefined, State) -> State; init_overflow(Overflow, State) -> diff --git a/src/rabbit_priority_queue.erl b/src/rabbit_priority_queue.erl index b1eb83dddc..481fd9a390 100644 --- a/src/rabbit_priority_queue.erl +++ b/src/rabbit_priority_queue.erl @@ -28,6 +28,8 @@ {requires, pre_boot}, {enables, kernel_ready}]}). +-import(rabbit_misc, [pget/2]). + -export([enable/0]). -export([start/2, stop/1]). @@ -43,6 +45,8 @@ info/2, invoke/3, is_duplicate/2, set_queue_mode/2, zip_msgs_and_acks/4, handle_info/2]). +-export([max_priority/1, priorities/1]). + -record(state, {bq, bqss, max_priority}). -record(passthrough, {bq, bqs}). @@ -125,9 +129,19 @@ collapse_recovery(QNames, DupNames, Recovery) -> end, dict:new(), lists:zip(DupNames, Recovery)), [dict:fetch(Name, NameToTerms) || Name <- QNames]. -priorities(#amqqueue{arguments = Args}) -> - Ints = [long, short, signedint, byte, unsignedbyte, unsignedshort, unsignedint], +max_priority(Q = #amqqueue{arguments = Args}) -> case rabbit_misc:table_lookup(Args, <<"x-max-priority">>) of + {Type, RequestedMax} -> {Type, RequestedMax}; + undefined -> + case rabbit_policy:effective_definition(Q) of + undefined -> undefined; + Proplist -> {unsignedbyte, pget(<<"max-priority">>, Proplist)} + end + end. + +priorities(Q) -> + Ints = [long, short, signedint, byte, unsignedbyte, unsignedshort, unsignedint], + case max_priority(Q) of {Type, RequestedMax} -> case lists:member(Type, Ints) of false -> none; From 5cdee1530d5002b316b80f488a5d87417e1d0db0 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Mon, 14 May 2018 16:03:08 +0100 Subject: [PATCH 06/35] Do not lock entire routing table when cleaning up bindings. Instead of locking entire table we can use a custom global lock on the affected resource (source or destination). This can improve performance when multiple records deleted at the same time, for example when connection with exclusive queues closes. Resource lock also aquired when adding or removing a binding, so it won't conflict with bulk removal. Addresses #1566 [#156352963] --- src/rabbit_amqqueue.erl | 2 +- src/rabbit_amqqueue_process.erl | 10 ++++-- src/rabbit_binding.erl | 64 ++++++++++++++++----------------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index ac79d56358..f4f6d49c72 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -981,7 +981,7 @@ internal_delete(QueueName, ActingUser) -> ?INTERNAL_USER), fun() -> ok = T(), - rabbit_core_metrics:queue_deleted(QueueName), + rabbit_core_metrics:queue_deleted(QueueName), ok = rabbit_event:notify(queue_deleted, [{name, QueueName}, {user_who_performed_action, ActingUser}]) diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index a3c8f99519..11f272073b 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -311,8 +311,14 @@ terminate_delete(EmitStats, Reason, fun() -> emit_stats(State) end); true -> ok end, - %% don't care if the internal delete doesn't return 'ok'. - rabbit_amqqueue:internal_delete(QName, ActingUser), + %% This try-catch block transforms throws to errors since throws are not + %% logged. + try + %% don't care if the internal delete doesn't return 'ok'. + rabbit_amqqueue:internal_delete(QName, ActingUser) + catch + {error, Reason} -> error(Reason) + end, BQS1 end. diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl index 7498d6b765..06a71ce995 100644 --- a/src/rabbit_binding.erl +++ b/src/rabbit_binding.erl @@ -166,6 +166,8 @@ add(Binding, InnerFun, ActingUser) -> fun (Src, Dst, B) -> case rabbit_exchange:validate_binding(Src, B) of ok -> + lock_resource(Src), + lock_resource(Dst), %% this argument is used to check queue exclusivity; %% in general, we want to fail on that in preference to %% anything else @@ -184,6 +186,8 @@ add(Binding, InnerFun, ActingUser) -> end, fun not_found_or_absent_errs/1). add(Src, Dst, B, ActingUser) -> + lock_resource(Src), + lock_resource(Dst), [SrcDurable, DstDurable] = [durable(E) || E <- [Src, Dst]], case (SrcDurable andalso DstDurable andalso mnesia:read({rabbit_durable_route, B}) =/= []) of @@ -206,6 +210,8 @@ remove(Binding, InnerFun, ActingUser) -> binding_action( Binding, fun (Src, Dst, B) -> + lock_resource(Src), + lock_resource(Dst), case mnesia:read(rabbit_route, B, write) of [] -> case mnesia:read(rabbit_durable_route, B, write) of [] -> rabbit_misc:const(ok); @@ -219,6 +225,8 @@ remove(Binding, InnerFun, ActingUser) -> end, fun absent_errs_only/1). remove(Src, Dst, B, ActingUser) -> + lock_resource(Src), + lock_resource(Dst), ok = sync_route(#route{binding = B}, durable(Src), durable(Dst), fun mnesia:delete_object/3), Deletions = maybe_auto_delete( @@ -303,12 +311,12 @@ has_for_source(SrcName) -> contains(rabbit_semi_durable_route, Match). remove_for_source(SrcName) -> - lock_route_tables(), + lock_resource(SrcName), Match = #route{binding = #binding{source = SrcName, _ = '_'}}, remove_routes( lists:usort( - mnesia:match_object(rabbit_route, Match, write) ++ - mnesia:match_object(rabbit_semi_durable_route, Match, write))). + mnesia:dirty_match_object(rabbit_route, Match) ++ + mnesia:dirty_match_object(rabbit_semi_durable_route, Match))). remove_for_destination(DstName, OnlyDurable) -> remove_for_destination(DstName, OnlyDurable, fun remove_routes/1). @@ -393,32 +401,12 @@ continue('$end_of_table') -> false; continue({[_|_], _}) -> true; continue({[], Continuation}) -> continue(mnesia:select(Continuation)). -%% For bulk operations we lock the tables we are operating on in order -%% to reduce the time complexity. Without the table locks we end up -%% with num_tables*num_bulk_bindings row-level locks. Taking each lock -%% takes time proportional to the number of existing locks, thus -%% resulting in O(num_bulk_bindings^2) complexity. -%% -%% The locks need to be write locks since ultimately we end up -%% removing all these rows. -%% -%% The downside of all this is that no other binding operations except -%% lookup/routing (which uses dirty ops) can take place -%% concurrently. However, that is the case already since the bulk -%% operations involve mnesia:match_object calls with a partial key, -%% which entails taking a table lock. -lock_route_tables() -> - [mnesia:lock({table, T}, write) || T <- [rabbit_route, - rabbit_reverse_route, - rabbit_semi_durable_route, - rabbit_durable_route]]. - remove_routes(Routes) -> %% This partitioning allows us to suppress unnecessary delete %% operations on disk tables, which require an fsync. {RamRoutes, DiskRoutes} = - lists:partition(fun (R) -> mnesia:match_object( - rabbit_durable_route, R, write) == [] end, + lists:partition(fun (R) -> mnesia:dirty_match_object( + rabbit_durable_route, R) == [] end, Routes), %% Of course the destination might not really be durable but it's %% just as easy to try to delete it from the semi-durable table @@ -436,23 +424,33 @@ remove_transient_routes(Routes) -> end || R <- Routes]. remove_for_destination(DstName, OnlyDurable, Fun) -> - lock_route_tables(), + lock_resource(DstName), MatchFwd = #route{binding = #binding{destination = DstName, _ = '_'}}, MatchRev = reverse_route(MatchFwd), Routes = case OnlyDurable of - false -> [reverse_route(R) || - R <- mnesia:match_object( - rabbit_reverse_route, MatchRev, write)]; + false -> + [reverse_route(R) || + R <- mnesia:dirty_match_object( + rabbit_reverse_route, MatchRev)]; true -> lists:usort( - mnesia:match_object( - rabbit_durable_route, MatchFwd, write) ++ - mnesia:match_object( - rabbit_semi_durable_route, MatchFwd, write)) + mnesia:dirty_match_object( + rabbit_durable_route, MatchFwd) ++ + mnesia:dirty_match_object( + rabbit_semi_durable_route, MatchFwd)) end, Bindings = Fun(Routes), group_bindings_fold(fun maybe_auto_delete/4, new_deletions(), lists:keysort(#binding.source, Bindings), OnlyDurable). +%% Instead of locking entire table on remove operations we can lock the +%% affected resource only. This will allow us to use dirty_match_object for +%% do faster search of records to delete. +%% This works better when there are multiple resources deleted at once, for +%% example when exclusive queues are deleted. +lock_resource(Name) -> + mnesia:lock({global, Name, mnesia:table_info(rabbit_route, where_to_write)}, + write). + %% Requires that its input binding list is sorted in exchange-name %% order, so that the grouping of bindings (for passing to %% group_bindings_and_auto_delete1) works properly. From 44500061d465a8552a07af5a3a519d2263f94f8a Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Mon, 14 May 2018 12:37:39 -0300 Subject: [PATCH 07/35] Use the constant across the board Part of #1590. --- src/rabbit_policies.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rabbit_policies.erl b/src/rabbit_policies.erl index 0faad7259d..3ed6e33279 100644 --- a/src/rabbit_policies.erl +++ b/src/rabbit_policies.erl @@ -102,10 +102,10 @@ validate_policy0(<<"max-length-bytes">>, Value) -> {error, "~p is not a valid maximum length in bytes", [Value]}; validate_policy0(<<"max-priority">>, Value) - when is_integer(Value), Value >= 0, Value =< 255 -> + when is_integer(Value), Value >= 0, Value =< ?MAX_SUPPORTED_PRIORITY -> ok; validate_policy0(<<"max-priority">>, Value) -> - {error, "~p is not a valid max priority (must be an integer in the 1-255 range)", [Value]}; + {error, "~p is not a valid max priority (must be an integer in the 1-~p range)", [Value, ?MAX_SUPPORTED_PRIORITY]}; validate_policy0(<<"queue-mode">>, <<"default">>) -> ok; From 7705d4e682b780575b6697d30781e0a9493b0db6 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Mon, 14 May 2018 17:49:12 +0100 Subject: [PATCH 08/35] Do dirty deletes when cleaning up bindings. Dirty deletes are faster and idempotent, which means that it can be run in transaction as long as it's locked in the begining of transaction, which is done in `lock_resource`. Speed improvement is aquired by not setting record locks for each record, since we already have a record lock Addresses #1566 [#156352963] --- src/rabbit_binding.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl index 06a71ce995..ba28cbeb70 100644 --- a/src/rabbit_binding.erl +++ b/src/rabbit_binding.erl @@ -339,8 +339,8 @@ binding_action(Binding = #binding{source = SrcName, Fun(Src, Dst, Binding#binding{args = SortedArgs}) end, ErrFun). -delete_object(Table, Record, LockKind) -> - mnesia:delete_object(Table, Record, LockKind). +dirty_delete_object(Table, Record, LockKind) -> + mnesia:dirty_delete_object(Table, Record). sync_route(Route, true, true, Fun) -> ok = Fun(rabbit_durable_route, Route, write), @@ -411,15 +411,15 @@ remove_routes(Routes) -> %% Of course the destination might not really be durable but it's %% just as easy to try to delete it from the semi-durable table %% than check first - [ok = sync_route(R, false, true, fun mnesia:delete_object/3) || + [ok = sync_route(R, false, true, fun dirty_delete_object/3) || R <- RamRoutes], - [ok = sync_route(R, true, true, fun mnesia:delete_object/3) || + [ok = sync_route(R, true, true, fun dirty_delete_object/3) || R <- DiskRoutes], [R#route.binding || R <- Routes]. remove_transient_routes(Routes) -> [begin - ok = sync_transient_route(R, fun delete_object/3), + ok = sync_transient_route(R, fun dirty_delete_object/3), R#route.binding end || R <- Routes]. From 08168de6b0c3c4d7f6429f2c3d098e5080ff7b1a Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Mon, 14 May 2018 15:06:52 -0300 Subject: [PATCH 09/35] Revert "Take policy-configured max-priority into account" This reverts commit f5aa1fbe043395806d9b9ed8780892924431466c. This feature wasn't available in the original implementation for a reason: policies are dynamic and can change after a queue's been declared. However, queue priorities are (at least currently) set in stone from the moment of queue creation. This was mentioned in the docs but not explicitly enough and got overlooked. Credit for the [re-]discovery goes to @acogoluegnes :) References #1590. [#157380396] --- src/rabbit_amqqueue_process.erl | 4 ---- src/rabbit_priority_queue.erl | 18 ++---------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index 746679168f..a3c8f99519 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -382,7 +382,6 @@ process_args_policy(State = #q{q = Q, {<<"message-ttl">>, fun res_min/2, fun init_ttl/2}, {<<"max-length">>, fun res_min/2, fun init_max_length/2}, {<<"max-length-bytes">>, fun res_min/2, fun init_max_bytes/2}, - {<<"max-priority">>, fun res_arg/2, fun init_max_priority/2}, {<<"overflow">>, fun res_arg/2, fun init_overflow/2}, {<<"queue-mode">>, fun res_arg/2, fun init_queue_mode/2}], drop_expired_msgs( @@ -427,9 +426,6 @@ init_max_bytes(MaxBytes, State) -> {_Dropped, State1} = maybe_drop_head(State#q{max_bytes = MaxBytes}), State1. -init_max_priority(_MaxPriority, State) -> - State. - init_overflow(undefined, State) -> State; init_overflow(Overflow, State) -> diff --git a/src/rabbit_priority_queue.erl b/src/rabbit_priority_queue.erl index 481fd9a390..b1eb83dddc 100644 --- a/src/rabbit_priority_queue.erl +++ b/src/rabbit_priority_queue.erl @@ -28,8 +28,6 @@ {requires, pre_boot}, {enables, kernel_ready}]}). --import(rabbit_misc, [pget/2]). - -export([enable/0]). -export([start/2, stop/1]). @@ -45,8 +43,6 @@ info/2, invoke/3, is_duplicate/2, set_queue_mode/2, zip_msgs_and_acks/4, handle_info/2]). --export([max_priority/1, priorities/1]). - -record(state, {bq, bqss, max_priority}). -record(passthrough, {bq, bqs}). @@ -129,19 +125,9 @@ collapse_recovery(QNames, DupNames, Recovery) -> end, dict:new(), lists:zip(DupNames, Recovery)), [dict:fetch(Name, NameToTerms) || Name <- QNames]. -max_priority(Q = #amqqueue{arguments = Args}) -> - case rabbit_misc:table_lookup(Args, <<"x-max-priority">>) of - {Type, RequestedMax} -> {Type, RequestedMax}; - undefined -> - case rabbit_policy:effective_definition(Q) of - undefined -> undefined; - Proplist -> {unsignedbyte, pget(<<"max-priority">>, Proplist)} - end - end. - -priorities(Q) -> +priorities(#amqqueue{arguments = Args}) -> Ints = [long, short, signedint, byte, unsignedbyte, unsignedshort, unsignedint], - case max_priority(Q) of + case rabbit_misc:table_lookup(Args, <<"x-max-priority">>) of {Type, RequestedMax} -> case lists:member(Type, Ints) of false -> none; From 9f9a414bef538f86f64b79eb46f60cd5d2c0483a Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Mon, 14 May 2018 15:44:25 -0300 Subject: [PATCH 10/35] Follow-up to 08168de6b0c3c4d7f6429f2c3d098e5080ff7b1a References #1590. [#157380396] --- src/rabbit_policies.erl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/rabbit_policies.erl b/src/rabbit_policies.erl index 3ed6e33279..43f59f17d5 100644 --- a/src/rabbit_policies.erl +++ b/src/rabbit_policies.erl @@ -41,7 +41,6 @@ register() -> {policy_validator, <<"expires">>}, {policy_validator, <<"max-length">>}, {policy_validator, <<"max-length-bytes">>}, - {policy_validator, <<"max-priority">>}, {policy_validator, <<"queue-mode">>}, {policy_validator, <<"overflow">>}, {operator_policy_validator, <<"expires">>}, @@ -101,12 +100,6 @@ validate_policy0(<<"max-length-bytes">>, Value) validate_policy0(<<"max-length-bytes">>, Value) -> {error, "~p is not a valid maximum length in bytes", [Value]}; -validate_policy0(<<"max-priority">>, Value) - when is_integer(Value), Value >= 0, Value =< ?MAX_SUPPORTED_PRIORITY -> - ok; -validate_policy0(<<"max-priority">>, Value) -> - {error, "~p is not a valid max priority (must be an integer in the 1-~p range)", [Value, ?MAX_SUPPORTED_PRIORITY]}; - validate_policy0(<<"queue-mode">>, <<"default">>) -> ok; validate_policy0(<<"queue-mode">>, <<"lazy">>) -> From 1709368d2aa2d6aa3e70339cd391dad1c1858e47 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Tue, 15 May 2018 12:41:03 +0100 Subject: [PATCH 11/35] Ignore lock kind argument when dirty-deleting a route --- src/rabbit_binding.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl index ba28cbeb70..5e6cd48ce9 100644 --- a/src/rabbit_binding.erl +++ b/src/rabbit_binding.erl @@ -339,7 +339,7 @@ binding_action(Binding = #binding{source = SrcName, Fun(Src, Dst, Binding#binding{args = SortedArgs}) end, ErrFun). -dirty_delete_object(Table, Record, LockKind) -> +dirty_delete_object(Table, Record, _LockKind) -> mnesia:dirty_delete_object(Table, Record). sync_route(Route, true, true, Fun) -> From 75cee2c119fe257cf70e876955c9bb201a9d2b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20Cogolu=C3=A8gnes?= Date: Tue, 15 May 2018 15:51:20 +0200 Subject: [PATCH 12/35] Use MAX_SUPPORTED_PRIORITY constant for argument validation Part of #1590. --- src/rabbit_amqqueue.erl | 6 +- src/rabbit_ctl_usage.erl | 135 +++++++++++++++++++++++++++++++++++ src/rabbit_plugins_usage.erl | 14 ++++ 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 src/rabbit_ctl_usage.erl create mode 100644 src/rabbit_plugins_usage.erl diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index eb00729baa..381f733763 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -613,9 +613,9 @@ check_message_ttl_arg({Type, Val}, Args) -> check_max_priority_arg({Type, Val}, Args) -> case check_non_neg_int_arg({Type, Val}, Args) of - ok when Val =< 255 -> ok; - ok -> {error, {max_value_exceeded, Val}}; - Error -> Error + ok when Val =< ?MAX_SUPPORTED_PRIORITY -> ok; + ok -> {error, {max_value_exceeded, Val}}; + Error -> Error end. %% Note that the validity of x-dead-letter-exchange is already verified diff --git a/src/rabbit_ctl_usage.erl b/src/rabbit_ctl_usage.erl new file mode 100644 index 0000000000..0780705a82 --- /dev/null +++ b/src/rabbit_ctl_usage.erl @@ -0,0 +1,135 @@ +%% Generated, do not edit! +-module(rabbit_ctl_usage). +-export([usage/0]). +usage() -> "Usage: +rabbitmqctl [-n ] [-t ] [-q] [] + +Options: + -n node + -q + -t timeout + +Default node is \"rabbit@server\", where server is the local host. On a host +named \"server.example.com\", the node name of the RabbitMQ Erlang node will +usually be rabbit@server (unless RABBITMQ_NODENAME has been set to some +non-default value at broker startup time). The output of hostname -s is usually +the correct suffix to use after the \"@\" sign. See rabbitmq-server(1) for +details of configuring the RabbitMQ broker. + +Quiet output mode is selected with the \"-q\" flag. Informational messages are +suppressed when quiet mode is in effect. + +Operation timeout in seconds. Only applicable to \"list\" commands. Default is +\"infinity\". + +Commands: + stop [] + shutdown + stop_app + start_app + wait + reset + force_reset + rotate_logs + hipe_compile + + join_cluster [--ram] + cluster_status + change_cluster_node_type disc | ram + forget_cluster_node [--offline] + rename_cluster_node oldnode1 newnode1 [oldnode2] [newnode2 ...] + update_cluster_nodes clusternode + force_boot + sync_queue [-p ] queue + cancel_sync_queue [-p ] queue + purge_queue [-p ] queue + set_cluster_name name + + add_user + delete_user + change_password + clear_password + authenticate_user + set_user_tags ... + list_users + + add_vhost + delete_vhost + list_vhosts [ ...] + set_permissions [-p ] + clear_permissions [-p ] + list_permissions [-p ] + list_user_permissions + + set_parameter [-p ] + clear_parameter [-p ] + list_parameters [-p ] + set_global_parameter + clear_global_parameter + list_global_parameters + + set_policy [-p ] [--priority ] [--apply-to ] + + clear_policy [-p ] + list_policies [-p ] + + list_queues [-p ] [--offline|--online|--local] [ ...] + list_exchanges [-p ] [ ...] + list_bindings [-p ] [ ...] + list_connections [ ...] + list_channels [ ...] + list_consumers [-p ] + status + node_health_check + environment + report + eval + + close_connection + trace_on [-p ] + trace_off [-p ] + set_vm_memory_high_watermark + set_vm_memory_high_watermark absolute + set_disk_free_limit + set_disk_free_limit mem_relative + encode [--decode] [] [] [--list-ciphers] [--list-hashes] +[--cipher ] [--hash ] [--iterations ] + decode [] [][--cipher ] [--hash ] +[--iterations ] + list_hashes + list_ciphers + + must be a member of the list [name, tracing]. + +The list_queues, list_exchanges and list_bindings commands accept an optional +virtual host parameter for which to display results. The default value is \"/\". + + must be a member of the list [name, durable, auto_delete, +arguments, policy, pid, owner_pid, exclusive, exclusive_consumer_pid, +exclusive_consumer_tag, messages_ready, messages_unacknowledged, messages, +messages_ready_ram, messages_unacknowledged_ram, messages_ram, +messages_persistent, message_bytes, message_bytes_ready, +message_bytes_unacknowledged, message_bytes_ram, message_bytes_persistent, +head_message_timestamp, disk_reads, disk_writes, consumers, +consumer_utilisation, memory, slave_pids, synchronised_slave_pids, state]. + + must be a member of the list [name, type, durable, +auto_delete, internal, arguments, policy]. + + must be a member of the list [source_name, source_kind, +destination_name, destination_kind, routing_key, arguments]. + + must be a member of the list [pid, name, port, host, +peer_port, peer_host, ssl, ssl_protocol, ssl_key_exchange, ssl_cipher, +ssl_hash, peer_cert_subject, peer_cert_issuer, peer_cert_validity, state, +channels, protocol, auth_mechanism, user, vhost, timeout, frame_max, +channel_max, client_properties, recv_oct, recv_cnt, send_oct, send_cnt, +send_pend, connected_at]. + + must be a member of the list [pid, connection, name, number, +user, vhost, transactional, confirm, consumer_count, messages_unacknowledged, +messages_uncommitted, acks_uncommitted, messages_unconfirmed, prefetch_count, +global_prefetch_count]. + + +". diff --git a/src/rabbit_plugins_usage.erl b/src/rabbit_plugins_usage.erl new file mode 100644 index 0000000000..6d93cf58e2 --- /dev/null +++ b/src/rabbit_plugins_usage.erl @@ -0,0 +1,14 @@ +%% Generated, do not edit! +-module(rabbit_plugins_usage). +-export([usage/0]). +usage() -> "Usage: +rabbitmq-plugins [-n ] [] + +Commands: + list [-v] [-m] [-E] [-e] [] + enable [--offline] [--online] ... + disable [--offline] [--online] ... + set [--offline] [--online] ... + + +". From b89f9cb4d8f42187581a46a4f9a650b9a11ed36e Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Wed, 16 May 2018 18:17:49 +0100 Subject: [PATCH 13/35] Integrate syslog library into rabbitmq logging --- Makefile | 4 +++- src/rabbit_lager.erl | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index da56b2ad9c..389de8d925 100644 --- a/Makefile +++ b/Makefile @@ -132,10 +132,12 @@ define PROJECT_ENV endef LOCAL_DEPS = sasl mnesia os_mon inets -BUILD_DEPS = rabbitmq_cli +BUILD_DEPS = rabbitmq_cli syslog DEPS = ranch lager rabbit_common TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper +dep_syslog = git https://github.com/schlagert/syslog master + define usage_xml_to_erl $(subst __,_,$(patsubst $(DOCS_DIR)/rabbitmq%.1.xml, src/rabbit_%_usage.erl, $(subst -,_,$(1)))) endef diff --git a/src/rabbit_lager.erl b/src/rabbit_lager.erl index 9af1ed198b..8575e6fe6a 100644 --- a/src/rabbit_lager.erl +++ b/src/rabbit_lager.erl @@ -27,7 +27,9 @@ start_logger() -> application:stop(lager), + application:stop(syslog), ensure_lager_configured(), + application:ensure_all_started(syslog), lager:start(), fold_sinks( fun @@ -163,15 +165,19 @@ ensure_lager_configured() -> end. %% Lager should have handlers and sinks +%% Error logger forwarding to syslog should be disabled lager_configured() -> Sinks = lager:list_all_sinks(), ExpectedSinks = list_expected_sinks(), application:get_env(lager, handlers) =/= undefined andalso - lists:all(fun(S) -> lists:member(S, Sinks) end, ExpectedSinks). + lists:all(fun(S) -> lists:member(S, Sinks) end, ExpectedSinks) + andalso + application:get_env(syslog, syslog_error_logger) =/= undefined. configure_lager() -> application:load(lager), + application:load(syslog), %% Turn off reformatting for error_logger messages case application:get_env(lager, error_logger_format_raw) of undefined -> application:set_env(lager, error_logger_format_raw, true); @@ -192,6 +198,8 @@ configure_lager() -> end, %% Set rabbit.log config variable based on environment. prepare_rabbit_log_config(), + %% Configure syslog library. + configure_syslog(), %% At this point we should have rabbit.log application variable %% configured to generate RabbitMQ log handlers. GeneratedHandlers = generate_lager_handlers(), @@ -261,6 +269,23 @@ configure_lager() -> end, ok. +configure_syslog() -> + %% Disable error_logger forwarding to syslog if it's not configured + case application:get_env(syslog, syslog_error_logger) of + undefined -> application:set_env(syslog, syslog_error_logger, false); + _ -> ok + end, + LogConfig = application:get_env(rabbit, log, []), + case proplists:get_value(syslog, LogConfig, undefined) of + undefined -> ok; + SyslogConfig when is_list(SyslogConfig) -> + Identity = proplists:get_value(identity, SyslogConfig, "rabbitmq"), + Facility = proplists:get_value(facility, SyslogConfig, daemon), + application:set_env(syslog, app_name, Identity), + application:set_env(syslog, facility, Facility) + end. + + remove_rabbit_handlers(Handlers, FormerHandlers) -> lists:filter(fun(Handler) -> not lists:member(Handler, FormerHandlers) @@ -296,19 +321,23 @@ generate_lager_handlers(LogHandlersConfig) -> lager_backend(file) -> lager_file_backend; lager_backend(console) -> lager_console_backend; -lager_backend(syslog) -> lager_syslog_backend; +lager_backend(syslog) -> syslog_lager_backend; lager_backend(exchange) -> lager_exchange_backend. +%% Syslog backend is using an old API for configuration and +%% does not support proplists. +generate_handler(syslog_lager_backend, HandlerConfig) -> + Level = proplists:get_value(level, HandlerConfig, + default_config_value(level)), + [{syslog_lager_backend, + [Level, + {}, + {lager_default_formatter, default_config_value(formatter_config)}]}]; generate_handler(Backend, HandlerConfig) -> [{Backend, lists:ukeymerge(1, lists:ukeysort(1, HandlerConfig), lists:ukeysort(1, default_handler_config(Backend)))}]. -default_handler_config(lager_syslog_backend) -> - [{level, default_config_value(level)}, - {identity, "rabbitmq"}, - {facility, daemon}, - {formatter_config, default_config_value(formatter_config)}]; default_handler_config(lager_console_backend) -> [{level, default_config_value(level)}, {formatter_config, default_config_value(formatter_config)}]; From 88d51fae8929fbb8a9634825b6342c4aba237b55 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 16 May 2018 16:02:19 -0300 Subject: [PATCH 14/35] Remove two files that were not meant to be committed in ee9a02429 They are no longer used in 3.7.x. (cherry picked from commit 20de46c1df013874147835ac8bdc1e707ac95030) --- src/rabbit_ctl_usage.erl | 135 ----------------------------------- src/rabbit_plugins_usage.erl | 14 ---- 2 files changed, 149 deletions(-) delete mode 100644 src/rabbit_ctl_usage.erl delete mode 100644 src/rabbit_plugins_usage.erl diff --git a/src/rabbit_ctl_usage.erl b/src/rabbit_ctl_usage.erl deleted file mode 100644 index 0780705a82..0000000000 --- a/src/rabbit_ctl_usage.erl +++ /dev/null @@ -1,135 +0,0 @@ -%% Generated, do not edit! --module(rabbit_ctl_usage). --export([usage/0]). -usage() -> "Usage: -rabbitmqctl [-n ] [-t ] [-q] [] - -Options: - -n node - -q - -t timeout - -Default node is \"rabbit@server\", where server is the local host. On a host -named \"server.example.com\", the node name of the RabbitMQ Erlang node will -usually be rabbit@server (unless RABBITMQ_NODENAME has been set to some -non-default value at broker startup time). The output of hostname -s is usually -the correct suffix to use after the \"@\" sign. See rabbitmq-server(1) for -details of configuring the RabbitMQ broker. - -Quiet output mode is selected with the \"-q\" flag. Informational messages are -suppressed when quiet mode is in effect. - -Operation timeout in seconds. Only applicable to \"list\" commands. Default is -\"infinity\". - -Commands: - stop [] - shutdown - stop_app - start_app - wait - reset - force_reset - rotate_logs - hipe_compile - - join_cluster [--ram] - cluster_status - change_cluster_node_type disc | ram - forget_cluster_node [--offline] - rename_cluster_node oldnode1 newnode1 [oldnode2] [newnode2 ...] - update_cluster_nodes clusternode - force_boot - sync_queue [-p ] queue - cancel_sync_queue [-p ] queue - purge_queue [-p ] queue - set_cluster_name name - - add_user - delete_user - change_password - clear_password - authenticate_user - set_user_tags ... - list_users - - add_vhost - delete_vhost - list_vhosts [ ...] - set_permissions [-p ] - clear_permissions [-p ] - list_permissions [-p ] - list_user_permissions - - set_parameter [-p ] - clear_parameter [-p ] - list_parameters [-p ] - set_global_parameter - clear_global_parameter - list_global_parameters - - set_policy [-p ] [--priority ] [--apply-to ] - - clear_policy [-p ] - list_policies [-p ] - - list_queues [-p ] [--offline|--online|--local] [ ...] - list_exchanges [-p ] [ ...] - list_bindings [-p ] [ ...] - list_connections [ ...] - list_channels [ ...] - list_consumers [-p ] - status - node_health_check - environment - report - eval - - close_connection - trace_on [-p ] - trace_off [-p ] - set_vm_memory_high_watermark - set_vm_memory_high_watermark absolute - set_disk_free_limit - set_disk_free_limit mem_relative - encode [--decode] [] [] [--list-ciphers] [--list-hashes] -[--cipher ] [--hash ] [--iterations ] - decode [] [][--cipher ] [--hash ] -[--iterations ] - list_hashes - list_ciphers - - must be a member of the list [name, tracing]. - -The list_queues, list_exchanges and list_bindings commands accept an optional -virtual host parameter for which to display results. The default value is \"/\". - - must be a member of the list [name, durable, auto_delete, -arguments, policy, pid, owner_pid, exclusive, exclusive_consumer_pid, -exclusive_consumer_tag, messages_ready, messages_unacknowledged, messages, -messages_ready_ram, messages_unacknowledged_ram, messages_ram, -messages_persistent, message_bytes, message_bytes_ready, -message_bytes_unacknowledged, message_bytes_ram, message_bytes_persistent, -head_message_timestamp, disk_reads, disk_writes, consumers, -consumer_utilisation, memory, slave_pids, synchronised_slave_pids, state]. - - must be a member of the list [name, type, durable, -auto_delete, internal, arguments, policy]. - - must be a member of the list [source_name, source_kind, -destination_name, destination_kind, routing_key, arguments]. - - must be a member of the list [pid, name, port, host, -peer_port, peer_host, ssl, ssl_protocol, ssl_key_exchange, ssl_cipher, -ssl_hash, peer_cert_subject, peer_cert_issuer, peer_cert_validity, state, -channels, protocol, auth_mechanism, user, vhost, timeout, frame_max, -channel_max, client_properties, recv_oct, recv_cnt, send_oct, send_cnt, -send_pend, connected_at]. - - must be a member of the list [pid, connection, name, number, -user, vhost, transactional, confirm, consumer_count, messages_unacknowledged, -messages_uncommitted, acks_uncommitted, messages_unconfirmed, prefetch_count, -global_prefetch_count]. - - -". diff --git a/src/rabbit_plugins_usage.erl b/src/rabbit_plugins_usage.erl deleted file mode 100644 index 6d93cf58e2..0000000000 --- a/src/rabbit_plugins_usage.erl +++ /dev/null @@ -1,14 +0,0 @@ -%% Generated, do not edit! --module(rabbit_plugins_usage). --export([usage/0]). -usage() -> "Usage: -rabbitmq-plugins [-n ] [] - -Commands: - list [-v] [-m] [-E] [-e] [] - enable [--offline] [--online] ... - disable [--offline] [--online] ... - set [--offline] [--online] ... - - -". From c39524625d0f3dc3cb46b9737093144ee6b21114 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 16 May 2018 16:30:16 -0300 Subject: [PATCH 15/35] Typo/weird wording (cherry picked from commit ad5abba68c3c335a856ade2a2a38ba2c1de871fa) --- test/priority_queue_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/priority_queue_SUITE.erl b/test/priority_queue_SUITE.erl index cfa98b94c7..33906ee70b 100644 --- a/test/priority_queue_SUITE.erl +++ b/test/priority_queue_SUITE.erl @@ -200,7 +200,7 @@ max_priorities_above_hard_limit(Config) -> Q = <<"max_priorities_above_hard_limit">>, ?assertExit( {{shutdown, {server_initiated_close, 406, _}}, _}, - %% Note that lower values (e.g. 300) will cause overflow the byte type here. + %% Note that lower values (e.g. 300) will overflow the byte type here. %% However, values >= 256 would still be rejected when used by %% other clients declare(Ch, Q, 3000)), From 0fe2422741d1c726de1564b8c4aa9a909191b8b0 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Thu, 17 May 2018 15:35:41 +0100 Subject: [PATCH 16/35] Cuttlefish schema to configure syslog application. Configure the syslog application directly instead of relying on rabbit_lager module. --- priv/schema/rabbit.schema | 168 +++++++++++++++++- src/rabbit_lager.erl | 10 -- test/config_schema_SUITE_data/rabbit.snippets | 51 +++++- 3 files changed, 215 insertions(+), 14 deletions(-) diff --git a/priv/schema/rabbit.schema b/priv/schema/rabbit.schema index 181e20447e..6d82cdc1f5 100644 --- a/priv/schema/rabbit.schema +++ b/priv/schema/rabbit.schema @@ -1070,13 +1070,175 @@ end}. {mapping, "log.syslog.level", "rabbit.log.syslog.level", [ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. -{mapping, "log.syslog.identity", "rabbit.log.syslog.identity", [ + +{mapping, "log.syslog.identity", "syslog.app_name", [ {datatype, string} ]}. -{mapping, "log.syslog.facility", "rabbit.log.syslog.facility", [ - {datatype, atom} + +{mapping, "log.syslog.facility", "syslog.facility", [ + {datatype, {enum, [kern, kernel, user, mail, daemon, auth, syslog, lpr, + news, uucp, cron, authpriv, ftp, ntp, audit, alert, + clock, local0, local1, local2, local3, local4, + local5, local6, local7]}} ]}. +{mapping, "log.syslog.multiline_mode", "syslog.multiline_mode", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "log.syslog.ip", "syslog.dest_host", [ + {datatype, string}, + {validators, ["is_ip"]} +]}. + +{translation, "syslog.dest_host", +fun(Conf) -> + IpString = cuttlefish:conf_get("log.syslog.ip", Conf), + {ok, IP} = inet:parse_address(IpString), + IP +end}. + +{mapping, "log.syslog.port", "syslog.dest_port", [ + {datatype, integer} +]}. + +{mapping, "log.syslog.transport", "syslog.protocol", [ + {datatype, {enum, [udp, tcp, tls, ssl]}} +]}. +{mapping, "log.syslog.protocol", "syslog.protocol", [ + {datatype, {enum, [rfc3164, rfc5424]}} +]}. +{mapping, "log.syslog.ssl_options.verify", "syslog.protocol", [ + {datatype, {enum, [verify_peer, verify_none]}}]}. + +{mapping, "log.syslog.ssl_options.fail_if_no_peer_cert", "syslog.protocol", [ + {datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.cacertfile", "syslog.protocol", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "log.syslog.ssl_options.certfile", "syslog.protocol", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "log.syslog.ssl_options.cacerts.$name", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.cert", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.client_renegotiation", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.crl_check", "syslog.protocol", + [{datatype, [{enum, [true, false, peer, best_effort]}]}]}. + +{mapping, "log.syslog.ssl_options.depth", "syslog.protocol", + [{datatype, integer}, {validators, ["byte"]}]}. + +{mapping, "log.syslog.ssl_options.dh", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.dhfile", "syslog.protocol", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "log.syslog.ssl_options.honor_cipher_order", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.honor_ecc_order", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.key.RSAPrivateKey", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.key.DSAPrivateKey", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.key.PrivateKeyInfo", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.keyfile", "syslog.protocol", + [{datatype, string}, {validators, ["file_accessible"]}]}. + +{mapping, "log.syslog.ssl_options.log_alert", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.password", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.psk_identity", "syslog.protocol", + [{datatype, string}]}. + +{mapping, "log.syslog.ssl_options.reuse_sessions", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.secure_renegotiate", "syslog.protocol", + [{datatype, {enum, [true, false]}}]}. + +{mapping, "log.syslog.ssl_options.versions.$version", "syslog.protocol", + [{datatype, atom}]}. + +{translation, "syslog.protocol", +fun(Conf) -> + ParseSslOptions = fun() -> + RawSettings = [ + {verify, cuttlefish:conf_get("log.syslog.ssl_options.verify", Conf, undefined)}, + {fail_if_no_peer_cert, cuttlefish:conf_get("log.syslog.ssl_options.fail_if_no_peer_cert", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get("log.syslog.ssl_options.cacertfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get("log.syslog.ssl_options.certfile", Conf, undefined)}, + {cert, cuttlefish:conf_get("log.syslog.ssl_options.cert", Conf, undefined)}, + {client_renegotiation, cuttlefish:conf_get("log.syslog.ssl_options.client_renegotiation", Conf, undefined)}, + {crl_check, cuttlefish:conf_get("log.syslog.ssl_options.crl_check", Conf, undefined)}, + {depth, cuttlefish:conf_get("log.syslog.ssl_options.depth", Conf, undefined)}, + {dh, cuttlefish:conf_get("log.syslog.ssl_options.dh", Conf, undefined)}, + {dhfile, cuttlefish:conf_get("log.syslog.ssl_options.dhfile", Conf, undefined)}, + {honor_cipher_order, cuttlefish:conf_get("log.syslog.ssl_options.honor_cipher_order", Conf, undefined)}, + {honor_ecc_order, cuttlefish:conf_get("log.syslog.ssl_options.honor_ecc_order", Conf, undefined)}, + + {keyfile, cuttlefish:conf_get("log.syslog.ssl_options.keyfile", Conf, undefined)}, + {log_alert, cuttlefish:conf_get("log.syslog.ssl_options.log_alert", Conf, undefined)}, + {password, cuttlefish:conf_get("log.syslog.ssl_options.password", Conf, undefined)}, + {psk_identity, cuttlefish:conf_get("log.syslog.ssl_options.psk_identity", Conf, undefined)}, + {reuse_sessions, cuttlefish:conf_get("log.syslog.ssl_options.reuse_sessions", Conf, undefined)}, + {secure_renegotiate, cuttlefish:conf_get("log.syslog.ssl_options.secure_renegotiate", Conf, undefined)} + ], + DefinedSettings = [{K, V} || {K, V} <- RawSettings, V =/= undefined], + + lists:map( + fun({K, Val}) when K == dh; K == cert -> {K, list_to_binary(Val)}; + ({K, Val}) -> {K, Val} + end, + DefinedSettings) ++ + [ {K, V} + || {K, V} <- + [{cacerts, [ list_to_binary(V) || {_, V} <- cuttlefish_variable:filter_by_prefix("log.syslog.ssl_options.cacerts", Conf)]}, + {versions, [ V || {_, V} <- cuttlefish_variable:filter_by_prefix("log.syslog.ssl_options.versions", Conf) ]}, + {key, case cuttlefish_variable:filter_by_prefix("log.syslog.ssl_options.key", Conf) of + [{[_,_,Key], Val}|_] -> {list_to_atom(Key), list_to_binary(Val)}; + _ -> undefined + end}], + V =/= undefined, + V =/= []] + end, + + Proto = cuttlefish:conf_get("log.syslog.protocol", Conf, undefined), + Transport = cuttlefish:conf_get("log.syslog.transport", Conf, udp), + case Transport of + TLS when TLS == tls; TLS == ssl -> + case Proto of + rfc3164 -> + cuttlefish:invalid("Syslog protocol rfc3164 is not compatible with TLS"); + _ -> + {rfc5424, tls, ParseSslOptions()} + end; + _ when Transport == udp; Transport == tcp -> + case Proto of + undefined -> {rfc3164, Transport}; + _ -> {Proto, Transport} + end; + _ -> cuttlefish:invalid("Invalid syslog transport ~p~n", [Transport]) + end +end}. + {mapping, "log.file", "rabbit.log.file.file", [ {datatype, [{enum, [false]}, string]} ]}. diff --git a/src/rabbit_lager.erl b/src/rabbit_lager.erl index 8575e6fe6a..0003b74ea8 100644 --- a/src/rabbit_lager.erl +++ b/src/rabbit_lager.erl @@ -274,18 +274,8 @@ configure_syslog() -> case application:get_env(syslog, syslog_error_logger) of undefined -> application:set_env(syslog, syslog_error_logger, false); _ -> ok - end, - LogConfig = application:get_env(rabbit, log, []), - case proplists:get_value(syslog, LogConfig, undefined) of - undefined -> ok; - SyslogConfig when is_list(SyslogConfig) -> - Identity = proplists:get_value(identity, SyslogConfig, "rabbitmq"), - Facility = proplists:get_value(facility, SyslogConfig, daemon), - application:set_env(syslog, app_name, Identity), - application:set_env(syslog, facility, Facility) end. - remove_rabbit_handlers(Handlers, FormerHandlers) -> lists:filter(fun(Handler) -> not lists:member(Handler, FormerHandlers) diff --git a/test/config_schema_SUITE_data/rabbit.snippets b/test/config_schema_SUITE_data/rabbit.snippets index e4dda9f368..93546cb1c6 100644 --- a/test/config_schema_SUITE_data/rabbit.snippets +++ b/test/config_schema_SUITE_data/rabbit.snippets @@ -534,5 +534,54 @@ credential_validator.regexp = ^abc\\d+", [{kernel, [ {net_ticktime, 20} ]}], - []} + []}, + {log_syslog_settings, + "log.syslog = true + log.syslog.identity = rabbitmq + log.syslog.facility = user + log.syslog.multiline_mode = true + log.syslog.ip = 10.10.10.10 + log.syslog.port = 123", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{app_name, "rabbitmq"}, + {facility, user}, + {multiline_mode, true}, + {dest_host, {10, 10, 10, 10}}, + {dest_port, 123}]} + ], + []}, + {log_syslog_tcp, + "log.syslog = true + log.syslog.transport = tcp + log.syslog.protocol = rfc5424", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc5424, tcp}}]} + ], + []}, + {log_syslog_udp_default, + "log.syslog = true + log.syslog.protocol = rfc3164", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc3164, udp}}]} + ], + []}, + {log_syslog_tls, + "log.syslog = true + log.syslog.transport = tls + log.syslog.ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + log.syslog.ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + log.syslog.ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + log.syslog.ssl_options.verify = verify_peer + log.syslog.ssl_options.fail_if_no_peer_cert = false", + [{rabbit, [{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc5424, tls, + [{verify,verify_peer}, + {fail_if_no_peer_cert,false}, + {cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}]}}]}], + []} ]. From 0f626e08eb4a70d1674668c82c2a55fdcf59c17d Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Thu, 17 May 2018 17:07:16 +0100 Subject: [PATCH 17/35] Do not parse syslog IP in schema transformer. String IPs are supported anyway. --- priv/schema/rabbit.schema | 7 ------- test/config_schema_SUITE_data/rabbit.snippets | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/priv/schema/rabbit.schema b/priv/schema/rabbit.schema index 6d82cdc1f5..862537bf61 100644 --- a/priv/schema/rabbit.schema +++ b/priv/schema/rabbit.schema @@ -1091,13 +1091,6 @@ end}. {validators, ["is_ip"]} ]}. -{translation, "syslog.dest_host", -fun(Conf) -> - IpString = cuttlefish:conf_get("log.syslog.ip", Conf), - {ok, IP} = inet:parse_address(IpString), - IP -end}. - {mapping, "log.syslog.port", "syslog.dest_port", [ {datatype, integer} ]}. diff --git a/test/config_schema_SUITE_data/rabbit.snippets b/test/config_schema_SUITE_data/rabbit.snippets index 93546cb1c6..3dce388b79 100644 --- a/test/config_schema_SUITE_data/rabbit.snippets +++ b/test/config_schema_SUITE_data/rabbit.snippets @@ -547,7 +547,7 @@ credential_validator.regexp = ^abc\\d+", {syslog, [{app_name, "rabbitmq"}, {facility, user}, {multiline_mode, true}, - {dest_host, {10, 10, 10, 10}}, + {dest_host, "10.10.10.10"}, {dest_port, 123}]} ], []}, From 99ea452490a8bed98a8840c604759e189edf0f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20Cogolu=C3=A8gnes?= Date: Fri, 18 May 2018 11:02:05 +0200 Subject: [PATCH 18/35] Add client properties to connection.closed events This commit adds client properties to connection.closed events. The original need was to have only the optional user-provided connection name added to correlate connections between created and closed events, but all the client properties are finally added for the sake of consistency between the 2 events. This commit uses the process dictionary to convey the client properties, as they're not yet available in the connection state when the call to send the closed event is made (in the after block, just after the network connection has been established). Fixes #1596 [#157500358] --- src/rabbit_reader.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index bae59bd2f2..8b6d640dec 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -387,10 +387,17 @@ start_connection(Parent, HelperSup, Deb, Sock) -> %% socket w/o delay before termination. rabbit_net:fast_close(RealSocket), rabbit_networking:unregister_connection(self()), - rabbit_core_metrics:connection_closed(self()), + rabbit_core_metrics:connection_closed(self()), + ClientProperties = case get(client_properties) of + undefined -> + []; + Properties -> + Properties + end, rabbit_event:notify(connection_closed, [{name, Name}, {pid, self()}, - {node, node()}]) + {node, node()}, + {client_properties, ClientProperties}]) end, done. @@ -1130,6 +1137,9 @@ handle_method0(#'connection.start_ok'{mechanism = Mechanism, Connection2 = augment_connection_log_name(Connection1), State = State0#v1{connection_state = securing, connection = Connection2}, + % adding client properties to process dictionary to send them later + % in the connection_closed event + put(client_properties, ClientProperties), auth_phase(Response, State); handle_method0(#'connection.secure_ok'{response = Response}, From d0423f958599cba5148864997aa8446c873dad00 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Fri, 18 May 2018 12:31:28 +0100 Subject: [PATCH 19/35] Bring back transactional match_object for bindings cleanup. Follow up to 5cdee1530d5002b316b80f488a5d87417e1d0db0 dirty_match_object does not provide much performance improvement while it's breaking auto-delete exchanges cleanup. A transaction with a binding deletion will call auto-delete exchange removal, which will call a cleanup. On this cleanup the deleted binding should not be dirty-deleted again. Follow-up to #1589 [#156352963] --- src/rabbit_binding.erl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl index 5e6cd48ce9..090e30b2b4 100644 --- a/src/rabbit_binding.erl +++ b/src/rabbit_binding.erl @@ -315,8 +315,8 @@ remove_for_source(SrcName) -> Match = #route{binding = #binding{source = SrcName, _ = '_'}}, remove_routes( lists:usort( - mnesia:dirty_match_object(rabbit_route, Match) ++ - mnesia:dirty_match_object(rabbit_semi_durable_route, Match))). + mnesia:match_object(rabbit_route, Match, read) ++ + mnesia:match_object(rabbit_semi_durable_route, Match, read))). remove_for_destination(DstName, OnlyDurable) -> remove_for_destination(DstName, OnlyDurable, fun remove_routes/1). @@ -405,8 +405,8 @@ remove_routes(Routes) -> %% This partitioning allows us to suppress unnecessary delete %% operations on disk tables, which require an fsync. {RamRoutes, DiskRoutes} = - lists:partition(fun (R) -> mnesia:dirty_match_object( - rabbit_durable_route, R) == [] end, + lists:partition(fun (R) -> mnesia:match_object( + rabbit_durable_route, R, read) == [] end, Routes), %% Of course the destination might not really be durable but it's %% just as easy to try to delete it from the semi-durable table @@ -430,13 +430,13 @@ remove_for_destination(DstName, OnlyDurable, Fun) -> Routes = case OnlyDurable of false -> [reverse_route(R) || - R <- mnesia:dirty_match_object( - rabbit_reverse_route, MatchRev)]; + R <- mnesia:match_object( + rabbit_reverse_route, MatchRev, read)]; true -> lists:usort( - mnesia:dirty_match_object( - rabbit_durable_route, MatchFwd) ++ - mnesia:dirty_match_object( - rabbit_semi_durable_route, MatchFwd)) + mnesia:match_object( + rabbit_durable_route, MatchFwd, read) ++ + mnesia:match_object( + rabbit_semi_durable_route, MatchFwd, read)) end, Bindings = Fun(Routes), group_bindings_fold(fun maybe_auto_delete/4, new_deletions(), From 1b38416a20f7b078aefd8defabf40bcaed5aa79a Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Fri, 18 May 2018 15:19:32 +0100 Subject: [PATCH 20/35] Fix syslog config translation tests. Syslog backend is no longer configured via lager handler, facility and identity options are in the syslog application config now. --- test/unit_log_config_SUITE.erl | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/test/unit_log_config_SUITE.erl b/test/unit_log_config_SUITE.erl index a09143cd97..1cfa6aa6c1 100644 --- a/test/unit_log_config_SUITE.erl +++ b/test/unit_log_config_SUITE.erl @@ -394,7 +394,7 @@ config_multiple_handlers(_) -> ConsoleHandlers = expected_console_handler(), RabbitHandlers = expected_rabbit_handler(), - SyslogHandlers = expected_syslog_handler(error, "rabbitmq", daemon), + SyslogHandlers = expected_syslog_handler(error), ExpectedHandlers = sort_handlers(SyslogHandlers ++ ConsoleHandlers ++ RabbitHandlers), @@ -464,14 +464,12 @@ config_syslog_handler_options(_) -> DefaultLogFile = "rabbit_default.log", application:set_env(rabbit, lager_default_file, DefaultLogFile), application:set_env(rabbit, log, [{syslog, [{enabled, true}, - {identity, "foo"}, - {facility, local1}, {level, warning}]}]), rabbit_lager:configure_lager(), FileHandlers = default_expected_handlers(DefaultLogFile), - SyslogHandlers = expected_syslog_handler(warning, "foo", local1), + SyslogHandlers = expected_syslog_handler(warning), ExpectedHandlers = sort_handlers(FileHandlers ++ SyslogHandlers), @@ -479,13 +477,12 @@ config_syslog_handler_options(_) -> ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). expected_syslog_handler() -> - expected_syslog_handler(info, "rabbitmq", daemon). + expected_syslog_handler(info). -expected_syslog_handler(Level, Identity, Facility) -> - [{lager_syslog_backend, [{level, Level}, - {facility, Facility}, - {formatter_config, formatter_config()}, - {identity, Identity}]}]. +expected_syslog_handler(Level) -> + [{syslog_lager_backend, [Level, + {}, + {lager_default_formatter, formatter_config()}]}]. env_var_overrides_config(_) -> EnvLogFile = "rabbit_default.log", From 9eb0dfd6727a1737b2844c105b05e17e13abae2e Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Mon, 21 May 2018 14:12:46 +0100 Subject: [PATCH 21/35] Pin syslog library version to 3.4.2 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 389de8d925..671e2daf89 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ BUILD_DEPS = rabbitmq_cli syslog DEPS = ranch lager rabbit_common TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper -dep_syslog = git https://github.com/schlagert/syslog master +dep_syslog = git https://github.com/schlagert/syslog 3.4.2 define usage_xml_to_erl $(subst __,_,$(patsubst $(DOCS_DIR)/rabbitmq%.1.xml, src/rabbit_%_usage.erl, $(subst -,_,$(1)))) From 66c1179d2c6e74e25481397de3c6e251904c0110 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Mon, 21 May 2018 16:01:07 +0100 Subject: [PATCH 22/35] Remove date and time from lager formatter when logging to syslog. --- src/rabbit_lager.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rabbit_lager.erl b/src/rabbit_lager.erl index 0003b74ea8..e510db89b3 100644 --- a/src/rabbit_lager.erl +++ b/src/rabbit_lager.erl @@ -322,7 +322,7 @@ generate_handler(syslog_lager_backend, HandlerConfig) -> [{syslog_lager_backend, [Level, {}, - {lager_default_formatter, default_config_value(formatter_config)}]}]; + {lager_default_formatter, syslog_formatter_config()}]}]; generate_handler(Backend, HandlerConfig) -> [{Backend, lists:ukeymerge(1, lists:ukeysort(1, HandlerConfig), @@ -346,6 +346,11 @@ default_config_value(formatter_config) -> {pid, ""}, " ", message, "\n"]. +syslog_formatter_config() -> + [color, "[", severity, "] ", + {pid, ""}, + " ", message, "\n"]. + prepare_rabbit_log_config() -> %% If RABBIT_LOGS is not set, we should ignore it. DefaultFile = application:get_env(rabbit, lager_default_file, undefined), From c526e453d3749f8d3b91601aa3376d867eb5c3b4 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Mon, 21 May 2018 16:53:53 +0100 Subject: [PATCH 23/35] Make syslog a proper dependency of rabbit application, just like lager. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 671e2daf89..7a7331bb0b 100644 --- a/Makefile +++ b/Makefile @@ -132,8 +132,8 @@ define PROJECT_ENV endef LOCAL_DEPS = sasl mnesia os_mon inets -BUILD_DEPS = rabbitmq_cli syslog -DEPS = ranch lager rabbit_common +BUILD_DEPS = rabbitmq_cli +DEPS = ranch syslog lager rabbit_common TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper dep_syslog = git https://github.com/schlagert/syslog 3.4.2 From 8b062c4fdea4b5df7d897d52e04c6a46f84cdc8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20Cogolu=C3=A8gnes?= Date: Mon, 21 May 2018 17:39:52 +0100 Subject: [PATCH 24/35] Add connection_user_provided_name to connection created/closed events Fixes #1596 [#157500358] --- src/rabbit_reader.erl | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 8b6d640dec..4e6ef103b0 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -134,7 +134,7 @@ peer_cert_validity, auth_mechanism, ssl_protocol, ssl_key_exchange, ssl_cipher, ssl_hash, protocol, user, vhost, timeout, frame_max, channel_max, client_properties, connected_at, - node, user_who_performed_action]). + node, user_who_performed_action, connection_user_provided_name]). -define(INFO_KEYS, ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [pid]). @@ -394,10 +394,17 @@ start_connection(Parent, HelperSup, Deb, Sock) -> Properties -> Properties end, + ConnectionUserProvidedName = case get(connection_user_provided_name) of + undefined -> + ''; + ConnectionName -> + ConnectionName + end, rabbit_event:notify(connection_closed, [{name, Name}, {pid, self()}, {node, node()}, - {client_properties, ClientProperties}]) + {client_properties, ClientProperties}, + {connection_user_provided_name, ConnectionUserProvidedName}]) end, done. @@ -1140,6 +1147,7 @@ handle_method0(#'connection.start_ok'{mechanism = Mechanism, % adding client properties to process dictionary to send them later % in the connection_closed event put(client_properties, ClientProperties), + put(connection_user_provided_name, user_provided_connection_name(Connection2)), auth_phase(Response, State); handle_method0(#'connection.secure_ok'{response = Response}, @@ -1469,6 +1477,13 @@ ic(client_properties, #connection{client_properties = CP}) -> CP; ic(auth_mechanism, #connection{auth_mechanism = none}) -> none; ic(auth_mechanism, #connection{auth_mechanism = {Name, _Mod}}) -> Name; ic(connected_at, #connection{connected_at = T}) -> T; +ic(connection_user_provided_name, C) -> + case user_provided_connection_name(C) of + undefined -> + ''; + ConnectionUserProvidedName -> + ConnectionUserProvidedName + end; ic(Item, #connection{}) -> throw({bad_argument, Item}). socket_info(Get, Select, #v1{sock = Sock}) -> @@ -1671,16 +1686,23 @@ control_throttle(State = #v1{connection_state = CS, _ -> State1 end. -augment_connection_log_name(#connection{client_properties = ClientProperties, - name = Name} = Connection) -> - case rabbit_misc:table_lookup(ClientProperties, <<"connection_name">>) of - {longstr, UserSpecifiedName} -> +augment_connection_log_name(#connection{name = Name} = Connection) -> + case user_provided_connection_name(Connection) of + undefined -> + Connection; + UserSpecifiedName -> LogName = <>, rabbit_log_connection:info("Connection ~p (~s) has a client-provided name: ~s~n", [self(), Name, UserSpecifiedName]), ?store_proc_name(LogName), - Connection#connection{log_name = LogName}; + Connection#connection{log_name = LogName} + end. + +user_provided_connection_name(#connection{client_properties = ClientProperties}) -> + case rabbit_misc:table_lookup(ClientProperties, <<"connection_name">>) of + {longstr, UserSpecifiedName} -> + UserSpecifiedName; _ -> - Connection + undefined end. dynamic_connection_name(Default) -> From e8920ec0c59639c325f18ec7a9b9aa2999aa2c80 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Tue, 22 May 2018 12:03:11 +0100 Subject: [PATCH 25/35] Syslog formatter is different from e.g. the file one now, update the tests --- test/unit_log_config_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/unit_log_config_SUITE.erl b/test/unit_log_config_SUITE.erl index 1cfa6aa6c1..b753c4075c 100644 --- a/test/unit_log_config_SUITE.erl +++ b/test/unit_log_config_SUITE.erl @@ -482,7 +482,7 @@ expected_syslog_handler() -> expected_syslog_handler(Level) -> [{syslog_lager_backend, [Level, {}, - {lager_default_formatter, formatter_config()}]}]. + {lager_default_formatter, syslog_formatter_config()}]}]. env_var_overrides_config(_) -> EnvLogFile = "rabbit_default.log", @@ -691,3 +691,6 @@ sort_handlers(Handlers) -> formatter_config() -> [date," ",time," ",color,"[",severity, "] ", {pid,[]}, " ",message,"\n"]. + +syslog_formatter_config() -> + [color,"[",severity, "] ", {pid,[]}, " ",message,"\n"]. From f7fc5b1f28d2bf6f02d20a20042e4d75e5b2247f Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Thu, 24 May 2018 07:51:06 +0100 Subject: [PATCH 26/35] Introduce rabbit_vhost:await_running_on_all_nodes/2 Part of rabbitmq/rabbitmq-management#575. [#157817330] --- src/rabbit_vhost.erl | 57 ++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/rabbit_vhost.erl b/src/rabbit_vhost.erl index 73c05389be..7360f8bc95 100644 --- a/src/rabbit_vhost.erl +++ b/src/rabbit_vhost.erl @@ -22,7 +22,7 @@ -export([recover/0, recover/1]). -export([add/2, delete/2, exists/1, list/0, with/2, with_user_and_vhost/3, assert/1, update/2, - set_limits/2, limits_of/1]). + set_limits/2, limits_of/1, vhost_cluster_state/1, is_running_on_all_nodes/1, await_running_on_all_nodes/2]). -export([info/1, info/2, info_all/0, info_all/1, info_all/2, info_all/3]). -export([dir/1, msg_store_dir_path/1, msg_store_dir_wildcard/0]). -export([delete_storage/1]). @@ -93,7 +93,7 @@ add(VHostPath, ActingUser) -> fun (ok, true) -> ok; (ok, false) -> - [rabbit_exchange:declare( + [_ = rabbit_exchange:declare( rabbit_misc:r(VHostPath, exchange, Name), Type, true, false, Internal, [], ActingUser) || {Name, Type, Internal} <- @@ -148,6 +148,45 @@ delete(VHostPath, ActingUser) -> rabbit_vhost_sup_sup:delete_on_all_nodes(VHostPath), ok. +%% 50 ms +-define(AWAIT_SAMPLE_INTERVAL, 50). + +-spec await_running_on_all_nodes(rabbit_types:vhost(), integer()) -> ok | {error, timeout}. +await_running_on_all_nodes(VHost, Timeout) -> + Attempts = ceil(Timeout / ?AWAIT_SAMPLE_INTERVAL), + await_running_on_all_nodes0(VHost, Attempts). + +await_running_on_all_nodes0(_VHost, 0) -> + {error, timeout}; +await_running_on_all_nodes0(VHost, Attempts) -> + case is_running_on_all_nodes(VHost) of + true -> ok; + _ -> + timer:sleep(?AWAIT_SAMPLE_INTERVAL), + await_running_on_all_nodes0(VHost, Attempts - 1) + end. + +-spec is_running_on_all_nodes(rabbit_types:vhost()) -> boolean(). +is_running_on_all_nodes(VHost) -> + States = vhost_cluster_state(VHost), + lists:all(fun ({_Node, State}) -> State =:= running end, + States). + +-spec vhost_cluster_state(rabbit_types:vhost()) -> [{atom(), atom()}]. +vhost_cluster_state(VHost) -> + Nodes = rabbit_nodes:all_running(), + lists:map(fun(Node) -> + State = case rabbit_misc:rpc_call(Node, + rabbit_vhost_sup_sup, is_vhost_alive, + [VHost]) of + {badrpc, nodedown} -> nodedown; + true -> running; + false -> stopped + end, + {Node, State} + end, + Nodes). + vhost_down(VHostPath) -> ok = rabbit_event:notify(vhost_down, [{name, VHostPath}, @@ -263,19 +302,7 @@ infos(Items, X) -> [{Item, i(Item, X)} || Item <- Items]. i(name, VHost) -> VHost; i(tracing, VHost) -> rabbit_trace:enabled(VHost); -i(cluster_state, VHost) -> - Nodes = rabbit_nodes:all_running(), - lists:map(fun(Node) -> - State = case rabbit_misc:rpc_call(Node, - rabbit_vhost_sup_sup, is_vhost_alive, - [VHost]) of - {badrpc, nodedown} -> nodedown; - true -> running; - false -> stopped - end, - {Node, State} - end, - Nodes); +i(cluster_state, VHost) -> vhost_cluster_state(VHost); i(Item, _) -> throw({bad_argument, Item}). info(VHost) -> infos(?INFO_KEYS, VHost). From dd186dbf5e8cfa9b6b3efe4902fb27a07463b751 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 25 May 2018 07:27:09 +0100 Subject: [PATCH 27/35] Clarify what MPL version is used, references #1609 --- LICENSE | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 8964048501..2427c5197d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ -This package, the RabbitMQ server is licensed under the MPL. For the -MPL, please see LICENSE-MPL-RabbitMQ. +This package, the RabbitMQ server is licensed under the MPL 1.1. For the +MPL 1.1, please see LICENSE-MPL-RabbitMQ. If you have any questions regarding licensing, please contact us at info@rabbitmq.com. diff --git a/README.md b/README.md index bf04218909..dde14f9e55 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) and our [development process overview](https://rabbitmq.com/github.html). -## License +## Licensing -RabbitMQ server is [licensed under the MPL](LICENSE-MPL-RabbitMQ). +RabbitMQ server is [licensed under the MPL 1.1](LICENSE-MPL-RabbitMQ). ## Building From Source and Packaging From 1e97bce7cdf0b7e55198ebddaf910207c9bbf158 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sat, 26 May 2018 16:59:05 +0300 Subject: [PATCH 28/35] Restore Erlang/OTP 19.3 compatibility erlang:ceil/1 is not available in 19.3. Part of rabbitmq/rabbitmq-management#575. [#157817330] --- src/rabbit_vhost.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_vhost.erl b/src/rabbit_vhost.erl index 7360f8bc95..e97af7a757 100644 --- a/src/rabbit_vhost.erl +++ b/src/rabbit_vhost.erl @@ -153,7 +153,7 @@ delete(VHostPath, ActingUser) -> -spec await_running_on_all_nodes(rabbit_types:vhost(), integer()) -> ok | {error, timeout}. await_running_on_all_nodes(VHost, Timeout) -> - Attempts = ceil(Timeout / ?AWAIT_SAMPLE_INTERVAL), + Attempts = round(Timeout / ?AWAIT_SAMPLE_INTERVAL), await_running_on_all_nodes0(VHost, Attempts). await_running_on_all_nodes0(_VHost, 0) -> From 2df7b4cfad562dedaefa4bca908b04aef013e7c2 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Tue, 22 May 2018 04:18:11 +0100 Subject: [PATCH 29/35] Introduce rabbit_nodes:await_running_count/2 It will wait until the cluster has N members, up to so many seconds. The function will return immediately for the value of 1. Part of rabbitmq/rabbitmq-cli#235. --- src/rabbit_nodes.erl | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/rabbit_nodes.erl b/src/rabbit_nodes.erl index 5d554f38b6..daee4259e0 100644 --- a/src/rabbit_nodes.erl +++ b/src/rabbit_nodes.erl @@ -20,10 +20,13 @@ -export([names/1, diagnostics/1, make/1, parts/1, cookie_hash/0, is_running/2, is_process_running/2, cluster_name/0, set_cluster_name/2, ensure_epmd/0, - all_running/0, name_type/0]). + all_running/0, name_type/0, running_count/0, + await_running_count/2]). -include_lib("kernel/include/inet.hrl"). +-define(SAMPLING_INTERVAL, 1000). + %%---------------------------------------------------------------------------- %% Specs %%---------------------------------------------------------------------------- @@ -37,6 +40,9 @@ -spec cluster_name() -> binary(). -spec set_cluster_name(binary(), rabbit_types:username()) -> 'ok'. -spec all_running() -> [node()]. +-spec running_count() -> integer(). + +-spec await_running_count(integer(), integer()) -> 'ok,' | {'error', atom()}. %%---------------------------------------------------------------------------- @@ -85,3 +91,20 @@ ensure_epmd() -> rabbit_nodes_common:ensure_epmd(). all_running() -> rabbit_mnesia:cluster_nodes(running). + +running_count() -> length(all_running()). + +await_running_count(TargetCount, Timeout) -> + Retries = floor(Timeout/?SAMPLING_INTERVAL), + await_running_count_with_retries(TargetCount, Retries). + +await_running_count_with_retries(1, _Retries) -> true; +await_running_count_with_retries(_TargetCount, Retries) when Retries =:= 0 -> + {error, timeout}; +await_running_count_with_retries(TargetCount, Retries) -> + case running_count() >= TargetCount of + true -> ok; + false -> + timer:sleep(?SAMPLING_INTERVAL), + await_running_count_with_retries(TargetCount, Retries - 1) + end. From e48ac193d7cd2d6968c90431b5f408b403150fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20Cogolu=C3=A8gnes?= Date: Mon, 28 May 2018 14:28:38 +0200 Subject: [PATCH 30/35] Add connection name to connection created/closed events if necessary Fixes #1596 [#157500358] --- src/rabbit_reader.erl | 53 ++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 4e6ef103b0..9127fa530d 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -134,7 +134,7 @@ peer_cert_validity, auth_mechanism, ssl_protocol, ssl_key_exchange, ssl_cipher, ssl_hash, protocol, user, vhost, timeout, frame_max, channel_max, client_properties, connected_at, - node, user_who_performed_action, connection_user_provided_name]). + node, user_who_performed_action]). -define(INFO_KEYS, ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [pid]). @@ -394,17 +394,17 @@ start_connection(Parent, HelperSup, Deb, Sock) -> Properties -> Properties end, - ConnectionUserProvidedName = case get(connection_user_provided_name) of - undefined -> - ''; - ConnectionName -> - ConnectionName + EventProperties = [{name, Name}, + {pid, self()}, + {node, node()}, + {client_properties, ClientProperties}], + EventProperties1 = case get(connection_user_provided_name) of + undefined -> + EventProperties; + ConnectionUserProvidedName -> + [{user_provided_name, ConnectionUserProvidedName} | EventProperties] end, - rabbit_event:notify(connection_closed, [{name, Name}, - {pid, self()}, - {node, node()}, - {client_properties, ClientProperties}, - {connection_user_provided_name, ConnectionUserProvidedName}]) + rabbit_event:notify(connection_closed, EventProperties1) end, done. @@ -621,7 +621,9 @@ handle_other({'$gen_cast', {force_event_refresh, Ref}}, State) when ?IS_RUNNING(State) -> rabbit_event:notify( connection_created, - [{type, network} | infos(?CREATION_EVENT_KEYS, State)], Ref), + augment_infos_with_user_provided_connection_name( + [{type, network} | infos(?CREATION_EVENT_KEYS, State)], State), + Ref), rabbit_event:init_stats_timer(State, #v1.stats_timer); handle_other({'$gen_cast', {force_event_refresh, _Ref}}, State) -> %% Ignore, we will emit a created event once we start running. @@ -1147,7 +1149,12 @@ handle_method0(#'connection.start_ok'{mechanism = Mechanism, % adding client properties to process dictionary to send them later % in the connection_closed event put(client_properties, ClientProperties), - put(connection_user_provided_name, user_provided_connection_name(Connection2)), + case user_provided_connection_name(Connection2) of + undefined -> + undefined; + UserProvidedConnectionName -> + put(connection_user_provided_name, UserProvidedConnectionName) + end, auth_phase(Response, State); handle_method0(#'connection.secure_ok'{response = Response}, @@ -1220,7 +1227,10 @@ handle_method0(#'connection.open'{virtual_host = VHost}, connection = NewConnection, channel_sup_sup_pid = ChannelSupSupPid, throttle = Throttle1}), - Infos = [{type, network} | infos(?CREATION_EVENT_KEYS, State1)], + Infos = augment_infos_with_user_provided_connection_name( + [{type, network} | infos(?CREATION_EVENT_KEYS, State1)], + State1 + ), rabbit_core_metrics:connection_created(proplists:get_value(pid, Infos), Infos), rabbit_event:notify(connection_created, Infos), @@ -1477,13 +1487,6 @@ ic(client_properties, #connection{client_properties = CP}) -> CP; ic(auth_mechanism, #connection{auth_mechanism = none}) -> none; ic(auth_mechanism, #connection{auth_mechanism = {Name, _Mod}}) -> Name; ic(connected_at, #connection{connected_at = T}) -> T; -ic(connection_user_provided_name, C) -> - case user_provided_connection_name(C) of - undefined -> - ''; - ConnectionUserProvidedName -> - ConnectionUserProvidedName - end; ic(Item, #connection{}) -> throw({bad_argument, Item}). socket_info(Get, Select, #v1{sock = Sock}) -> @@ -1697,6 +1700,14 @@ augment_connection_log_name(#connection{name = Name} = Connection) -> Connection#connection{log_name = LogName} end. +augment_infos_with_user_provided_connection_name(Infos, #v1{connection = Connection}) -> + case user_provided_connection_name(Connection) of + undefined -> + Infos; + UserProvidedConnectionName -> + [{user_provided_name, UserProvidedConnectionName} | Infos] + end. + user_provided_connection_name(#connection{client_properties = ClientProperties}) -> case rabbit_misc:table_lookup(ClientProperties, <<"connection_name">>) of {longstr, UserSpecifiedName} -> From ced1c03338dbb0e19a7ceccd1874432d4235e15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 21 May 2018 13:26:01 +0200 Subject: [PATCH 31/35] Use better default memory allocators We have settled on the following configuration for memory allocators after testing many different combinations: +MBas ageffcbf +MHas ageffcbf +MBlmbcs 512 +MHlmbcs 512 +MMmcs 30 They reduce the memory usage and help RabbitMQ reclaim memory, at the cost of a slight decrease in performance due to an increased number of memory operations. We need to start Erlang with these values in order to figure out whether they are supported. The allocator strategies we recommend were introduced in Erlang/OTP 20.2.3. The values can be overriden using RABBITMQ_SERVER_ERL_ARGS. cc @gerhard --- scripts/rabbitmq-server | 20 ++++++++++++++++++++ scripts/rabbitmq-server.bat | 22 ++++++++++++++++++++++ scripts/rabbitmq-service.bat | 22 ++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/scripts/rabbitmq-server b/scripts/rabbitmq-server index 27948ea6b7..f8b868c039 100755 --- a/scripts/rabbitmq-server +++ b/scripts/rabbitmq-server @@ -120,6 +120,25 @@ if [ ! -f "${RABBITMQ_SCHEMA_DIR}/rabbit.schema" ]; then cp "${RABBITMQ_HOME}/priv/schema/rabbit.schema" "${RABBITMQ_SCHEMA_DIR}" fi +# The default allocation strategy RabbitMQ is using was introduced +# in Erlang/OTP 20.2.3. Earlier Erlang versions fail to start with +# this configuration. We therefore need to ensure that erl accepts +# these values before we can use them. +# +# The defaults are meant to reduce RabbitMQ's memory usage and help +# it reclaim memory at the cost of a slight decrease in performance +# (due to an increase in memory operations). These defaults can be +# overriden using the RABBITMQ_SERVER_ERL_ARGS variable. +RABBITMQ_DEFAULT_ALLOC_ARGS="+MBas ageffcbf +MHas ageffcbf +MBlmbcs 512 +MHlmbcs 512 +MMmcs 30" + +${ERL_DIR}erl ${RABBITMQ_DEFAULT_ALLOC_ARGS} \ + -boot "${CLEAN_BOOT_FILE}" \ + -noinput -eval 'halt(0)' 2>/dev/null + +if [ $? != 0 ] ; then + RABBITMQ_DEFAULT_ALLOC_ARGS= +fi + set -e RABBITMQ_CONFIG_FILE_NOEX="${RABBITMQ_CONFIG_FILE%.*}" @@ -213,6 +232,7 @@ start_rabbitmq_server() { ${RABBITMQ_CONFIG_ARG} \ +W w \ +A ${RABBITMQ_IO_THREAD_POOL_SIZE} \ + ${RABBITMQ_DEFAULT_ALLOC_ARGS} \ ${RABBITMQ_SERVER_ERL_ARGS} \ +K true \ -kernel inet_default_connect_options "[{nodelay,true}]" \ diff --git a/scripts/rabbitmq-server.bat b/scripts/rabbitmq-server.bat index 7f3751f298..f6a61842fd 100644 --- a/scripts/rabbitmq-server.bat +++ b/scripts/rabbitmq-server.bat @@ -72,6 +72,27 @@ if ERRORLEVEL 2 ( set RABBITMQ_DIST_ARG=-kernel inet_dist_listen_min !RABBITMQ_DIST_PORT! -kernel inet_dist_listen_max !RABBITMQ_DIST_PORT! ) +rem The default allocation strategy RabbitMQ is using was introduced +rem in Erlang/OTP 20.2.3. Earlier Erlang versions fail to start with +rem this configuration. We therefore need to ensure that erl accepts +rem these values before we can use them. +rem +rem The defaults are meant to reduce RabbitMQ's memory usage and help +rem it reclaim memory at the cost of a slight decrease in performance +rem (due to an increase in memory operations). These defaults can be +rem overriden using the RABBITMQ_SERVER_ERL_ARGS variable. + +set RABBITMQ_DEFAULT_ALLOC_ARGS=+MBas ageffcbf +MHas ageffcbf +MBlmbcs 512 +MHlmbcs 512 +MMmcs 30 + +"!ERLANG_HOME!\bin\erl.exe" ^ + !RABBITMQ_DEFAULT_ALLOC_ARGS! ^ + -boot !CLEAN_BOOT_FILE! ^ + -noinput -eval "halt(0)" + +if ERRORLEVEL 1 ( + set RABBITMQ_DEFAULT_ALLOC_ARGS= +) + if not exist "!RABBITMQ_SCHEMA_DIR!" ( mkdir "!RABBITMQ_SCHEMA_DIR!" ) @@ -173,6 +194,7 @@ if "!ENV_OK!"=="false" ( !RABBITMQ_NAME_TYPE! !RABBITMQ_NODENAME! ^ +W w ^ +A "!RABBITMQ_IO_THREAD_POOL_SIZE!" ^ +!RABBITMQ_DEFAULT_ALLOC_ARGS! ^ !RABBITMQ_SERVER_ERL_ARGS! ^ !RABBITMQ_LISTEN_ARG! ^ -kernel inet_default_connect_options "[{nodelay, true}]" ^ diff --git a/scripts/rabbitmq-service.bat b/scripts/rabbitmq-service.bat index cef481c252..0a93462359 100644 --- a/scripts/rabbitmq-service.bat +++ b/scripts/rabbitmq-service.bat @@ -160,6 +160,27 @@ if ERRORLEVEL 3 ( set RABBITMQ_DIST_ARG=-kernel inet_dist_listen_min !RABBITMQ_DIST_PORT! -kernel inet_dist_listen_max !RABBITMQ_DIST_PORT! ) +rem The default allocation strategy RabbitMQ is using was introduced +rem in Erlang/OTP 20.2.3. Earlier Erlang versions fail to start with +rem this configuration. We therefore need to ensure that erl accepts +rem these values before we can use them. +rem +rem The defaults are meant to reduce RabbitMQ's memory usage and help +rem it reclaim memory at the cost of a slight decrease in performance +rem (due to an increase in memory operations). These defaults can be +rem overriden using the RABBITMQ_SERVER_ERL_ARGS variable. + +set RABBITMQ_DEFAULT_ALLOC_ARGS=+MBas ageffcbf +MHas ageffcbf +MBlmbcs 512 +MHlmbcs 512 +MMmcs 30 + +"!ERLANG_HOME!\bin\erl.exe" ^ + !RABBITMQ_DEFAULT_ALLOC_ARGS! ^ + -boot !CLEAN_BOOT_FILE! ^ + -noinput -eval "halt(0)" + +if ERRORLEVEL 1 ( + set RABBITMQ_DEFAULT_ALLOC_ARGS= +) + if not exist "!RABBITMQ_SCHEMA_DIR!" ( mkdir "!RABBITMQ_SCHEMA_DIR!" ) @@ -254,6 +275,7 @@ set ERLANG_SERVICE_ARGUMENTS= ^ !RABBITMQ_CONFIG_ARG! ^ +W w ^ +A "!RABBITMQ_IO_THREAD_POOL_SIZE!" ^ +!RABBITMQ_DEFAULT_ALLOC_ARGS! ^ !RABBITMQ_SERVER_ERL_ARGS! ^ !RABBITMQ_LISTEN_ARG! ^ -kernel inet_default_connect_options "[{nodelay,true}]" ^ From a604fa6668fc5bcb2fbc67fdf3cd6920741a9b87 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Tue, 29 May 2018 15:44:09 +0300 Subject: [PATCH 32/35] rabbit_nodes:await_running_count_with_retries/2: correct type spec and base clause return value [#156729133] --- src/rabbit_nodes.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rabbit_nodes.erl b/src/rabbit_nodes.erl index daee4259e0..d3ec06b05a 100644 --- a/src/rabbit_nodes.erl +++ b/src/rabbit_nodes.erl @@ -42,8 +42,6 @@ -spec all_running() -> [node()]. -spec running_count() -> integer(). --spec await_running_count(integer(), integer()) -> 'ok,' | {'error', atom()}. - %%---------------------------------------------------------------------------- name_type() -> @@ -94,11 +92,13 @@ all_running() -> rabbit_mnesia:cluster_nodes(running). running_count() -> length(all_running()). +-spec await_running_count(integer(), integer()) -> 'ok' | {'error', atom()}. + await_running_count(TargetCount, Timeout) -> Retries = floor(Timeout/?SAMPLING_INTERVAL), await_running_count_with_retries(TargetCount, Retries). -await_running_count_with_retries(1, _Retries) -> true; +await_running_count_with_retries(1, _Retries) -> ok; await_running_count_with_retries(_TargetCount, Retries) when Retries =:= 0 -> {error, timeout}; await_running_count_with_retries(TargetCount, Retries) -> From 7462d83b2417fae9b2182c734c384fb8e7d289ff Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 30 May 2018 05:11:08 +0300 Subject: [PATCH 33/35] An integration test for rabbit_nodes:await_running_count/2 [#156729133] --- test/clustering_management_SUITE.erl | 46 ++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/test/clustering_management_SUITE.erl b/test/clustering_management_SUITE.erl index e04fff2182..54f4bafbfd 100644 --- a/test/clustering_management_SUITE.erl +++ b/test/clustering_management_SUITE.erl @@ -18,6 +18,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). -compile(export_all). @@ -53,7 +54,8 @@ groups() -> forget_offline_removes_things, force_boot, status_with_alarm, - wait_fails_when_cluster_fails + pid_file_and_await_node_startup, + await_running_count ]}, {cluster_size_4, [], [ forget_promotes_offline_slave @@ -611,7 +613,7 @@ status_with_alarm(Config) -> ok = alarm_information_on_each_node(R, Rabbit, Hare). -wait_fails_when_cluster_fails(Config) -> +pid_file_and_await_node_startup(Config) -> [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), RabbitConfig = rabbit_ct_broker_helpers:get_node_config(Config,Rabbit), @@ -636,6 +638,46 @@ wait_fails_when_cluster_fails(Config) -> {error, _, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, ["wait", RabbitPidFile]). +await_running_count(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + RabbitConfig = rabbit_ct_broker_helpers:get_node_config(Config,Rabbit), + RabbitPidFile = ?config(pid_file, RabbitConfig), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]), + %% stop both nodes + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit), + %% start one node in the background + rabbit_ct_broker_helpers:start_node(Config, Rabbit), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]), + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [1, 30000])), + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [2, 1000])), + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [5, 1000])), + rabbit_ct_broker_helpers:start_node(Config, Hare), + %% this now succeeds + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [2, 30000])), + %% this still succeeds + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [1, 30000])), + %% this still fails + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [5, 1000])). + %% ---------------------------------------------------------------------------- %% Internal utils %% ---------------------------------------------------------------------------- From 4e4624208fd30bac46c280caca7c0f12a2c4aa6b Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 30 May 2018 15:52:05 +0300 Subject: [PATCH 34/35] Erlang/OTP 19.3 compatibility `erlang:floor/1` is not available in 19.3. [#156729133] --- src/rabbit_nodes.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_nodes.erl b/src/rabbit_nodes.erl index d3ec06b05a..e6131265b0 100644 --- a/src/rabbit_nodes.erl +++ b/src/rabbit_nodes.erl @@ -95,7 +95,7 @@ running_count() -> length(all_running()). -spec await_running_count(integer(), integer()) -> 'ok' | {'error', atom()}. await_running_count(TargetCount, Timeout) -> - Retries = floor(Timeout/?SAMPLING_INTERVAL), + Retries = round(Timeout/?SAMPLING_INTERVAL), await_running_count_with_retries(TargetCount, Retries). await_running_count_with_retries(1, _Retries) -> ok; From 19337af368b53c85a3f739cbb178a2b33fbb899c Mon Sep 17 00:00:00 2001 From: Nathan Oyler Date: Fri, 1 Jun 2018 13:14:21 -0700 Subject: [PATCH 35/35] fix default heartbeat in example conf file to 60 listed as 600, not the true default of 60 --- docs/rabbitmq.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rabbitmq.conf.example b/docs/rabbitmq.conf.example index 39722505f9..f5bf5dbf9d 100644 --- a/docs/rabbitmq.conf.example +++ b/docs/rabbitmq.conf.example @@ -213,7 +213,7 @@ ## * http://rabbitmq.com/heartbeats.html ## * http://rabbitmq.com/networking.html ## -# heartbeat = 600 +# heartbeat = 60 ## Set the max permissible size of an AMQP frame (in bytes). ##