diff --git a/deps/rabbit/src/rabbit_amqqueue.erl b/deps/rabbit/src/rabbit_amqqueue.erl index 05dbc71378..ec73770faa 100644 --- a/deps/rabbit/src/rabbit_amqqueue.erl +++ b/deps/rabbit/src/rabbit_amqqueue.erl @@ -778,6 +778,7 @@ declare_args() -> {<<"x-message-ttl">>, fun check_message_ttl_arg/2}, {<<"x-dead-letter-exchange">>, fun check_dlxname_arg/2}, {<<"x-dead-letter-routing-key">>, fun check_dlxrk_arg/2}, + {<<"x-dead-letter-strategy">>, fun check_dlxstrategy_arg/2}, {<<"x-max-length">>, fun check_non_neg_int_arg/2}, {<<"x-max-length-bytes">>, fun check_non_neg_int_arg/2}, {<<"x-max-in-memory-length">>, fun check_non_neg_int_arg/2}, @@ -946,6 +947,22 @@ check_dlxrk_arg(Val, Args) when is_binary(Val) -> check_dlxrk_arg(_Val, _Args) -> {error, {unacceptable_type, "expected a string"}}. +-define(KNOWN_DLX_STRATEGIES, [<<"at-most-once">>, <<"at-least-once">>]). +check_dlxstrategy_arg({longstr, Val}, _Args) -> + case lists:member(Val, ?KNOWN_DLX_STRATEGIES) of + true -> ok; + false -> {error, invalid_dlx_strategy} + end; +check_dlxstrategy_arg({Type, _}, _Args) -> + {error, {unacceptable_type, Type}}; +check_dlxstrategy_arg(Val, _Args) when is_binary(Val) -> + case lists:member(Val, ?KNOWN_DLX_STRATEGIES) of + true -> ok; + false -> {error, invalid_dlx_strategy} + end; +check_dlxstrategy_arg(_Val, _Args) -> + {error, invalid_dlx_strategy}. + -define(KNOWN_OVERFLOW_MODES, [<<"drop-head">>, <<"reject-publish">>, <<"reject-publish-dlx">>]). check_overflow({longstr, Val}, _Args) -> case lists:member(Val, ?KNOWN_OVERFLOW_MODES) of @@ -1657,8 +1674,8 @@ credit(Q, CTag, Credit, Drain, QStates) -> {'ok', non_neg_integer(), qmsg(), rabbit_queue_type:state()} | {'empty', rabbit_queue_type:state()} | {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -basic_get(Q, NoAck, LimiterPid, CTag, QStates0) -> - rabbit_queue_type:dequeue(Q, NoAck, LimiterPid, CTag, QStates0). +basic_get(Q, NoAck, LimiterPid, CTag, QStates) -> + rabbit_queue_type:dequeue(Q, NoAck, LimiterPid, CTag, QStates). -spec basic_consume(amqqueue:amqqueue(), boolean(), pid(), pid(), boolean(), @@ -1670,7 +1687,7 @@ basic_get(Q, NoAck, LimiterPid, CTag, QStates0) -> {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. basic_consume(Q, NoAck, ChPid, LimiterPid, LimiterActive, ConsumerPrefetchCount, ConsumerTag, - ExclusiveConsume, Args, OkMsg, ActingUser, Contexts) -> + ExclusiveConsume, Args, OkMsg, ActingUser, QStates) -> QName = amqqueue:get_name(Q), %% first phase argument validation @@ -1686,7 +1703,7 @@ basic_consume(Q, NoAck, ChPid, LimiterPid, args => Args, ok_msg => OkMsg, acting_user => ActingUser}, - rabbit_queue_type:consume(Q, Spec, Contexts). + rabbit_queue_type:consume(Q, Spec, QStates). -spec basic_cancel(amqqueue:amqqueue(), rabbit_types:ctag(), any(), rabbit_types:username(), diff --git a/deps/rabbit/src/rabbit_basic.erl b/deps/rabbit/src/rabbit_basic.erl index cc7c00047e..b42e832f71 100644 --- a/deps/rabbit/src/rabbit_basic.erl +++ b/deps/rabbit/src/rabbit_basic.erl @@ -12,7 +12,8 @@ -export([publish/4, publish/5, publish/1, message/3, message/4, properties/1, prepend_table_header/3, extract_headers/1, extract_timestamp/1, map_headers/2, delivery/4, - header_routes/1, parse_expiration/1, header/2, header/3]). + header_routes/1, parse_expiration/1, header/2, header/3, + is_message_persistent/1]). -export([build_content/2, from_content/1, msg_size/1, maybe_gc_large_msg/1, maybe_gc_large_msg/2]). -export([add_header/4, diff --git a/deps/rabbit/src/rabbit_classic_queue.erl b/deps/rabbit/src/rabbit_classic_queue.erl index 20edb7872d..f4c52d44d6 100644 --- a/deps/rabbit/src/rabbit_classic_queue.erl +++ b/deps/rabbit/src/rabbit_classic_queue.erl @@ -445,8 +445,10 @@ recover_durable_queues(QueuesAndRecoveryTerms) -> capabilities() -> #{unsupported_policies => [ %% Stream policies - <<"max-age">>, <<"stream-max-segment-size-bytes">>, - <<"queue-leader-locator">>, <<"initial-cluster-size">>], + <<"max-age">>, <<"stream-max-segment-size-bytes">>, + <<"queue-leader-locator">>, <<"initial-cluster-size">>, + %% Quorum policies + <<"dead-letter-strategy">>], queue_arguments => [<<"x-expires">>, <<"x-message-ttl">>, <<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, <<"x-max-length">>, <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, diff --git a/deps/rabbit/src/rabbit_dead_letter.erl b/deps/rabbit/src/rabbit_dead_letter.erl index f13b409dce..c3865d31b6 100644 --- a/deps/rabbit/src/rabbit_dead_letter.erl +++ b/deps/rabbit/src/rabbit_dead_letter.erl @@ -7,7 +7,9 @@ -module(rabbit_dead_letter). --export([publish/5]). +-export([publish/5, + make_msg/5, + detect_cycles/3]). -include_lib("rabbit_common/include/rabbit.hrl"). -include_lib("rabbit_common/include/rabbit_framing.hrl"). @@ -39,7 +41,7 @@ make_msg(Msg = #basic_message{content = Content, undefined -> {RoutingKeys, fun (H) -> H end}; _ -> {[RK], fun (H) -> lists:keydelete(<<"CC">>, 1, H) end} end, - ReasonBin = list_to_binary(atom_to_list(Reason)), + ReasonBin = atom_to_binary(Reason), TimeSec = os:system_time(seconds), PerMsgTTL = per_msg_ttl_header(Content#content.properties), HeadersFun2 = diff --git a/deps/rabbit/src/rabbit_fifo.erl b/deps/rabbit/src/rabbit_fifo.erl index cb2fe7fd78..827d62dccd 100644 --- a/deps/rabbit/src/rabbit_fifo.erl +++ b/deps/rabbit/src/rabbit_fifo.erl @@ -20,11 +20,13 @@ -include_lib("rabbit_common/include/rabbit.hrl"). -export([ + %% ra_machine callbacks init/1, apply/3, state_enter/2, tick/2, overview/1, + get_checked_out/4, %% versioning version/0, @@ -41,6 +43,7 @@ query_consumer_count/1, query_consumers/1, query_stat/1, + query_stat_dlx/1, query_single_active_consumer/1, query_in_memory_usage/1, query_peek/2, @@ -51,7 +54,10 @@ %% misc dehydrate_state/1, + dehydrate_message/1, normalize/1, + get_msg_header/1, + get_header/2, %% protocol helpers make_enqueue/3, @@ -103,7 +109,7 @@ #update_config{} | #garbage_collection{}. --type command() :: protocol() | ra_machine:builtin_command(). +-type command() :: protocol() | rabbit_fifo_dlx:protocol() | ra_machine:builtin_command(). %% all the command types supported by ra fifo -type client_msg() :: delivery(). @@ -126,6 +132,8 @@ state/0, config/0]). +%% This function is never called since only rabbit_fifo_v0:init/1 is called. +%% See https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 -spec init(config()) -> state(). init(#{name := Name, queue_resource := Resource} = Conf) -> @@ -143,6 +151,7 @@ update_config(Conf, State) -> MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), DeliveryLimit = maps:get(delivery_limit, Conf, undefined), Expires = maps:get(expires, Conf, undefined), + MsgTTL = maps:get(msg_ttl, Conf, undefined), ConsumerStrategy = case maps:get(single_active_consumer_on, Conf, false) of true -> single_active; @@ -153,6 +162,7 @@ update_config(Conf, State) -> RCISpec = {RCI, RCI}, LastActive = maps:get(created, Conf, undefined), + MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), State#?MODULE{cfg = Cfg#cfg{release_cursor_interval = RCISpec, dead_letter_handler = DLH, become_leader_handler = BLH, @@ -163,8 +173,9 @@ update_config(Conf, State) -> max_in_memory_bytes = MaxMemoryBytes, consumer_strategy = ConsumerStrategy, delivery_limit = DeliveryLimit, - expires = Expires}, - last_active = LastActive}. + expires = Expires, + msg_ttl = MsgTTL}, + last_active = LastActive}. zero(_) -> 0. @@ -201,30 +212,46 @@ apply(Meta, case Cons0 of #{ConsumerId := Con0} -> complete_and_checkout(Meta, MsgIds, ConsumerId, - Con0, [], State); + Con0, [], State, true); _ -> {State, ok} end; apply(Meta, #discard{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0} = State0) -> - case Cons0 of - #{ConsumerId := #consumer{checked_out = Checked} = Con0} -> - % Discarded maintains same order as MsgIds (so that publishing to - % dead-letter exchange will be in same order as messages got rejected) - Discarded = lists:filtermap(fun(Id) -> - case maps:find(Id, Checked) of - {ok, Msg} -> - {true, Msg}; - error -> - false - end - end, MsgIds), - Effects = dead_letter_effects(rejected, Discarded, State0, []), - complete_and_checkout(Meta, MsgIds, ConsumerId, Con0, - Effects, State0); + #?MODULE{consumers = Cons, + dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}} = State) -> + case Cons of + #{ConsumerId := #consumer{checked_out = Checked} = Con} -> + case DLH of + at_least_once -> + DlxState = lists:foldl(fun(MsgId, S) -> + case maps:find(MsgId, Checked) of + {ok, Msg} -> + rabbit_fifo_dlx:discard(Msg, rejected, S); + error -> + S + end + end, DlxState0, MsgIds), + complete_and_checkout(Meta, MsgIds, ConsumerId, Con, + [], State#?MODULE{dlx = DlxState}, false); + _ -> + % Discarded maintains same order as MsgIds (so that publishing to + % dead-letter exchange will be in same order as messages got rejected) + Discarded = lists:filtermap(fun(Id) -> + case maps:find(Id, Checked) of + {ok, Msg} -> + {true, Msg}; + error -> + false + end + end, MsgIds), + Effects = dead_letter_effects(rejected, Discarded, State, []), + complete_and_checkout(Meta, MsgIds, ConsumerId, Con, + Effects, State, true) + end; _ -> - {State0, ok} + {State, ok} end; apply(Meta, #return{msg_ids = MsgIds, consumer_id = ConsumerId}, #?MODULE{consumers = Cons0} = State) -> @@ -319,17 +346,17 @@ apply(#{index := Index, State1 = update_consumer(ConsumerId, ConsumerMeta, {once, 1, simple_prefetch}, 0, State0), - {success, _, MsgId, Msg, State2} = checkout_one(Meta, State1), + {success, _, MsgId, Msg, State2, Effects0} = checkout_one(Meta, State1, []), {State4, Effects1} = case Settlement of unsettled -> {_, Pid} = ConsumerId, - {State2, [{monitor, process, Pid}]}; + {State2, [{monitor, process, Pid} | Effects0]}; settled -> %% immediately settle the checkout - {State3, _, Effects0} = + {State3, _, SettleEffects} = apply(Meta, make_settle(ConsumerId, [MsgId]), State2), - {State3, Effects0} + {State3, SettleEffects ++ Effects0} end, {Reply, Effects2} = case Msg of @@ -366,34 +393,45 @@ apply(#{index := Index}, #purge{}, #?MODULE{messages_total = Tot, returns = Returns, messages = Messages, - ra_indexes = Indexes0} = State0) -> - Total = messages_ready(State0), - Indexes1 = lists:foldl(fun (?INDEX_MSG(I, _), Acc0) when is_integer(I) -> + ra_indexes = Indexes0, + dlx = DlxState0} = State0) -> + NumReady = messages_ready(State0), + Indexes1 = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> rabbit_fifo_index:delete(I, Acc0); (_, Acc) -> Acc end, Indexes0, lqueue:to_list(Returns)), - Indexes = lists:foldl(fun (?INDEX_MSG(I, _), Acc0) when is_integer(I) -> + Indexes2 = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> rabbit_fifo_index:delete(I, Acc0); (_, Acc) -> Acc end, Indexes1, lqueue:to_list(Messages)), + {DlxState, DiscardMsgs} = rabbit_fifo_dlx:purge(DlxState0), + Indexes = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> + rabbit_fifo_index:delete(I, Acc0); + (_, Acc) -> + Acc + end, Indexes2, DiscardMsgs), + NumPurged = NumReady + length(DiscardMsgs), State1 = State0#?MODULE{ra_indexes = Indexes, messages = lqueue:new(), - messages_total = Tot - Total, + messages_total = Tot - NumPurged, returns = lqueue:new(), + dlx = DlxState, msg_bytes_enqueue = 0, prefix_msgs = {0, [], 0, []}, msg_bytes_in_memory = 0, msgs_ready_in_memory = 0}, Effects0 = [garbage_collection], - Reply = {purge, Total}, + Reply = {purge, NumPurged}, {State, _, Effects} = evaluate_limit(Index, false, State0, State1, Effects0), update_smallest_raft_index(Index, Reply, State, Effects); apply(#{index := Idx}, #garbage_collection{}, State) -> update_smallest_raft_index(Idx, ok, State, [{aux, garbage_collection}]); +apply(Meta, {timeout, expire_msgs}, State) -> + checkout(Meta, State, State, [], false); apply(#{system_time := Ts} = Meta, {down, Pid, noconnection}, #?MODULE{consumers = Cons0, cfg = #cfg{consumer_strategy = single_active}, @@ -531,12 +569,73 @@ apply(#{index := Idx} = Meta, #purge_nodes{nodes = Nodes}, State0) -> purge_node(Meta, Node, S, E) end, {State0, []}, Nodes), update_smallest_raft_index(Idx, ok, State, Effects); -apply(#{index := Idx} = Meta, #update_config{config = Conf}, State0) -> - {State, Reply, Effects} = checkout(Meta, State0, update_config(Conf, State0), []), +apply(#{index := Idx} = Meta, #update_config{config = Conf}, + #?MODULE{cfg = #cfg{dead_letter_handler = Old_DLH}} = State0) -> + #?MODULE{cfg = #cfg{dead_letter_handler = DLH}, + dlx = DlxState, + ra_indexes = Indexes0, + messages_total = Tot} = State1 = update_config(Conf, State0), + %%TODO return aux effect here and move logic over to handle_aux/6 which can return effects as last arguments. + {State4, Effects1} = case DLH of + at_least_once -> + case rabbit_fifo_dlx:consumer_pid(DlxState) of + undefined -> + %% Policy changed from at-most-once to at-least-once. + %% Therefore, start rabbit_fifo_dlx_worker on leader. + {State1, [{aux, start_dlx_worker}]}; + DlxWorkerPid -> + %% Leader already exists. + %% Notify leader of new policy. + Effect = {send_msg, DlxWorkerPid, lookup_topology, ra_event}, + {State1, [Effect]} + end; + _ when Old_DLH =:= at_least_once -> + %% Cleanup any remaining messages stored by rabbit_fifo_dlx + %% by either dropping or at-most-once dead-lettering. + ReasonMsgs = rabbit_fifo_dlx:cleanup(DlxState), + Len = length(ReasonMsgs), + rabbit_log:debug("Cleaning up ~b dead-lettered messages " + "since dead_letter_handler changed from ~s to ~p", + [Len, Old_DLH, DLH]), + Effects0 = dead_letter_effects(undefined, ReasonMsgs, State1, []), + {_, Msgs} = lists:unzip(ReasonMsgs), + Indexes = delete_indexes(Msgs, Indexes0), + State2 = subtract_in_memory(Msgs, State1), + State3 = State2#?MODULE{dlx = rabbit_fifo_dlx:init(), + ra_indexes = Indexes, + messages_total = Tot - Len}, + {State3, Effects0}; + _ -> + {State1, []} + end, + {State, Reply, Effects} = checkout(Meta, State0, State4, Effects1), update_smallest_raft_index(Idx, Reply, State, Effects); apply(_Meta, {machine_version, FromVersion, ToVersion}, V0State) -> State = convert(FromVersion, ToVersion, V0State), - {State, ok, []}; + {State, ok, [{aux, start_dlx_worker}]}; +%%TODO are there better approach to +%% 1. matching against opaque rabbit_fifo_dlx:protocol / record (without exposing all the protocol details), and +%% 2. Separate the logic running in rabbit_fifo and rabbit_fifo_dlx when dead-letter messages is acked? +apply(#{index := IncomingRaftIdx} = Meta, {dlx, Cmd}, + #?MODULE{dlx = DlxState0, + messages_total = Total0, + ra_indexes = Indexes0} = State0) when element(1, Cmd) =:= settle -> + {DlxState, AckedMsgs} = rabbit_fifo_dlx:apply(Cmd, DlxState0), + Indexes = delete_indexes(AckedMsgs, Indexes0), + Total = Total0 - length(AckedMsgs), + State1 = subtract_in_memory(AckedMsgs, State0), + State2 = State1#?MODULE{dlx = DlxState, + messages_total = Total, + ra_indexes = Indexes}, + {State, ok, Effects} = checkout(Meta, State0, State2, [], false), + update_smallest_raft_index(IncomingRaftIdx, State, Effects); +apply(Meta, {dlx, Cmd}, + #?MODULE{dlx = DlxState0} = State0) -> + {DlxState, ok} = rabbit_fifo_dlx:apply(Cmd, DlxState0), + State1 = State0#?MODULE{dlx = DlxState}, + %% Run a checkout so that a new DLX consumer will be delivered discarded messages + %% directly after it subscribes. + checkout(Meta, State0, State1, [], false); apply(_Meta, Cmd, State) -> %% handle unhandled commands gracefully rabbit_log:debug("rabbit_fifo: unhandled command ~W", [Cmd, 10]), @@ -627,11 +726,31 @@ convert_v1_to_v2(V1State) -> end, Ch)} end, ConsumersV1), + %% The (old) format of dead_letter_handler in RMQ < v3.10 is: + %% {Module, Function, Args} + %% The (new) format of dead_letter_handler in RMQ >= v3.10 is: + %% undefined | {at_most_once, {Module, Function, Args}} | at_least_once + %% + %% Note that the conversion must convert both from old format to new format + %% as well as from new format to new format. The latter is because quorum queues + %% created in RMQ >= v3.10 are still initialised with rabbit_fifo_v0 as described in + %% https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 + DLH = case rabbit_fifo_v1:get_cfg_field(dead_letter_handler, V1State) of + {_M, _F, _A = [_DLX = undefined|_]} -> + %% queue was declared in RMQ < v3.10 and no DLX configured + undefined; + {_M, _F, _A} = MFA -> + %% queue was declared in RMQ < v3.10 and DLX configured + {at_most_once, MFA}; + Other -> + Other + end, + %% Then add all pending messages back into the index Cfg = #cfg{name = rabbit_fifo_v1:get_cfg_field(name, V1State), resource = rabbit_fifo_v1:get_cfg_field(resource, V1State), release_cursor_interval = rabbit_fifo_v1:get_cfg_field(release_cursor_interval, V1State), - dead_letter_handler = rabbit_fifo_v1:get_cfg_field(dead_letter_handler, V1State), + dead_letter_handler = DLH, become_leader_handler = rabbit_fifo_v1:get_cfg_field(become_leader_handler, V1State), %% TODO: what if policy enabling reject_publish was applied before conversion? overflow_strategy = rabbit_fifo_v1:get_cfg_field(overflow_strategy, V1State), @@ -670,14 +789,20 @@ purge_node(Meta, Node, State, Effects) -> end, {State, Effects}, all_pids_for(Node, State)). %% any downs that re not noconnection -handle_down(Meta, Pid, #?MODULE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> +handle_down(#{system_time := DownTs} = Meta, Pid, #?MODULE{consumers = Cons0, + enqueuers = Enqs0} = State0) -> % Remove any enqueuer for the same pid and enqueue any pending messages % This should be ok as we won't see any more enqueues from this pid State1 = case maps:take(Pid, Enqs0) of {#enqueuer{pending = Pend}, Enqs} -> - lists:foldl(fun ({_, RIdx, RawMsg}, S) -> - enqueue(RIdx, RawMsg, S) + lists:foldl(fun ({_, RIdx, Ts, RawMsg}, S) -> + enqueue(RIdx, Ts, RawMsg, S); + ({_, RIdx, RawMsg}, S) -> + %% This is an edge case: It is an out-of-order delivery + %% from machine version 1. + %% If message TTL is configured, expiration will be delayed + %% for the time the message has been pending. + enqueue(RIdx, DownTs, RawMsg, S) end, State0#?MODULE{enqueuers = Enqs}, Pend); error -> State0 @@ -738,7 +863,16 @@ update_waiting_consumer_status(Node, Consumer#consumer.status =/= cancelled]. -spec state_enter(ra_server:ra_state(), state()) -> ra_machine:effects(). -state_enter(leader, #?MODULE{consumers = Cons, +state_enter(RaState, #?MODULE{cfg = #cfg{dead_letter_handler = at_least_once, + resource = QRef, + name = QName}, + dlx = DlxState} = State) -> + rabbit_fifo_dlx:state_enter(RaState, QRef, QName, DlxState), + state_enter0(RaState, State); +state_enter(RaState, State) -> + state_enter0(RaState, State). + +state_enter0(leader, #?MODULE{consumers = Cons, enqueuers = Enqs, waiting_consumers = WaitingConsumers, cfg = #cfg{name = Name, @@ -753,6 +887,7 @@ state_enter(leader, #?MODULE{consumers = Cons, Mons = [{monitor, process, P} || P <- Pids], Nots = [{send_msg, P, leader_change, ra_event} || P <- Pids], NodeMons = lists:usort([{monitor, node, node(P)} || P <- Pids]), + %% TODO reissue timer effect if head of message queue has expiry header set FHReservation = [{mod_call, rabbit_quorum_queue, file_handle_leader_reservation, [Resource]}], Effects = Mons ++ Nots ++ NodeMons ++ FHReservation, case BLH of @@ -761,7 +896,7 @@ state_enter(leader, #?MODULE{consumers = Cons, {Mod, Fun, Args} -> [{mod_call, Mod, Fun, Args ++ [Name]} | Effects] end; -state_enter(eol, #?MODULE{enqueuers = Enqs, +state_enter0(eol, #?MODULE{enqueuers = Enqs, consumers = Custs0, waiting_consumers = WaitingConsumers0}) -> Custs = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Custs0), @@ -772,30 +907,32 @@ state_enter(eol, #?MODULE{enqueuers = Enqs, || P <- maps:keys(maps:merge(Enqs, AllConsumers))] ++ [{aux, eol}, {mod_call, rabbit_quorum_queue, file_handle_release_reservation, []}]; -state_enter(State, #?MODULE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> +state_enter0(State, #?MODULE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> FHReservation = {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []}, [FHReservation]; - state_enter(_, _) -> +state_enter0(_, _) -> %% catch all as not handling all states []. - -spec tick(non_neg_integer(), state()) -> ra_machine:effects(). tick(Ts, #?MODULE{cfg = #cfg{name = Name, resource = QName}, msg_bytes_enqueue = EnqueueBytes, - msg_bytes_checkout = CheckoutBytes} = State) -> + msg_bytes_checkout = CheckoutBytes, + dlx = DlxState} = State) -> case is_expired(Ts, State) of true -> [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}]; false -> + {_, MsgBytesDiscard} = rabbit_fifo_dlx:stat(DlxState), Metrics = {Name, messages_ready(State), num_checked_out(State), % checked out messages_total(State), query_consumer_count(State), % Consumers EnqueueBytes, - CheckoutBytes}, + CheckoutBytes, + MsgBytesDiscard}, [{mod_call, rabbit_quorum_queue, handle_tick, [QName, Metrics, all_nodes(State)]}] end. @@ -805,6 +942,9 @@ overview(#?MODULE{consumers = Cons, enqueuers = Enqs, release_cursors = Cursors, enqueue_count = EnqCount, + dlx = DlxState, + msgs_ready_in_memory = InMemReady, + msg_bytes_in_memory = InMemBytes, msg_bytes_enqueue = EnqueueBytes, msg_bytes_checkout = CheckoutBytes, cfg = Cfg} = State) -> @@ -818,23 +958,28 @@ overview(#?MODULE{consumers = Cons, max_in_memory_length => Cfg#cfg.max_in_memory_length, max_in_memory_bytes => Cfg#cfg.max_in_memory_bytes, expires => Cfg#cfg.expires, + msg_ttl => Cfg#cfg.msg_ttl, delivery_limit => Cfg#cfg.delivery_limit - }, + }, {Smallest, _} = smallest_raft_index(State), - #{type => ?MODULE, - config => Conf, - num_consumers => maps:size(Cons), - num_checked_out => num_checked_out(State), - num_enqueuers => maps:size(Enqs), - num_ready_messages => messages_ready(State), - num_pending_messages => messages_pending(State), - num_messages => messages_total(State), - num_release_cursors => lqueue:len(Cursors), - release_cursors => [{I, messages_total(S)} || {_, I, S} <- lqueue:to_list(Cursors)], - release_cursor_enqueue_counter => EnqCount, - enqueue_message_bytes => EnqueueBytes, - checkout_message_bytes => CheckoutBytes, - smallest_raft_index => Smallest}. + Overview = #{type => ?MODULE, + config => Conf, + num_consumers => maps:size(Cons), + num_checked_out => num_checked_out(State), + num_enqueuers => maps:size(Enqs), + num_ready_messages => messages_ready(State), + num_in_memory_ready_messages => InMemReady, + num_pending_messages => messages_pending(State), + num_messages => messages_total(State), + num_release_cursors => lqueue:len(Cursors), + release_cursors => [{I, messages_total(S)} || {_, I, S} <- lqueue:to_list(Cursors)], + release_cursor_enqueue_counter => EnqCount, + enqueue_message_bytes => EnqueueBytes, + checkout_message_bytes => CheckoutBytes, + in_memory_message_bytes => InMemBytes, + smallest_raft_index => Smallest}, + DlxOverview = rabbit_fifo_dlx:overview(DlxState), + maps:merge(Overview, DlxOverview). -spec get_checked_out(consumer_id(), msg_id(), msg_id(), state()) -> [delivery_msg()]. @@ -917,8 +1062,15 @@ handle_aux(_RaState, {call, _From}, {peek, Pos}, Aux0, {reply, {ok, {Header, Msg}}, Aux0, Log0}; Err -> {reply, Err, Aux0, Log0} - end. - + end; +handle_aux(leader, _, start_dlx_worker, Aux, Log, + #?MODULE{cfg = #cfg{resource = QRef, + name = QName, + dead_letter_handler = at_least_once}}) -> + rabbit_fifo_dlx:start_worker(QRef, QName), + {no_reply, Aux, Log}; +handle_aux(_, _, start_dlx_worker, Aux, Log, _) -> + {no_reply, Aux, Log}. eval_gc(Log, #?MODULE{cfg = #cfg{resource = QR}} = MacState, #aux{gc = #aux_gc{last_raft_idx = LastGcIdx} = Gc} = AuxState) -> @@ -1063,6 +1215,9 @@ query_in_memory_usage(#?MODULE{msg_bytes_in_memory = Bytes, msgs_ready_in_memory = Length}) -> {Length, Bytes}. +query_stat_dlx(#?MODULE{dlx = DlxState}) -> + rabbit_fifo_dlx:stat(DlxState). + query_peek(Pos, State0) when Pos > 0 -> case take_next_msg(State0) of empty -> @@ -1113,8 +1268,15 @@ messages_total(#?MODULE{messages = _M, messages_total = Total, ra_indexes = _Indexes, prefix_msgs = {_RCnt, _R, _PCnt, _P}}) -> - Total. % lqueue:len(M) + rabbit_fifo_index:size(Indexes) + RCnt + PCnt. + Total; +%% release cursors might be old state (e.g. after recent upgrade) +messages_total(State) + when element(1, State) =:= rabbit_fifo_v1 -> + rabbit_fifo_v1:query_messages_total(State); +messages_total(State) + when element(1, State) =:= rabbit_fifo_v0 -> + rabbit_fifo_v0:query_messages_total(State). update_use({inactive, _, _, _} = CUInfo, inactive) -> CUInfo; @@ -1265,8 +1427,9 @@ maybe_return_all(#{system_time := Ts} = Meta, ConsumerId, Consumer, S0, Effects0 Effects1} end. -apply_enqueue(#{index := RaftIdx} = Meta, From, Seq, RawMsg, State0) -> - case maybe_enqueue(RaftIdx, From, Seq, RawMsg, [], State0) of +apply_enqueue(#{index := RaftIdx, + system_time := Ts} = Meta, From, Seq, RawMsg, State0) -> + case maybe_enqueue(RaftIdx, Ts, From, Seq, RawMsg, [], State0) of {ok, State1, Effects1} -> State2 = incr_enqueue_count(incr_total(State1)), {State, ok, Effects} = checkout(Meta, State0, State2, Effects1, false), @@ -1305,10 +1468,11 @@ drop_head(#?MODULE{ra_indexes = Indexes0} = State0, Effects0) -> {State0, Effects0} end. -enqueue(RaftIdx, RawMsg, #?MODULE{messages = Messages} = State0) -> +enqueue(RaftIdx, Ts, RawMsg, #?MODULE{messages = Messages} = State0) -> %% the initial header is an integer only - it will get expanded to a map %% when the next required key is added - Header = message_size(RawMsg), + Header0 = message_size(RawMsg), + Header = maybe_set_msg_ttl(RawMsg, Ts, Header0, State0), {State1, Msg} = case evaluate_memory_limit(Header, State0) of true -> @@ -1322,6 +1486,39 @@ enqueue(RaftIdx, RawMsg, #?MODULE{messages = Messages} = State0) -> State = add_bytes_enqueue(Header, State1), State#?MODULE{messages = lqueue:in(Msg, Messages)}. +maybe_set_msg_ttl(#basic_message{content = #content{properties = none}}, + _, Header, + #?MODULE{cfg = #cfg{msg_ttl = undefined}}) -> + Header; +maybe_set_msg_ttl(#basic_message{content = #content{properties = none}}, + RaCmdTs, Header, + #?MODULE{cfg = #cfg{msg_ttl = PerQueueMsgTTL}}) -> + update_expiry_header(RaCmdTs, PerQueueMsgTTL, Header); +maybe_set_msg_ttl(#basic_message{content = #content{properties = Props}}, + RaCmdTs, Header, + #?MODULE{cfg = #cfg{msg_ttl = PerQueueMsgTTL}}) -> + %% rabbit_quorum_queue will leave the properties decoded if and only if + %% per message message TTL is set. + %% We already check in the channel that expiration must be valid. + {ok, PerMsgMsgTTL} = rabbit_basic:parse_expiration(Props), + TTL = min(PerMsgMsgTTL, PerQueueMsgTTL), + update_expiry_header(RaCmdTs, TTL, Header). + +update_expiry_header(_, undefined, Header) -> + Header; +update_expiry_header(RaCmdTs, 0, Header) -> + %% We do not comply exactly with the "TTL=0 models AMQP immediate flag" semantics + %% as done for classic queues where the message is discarded if it cannot be + %% consumed immediately. + %% Instead, we discard the message if it cannot be consumed within the same millisecond + %% when it got enqueued. This behaviour should be good enough. + update_expiry_header(RaCmdTs + 1, Header); +update_expiry_header(RaCmdTs, TTL, Header) -> + update_expiry_header(RaCmdTs + TTL, Header). + +update_expiry_header(ExpiryTs, Header) -> + update_header(expiry, fun(Ts) -> Ts end, ExpiryTs, Header). + incr_enqueue_count(#?MODULE{enqueue_count = EC, cfg = #cfg{release_cursor_interval = {_Base, C}} } = State0) when EC >= C -> @@ -1363,39 +1560,39 @@ maybe_store_dehydrated_state(_RaftIdx, State) -> enqueue_pending(From, #enqueuer{next_seqno = Next, - pending = [{Next, RaftIdx, RawMsg} | Pending]} = Enq0, + pending = [{Next, RaftIdx, Ts, RawMsg} | Pending]} = Enq0, State0) -> - State = enqueue(RaftIdx, RawMsg, State0), + State = enqueue(RaftIdx, Ts, RawMsg, State0), Enq = Enq0#enqueuer{next_seqno = Next + 1, pending = Pending}, enqueue_pending(From, Enq, State); enqueue_pending(From, Enq, #?MODULE{enqueuers = Enqueuers0} = State) -> State#?MODULE{enqueuers = Enqueuers0#{From => Enq}}. -maybe_enqueue(RaftIdx, undefined, undefined, RawMsg, Effects, State0) -> +maybe_enqueue(RaftIdx, Ts, undefined, undefined, RawMsg, Effects, State0) -> % direct enqueue without tracking - State = enqueue(RaftIdx, RawMsg, State0), + State = enqueue(RaftIdx, Ts, RawMsg, State0), {ok, State, Effects}; -maybe_enqueue(RaftIdx, From, MsgSeqNo, RawMsg, Effects0, +maybe_enqueue(RaftIdx, Ts, From, MsgSeqNo, RawMsg, Effects0, #?MODULE{enqueuers = Enqueuers0, ra_indexes = Indexes0} = State0) -> case maps:get(From, Enqueuers0, undefined) of undefined -> State1 = State0#?MODULE{enqueuers = Enqueuers0#{From => #enqueuer{}}}, - {ok, State, Effects} = maybe_enqueue(RaftIdx, From, MsgSeqNo, + {ok, State, Effects} = maybe_enqueue(RaftIdx, Ts, From, MsgSeqNo, RawMsg, Effects0, State1), {ok, State, [{monitor, process, From} | Effects]}; #enqueuer{next_seqno = MsgSeqNo} = Enq0 -> % it is the next expected seqno - State1 = enqueue(RaftIdx, RawMsg, State0), + State1 = enqueue(RaftIdx, Ts, RawMsg, State0), Enq = Enq0#enqueuer{next_seqno = MsgSeqNo + 1}, State = enqueue_pending(From, Enq, State1), {ok, State, Effects0}; #enqueuer{next_seqno = Next, pending = Pending0} = Enq0 when MsgSeqNo > Next -> - % out of order enqueue - Pending = [{MsgSeqNo, RaftIdx, RawMsg} | Pending0], + % out of order delivery + Pending = [{MsgSeqNo, RaftIdx, Ts, RawMsg} | Pending0], Enq = Enq0#enqueuer{pending = lists:sort(Pending)}, %% if the enqueue it out of order we need to mark it in the %% index @@ -1426,29 +1623,39 @@ return(#{index := IncomingRaftIdx} = Meta, ConsumerId, Returned, {State, ok, Effects} = checkout(Meta, State0, State2, Effects1, false), update_smallest_raft_index(IncomingRaftIdx, State, Effects). -% used to processes messages that are finished +% used to process messages that are finished complete(Meta, ConsumerId, DiscardedMsgIds, - #consumer{checked_out = Checked} = Con0, Effects, + #consumer{checked_out = Checked} = Con0, #?MODULE{messages_total = Tot, - ra_indexes = Indexes0} = State0) -> + ra_indexes = Indexes0} = State0, Delete) -> %% credit_mode = simple_prefetch should automatically top-up credit %% as messages are simple_prefetch or otherwise returned Discarded = maps:with(DiscardedMsgIds, Checked), + DiscardedMsgs = maps:values(Discarded), + Len = length(DiscardedMsgs), Con = Con0#consumer{checked_out = maps:without(DiscardedMsgIds, Checked), - credit = increase_credit(Con0, map_size(Discarded))}, + credit = increase_credit(Con0, Len)}, State1 = update_or_remove_sub(Meta, ConsumerId, Con, State0), + State = lists:foldl(fun(Msg, Acc) -> + add_bytes_settle( + get_msg_header(Msg), Acc) + end, State1, DiscardedMsgs), + case Delete of + true -> + Indexes = delete_indexes(DiscardedMsgs, Indexes0), + State#?MODULE{messages_total = Tot - Len, + ra_indexes = Indexes}; + false -> + State + end. + +delete_indexes(Msgs, Indexes) -> %% TODO: optimise by passing a list to rabbit_fifo_index - Indexes = maps:fold(fun (_, ?INDEX_MSG(I, _), Acc0) when is_integer(I) -> - rabbit_fifo_index:delete(I, Acc0); - (_, _, Acc) -> - Acc - end, Indexes0, Discarded), - State = maps:fold(fun(_, Msg, Acc) -> - add_bytes_settle( - get_msg_header(Msg), Acc) - end, State1, Discarded), - {State#?MODULE{messages_total = Tot - length(DiscardedMsgIds), - ra_indexes = Indexes}, Effects}. + lists:foldl(fun (?INDEX_MSG(I, ?MSG(_,_)), Acc) when is_integer(I) -> + rabbit_fifo_index:delete(I, Acc); + (_, Acc) -> + Acc + end, Indexes, Msgs). increase_credit(#consumer{lifetime = once, credit = Credit}, _) -> @@ -1464,10 +1671,9 @@ increase_credit(#consumer{credit = Current}, Credit) -> complete_and_checkout(#{index := IncomingRaftIdx} = Meta, MsgIds, ConsumerId, #consumer{} = Con0, - Effects0, State0) -> - {State1, Effects1} = complete(Meta, ConsumerId, MsgIds, Con0, - Effects0, State0), - {State, ok, Effects} = checkout(Meta, State0, State1, Effects1, false), + Effects0, State0, Delete) -> + State1 = complete(Meta, ConsumerId, MsgIds, Con0, State0, Delete), + {State, ok, Effects} = checkout(Meta, State0, State1, Effects0, false), update_smallest_raft_index(IncomingRaftIdx, State, Effects). dead_letter_effects(_Reason, _Discarded, @@ -1475,12 +1681,14 @@ dead_letter_effects(_Reason, _Discarded, Effects) -> Effects; dead_letter_effects(Reason, Discarded, - #?MODULE{cfg = #cfg{dead_letter_handler = {Mod, Fun, Args}}}, + #?MODULE{cfg = #cfg{dead_letter_handler = {at_most_once, {Mod, Fun, Args}}}}, Effects) -> RaftIdxs = lists:filtermap( fun (?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))) -> {true, RaftIdx}; - (_) -> + ({_PerMsgReason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))}) when Reason =:= undefined -> + {true, RaftIdx}; + (_IgnorePrefixMessage) -> false end, Discarded), [{log, RaftIdxs, @@ -1492,7 +1700,12 @@ dead_letter_effects(Reason, Discarded, {true, {Reason, Msg}}; (?INDEX_MSG(_, ?MSG(_Header, Msg))) -> {true, {Reason, Msg}}; - (_) -> + ({PerMsgReason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))}) when Reason =:= undefined -> + {enqueue, _, _, Msg} = maps:get(RaftIdx, Lookup), + {true, {PerMsgReason, Msg}}; + ({PerMsgReason, ?INDEX_MSG(_, ?MSG(_Header, Msg))}) when Reason =:= undefined -> + {true, {PerMsgReason, Msg}}; + (_IgnorePrefixMessage) -> false end, Discarded), [{mod_call, Mod, Fun, Args ++ [DeadLetters]}] @@ -1592,7 +1805,9 @@ get_header(Key, Header) when is_map(Header) -> return_one(Meta, MsgId, Msg0, #?MODULE{returns = Returns, consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, + dlx = DlxState0, + cfg = #cfg{delivery_limit = DeliveryLimit, + dead_letter_handler = DLH}} = State0, Effects0, ConsumerId) -> #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), Msg = update_msg_header(delivery_count, fun (C) -> C + 1 end, 1, Msg0), @@ -1600,9 +1815,17 @@ return_one(Meta, MsgId, Msg0, case get_header(delivery_count, Header) of DeliveryCount when DeliveryCount > DeliveryLimit -> %% TODO: don't do for prefix msgs - Effects = dead_letter_effects(delivery_limit, [Msg], - State0, Effects0), - complete(Meta, ConsumerId, [MsgId], Con0, Effects, State0); + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(Msg, delivery_limit, DlxState0), + State = complete(Meta, ConsumerId, [MsgId], Con0, State0#?MODULE{dlx = DlxState}, false), + {State, Effects0}; + _ -> + Effects = dead_letter_effects(delivery_limit, [Msg], + State0, Effects0), + State = complete(Meta, ConsumerId, [MsgId], Con0, State0, true), + {State, Effects} + end; _ -> Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, @@ -1649,11 +1872,15 @@ return_all(Meta, #?MODULE{consumers = Cons} = State0, Effects0, ConsumerId, checkout(Meta, OldState, State, Effects) -> checkout(Meta, OldState, State, Effects, true). -checkout(#{index := Index} = Meta, #?MODULE{cfg = #cfg{resource = QName}} = OldState, +checkout(#{index := Index} = Meta, + #?MODULE{cfg = #cfg{resource = QName}} = OldState, State0, Effects0, HandleConsumerChanges) -> - {State1, _Result, Effects1} = checkout0(Meta, checkout_one(Meta, State0), - Effects0, #{}), - case evaluate_limit(Index, false, OldState, State1, Effects1) of + {#?MODULE{dlx = DlxState0} = State1, _Result, Effects1} = checkout0(Meta, checkout_one(Meta, State0, Effects0), #{}), + %%TODO For now we checkout the discards queue here. Move it to a better place + {DlxState1, DlxDeliveryEffects} = rabbit_fifo_dlx:checkout(DlxState0), + State2 = State1#?MODULE{dlx = DlxState1}, + Effects2 = DlxDeliveryEffects ++ Effects1, + case evaluate_limit(Index, false, OldState, State2, Effects2) of {State, true, Effects} -> case maybe_notify_decorators(State, HandleConsumerChanges) of {true, {MaxActivePriority, IsEmpty}} -> @@ -1673,27 +1900,31 @@ checkout(#{index := Index} = Meta, #?MODULE{cfg = #cfg{resource = QName}} = OldS end. checkout0(Meta, {success, ConsumerId, MsgId, - ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header)), State}, - Effects, SendAcc0) when is_integer(RaftIdx) -> + ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header)), State, Effects}, + SendAcc0) when is_integer(RaftIdx) -> DelMsg = {RaftIdx, {MsgId, Header}}, SendAcc = maps:update_with(ConsumerId, - fun ({InMem, LogMsgs}) -> - {InMem, [DelMsg | LogMsgs]} - end, {[], [DelMsg]}, SendAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); + fun ({InMem, LogMsgs}) -> + {InMem, [DelMsg | LogMsgs]} + end, {[], [DelMsg]}, SendAcc0), + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); checkout0(Meta, {success, ConsumerId, MsgId, - ?INDEX_MSG(Idx, ?MSG(Header, Msg)), State}, Effects, + ?INDEX_MSG(Idx, ?MSG(Header, Msg)), State, Effects}, SendAcc0) when is_integer(Idx) -> DelMsg = {MsgId, {Header, Msg}}, SendAcc = maps:update_with(ConsumerId, - fun ({InMem, LogMsgs}) -> - {[DelMsg | InMem], LogMsgs} - end, {[DelMsg], []}, SendAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); -checkout0(Meta, {success, _ConsumerId, _MsgId, ?TUPLE(_, _), State}, Effects, + fun ({InMem, LogMsgs}) -> + {[DelMsg | InMem], LogMsgs} + end, {[DelMsg], []}, SendAcc0), + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); +checkout0(Meta, {success, _ConsumerId, _MsgId, ?TUPLE(_, _), State, Effects}, SendAcc) -> - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); -checkout0(_Meta, {Activity, State0}, Effects0, SendAcc) -> + %% Do not append delivery effect for prefix messages. + %% Prefix messages do not exist anymore, but they still go through the + %% normal checkout flow to derive correct consumer states + %% after recovery and will still be settled or discarded later on. + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); +checkout0(_Meta, {Activity, State0, Effects0}, SendAcc) -> Effects1 = case Activity of nochange -> append_delivery_effects(Effects0, SendAcc); @@ -1844,9 +2075,12 @@ reply_log_effect(RaftIdx, MsgId, Header, Ready, From) -> {dequeue, {MsgId, {Header, Msg}}, Ready}}}] end}. -checkout_one(Meta, #?MODULE{service_queue = SQ0, - messages = Messages0, - consumers = Cons0} = InitState) -> +checkout_one(#{system_time := Ts} = Meta, InitState0, Effects0) -> + %% Before checking out any messsage to any consumer, + %% first remove all expired messages from the head of the queue. + {#?MODULE{service_queue = SQ0, + messages = Messages0, + consumers = Cons0} = InitState, Effects1} = expire_msgs(Ts, InitState0, Effects0), case priority_queue:out(SQ0) of {{value, ConsumerId}, SQ1} when is_map_key(ConsumerId, Cons0) -> @@ -1859,11 +2093,11 @@ checkout_one(Meta, #?MODULE{service_queue = SQ0, %% no credit but was still on queue %% can happen when draining %% recurse without consumer on queue - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{status = cancelled} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{status = suspected_down} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{checked_out = Checked0, next_msg_id = Next, credit = Credit, @@ -1881,27 +2115,101 @@ checkout_one(Meta, #?MODULE{service_queue = SQ0, true -> add_bytes_checkout(Header, State1); false -> + %% TODO do not subtract from memory here since + %% messages are still in memory when checked out subtract_in_memory_counts( Header, add_bytes_checkout(Header, State1)) end, - {success, ConsumerId, Next, ConsumerMsg, State}; + {success, ConsumerId, Next, ConsumerMsg, State, Effects1}; error -> %% consumer did not exist but was queued, recurse - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}) + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1) end; empty -> - {nochange, InitState} + {nochange, InitState, Effects1} end; {{value, _ConsumerId}, SQ1} -> %% consumer did not exist but was queued, recurse - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); {empty, _} -> + Effects = timer_effect(Ts, InitState, Effects1), case lqueue:len(Messages0) of - 0 -> {nochange, InitState}; - _ -> {inactive, InitState} + 0 -> + {nochange, InitState, Effects}; + _ -> + {inactive, InitState, Effects} end end. +%% dequeue all expired messages +expire_msgs(RaCmdTs, State0, Effects0) -> + case take_next_msg(State0) of + {?INDEX_MSG(Idx, ?MSG(#{expiry := Expiry} = Header, _) = Msg) = FullMsg, State1} + when RaCmdTs >= Expiry -> + #?MODULE{dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}, + ra_indexes = Indexes0} = State2 = add_bytes_drop(Header, State1), + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(FullMsg, expired, DlxState0), + State = State2#?MODULE{dlx = DlxState}, + expire_msgs(RaCmdTs, State, Effects0); + _ -> + Indexes = rabbit_fifo_index:delete(Idx, Indexes0), + State3 = decr_total(State2), + State4 = case Msg of + ?DISK_MSG(_) -> + State3; + _ -> + subtract_in_memory_counts(Header, State3) + end, + Effects = dead_letter_effects(expired, [FullMsg], + State4, Effects0), + State = State4#?MODULE{ra_indexes = Indexes}, + expire_msgs(RaCmdTs, State, Effects) + end; + {?PREFIX_MEM_MSG(#{expiry := Expiry} = Header) = Msg, State1} + when RaCmdTs >= Expiry -> + State2 = expire_prefix_msg(Msg, Header, State1), + expire_msgs(RaCmdTs, State2, Effects0); + {?DISK_MSG(#{expiry := Expiry} = Header) = Msg, State1} + when RaCmdTs >= Expiry -> + State2 = expire_prefix_msg(Msg, Header, State1), + expire_msgs(RaCmdTs, State2, Effects0); + _ -> + {State0, Effects0} + end. + +expire_prefix_msg(Msg, Header, State0) -> + #?MODULE{dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}} = State1 = add_bytes_drop(Header, State0), + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(Msg, expired, DlxState0), + State1#?MODULE{dlx = DlxState}; + _ -> + State2 = case Msg of + ?DISK_MSG(_) -> + State1; + _ -> + subtract_in_memory_counts(Header, State1) + end, + decr_total(State2) + end. + +timer_effect(RaCmdTs, State, Effects) -> + T = case take_next_msg(State) of + {?INDEX_MSG(_, ?MSG(#{expiry := Expiry}, _)), _} when is_number(Expiry) -> + %% Next message contains 'expiry' header. + %% (Re)set timer so that mesage will be dropped or dead-lettered on time. + Expiry - RaCmdTs; + _ -> + %% Next message does not contain 'expiry' header. + %% Therefore, do not set timer or cancel timer if it was set. + infinity + end, + [{timer, expire_msgs, T} | Effects]. + update_or_remove_sub(_Meta, ConsumerId, #consumer{lifetime = auto, credit = 0} = Con, #?MODULE{consumers = Cons} = State) -> @@ -1996,21 +2304,22 @@ maybe_queue_consumer(ConsumerId, #consumer{credit = Credit} = Con, %% creates a dehydrated version of the current state to be cached and %% potentially used to for a snaphot at a later point dehydrate_state(#?MODULE{msg_bytes_in_memory = 0, - cfg = #cfg{max_length = 0}, + cfg = #cfg{max_in_memory_length = 0}, consumers = Consumers} = State) -> - %% no messages are kept in memory, no need to - %% overly mutate the current state apart from removing indexes and cursors + % no messages are kept in memory, no need to + % overly mutate the current state apart from removing indexes and cursors State#?MODULE{ - ra_indexes = rabbit_fifo_index:empty(), - consumers = maps:map(fun (_, C) -> - dehydrate_consumer(C) - end, Consumers), - release_cursors = lqueue:new()}; + ra_indexes = rabbit_fifo_index:empty(), + consumers = maps:map(fun (_, C) -> + dehydrate_consumer(C) + end, Consumers), + release_cursors = lqueue:new()}; dehydrate_state(#?MODULE{messages = Messages, consumers = Consumers, returns = Returns, prefix_msgs = {PRCnt, PrefRet0, PPCnt, PrefMsg0}, - waiting_consumers = Waiting0} = State) -> + waiting_consumers = Waiting0, + dlx = DlxState} = State) -> RCnt = lqueue:len(Returns), %% TODO: optimise this function as far as possible PrefRet1 = lists:foldr(fun (M, Acc) -> @@ -2031,7 +2340,8 @@ dehydrate_state(#?MODULE{messages = Messages, returns = lqueue:new(), prefix_msgs = {PRCnt + RCnt, PrefRet, PPCnt + lqueue:len(Messages), PrefMsgs}, - waiting_consumers = Waiting}. + waiting_consumers = Waiting, + dlx = rabbit_fifo_dlx:dehydrate(DlxState)}. dehydrate_messages(Msgs0) -> {OutRes, Msgs} = lqueue:out(Msgs0), @@ -2053,7 +2363,8 @@ dehydrate_message(?PREFIX_MEM_MSG(_) = M) -> dehydrate_message(?DISK_MSG(_) = M) -> M; dehydrate_message(?INDEX_MSG(_Idx, ?DISK_MSG(_Header) = Msg)) -> - %% use disk msgs directly as prefix messages + %% Use disk msgs directly as prefix messages. + %% This avoids memory allocation since we do not convert. Msg; dehydrate_message(?INDEX_MSG(Idx, ?MSG(Header, _))) when is_integer(Idx) -> ?PREFIX_MEM_MSG(Header). @@ -2062,11 +2373,13 @@ dehydrate_message(?INDEX_MSG(Idx, ?MSG(Header, _))) when is_integer(Idx) -> normalize(#?MODULE{ra_indexes = _Indexes, returns = Returns, messages = Messages, - release_cursors = Cursors} = State) -> + release_cursors = Cursors, + dlx = DlxState} = State) -> State#?MODULE{ returns = lqueue:from_list(lqueue:to_list(Returns)), messages = lqueue:from_list(lqueue:to_list(Messages)), - release_cursors = lqueue:from_list(lqueue:to_list(Cursors))}. + release_cursors = lqueue:from_list(lqueue:to_list(Cursors)), + dlx = rabbit_fifo_dlx:normalize(DlxState)}. is_over_limit(#?MODULE{cfg = #cfg{max_length = undefined, max_bytes = undefined}}) -> @@ -2314,3 +2627,14 @@ smallest_raft_index(#?MODULE{cfg = _Cfg, {undefined, State} end end. + +subtract_in_memory(Msgs, State) -> + lists:foldl(fun(?INDEX_MSG(_, ?DISK_MSG(_)), S) -> + S; + (?INDEX_MSG(_, ?MSG(H, _)), S) -> + subtract_in_memory_counts(H, S); + (?DISK_MSG(_), S) -> + S; + (?PREFIX_MEM_MSG(H), S) -> + subtract_in_memory_counts(H, S) + end, State, Msgs). diff --git a/deps/rabbit/src/rabbit_fifo.hrl b/deps/rabbit/src/rabbit_fifo.hrl index c797c9d9bd..ca37fbca79 100644 --- a/deps/rabbit/src/rabbit_fifo.hrl +++ b/deps/rabbit/src/rabbit_fifo.hrl @@ -2,16 +2,21 @@ %% macros for memory optimised tuple structures -define(TUPLE(A, B), [A | B]). --define(DISK_MSG_TAG, '$disk'). -% -define(PREFIX_DISK_MSG_TAG, '$prefix_disk'). --define(PREFIX_MEM_MSG_TAG, '$prefix_inmem'). +%% We want short atoms since their binary representations will get +%% persisted in a snapshot for every message. +%% '$d' stand for 'disk'. +-define(DISK_MSG_TAG, '$d'). +%% '$m' stand for 'memory'. +-define(PREFIX_MEM_MSG_TAG, '$m'). -define(DISK_MSG(Header), [Header | ?DISK_MSG_TAG]). -define(MSG(Header, RawMsg), [Header | RawMsg]). -define(INDEX_MSG(Index, Msg), [Index | Msg]). +-define(PREFIX_MEM_MSG(Header), [Header | ?PREFIX_MEM_MSG_TAG]). + +% -define(PREFIX_DISK_MSG_TAG, '$prefix_disk'). % -define(PREFIX_DISK_MSG(Header), [?PREFIX_DISK_MSG_TAG | Header]). % -define(PREFIX_DISK_MSG(Header), ?DISK_MSG(Header)). --define(PREFIX_MEM_MSG(Header), [?PREFIX_MEM_MSG_TAG | Header]). -type option(T) :: undefined | T. @@ -32,11 +37,14 @@ %% same process -type msg_header() :: msg_size() | - #{size := msg_size(), - delivery_count => non_neg_integer()}. +#{size := msg_size(), + delivery_count => non_neg_integer(), + expiry => milliseconds()}. %% The message header: %% delivery_count: the number of unsuccessful delivery attempts. %% A non-zero value indicates a previous attempt. +%% expiry: Epoch time in ms when a message expires. Set during enqueue. +%% Value is determined by per-queue or per-message message TTL. %% If it only contains the size it can be condensed to an integer only -type msg() :: ?MSG(msg_header(), raw_msg()) | @@ -122,7 +130,7 @@ -record(enqueuer, {next_seqno = 1 :: msg_seqno(), % out of order enqueues - sorted list - pending = [] :: [{msg_seqno(), ra:index(), raw_msg()}], + pending = [] :: [{msg_seqno(), ra:index(), milliseconds(), raw_msg()}], status = up :: up | suspected_down, %% it is useful to have a record of when this was blocked @@ -137,7 +145,7 @@ {name :: atom(), resource :: rabbit_types:r('queue'), release_cursor_interval :: option({non_neg_integer(), non_neg_integer()}), - dead_letter_handler :: option(applied_mfa()), + dead_letter_handler :: option({at_most_once, applied_mfa()} | at_least_once), become_leader_handler :: option(applied_mfa()), overflow_strategy = drop_head :: drop_head | reject_publish, max_length :: option(non_neg_integer()), @@ -149,6 +157,7 @@ max_in_memory_length :: option(non_neg_integer()), max_in_memory_bytes :: option(non_neg_integer()), expires :: undefined | milliseconds(), + msg_ttl :: undefined | milliseconds(), unused_1, unused_2 }). @@ -166,6 +175,7 @@ % queue of returned msg_in_ids - when checking out it picks from returns = lqueue:new() :: lqueue:lqueue(term()), % a counter of enqueues - used to trigger shadow copy points + % reset to 0 when release_cursor gets stored enqueue_count = 0 :: non_neg_integer(), % a map containing all the live processes that have ever enqueued % a message to this queue as well as a cached value of the smallest @@ -177,11 +187,19 @@ % index when there are large gaps but should be faster than gb_trees % for normal appending operations as it's backed by a map ra_indexes = rabbit_fifo_index:empty() :: rabbit_fifo_index:state(), + %% A release cursor is essentially a snapshot without message bodies + %% (aka. "dehydrated state") taken at time T in order to truncate + %% the log at some point in the future when all messages that were enqueued + %% up to time T have been removed (e.g. consumed, dead-lettered, or dropped). + %% This concept enables snapshots to not contain any message bodies. + %% Advantage: Smaller snapshots are sent between Ra nodes. + %% Working assumption: Messages are consumed in a FIFO-ish order because + %% the log is truncated only until the oldest message. release_cursors = lqueue:new() :: lqueue:lqueue({release_cursor, ra:index(), #rabbit_fifo{}}), % consumers need to reflect consumer state at time of snapshot % needs to be part of snapshot - consumers = #{} :: #{consumer_id() => #consumer{}}, + consumers = #{} :: #{consumer_id() => consumer()}, % consumers that require further service are queued here % needs to be part of snapshot service_queue = priority_queue:new() :: priority_queue:q(), @@ -194,7 +212,10 @@ %% overflow calculations). %% This is done so that consumers are still served in a deterministic %% order on recovery. + %% TODO Remove this field and store prefix messages in-place. This will + %% simplify the checkout logic. prefix_msgs = {0, [], 0, []} :: prefix_msgs(), + dlx = rabbit_fifo_dlx:init() :: rabbit_fifo_dlx:state(), msg_bytes_enqueue = 0 :: non_neg_integer(), msg_bytes_checkout = 0 :: non_neg_integer(), %% waiting consumers, one is picked active consumer is cancelled or dies @@ -209,7 +230,7 @@ -type config() :: #{name := atom(), queue_resource := rabbit_types:r('queue'), - dead_letter_handler => applied_mfa(), + dead_letter_handler => option({at_most_once, applied_mfa()} | at_least_once), become_leader_handler => applied_mfa(), release_cursor_interval => non_neg_integer(), max_length => non_neg_integer(), @@ -220,5 +241,6 @@ single_active_consumer_on => boolean(), delivery_limit => non_neg_integer(), expires => non_neg_integer(), + msg_ttl => non_neg_integer(), created => non_neg_integer() }. diff --git a/deps/rabbit/src/rabbit_fifo_client.erl b/deps/rabbit/src/rabbit_fifo_client.erl index 3f5315de08..0faf32fd30 100644 --- a/deps/rabbit/src/rabbit_fifo_client.erl +++ b/deps/rabbit/src/rabbit_fifo_client.erl @@ -531,7 +531,7 @@ update_machine_state(Server, Conf) -> %% `{internal, AppliedCorrelations, State}' if the event contained an internally %% handled event such as a notification and a correlation was included with %% the command (e.g. in a call to `enqueue/3' the correlation terms are returned -%% here. +%% here). %% %% `{RaFifoEvent, State}' if the event contained a client message generated by %% the `rabbit_fifo' state machine such as a delivery. diff --git a/deps/rabbit/src/rabbit_fifo_dlx.erl b/deps/rabbit/src/rabbit_fifo_dlx.erl new file mode 100644 index 0000000000..cc41733151 --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx.erl @@ -0,0 +1,324 @@ +-module(rabbit_fifo_dlx). + +-include("rabbit_fifo_dlx.hrl"). +-include("rabbit_fifo.hrl"). + +% client API, e.g. for rabbit_fifo_dlx_client +-export([make_checkout/2, + make_settle/1]). + +% called by rabbit_fifo delegating DLX handling to this module +-export([init/0, apply/2, discard/3, overview/1, + checkout/1, state_enter/4, + start_worker/2, terminate_worker/1, cleanup/1, purge/1, + consumer_pid/1, dehydrate/1, normalize/1, + stat/1]). + +%% This module handles the dead letter (DLX) part of the rabbit_fifo state machine. +%% This is a separate module to better unit test and provide separation of concerns. +%% This module maintains its own state: +%% a queue of DLX messages, a single node local DLX consumer, and some stats. +%% The state of this module is included into rabbit_fifo state because there can only by one Ra state machine. +%% The rabbit_fifo module forwards all DLX commands to this module where we then update the DLX specific state only: +%% e.g. DLX consumer subscribed, adding / removing discarded messages, stats +%% +%% It also runs its own checkout logic sending DLX messages to the DLX consumer. + +-record(checkout,{ + consumer :: atom(), + prefetch :: non_neg_integer() + }). +-record(settle, {msg_ids :: [msg_id()]}). +-opaque protocol() :: {dlx, #checkout{} | #settle{}}. +-opaque state() :: #?MODULE{}. +-export_type([state/0, protocol/0, reason/0]). + +init() -> + #?MODULE{}. + +make_checkout(RegName, NumUnsettled) -> + {dlx, #checkout{consumer = RegName, + prefetch = NumUnsettled + }}. + +make_settle(MessageIds) when is_list(MessageIds) -> + {dlx, #settle{msg_ids = MessageIds}}. + +overview(#?MODULE{consumer = undefined, + msg_bytes = MsgBytes, + msg_bytes_checkout = 0, + discards = Discards}) -> + overview0(Discards, #{}, MsgBytes, 0); +overview(#?MODULE{consumer = #dlx_consumer{checked_out = Checked}, + msg_bytes = MsgBytes, + msg_bytes_checkout = MsgBytesCheckout, + discards = Discards}) -> + overview0(Discards, Checked, MsgBytes, MsgBytesCheckout). + +overview0(Discards, Checked, MsgBytes, MsgBytesCheckout) -> + #{num_discarded => lqueue:len(Discards), + num_discard_checked_out => map_size(Checked), + discard_message_bytes => MsgBytes, + discard_checkout_message_bytes => MsgBytesCheckout}. + +stat(#?MODULE{consumer = Con, + discards = Discards, + msg_bytes = MsgBytes, + msg_bytes_checkout = MsgBytesCheckout}) -> + Num0 = lqueue:len(Discards), + Num = case Con of + undefined -> + Num0; + #dlx_consumer{checked_out = Checked} -> + Num0 + map_size(Checked) + end, + Bytes = MsgBytes + MsgBytesCheckout, + {Num, Bytes}. + +apply(#checkout{consumer = RegName, + prefetch = Prefetch}, + #?MODULE{consumer = undefined} = State0) -> + State = State0#?MODULE{consumer = #dlx_consumer{registered_name = RegName, + prefetch = Prefetch}}, + {State, ok}; +apply(#checkout{consumer = RegName, + prefetch = Prefetch}, + #?MODULE{consumer = #dlx_consumer{checked_out = CheckedOutOldConsumer}, + discards = Discards0, + msg_bytes = Bytes, + msg_bytes_checkout = BytesCheckout} = State0) -> + %% Since we allow only a single consumer, the new consumer replaces the old consumer. + %% All checked out messages to the old consumer need to be returned to the discards queue + %% such that these messages can be (eventually) re-delivered to the new consumer. + %% When inserting back into the discards queue, we respect the original order in which messages + %% were discarded. + Checked0 = maps:to_list(CheckedOutOldConsumer), + Checked1 = lists:keysort(1, Checked0), + {Discards, BytesMoved} = lists:foldr(fun({_Id, {_Reason, IdxMsg} = Msg}, {D, B}) -> + {lqueue:in_r(Msg, D), B + size_in_bytes(IdxMsg)} + end, {Discards0, 0}, Checked1), + State = State0#?MODULE{consumer = #dlx_consumer{registered_name = RegName, + prefetch = Prefetch}, + discards = Discards, + msg_bytes = Bytes + BytesMoved, + msg_bytes_checkout = BytesCheckout - BytesMoved}, + {State, ok}; +apply(#settle{msg_ids = MsgIds}, + #?MODULE{consumer = #dlx_consumer{checked_out = Checked} = C, + msg_bytes_checkout = BytesCheckout} = State0) -> + Acked = maps:with(MsgIds, Checked), + AckedRsnMsgs = maps:values(Acked), + AckedMsgs = lists:map(fun({_Reason, Msg}) -> Msg end, AckedRsnMsgs), + AckedBytes = lists:foldl(fun(Msg, Bytes) -> + Bytes + size_in_bytes(Msg) + end, 0, AckedMsgs), + Unacked = maps:without(MsgIds, Checked), + State = State0#?MODULE{consumer = C#dlx_consumer{checked_out = Unacked}, + msg_bytes_checkout = BytesCheckout - AckedBytes}, + {State, AckedMsgs}. + +%%TODO delete delivery_count header to save space? +%% It's not needed anymore. +discard(Msg, Reason, #?MODULE{discards = Discards0, + msg_bytes = MsgBytes0} = State) -> + Discards = lqueue:in({Reason, Msg}, Discards0), + MsgBytes = MsgBytes0 + size_in_bytes(Msg), + State#?MODULE{discards = Discards, + msg_bytes = MsgBytes}. + +checkout(#?MODULE{consumer = undefined, + discards = Discards} = State) -> + case lqueue:is_empty(Discards) of + true -> + ok; + false -> + rabbit_log:warning("there are dead-letter messages but no dead-letter consumer") + end, + {State, []}; +checkout(State) -> + checkout0(checkout_one(State), {[],[]}). + +checkout0({success, MsgId, {Reason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header))}, State}, {InMemMsgs, LogMsgs}) when is_integer(RaftIdx) -> + DelMsg = {RaftIdx, {Reason, MsgId, Header}}, + SendAcc = {InMemMsgs, [DelMsg|LogMsgs]}, + checkout0(checkout_one(State ), SendAcc); +checkout0({success, MsgId, {Reason, ?INDEX_MSG(Idx, ?MSG(Header, Msg))}, State}, {InMemMsgs, LogMsgs}) when is_integer(Idx) -> + DelMsg = {MsgId, {Reason, Header, Msg}}, + SendAcc = {[DelMsg|InMemMsgs], LogMsgs}, + checkout0(checkout_one(State), SendAcc); +checkout0({success, _MsgId, {_Reason, ?TUPLE(_, _)}, State}, SendAcc) -> + %% This is a prefix message which means we are recovering from a snapshot. + %% We know: + %% 1. This message was already delivered in the past, and + %% 2. The recovery Raft log ahead of this Raft command will defintely settle this message. + %% Therefore, here, we just check this message out to the consumer but do not re-deliver this message + %% so that we will end up with the correct and deterministic state once the whole recovery log replay is completed. + checkout0(checkout_one(State), SendAcc); +checkout0(#?MODULE{consumer = #dlx_consumer{registered_name = RegName}} = State, SendAcc) -> + Effects = delivery_effects(whereis(RegName), SendAcc), + {State, Effects}. + +checkout_one(#?MODULE{consumer = #dlx_consumer{checked_out = Checked, + prefetch = Prefetch}} = State) when map_size(Checked) >= Prefetch -> + State; +checkout_one(#?MODULE{consumer = #dlx_consumer{checked_out = Checked0, + next_msg_id = Next} = Con0} = State0) -> + case take_next_msg(State0) of + {{_, Msg} = ReasonMsg, State1} -> + Checked = maps:put(Next, ReasonMsg, Checked0), + State2 = State1#?MODULE{consumer = Con0#dlx_consumer{checked_out = Checked, + next_msg_id = Next + 1}}, + Bytes = size_in_bytes(Msg), + State = add_bytes_checkout(Bytes, State2), + {success, Next, ReasonMsg, State}; + empty -> + State0 + end. + +take_next_msg(#?MODULE{discards = Discards0} = State) -> + case lqueue:out(Discards0) of + {empty, _} -> + empty; + {{value, ReasonMsg}, Discards} -> + {ReasonMsg, State#?MODULE{discards = Discards}} + end. + +add_bytes_checkout(Size, #?MODULE{msg_bytes = Bytes, + msg_bytes_checkout = BytesCheckout} = State) -> + State#?MODULE{msg_bytes = Bytes - Size, + msg_bytes_checkout = BytesCheckout + Size}. + +size_in_bytes(Msg) -> + Header = rabbit_fifo:get_msg_header(Msg), + rabbit_fifo:get_header(size, Header). + +%% returns at most one delivery effect because there is only one consumer +delivery_effects(_CPid, {[], []}) -> + []; +delivery_effects(CPid, {InMemMsgs, []}) -> + [{send_msg, CPid, {dlx_delivery, lists:reverse(InMemMsgs)}, [ra_event]}]; +delivery_effects(CPid, {InMemMsgs, IdxMsgs0}) -> + IdxMsgs = lists:reverse(IdxMsgs0), + {RaftIdxs, Data} = lists:unzip(IdxMsgs), + [{log, RaftIdxs, + fun(Log) -> + Msgs0 = lists:zipwith(fun ({enqueue, _, _, Msg}, {Reason, MsgId, Header}) -> + {MsgId, {Reason, Header, Msg}} + end, Log, Data), + Msgs = case InMemMsgs of + [] -> + Msgs0; + _ -> + lists:sort(InMemMsgs ++ Msgs0) + end, + [{send_msg, CPid, {dlx_delivery, Msgs}, [ra_event]}] + end}]. + +state_enter(leader, QRef, QName, _State) -> + start_worker(QRef, QName); +state_enter(_, _, _, State) -> + terminate_worker(State). + +start_worker(QRef, QName) -> + RegName = registered_name(QName), + %% We must ensure that starting the rabbit_fifo_dlx_worker succeeds. + %% Therefore, we don't use an effect. + %% Also therefore, if starting the rabbit_fifo_dlx_worker fails, let the whole Ra server process crash + %% in which case another Ra node will become leader. + %% supervisor:start_child/2 blocks until rabbit_fifo_dlx_worker:init/1 returns (TODO check if this is correct). + %% That's okay since rabbit_fifo_dlx_worker:init/1 returns immediately by delegating + %% initial setup to handle_continue/2. + case whereis(RegName) of + undefined -> + {ok, Pid} = supervisor:start_child(rabbit_fifo_dlx_sup, [QRef, RegName]), + rabbit_log:debug("started rabbit_fifo_dlx_worker (~s ~p)", [RegName, Pid]); + Pid -> + rabbit_log:debug("rabbit_fifo_dlx_worker (~s ~p) already started", [RegName, Pid]) + end. + +terminate_worker(#?MODULE{consumer = #dlx_consumer{registered_name = RegName}}) -> + case whereis(RegName) of + undefined -> + ok; + Pid -> + %% Note that we can't return a mod_call effect here because mod_call is executed on the leader only. + ok = supervisor:terminate_child(rabbit_fifo_dlx_sup, Pid), + rabbit_log:debug("terminated rabbit_fifo_dlx_worker (~s ~p)", [RegName, Pid]) + end; +terminate_worker(_) -> + ok. + +%% TODO consider not registering the worker name at all +%% because if there is a new worker process, it will always subscribe and tell us its new pid +registered_name(QName) when is_atom(QName) -> + list_to_atom(atom_to_list(QName) ++ "_dlx"). + +consumer_pid(#?MODULE{consumer = #dlx_consumer{registered_name = Name}}) -> + whereis(Name); +consumer_pid(_) -> + undefined. + +%% called when switching from at-least-once to at-most-once +cleanup(#?MODULE{consumer = Consumer, + discards = Discards} = State) -> + terminate_worker(State), + %% Return messages in the order they got discarded originally + %% for the final at-most-once dead-lettering. + CheckedReasonMsgs = case Consumer of + #dlx_consumer{checked_out = Checked} when is_map(Checked) -> + L0 = maps:to_list(Checked), + L1 = lists:keysort(1, L0), + {_, L2} = lists:unzip(L1), + L2; + _ -> + [] + end, + DiscardReasonMsgs = lqueue:to_list(Discards), + CheckedReasonMsgs ++ DiscardReasonMsgs. + +purge(#?MODULE{consumer = Con0, + discards = Discards} = State0) -> + {Con, CheckedMsgs} = case Con0 of + #dlx_consumer{checked_out = Checked} when is_map(Checked) -> + L = maps:to_list(Checked), + {_, CheckedReasonMsgs} = lists:unzip(L), + {_, Msgs} = lists:unzip(CheckedReasonMsgs), + C = Con0#dlx_consumer{checked_out = #{}}, + {C, Msgs}; + _ -> + {Con0, []} + end, + DiscardReasonMsgs = lqueue:to_list(Discards), + {_, DiscardMsgs} = lists:unzip(DiscardReasonMsgs), + PurgedMsgs = CheckedMsgs ++ DiscardMsgs, + State = State0#?MODULE{consumer = Con, + discards = lqueue:new(), + msg_bytes = 0, + msg_bytes_checkout = 0 + }, + {State, PurgedMsgs}. + +%% TODO Consider alternative to not dehydrate at all +%% by putting messages to disk before enqueueing them in discards queue. +dehydrate(#?MODULE{discards = Discards, + consumer = Con} = State) -> + State#?MODULE{discards = dehydrate_messages(Discards), + consumer = dehydrate_consumer(Con)}. + +dehydrate_messages(Discards) -> + L0 = lqueue:to_list(Discards), + L1 = lists:map(fun({_Reason, Msg}) -> + {?NIL, rabbit_fifo:dehydrate_message(Msg)} + end, L0), + lqueue:from_list(L1). + +dehydrate_consumer(#dlx_consumer{checked_out = Checked0} = Con) -> + Checked = maps:map(fun (_, {_, Msg}) -> + {?NIL, rabbit_fifo:dehydrate_message(Msg)} + end, Checked0), + Con#dlx_consumer{checked_out = Checked}; +dehydrate_consumer(undefined) -> + undefined. + +normalize(#?MODULE{discards = Discards} = State) -> + State#?MODULE{discards = lqueue:from_list(lqueue:to_list(Discards))}. diff --git a/deps/rabbit/src/rabbit_fifo_dlx.hrl b/deps/rabbit/src/rabbit_fifo_dlx.hrl new file mode 100644 index 0000000000..5d8c023f9e --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx.hrl @@ -0,0 +1,30 @@ +-define(NIL, []). + +%% At-least-once dead-lettering does not support reason 'maxlen'. +%% Reason of prefix messages is [] because the message will not be +%% actually delivered and storing 2 bytes in the persisted snapshot +%% is less than the reason atom. +-type reason() :: 'expired' | 'rejected' | delivery_limit | ?NIL. + +% See snapshot scenarios in rabbit_fifo_prop_SUITE. Add dlx dehydrate tests. +-record(dlx_consumer,{ + %% We don't require a consumer tag because a consumer tag is a means to distinguish + %% multiple consumers in the same channel. The rabbit_fifo_dlx_worker channel like process however + %% creates only a single consumer to this quorum queue's discards queue. + registered_name :: atom(), + prefetch :: non_neg_integer(), + checked_out = #{} :: #{msg_id() => {reason(), indexed_msg()}}, + next_msg_id = 0 :: msg_id() % part of snapshot data + % total number of checked out messages - ever + % incremented for each delivery + % delivery_count = 0 :: non_neg_integer(), + % status = up :: up | suspected_down | cancelled + }). + +-record(rabbit_fifo_dlx,{ + consumer = undefined :: #dlx_consumer{} | undefined, + %% Queue of dead-lettered messages. + discards = lqueue:new() :: lqueue:lqueue({reason(), indexed_msg()}), + msg_bytes = 0 :: non_neg_integer(), + msg_bytes_checkout = 0 :: non_neg_integer() + }). diff --git a/deps/rabbit/src/rabbit_fifo_dlx_client.erl b/deps/rabbit/src/rabbit_fifo_dlx_client.erl new file mode 100644 index 0000000000..4b9733b769 --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_client.erl @@ -0,0 +1,93 @@ +-module(rabbit_fifo_dlx_client). + +-export([checkout/4, settle/2, handle_ra_event/3, + overview/1]). + +-record(state,{ + queue_resource :: rabbit_tyes:r(queue), + leader :: ra:server_id(), + last_msg_id :: non_neg_integer | -1 + }). +-opaque state() :: #state{}. +-export_type([state/0]). + +checkout(RegName, QResource, Leader, NumUnsettled) -> + Cmd = rabbit_fifo_dlx:make_checkout(RegName, NumUnsettled), + State = #state{queue_resource = QResource, + leader = Leader, + last_msg_id = -1}, + process_command(Cmd, State, 5). + +settle(MsgIds, State) when is_list(MsgIds) -> + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + %%TODO use pipeline_command without correlation ID, i.e. without notification + process_command(Cmd, State, 2). + +process_command(_Cmd, _State, 0) -> + {error, ra_command_failed}; +process_command(Cmd, #state{leader = Leader} = State, Tries) -> + case ra:process_command(Leader, Cmd, 60_000) of + {ok, ok, Leader} -> + {ok, State#state{leader = Leader}}; + {ok, ok, L} -> + rabbit_log:warning("Failed to process command ~p on quorum queue leader ~p because actual leader is ~p.", + [Cmd, Leader, L]), + {error, ra_command_failed}; + Err -> + rabbit_log:warning("Failed to process command ~p on quorum queue leader ~p: ~p~n" + "Trying ~b more time(s)...", + [Cmd, Leader, Err, Tries]), + process_command(Cmd, State, Tries - 1) + end. + +handle_ra_event(Leader, {machine, {dlx_delivery, _} = Del}, #state{leader = Leader} = State) -> + handle_delivery(Del, State); +handle_ra_event(_From, Evt, State) -> + rabbit_log:warning("~s received unknown ra event: ~p", [?MODULE, Evt]), + {ok, State, []}. + +handle_delivery({dlx_delivery, [{FstId, _} | _] = IdMsgs}, + #state{queue_resource = QRes, + last_msg_id = Prev} = State0) -> + %% format as a deliver action + {LastId, _} = lists:last(IdMsgs), + Del = {deliver, transform_msgs(QRes, IdMsgs)}, + case Prev of + Prev when FstId =:= Prev+1 -> + %% expected message ID(s) got delivered + State = State0#state{last_msg_id = LastId}, + {ok, State, [Del]}; + Prev when FstId > Prev+1 -> + %% messages ID(s) are missing, therefore fetch all checked-out discarded messages + %% TODO implement as done in + %% https://github.com/rabbitmq/rabbitmq-server/blob/b4eb5e2cfd7f85a1681617dc489dd347fa9aac72/deps/rabbit/src/rabbit_fifo_client.erl#L732-L744 + %% A: not needed because of local guarantees, let it crash + exit(not_implemented); + Prev when FstId =< Prev -> + rabbit_log:debug("dropping messages with duplicate IDs (~b to ~b) consumed from ~s", + [FstId, Prev, rabbit_misc:rs(QRes)]), + case lists:dropwhile(fun({Id, _}) -> Id =< Prev end, IdMsgs) of + [] -> + {ok, State0, []}; + IdMsgs2 -> + handle_delivery({dlx_delivery, IdMsgs2}, State0) + end; + _ when FstId =:= 0 -> + % the very first delivery + % TODO We init last_msg_id with -1. So, why would we ever run into this branch? + % A: can be a leftover + rabbit_log:debug("very first delivery consumed from ~s", [rabbit_misc:rs(QRes)]), + State = State0#state{last_msg_id = 0}, + {ok, State, [Del]} + end. + +transform_msgs(QRes, Msgs) -> + lists:map( + fun({MsgId, {Reason, _MsgHeader, Msg}}) -> + {QRes, MsgId, Msg, Reason} + end, Msgs). + +overview(#state{leader = Leader, + last_msg_id = LastMsgId}) -> + #{leader => Leader, + last_msg_id => LastMsgId}. diff --git a/deps/rabbit/src/rabbit_fifo_dlx_sup.erl b/deps/rabbit/src/rabbit_fifo_dlx_sup.erl new file mode 100644 index 0000000000..29043eec3f --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_sup.erl @@ -0,0 +1,37 @@ +-module(rabbit_fifo_dlx_sup). + +-behaviour(supervisor). + +-rabbit_boot_step({?MODULE, + [{description, "supervisor of quorum queue dead-letter workers"}, + {mfa, {rabbit_sup, start_supervisor_child, [?MODULE]}}, + {requires, kernel_ready}, + {enables, core_initialized}]}). + +%% supervisor callback +-export([init/1]). +%% client API +-export([start_link/0]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + FeatureFlag = quorum_queue, + %%TODO rabbit_feature_flags:is_enabled(FeatureFlag) ? + case rabbit_ff_registry:is_enabled(FeatureFlag) of + true -> + SupFlags = #{strategy => simple_one_for_one, + intensity => 1, + period => 5}, + Worker = rabbit_fifo_dlx_worker, + ChildSpec = #{id => Worker, + start => {Worker, start_link, []}, + type => worker, + modules => [Worker]}, + {ok, {SupFlags, [ChildSpec]}}; + false -> + rabbit_log:info("not starting supervisor ~s because feature flag ~s is disabled", + [?MODULE, FeatureFlag]), + ignore + end. diff --git a/deps/rabbit/src/rabbit_fifo_dlx_worker.erl b/deps/rabbit/src/rabbit_fifo_dlx_worker.erl new file mode 100644 index 0000000000..89c53533dc --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_worker.erl @@ -0,0 +1,571 @@ +%% This module consumes from a single quroum queue's discards queue (containing dead-letttered messages) +%% and forwards the DLX messages at least once to every target queue. +%% +%% Some parts of this module resemble the channel process in the sense that it needs to keep track what messages +%% are consumed but not acked yet and what messages are published but not confirmed yet. +%% Compared to the channel process, this module is protocol independent since it doesn't deal with AMQP clients. +%% +%% This module consumes directly from the rabbit_fifo_dlx_client bypassing the rabbit_queue_type interface, +%% but publishes via the rabbit_queue_type interface. +%% While consuming via rabbit_queue_type interface would have worked in practice (by using a special consumer argument, +%% e.g. {<<"x-internal-queue">>, longstr, <<"discards">>} ) using the rabbit_fifo_dlx_client directly provides +%% separation of concerns making things much easier to test, to debug, and to understand. + +-module(rabbit_fifo_dlx_worker). + +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + +-behaviour(gen_server2). + +-export([start_link/2]). +%% gen_server2 callbacks +-export([init/1, terminate/2, handle_continue/2, + handle_cast/2, handle_call/3, handle_info/2, + code_change/3, format_status/2]). + +%%TODO make configurable or leave at 0 which means 2000 as in +%% https://github.com/rabbitmq/rabbitmq-server/blob/1e7df8c436174735b1d167673afd3f1642da5cdc/deps/rabbit/src/rabbit_quorum_queue.erl#L726-L729 +-define(CONSUMER_PREFETCH_COUNT, 100). +-define(HIBERNATE_AFTER, 180_000). +%% If no publisher confirm was received for at least SETTLE_TIMEOUT, message will be redelivered. +%% To prevent duplicates in the target queue and to ensure message will eventually be acked to the source queue, +%% set this value higher than the maximum time it takes for a queue to settle a message. +-define(SETTLE_TIMEOUT, 120_000). + +-record(pending, { + %% consumed_msg_id is not to be confused with consumer delivery tag. + %% The latter represents a means for AMQP clients to (multi-)ack to a channel process. + %% However, queues are not aware of delivery tags. + %% This rabbit_fifo_dlx_worker does not have the concept of delivery tags because it settles (acks) + %% message IDs directly back to the queue (and there is no AMQP consumer). + consumed_msg_id :: non_neg_integer(), + content :: rabbit_types:decoded_content(), + %% TODO Reason is already stored in first x-death header of #content.properties.#'P_basic'.headers + %% So, we could remove this convenience field and lookup the 1st header when redelivering. + reason :: rabbit_fifo_dlx:reason(), + %% + %%TODO instead of using 'unsettled' and 'settled' fields, use rabbit_confirms because it handles many to one logic + %% in a generic way. Its API might need to be modified though if it is targeted only towards channel. + %% + %% target queues for which publisher confirm has not been received yet + unsettled = [] :: [rabbit_amqqueue:name()], + %% target queues for which publisher confirm was received + settled = [] :: [rabbit_amqqueue:name()], + %% Number of times the message was published (i.e. rabbit_queue_type:deliver/3 invoked). + %% Can be 0 if the message was never published (for example no route exists). + publish_count = 0 :: non_neg_integer(), + %% Epoch time in milliseconds when the message was last published (i.e. rabbit_queue_type:deliver/3 invoked). + %% It can be 'undefined' if the message was never published (for example no route exists). + last_published_at :: undefined | integer(), + %% Epoch time in milliseconds when the message was consumed from the source quorum queue. + %% This value never changes. + %% It's mainly informational and meant for debugging to understand for how long the message + %% is sitting around without having received all publisher confirms. + consumed_at :: integer() + }). + +-record(state, { + registered_name :: atom(), + %% There is one rabbit_fifo_dlx_worker per source quorum queue + %% (if dead-letter-strategy at-least-once is used). + queue_ref :: rabbit_amqqueue:name(), + %% configured (x-)dead-letter-exchange of source queue + exchange_ref, + %% configured (x-)dead-letter-routing-key of source queue + routing_key, + dlx_client_state :: rabbit_fifo_dlx_client:state(), + queue_type_state :: rabbit_queue_type:state(), + %% Consumed messages for which we have not received all publisher confirms yet. + %% Therefore, they have not been ACKed yet to the consumer queue. + %% This buffer contains at most CONSUMER_PREFETCH_COUNT pending messages at any given point in time. + pendings = #{} :: #{OutSeq :: non_neg_integer() => #pending{}}, + %% next publisher confirm delivery tag sequence number + next_out_seq = 1, + %% Timer firing every SETTLE_TIMEOUT milliseconds + %% redelivering messages for which not all publisher confirms were received. + %% If there are no pending messages, this timer will eventually be cancelled to allow + %% this worker to hibernate. + timer :: undefined | reference() + }). + +% -type state() :: #state{}. + +%%TODO add metrics like global counters for messages routed, delivered, etc. + +start_link(QRef, RegName) -> + gen_server:start_link({local, RegName}, + ?MODULE, {QRef, RegName}, + [{hibernate_after, ?HIBERNATE_AFTER}]). + +-spec init({rabbit_amqqueue:name(), atom()}) -> {ok, undefined, {continue, {rabbit_amqqueue:name(), atom()}}}. +init(Arg) -> + {ok, undefined, {continue, Arg}}. + +handle_continue({QRef, RegName}, undefined) -> + State = lookup_topology(#state{queue_ref = QRef}), + {ok, Q} = rabbit_amqqueue:lookup(QRef), + {ClusterName, _MaybeOldLeaderNode} = amqqueue:get_pid(Q), + {ok, ConsumerState} = rabbit_fifo_dlx_client:checkout(RegName, + QRef, + {ClusterName, node()}, + ?CONSUMER_PREFETCH_COUNT), + {noreply, State#state{registered_name = RegName, + dlx_client_state = ConsumerState, + queue_type_state = rabbit_queue_type:init()}}. + +terminate(_Reason, _State) -> + %%TODO cancel timer? + ok. + +handle_call(Request, From, State) -> + rabbit_log:warning("~s received unhandled call from ~p: ~p", [?MODULE, From, Request]), + {noreply, State}. + +handle_cast({queue_event, QRef, {_From, {machine, lookup_topology}}}, + #state{queue_ref = QRef} = State0) -> + State = lookup_topology(State0), + redeliver_and_ack(State); +handle_cast({queue_event, QRef, {From, Evt}}, + #state{queue_ref = QRef, + dlx_client_state = DlxState0} = State0) -> + %% received dead-letter messsage from source queue + % rabbit_log:debug("~s received queue event: ~p", [rabbit_misc:rs(QRef), E]), + {ok, DlxState, Actions} = rabbit_fifo_dlx_client:handle_ra_event(From, Evt, DlxState0), + State1 = State0#state{dlx_client_state = DlxState}, + State = handle_queue_actions(Actions, State1), + {noreply, State}; +handle_cast({queue_event, QRef, Evt}, + #state{queue_type_state = QTypeState0} = State0) -> + %% received e.g. confirm from target queue + case rabbit_queue_type:handle_event(QRef, Evt, QTypeState0) of + {ok, QTypeState1, Actions} -> + State1 = State0#state{queue_type_state = QTypeState1}, + State = handle_queue_actions(Actions, State1), + {noreply, State}; + %% TODO handle as done in + %% https://github.com/rabbitmq/rabbitmq-server/blob/9cf18e83f279408e20430b55428a2b19156c90d7/deps/rabbit/src/rabbit_channel.erl#L771-L783 + eol -> + {noreply, State0}; + {protocol_error, _Type, _Reason, _ReasonArgs} -> + {noreply, State0} + end; +handle_cast(settle_timeout, State0) -> + State = State0#state{timer = undefined}, + redeliver_and_ack(State); +handle_cast(Request, State) -> + rabbit_log:warning("~s received unhandled cast ~p", [?MODULE, Request]), + {noreply, State}. + +redeliver_and_ack(State0) -> + State1 = redeliver_messsages(State0), + %% Routes could have been changed dynamically. + %% If a publisher confirm timed out for a target queue to which we now don't route anymore, ack the message. + State2 = maybe_ack(State1), + State = maybe_set_timer(State2), + {noreply, State}. + +%%TODO monitor source quorum queue upon init / handle_continue and terminate ourself if source quorum queue is DOWN +%% since new leader will re-create a worker +handle_info({'DOWN', _MRef, process, QPid, Reason}, + #state{queue_type_state = QTypeState0} = State0) -> + %% received from target classic queue + State = case rabbit_queue_type:handle_down(QPid, Reason, QTypeState0) of + {ok, QTypeState, Actions} -> + State1 = State0#state{queue_type_state = QTypeState}, + handle_queue_actions(Actions, State1); + {eol, QTypeState1, QRef} -> + QTypeState = rabbit_queue_type:remove(QRef, QTypeState1), + State0#state{queue_type_state = QTypeState} + end, + {noreply, State}; +handle_info(Info, State) -> + rabbit_log:warning("~s received unhandled info ~p", [?MODULE, Info]), + {noreply, State}. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +lookup_topology(#state{queue_ref = {resource, Vhost, queue, _} = QRef} = State) -> + {ok, Q} = rabbit_amqqueue:lookup(QRef), + DLRKey = rabbit_queue_type_util:args_policy_lookup(<<"dead-letter-routing-key">>, fun(_Pol, QArg) -> QArg end, Q), + DLX = rabbit_queue_type_util:args_policy_lookup(<<"dead-letter-exchange">>, fun(_Pol, QArg) -> QArg end, Q), + DLXRef = rabbit_misc:r(Vhost, exchange, DLX), + State#state{exchange_ref = DLXRef, + routing_key = DLRKey}. + +%% https://github.com/rabbitmq/rabbitmq-server/blob/9cf18e83f279408e20430b55428a2b19156c90d7/deps/rabbit/src/rabbit_channel.erl#L2855-L2888 +handle_queue_actions(Actions, State0) -> + lists:foldl( + fun ({deliver, Msgs}, S0) -> + S1 = handle_deliver(Msgs, S0), + maybe_set_timer(S1); + ({settled, QRef, MsgSeqs}, S0) -> + S1 = handle_settled(QRef, MsgSeqs, S0), + S2 = maybe_ack(S1), + maybe_cancel_timer(S2); + ({rejected, QRef, MsgSeqNos}, S0) -> + rabbit_log:debug("Ignoring rejected messages ~p from ~s", [MsgSeqNos, rabbit_misc:rs(QRef)]), + S0; + ({queue_down, QRef}, S0) -> + %% target classic queue is down, but not deleted + rabbit_log:debug("Ignoring DOWN from ~s", [rabbit_misc:rs(QRef)]), + S0 + end, State0, Actions). + +handle_deliver(Msgs, #state{queue_ref = QRef} = State) when is_list(Msgs) -> + DLX = lookup_dlx(State), + lists:foldl(fun({_QRef, MsgId, Msg, Reason}, S) -> + forward(Msg, MsgId, QRef, DLX, Reason, S) + end, State, Msgs). + +lookup_dlx(#state{exchange_ref = DLXRef, + queue_ref = QRef}) -> + case rabbit_exchange:lookup(DLXRef) of + {error, not_found} -> + rabbit_log:warning("Cannot forward any dead-letter messages from source quorum ~s because its configured " + "dead-letter-exchange ~s does not exist. " + "Either create the configured dead-letter-exchange or re-configure " + "the dead-letter-exchange policy for the source quorum queue to prevent " + "dead-lettered messages from piling up in the source quorum queue.", + [rabbit_misc:rs(QRef), rabbit_misc:rs(DLXRef)]), + not_found; + {ok, X} -> + X + end. + +forward(ConsumedMsg, ConsumedMsgId, ConsumedQRef, DLX, Reason, + #state{next_out_seq = OutSeq, + pendings = Pendings, + exchange_ref = DLXRef, + routing_key = RKey} = State0) -> + #basic_message{content = Content, routing_keys = RKeys} = Msg = + rabbit_dead_letter:make_msg(ConsumedMsg, Reason, DLXRef, RKey, ConsumedQRef), + %% Field 'mandatory' is set to false because our module checks on its own whether the message is routable. + Delivery = rabbit_basic:delivery(_Mandatory = false, _Confirm = true, Msg, OutSeq), + TargetQs = case DLX of + not_found -> + []; + _ -> + RouteToQs = rabbit_exchange:route(DLX, Delivery), + case rabbit_dead_letter:detect_cycles(Reason, Msg, RouteToQs) of + {[], []} -> + rabbit_log:warning("Cannot deliver message with sequence number ~b " + "(for consumed message sequence number ~b) " + "because no queue is bound to dead-letter ~s with routing keys ~p.", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(DLXRef), RKeys]), + []; + {Qs, []} -> + %% the "normal" case, i.e. no dead-letter-topology misconfiguration + Qs; + {[], Cycles} -> + %%TODO introduce structured logging in rabbit_log by using type logger:report + rabbit_log:warning("Cannot route to any queues. Detected dead-letter queue cycles. " + "Fix the dead-letter routing topology to prevent dead-letter messages from " + "piling up in source quorum queue. " + "outgoing_sequene_number=~b " + "consumed_message_sequence_number=~b " + "consumed_queue=~s " + "dead_letter_exchange=~s " + "effective_dead_letter_routing_keys=~p " + "routed_to_queues=~s " + "dead_letter_queue_cycles=~p", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(ConsumedQRef), + rabbit_misc:rs(DLXRef), RKeys, strings(RouteToQs), Cycles]), + []; + {Qs, Cycles} -> + rabbit_log:warning("Detected dead-letter queue cycles. " + "Fix the dead-letter routing topology. " + "outgoing_sequene_number=~b " + "consumed_message_sequence_number=~b " + "consumed_queue=~s " + "dead_letter_exchange=~s " + "effective_dead_letter_routing_keys=~p " + "routed_to_queues_desired=~s " + "routed_to_queues_effective=~s " + "dead_letter_queue_cycles=~p", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(ConsumedQRef), + rabbit_misc:rs(DLXRef), RKeys, strings(RouteToQs), strings(Qs), Cycles]), + %% Ignore the target queues resulting in cycles. + %% We decide it's good enough to deliver to only routable target queues. + Qs + end + end, + Now = os:system_time(millisecond), + State1 = State0#state{next_out_seq = OutSeq + 1}, + Pend0 = #pending{ + consumed_msg_id = ConsumedMsgId, + consumed_at = Now, + content = Content, + reason = Reason + }, + case TargetQs of + [] -> + %% We can't deliver this message since there is no target queue we can route to. + %% Under no circumstances should we drop a message with dead-letter-strategy at-least-once. + %% We buffer this message and retry to send every SETTLE_TIMEOUT milliseonds + %% (until the user has fixed the dead-letter routing topology). + State1#state{pendings = maps:put(OutSeq, Pend0, Pendings)}; + _ -> + Pend = Pend0#pending{publish_count = 1, + last_published_at = Now, + unsettled = TargetQs}, + State = State1#state{pendings = maps:put(OutSeq, Pend, Pendings)}, + deliver_to_queues(Delivery, TargetQs, State) + end. + +deliver_to_queues(Delivery, RouteToQNames, #state{queue_type_state = QTypeState0} = State0) -> + Qs = rabbit_amqqueue:lookup(RouteToQNames), + {ok, QTypeState1, Actions} = rabbit_queue_type:deliver(Qs, Delivery, QTypeState0), + State = State0#state{queue_type_state = QTypeState1}, + handle_queue_actions(Actions, State). + +handle_settled(QRef, MsgSeqs, #state{pendings = Pendings0} = State) -> + Pendings = lists:foldl(fun (MsgSeq, P0) -> + handle_settled0(QRef, MsgSeq, P0) + end, Pendings0, MsgSeqs), + State#state{pendings = Pendings}. + +handle_settled0(QRef, MsgSeq, Pendings) -> + case maps:find(MsgSeq, Pendings) of + {ok, #pending{unsettled = Unset0, settled = Set0} = Pend0} -> + Unset = lists:delete(QRef, Unset0), + Set = [QRef | Set0], + Pend = Pend0#pending{unsettled = Unset, settled = Set}, + maps:update(MsgSeq, Pend, Pendings); + error -> + rabbit_log:warning("Ignoring publisher confirm for sequence number ~b " + "from target dead letter ~s after settle timeout of ~bms. " + "Troubleshoot why that queue confirms so slowly.", + [MsgSeq, rabbit_misc:rs(QRef), ?SETTLE_TIMEOUT]), + Pendings + end. + +maybe_ack(#state{pendings = Pendings0, + dlx_client_state = DlxState0} = State) -> + Settled = maps:filter(fun(_OutSeq, #pending{unsettled = [], settled = [_|_]}) -> + %% Ack because there is at least one target queue and all + %% target queues settled (i.e. combining publisher confirm + %% and mandatory flag semantics). + true; + (_, _) -> + false + end, Pendings0), + case maps:size(Settled) of + 0 -> + %% nothing to ack + State; + _ -> + Ids = lists:map(fun(#pending{consumed_msg_id = Id}) -> Id end, maps:values(Settled)), + case rabbit_fifo_dlx_client:settle(Ids, DlxState0) of + {ok, DlxState} -> + SettledOutSeqs = maps:keys(Settled), + Pendings = maps:without(SettledOutSeqs, Pendings0), + State#state{pendings = Pendings, + dlx_client_state = DlxState}; + {error, _Reason} -> + %% Failed to ack. Ack will be retried in the next maybe_ack/1 + State + end + end. + +%% Re-deliver messages that timed out waiting on publisher confirm and +%% messages that got never sent due to routing topology misconfiguration. +redeliver_messsages(#state{pendings = Pendings} = State) -> + case lookup_dlx(State) of + not_found -> + %% Configured dead-letter-exchange does (still) not exist. + %% Warning got already logged. + %% Keep the same Pendings in our state until user creates or re-configures the dead-letter-exchange. + State; + DLX -> + Now = os:system_time(millisecond), + maps:fold(fun(OutSeq, #pending{last_published_at = LastPub} = Pend, S0) + when LastPub + ?SETTLE_TIMEOUT =< Now -> + %% Publisher confirm timed out. + redeliver(Pend, DLX, OutSeq, S0); + (OutSeq, #pending{last_published_at = undefined} = Pend, S0) -> + %% Message was never published due to dead-letter routing topology misconfiguration. + redeliver(Pend, DLX, OutSeq, S0); + (_OutSeq, _Pending, S) -> + %% Publisher confirm did not time out. + S + end, State, Pendings) + end. + +redeliver(#pending{content = Content} = Pend, DLX, OldOutSeq, + #state{routing_key = undefined} = State) -> + %% No dead-letter-routing-key defined for source quorum queue. + %% Therefore use all of messages's original routing keys (which can include CC and BCC recipients). + %% This complies with the behaviour of the rabbit_dead_letter module. + %% We stored these original routing keys in the 1st (i.e. most recent) x-death entry. + #content{properties = #'P_basic'{headers = Headers}} = + rabbit_binary_parser:ensure_content_decoded(Content), + {array, [{table, MostRecentDeath}|_]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + {<<"routing-keys">>, array, Routes0} = lists:keyfind(<<"routing-keys">>, 1, MostRecentDeath), + Routes = [Route || {longstr, Route} <- Routes0], + redeliver0(Pend, DLX, Routes, OldOutSeq, State); +redeliver(Pend, DLX, OldOutSeq, #state{routing_key = DLRKey} = State) -> + redeliver0(Pend, DLX, [DLRKey], OldOutSeq, State). + +%% Quorum queues maintain their own Raft sequene number mapping to the message sequence number (= Raft correlation ID). +%% So, they would just send us a 'settled' queue action containing the correct message sequence number. +%% +%% Classic queues however maintain their state by mapping the message sequence number to pending and confirmed queues. +%% While re-using the same message sequence number could work there as well, it just gets unnecssary complicated when +%% different target queues settle two separate deliveries referring to the same message sequence number (and same basic message). +%% +%% Therefore, to keep things simple, create a brand new delivery, store it in our state and forget about the old delivery and +%% sequence number. +%% +%% If a sequene number gets settled after SETTLE_TIMEOUT, we can't map it anymore to the #pending{}. Hence, we ignore it. +%% +%% This can lead to issues when SETTLE_TIMEOUT is too low and time to settle takes too long. +%% For example, if SETTLE_TIMEOUT is set to only 10 seconds, but settling a message takes always longer than 10 seconds +%% (e.g. due to extremly slow hypervisor disks that ran out of credit), we will re-deliver the same message all over again +%% leading to many duplicates in the target queue without ever acking the message back to the source discards queue. +%% +%% Therefore, set SETTLE_TIMEOUT reasonably high (e.g. 2 minutes). +%% +%% TODO do not log per message? +redeliver0(#pending{consumed_msg_id = ConsumedMsgId, + content = Content, + unsettled = Unsettled, + settled = Settled, + publish_count = PublishCount, + reason = Reason} = Pend0, + DLX, DLRKeys, OldOutSeq, + #state{next_out_seq = OutSeq, + queue_ref = QRef, + pendings = Pendings0, + exchange_ref = DLXRef} = State0) when is_list(DLRKeys) -> + BasicMsg = #basic_message{exchange_name = DLXRef, + routing_keys = DLRKeys, + %% BCC Header was already stripped previously + content = Content, + id = rabbit_guid:gen(), + is_persistent = rabbit_basic:is_message_persistent(Content) + }, + %% Field 'mandatory' is set to false because our module checks on its own whether the message is routable. + Delivery = rabbit_basic:delivery(_Mandatory = false, _Confirm = true, BasicMsg, OutSeq), + RouteToQs0 = rabbit_exchange:route(DLX, Delivery), + %% Do not re-deliver to queues for which we already received a publisher confirm. + RouteToQs1 = RouteToQs0 -- Settled, + {RouteToQs, Cycles} = rabbit_dead_letter:detect_cycles(Reason, BasicMsg, RouteToQs1), + Prefix = io_lib:format("Message has not received required publisher confirm(s). " + "Received confirm from: [~s]. " + "Did not receive confirm from: [~s]. " + "timeout=~bms " + "message_sequence_number=~b " + "consumed_message_sequence_number=~b " + "publish_count=~b.", + [strings(Settled), strings(Unsettled), ?SETTLE_TIMEOUT, + OldOutSeq, ConsumedMsgId, PublishCount]), + case {RouteToQs, Cycles, Settled} of + {[], [], []} -> + rabbit_log:warning("~s Failed to re-deliver this message because no queue is bound " + "to dead-letter ~s with routing keys ~p.", + [Prefix, rabbit_misc:rs(DLXRef), DLRKeys]), + State0; + {[], [], [_|_]} -> + rabbit_log:debug("~s Routes changed dynamically so that this message does not need to be routed " + "to any queue anymore. This message will be acknowledged to the source ~s.", + [Prefix, rabbit_misc:rs(QRef)]), + State0; + {[], [_|_], []} -> + rabbit_log:warning("~s Failed to re-deliver this message because dead-letter queue cycles " + "got detected: ~p", + [Prefix, Cycles]), + State0; + {[], [_|_], [_|_]} -> + rabbit_log:warning("~s Dead-letter queue cycles detected: ~p. " + "This message will nevertheless be acknowledged to the source ~s " + "because it received at least one publisher confirm.", + [Prefix, Cycles, rabbit_misc:rs(QRef)]), + State0; + _ -> + case Cycles of + [] -> + rabbit_log:debug("~s Re-delivering this message to ~s", + [Prefix, strings(RouteToQs)]); + [_|_] -> + rabbit_log:warning("~s Dead-letter queue cycles detected: ~p. " + "Re-delivering this message only to ~s", + [Prefix, Cycles, strings(RouteToQs)]) + end, + Pend = Pend0#pending{publish_count = PublishCount + 1, + last_published_at = os:system_time(millisecond), + %% override 'unsettled' because topology could have changed + unsettled = RouteToQs}, + Pendings1 = maps:remove(OldOutSeq, Pendings0), + Pendings = maps:put(OutSeq, Pend, Pendings1), + State = State0#state{next_out_seq = OutSeq + 1, + pendings = Pendings}, + deliver_to_queues(Delivery, RouteToQs, State) + end. + +strings(QRefs) when is_list(QRefs) -> + L0 = lists:map(fun rabbit_misc:rs/1, QRefs), + L1 = lists:join(", ", L0), + lists:flatten(L1). + +maybe_set_timer(#state{timer = TRef} = State) when is_reference(TRef) -> + State; +maybe_set_timer(#state{timer = undefined, + pendings = Pendings} = State) when map_size(Pendings) =:= 0 -> + State; +maybe_set_timer(#state{timer = undefined} = State) -> + TRef = erlang:send_after(?SETTLE_TIMEOUT, self(), {'$gen_cast', settle_timeout}), + % rabbit_log:debug("set timer"), + State#state{timer = TRef}. + +maybe_cancel_timer(#state{timer = undefined} = State) -> + State; +maybe_cancel_timer(#state{timer = TRef, + pendings = Pendings} = State) -> + case maps:size(Pendings) of + 0 -> + erlang:cancel_timer(TRef, [{async, true}, {info, false}]), + % rabbit_log:debug("cancelled timer"), + State#state{timer = undefined}; + _ -> + State + end. + +%% Avoids large message contents being logged. +format_status(_Opt, [_PDict, #state{ + registered_name = RegisteredName, + queue_ref = QueueRef, + exchange_ref = ExchangeRef, + routing_key = RoutingKey, + dlx_client_state = DlxClientState, + queue_type_state = QueueTypeState, + pendings = Pendings, + next_out_seq = NextOutSeq, + timer = Timer + }]) -> + S = #{registered_name => RegisteredName, + queue_ref => QueueRef, + exchange_ref => ExchangeRef, + routing_key => RoutingKey, + dlx_client_state => rabbit_fifo_dlx_client:overview(DlxClientState), + queue_type_state => QueueTypeState, + pendings => maps:map(fun(_, P) -> format_pending(P) end, Pendings), + next_out_seq => NextOutSeq, + timer_is_active => Timer =/= undefined}, + [{data, [{"State", S}]}]. + +format_pending(#pending{consumed_msg_id = ConsumedMsgId, + reason = Reason, + unsettled = Unsettled, + settled = Settled, + publish_count = PublishCount, + last_published_at = LastPublishedAt, + consumed_at = ConsumedAt}) -> + #{consumed_msg_id => ConsumedMsgId, + reason => Reason, + unsettled => Unsettled, + settled => Settled, + publish_count => PublishCount, + last_published_at => LastPublishedAt, + consumed_at => ConsumedAt}. diff --git a/deps/rabbit/src/rabbit_fifo_v1.erl b/deps/rabbit/src/rabbit_fifo_v1.erl index a59a5c9250..51150b0f70 100644 --- a/deps/rabbit/src/rabbit_fifo_v1.erl +++ b/deps/rabbit/src/rabbit_fifo_v1.erl @@ -130,6 +130,8 @@ state/0, config/0]). +%% This function is never called since only rabbit_fifo_v0:init/1 is called. +%% See https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 -spec init(config()) -> state(). init(#{name := Name, queue_resource := Resource} = Conf) -> diff --git a/deps/rabbit/src/rabbit_policies.erl b/deps/rabbit/src/rabbit_policies.erl index 37a467ac75..2e8684e145 100644 --- a/deps/rabbit/src/rabbit_policies.erl +++ b/deps/rabbit/src/rabbit_policies.erl @@ -30,6 +30,7 @@ register() -> {Class, Name} <- [{policy_validator, <<"alternate-exchange">>}, {policy_validator, <<"dead-letter-exchange">>}, {policy_validator, <<"dead-letter-routing-key">>}, + {policy_validator, <<"dead-letter-strategy">>}, {policy_validator, <<"message-ttl">>}, {policy_validator, <<"expires">>}, {policy_validator, <<"max-length">>}, @@ -85,6 +86,13 @@ validate_policy0(<<"dead-letter-routing-key">>, Value) validate_policy0(<<"dead-letter-routing-key">>, Value) -> {error, "~p is not a valid dead letter routing key", [Value]}; +validate_policy0(<<"dead-letter-strategy">>, <<"at-most-once">>) -> + ok; +validate_policy0(<<"dead-letter-strategy">>, <<"at-least-once">>) -> + ok; +validate_policy0(<<"dead-letter-strategy">>, Value) -> + {error, "~p is not a valid dead letter strategy", [Value]}; + validate_policy0(<<"message-ttl">>, Value) when is_integer(Value), Value >= 0 -> ok; diff --git a/deps/rabbit/src/rabbit_quorum_queue.erl b/deps/rabbit/src/rabbit_quorum_queue.erl index a4c6d5dd5f..d95f8e8b69 100644 --- a/deps/rabbit/src/rabbit_quorum_queue.erl +++ b/deps/rabbit/src/rabbit_quorum_queue.erl @@ -71,6 +71,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). -include("amqqueue.hrl"). -type msg_id() :: non_neg_integer(). @@ -94,7 +95,9 @@ single_active_consumer_pid, single_active_consumer_ctag, messages_ram, - message_bytes_ram + message_bytes_ram, + messages_dlx, + message_bytes_dlx ]). -define(INFO_KEYS, [name, durable, auto_delete, arguments, pid, messages, messages_ready, @@ -227,18 +230,17 @@ ra_machine_config(Q) when ?is_amqqueue(Q) -> {Name, _} = amqqueue:get_pid(Q), %% take the minimum value of the policy and the queue arg if present MaxLength = args_policy_lookup(<<"max-length">>, fun min/2, Q), - %% prefer the policy defined strategy if available - Overflow = args_policy_lookup(<<"overflow">>, fun (A, _B) -> A end , Q), + OverflowBin = args_policy_lookup(<<"overflow">>, fun policyHasPrecedence/2, Q), + Overflow = overflow(OverflowBin, drop_head, QName), MaxBytes = args_policy_lookup(<<"max-length-bytes">>, fun min/2, Q), MaxMemoryLength = args_policy_lookup(<<"max-in-memory-length">>, fun min/2, Q), MaxMemoryBytes = args_policy_lookup(<<"max-in-memory-bytes">>, fun min/2, Q), DeliveryLimit = args_policy_lookup(<<"delivery-limit">>, fun min/2, Q), - Expires = args_policy_lookup(<<"expires">>, - fun (A, _B) -> A end, - Q), + Expires = args_policy_lookup(<<"expires">>, fun policyHasPrecedence/2, Q), + MsgTTL = args_policy_lookup(<<"message-ttl">>, fun min/2, Q), #{name => Name, queue_resource => QName, - dead_letter_handler => dlx_mfa(Q), + dead_letter_handler => dead_letter_handler(Q, Overflow), become_leader_handler => {?MODULE, become_leader, [QName]}, max_length => MaxLength, max_bytes => MaxBytes, @@ -246,11 +248,17 @@ ra_machine_config(Q) when ?is_amqqueue(Q) -> max_in_memory_bytes => MaxMemoryBytes, single_active_consumer_on => single_active_consumer_on(Q), delivery_limit => DeliveryLimit, - overflow_strategy => overflow(Overflow, drop_head, QName), + overflow_strategy => Overflow, created => erlang:system_time(millisecond), - expires => Expires + expires => Expires, + msg_ttl => MsgTTL }. +policyHasPrecedence(Policy, _QueueArg) -> + Policy. +queueArgHasPrecedence(_Policy, QueueArg) -> + QueueArg. + single_active_consumer_on(Q) -> QArguments = amqqueue:get_arguments(Q), case rabbit_misc:table_lookup(QArguments, <<"x-single-active-consumer">>) of @@ -293,7 +301,7 @@ become_leader(QName, Name) -> end, %% as this function is called synchronously when a ra node becomes leader %% we need to ensure there is no chance of blocking as else the ra node - %% may not be able to establish it's leadership + %% may not be able to establish its leadership spawn(fun() -> rabbit_misc:execute_mnesia_transaction( fun() -> @@ -377,19 +385,20 @@ filter_quorum_critical(Queues, ReplicaStates) -> capabilities() -> #{unsupported_policies => [ %% Classic policies - <<"message-ttl">>, <<"max-priority">>, <<"queue-mode">>, + <<"max-priority">>, <<"queue-mode">>, <<"single-active-consumer">>, <<"ha-mode">>, <<"ha-params">>, <<"ha-sync-mode">>, <<"ha-promote-on-shutdown">>, <<"ha-promote-on-failure">>, <<"queue-master-locator">>, %% Stream policies <<"max-age">>, <<"stream-max-segment-size-bytes">>, <<"queue-leader-locator">>, <<"initial-cluster-size">>], - queue_arguments => [<<"x-expires">>, <<"x-dead-letter-exchange">>, - <<"x-dead-letter-routing-key">>, <<"x-max-length">>, - <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, - <<"x-max-in-memory-bytes">>, <<"x-overflow">>, - <<"x-single-active-consumer">>, <<"x-queue-type">>, - <<"x-quorum-initial-group-size">>, <<"x-delivery-limit">>], + queue_arguments => [<<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, + <<"x-dead-letter-strategy">>, <<"x-expires">>, <<"x-max-length">>, + <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, + <<"x-max-in-memory-bytes">>, <<"x-overflow">>, + <<"x-single-active-consumer">>, <<"x-queue-type">>, + <<"x-quorum-initial-group-size">>, <<"x-delivery-limit">>, + <<"x-message-ttl">>], consumer_arguments => [<<"x-priority">>, <<"x-credit">>], server_named => false}. @@ -410,7 +419,7 @@ spawn_notify_decorators(QName, Fun, Args) -> end). handle_tick(QName, - {Name, MR, MU, M, C, MsgBytesReady, MsgBytesUnack}, + {Name, MR, MU, M, C, MsgBytesReady, MsgBytesUnack, MsgBytesDiscard}, Nodes) -> %% this makes calls to remote processes so cannot be run inside the %% ra server @@ -429,8 +438,8 @@ handle_tick(QName, {consumer_utilisation, Util}, {message_bytes_ready, MsgBytesReady}, {message_bytes_unacknowledged, MsgBytesUnack}, - {message_bytes, MsgBytesReady + MsgBytesUnack}, - {message_bytes_persistent, MsgBytesReady + MsgBytesUnack}, + {message_bytes, MsgBytesReady + MsgBytesUnack + MsgBytesDiscard}, + {message_bytes_persistent, MsgBytesReady + MsgBytesUnack + MsgBytesDiscard}, {messages_persistent, M} | infos(QName, ?STATISTICS_KEYS -- [consumers])], @@ -839,8 +848,11 @@ deliver(true, Delivery, QState0) -> rabbit_fifo_client:enqueue(Delivery#delivery.msg_seq_no, Delivery#delivery.message, QState0). -deliver(QSs, #delivery{confirm = Confirm} = Delivery0) -> - Delivery = clean_delivery(Delivery0), +deliver(QSs, #delivery{message = #basic_message{content = Content0} = Msg, + confirm = Confirm} = Delivery0) -> + %% TODO: we could also consider clearing out the message id here + Content = prepare_content(Content0), + Delivery = Delivery0#delivery{message = Msg#basic_message{content = Content}}, lists:foldl( fun({Q, stateless}, {Qs, Actions}) -> QRef = amqqueue:get_pid(Q), @@ -1253,20 +1265,46 @@ reclaim_memory(Vhost, QueueName) -> ra_log_wal:force_roll_over({?RA_WAL_NAME, Node}). %%---------------------------------------------------------------------------- -dlx_mfa(Q) -> - DLX = init_dlx(args_policy_lookup(<<"dead-letter-exchange">>, - fun res_arg/2, Q), Q), - DLXRKey = args_policy_lookup(<<"dead-letter-routing-key">>, - fun res_arg/2, Q), - {?MODULE, dead_letter_publish, [DLX, DLXRKey, amqqueue:get_name(Q)]}. - -init_dlx(undefined, _Q) -> - undefined; -init_dlx(DLX, Q) when ?is_amqqueue(Q) -> +dead_letter_handler(Q, Overflow) -> + %% Queue arg continues to take precedence to not break existing configurations + %% for queues upgraded from =v3.10 + Exchange = args_policy_lookup(<<"dead-letter-exchange">>, fun queueArgHasPrecedence/2, Q), + RoutingKey = args_policy_lookup(<<"dead-letter-routing-key">>, fun queueArgHasPrecedence/2, Q), + %% Policy takes precedence because it's a new key introduced in v3.10 and we want + %% users to use policies instead of queue args allowing dynamic reconfiguration. + %% TODO change to queueArgHasPrecedence for dead-letter-strategy + Strategy = args_policy_lookup(<<"dead-letter-strategy">>, fun policyHasPrecedence/2, Q), QName = amqqueue:get_name(Q), - rabbit_misc:r(QName, exchange, DLX). + dlh(Exchange, RoutingKey, Strategy, Overflow, QName). -res_arg(_PolVal, ArgVal) -> ArgVal. +dlh(undefined, undefined, undefined, _, _) -> + undefined; +dlh(undefined, RoutingKey, undefined, _, QName) -> + rabbit_log:warning("Disabling dead-lettering for ~s despite configured dead-letter-routing-key '~s' " + "because dead-letter-exchange is not configured.", + [rabbit_misc:rs(QName), RoutingKey]), + undefined; +dlh(undefined, _, Strategy, _, QName) -> + rabbit_log:warning("Disabling dead-lettering for ~s despite configured dead-letter-strategy '~s' " + "because dead-letter-exchange is not configured.", + [rabbit_misc:rs(QName), Strategy]), + undefined; +dlh(_, _, <<"at-least-once">>, reject_publish, _) -> + at_least_once; +dlh(Exchange, RoutingKey, <<"at-least-once">>, drop_head, QName) -> + rabbit_log:warning("Falling back to dead-letter-strategy at-most-once for ~s " + "because configured dead-letter-strategy at-least-once is incompatible with " + "effective overflow strategy drop-head. To enable dead-letter-strategy " + "at-least-once, set overflow strategy to reject-publish.", + [rabbit_misc:rs(QName)]), + dlh_at_most_once(Exchange, RoutingKey, QName); +dlh(Exchange, RoutingKey, _, _, QName) -> + dlh_at_most_once(Exchange, RoutingKey, QName). + +dlh_at_most_once(Exchange, RoutingKey, QName) -> + DLX = rabbit_misc:r(QName, exchange, Exchange), + MFA = {?MODULE, dead_letter_publish, [DLX, RoutingKey, QName]}, + {at_most_once, MFA}. dead_letter_publish(undefined, _, _, _) -> ok; @@ -1438,6 +1476,28 @@ i(message_bytes_ram, Q) when ?is_amqqueue(Q) -> {timeout, _} -> 0 end; +i(messages_dlx, Q) when ?is_amqqueue(Q) -> + QPid = amqqueue:get_pid(Q), + case ra:local_query(QPid, + fun rabbit_fifo:query_stat_dlx/1) of + {ok, {_, {Num, _}}, _} -> + Num; + {error, _} -> + 0; + {timeout, _} -> + 0 + end; +i(message_bytes_dlx, Q) when ?is_amqqueue(Q) -> + QPid = amqqueue:get_pid(Q), + case ra:local_query(QPid, + fun rabbit_fifo:query_stat_dlx/1) of + {ok, {_, {_, Bytes}}, _} -> + Bytes; + {error, _} -> + 0; + {timeout, _} -> + 0 + end; i(_K, _Q) -> ''. open_files(Name) -> @@ -1582,7 +1642,7 @@ overflow(undefined, Def, _QName) -> Def; overflow(<<"reject-publish">>, _Def, _QName) -> reject_publish; overflow(<<"drop-head">>, _Def, _QName) -> drop_head; overflow(<<"reject-publish-dlx">> = V, Def, QName) -> - rabbit_log:warning("Invalid overflow strategy ~p for quorum queue: ~p", + rabbit_log:warning("Invalid overflow strategy ~p for quorum queue: ~s", [V, rabbit_misc:rs(QName)]), Def. @@ -1626,19 +1686,15 @@ notify_decorators(QName, F, A) -> end. %% remove any data that a quorum queue doesn't need -clean_delivery(#delivery{message = - #basic_message{content = Content0} = Msg} = Delivery) -> - Content = case Content0 of - #content{properties = none} -> - Content0; - #content{protocol = none} -> - Content0; - #content{properties = Props, - protocol = Proto} -> - Content0#content{properties = none, - properties_bin = Proto:encode_properties(Props)} - end, - - %% TODO: we could also consider clearing out the message id here - Delivery#delivery{message = Msg#basic_message{content = Content}}. - +prepare_content(#content{properties = none} = Content) -> + Content; +prepare_content(#content{protocol = none} = Content) -> + Content; +prepare_content(#content{properties = #'P_basic'{expiration = undefined} = Props, + protocol = Proto} = Content) -> + Content#content{properties = none, + properties_bin = Proto:encode_properties(Props)}; +prepare_content(Content) -> + %% expiration is set. Therefore, leave properties decoded so that + %% rabbit_fifo can directly parse it without having to decode again. + Content. diff --git a/deps/rabbit/src/rabbit_stream_queue.erl b/deps/rabbit/src/rabbit_stream_queue.erl index 7ae66bfd9f..1717556e0b 100644 --- a/deps/rabbit/src/rabbit_stream_queue.erl +++ b/deps/rabbit/src/rabbit_stream_queue.erl @@ -957,7 +957,9 @@ capabilities() -> <<"single-active-consumer">>, <<"delivery-limit">>, <<"ha-mode">>, <<"ha-params">>, <<"ha-sync-mode">>, <<"ha-promote-on-shutdown">>, <<"ha-promote-on-failure">>, - <<"queue-master-locator">>], + <<"queue-master-locator">>, + %% Quorum policies + <<"dead-letter-strategy">>], queue_arguments => [<<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, <<"x-max-length">>, <<"x-max-length-bytes">>, <<"x-single-active-consumer">>, <<"x-queue-type">>, diff --git a/deps/rabbit/test/dead_lettering_SUITE.erl b/deps/rabbit/test/dead_lettering_SUITE.erl index 4c7e7968f9..ebd04bf720 100644 --- a/deps/rabbit/test/dead_lettering_SUITE.erl +++ b/deps/rabbit/test/dead_lettering_SUITE.erl @@ -93,7 +93,9 @@ init_per_group(quorum_queue, Config) -> ok -> rabbit_ct_helpers:set_config( Config, - [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + %%TODO add at-least-once tests + {<<"x-dead-letter-strategy">>, longstr, <<"at-most-once">>}]}, {queue_durable, true}]); Skip -> Skip @@ -708,7 +710,9 @@ dead_letter_policy(Config) -> {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), QName = ?config(queue_name, Config), DLXQName = ?config(queue_name_dlx, Config), - Args = ?config(queue_args, Config), + Args0 = ?config(queue_args, Config), + %% declaring a quorum queue with x-dead-letter-strategy without defining a DLX will fail + Args = proplists:delete(<<"x-dead-letter-strategy">>, Args0), Durable = ?config(queue_durable, Config), DLXExchange = ?config(dlx_exchange, Config), diff --git a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl index a22b0a286e..d4a061c1d5 100644 --- a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl +++ b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl @@ -10,6 +10,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("rabbit/src/rabbit_fifo.hrl"). +-include_lib("rabbit/src/rabbit_fifo_dlx.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + +-define(record_info(T,R),lists:zip(record_info(fields,T),tl(tuple_to_list(R)))). %%%=================================================================== %%% Common Test callbacks @@ -25,7 +30,6 @@ all_tests() -> [ test_run_log, snapshots, - scenario1, scenario2, scenario3, scenario4, @@ -69,7 +73,16 @@ all_tests() -> single_active_ordering_01, single_active_ordering_03, in_memory_limit, - max_length + max_length, + snapshots_dlx, + dlx_01, + dlx_02, + dlx_03, + dlx_04, + dlx_05, + dlx_06, + dlx_07, + dlx_08 % single_active_ordering_02 ]. @@ -103,32 +116,14 @@ end_per_testcase(_TestCase, _Config) -> % -type log_op() :: % {enqueue, pid(), maybe(msg_seqno()), Msg :: raw_msg()}. -scenario1(_Config) -> - C1 = {<<>>, c:pid(0,6723,1)}, - C2 = {<<0>>,c:pid(0,6723,1)}, - E = c:pid(0,6720,1), - - Commands = [ - make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,msg1), - make_enqueue(E,2,msg2), - make_checkout(C1, cancel), %% both on returns queue - make_checkout(C2, {auto,1,simple_prefetch}), - make_return(C2, [0]), %% E1 in returns, E2 with C2 - make_return(C2, [1]), %% E2 in returns E1 with C2 - make_settle(C2, [2]) %% E2 with C2 - ], - run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), - ok. - scenario2(_Config) -> C1 = {<<>>, c:pid(0,346,1)}, C2 = {<<>>,c:pid(0,379,1)}, E = c:pid(0,327,1), Commands = [make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,msg1), + make_enqueue(E,1,msg(<<"msg1">>)), make_checkout(C1, cancel), - make_enqueue(E,2,msg2), + make_enqueue(E,2,msg(<<"msg2">>)), make_checkout(C2, {auto,1,simple_prefetch}), make_settle(C1, [0]), make_settle(C2, [0]) @@ -140,10 +135,10 @@ scenario3(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,msg1), + make_enqueue(E,1,msg(<<"msg1">>)), make_return(C1, [0]), - make_enqueue(E,2,msg2), - make_enqueue(E,3,msg3), + make_enqueue(E,2,msg(<<"msg2">>)), + make_enqueue(E,3,msg(<<"msg3">>)), make_settle(C1, [1]), make_settle(C1, [2]) ], @@ -154,7 +149,7 @@ scenario4(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,msg), + make_enqueue(E,1,msg(<<"msg">>)), make_settle(C1, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), @@ -163,17 +158,17 @@ scenario4(_Config) -> scenario5(_Config) -> C1 = {<<>>, c:pid(0,505,0)}, E = c:pid(0,465,9), - Commands = [make_enqueue(E,1,<<0>>), + Commands = [make_enqueue(E,1,msg(<<0>>)), make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,2,<<>>), + make_enqueue(E,2,msg(<<>>)), make_settle(C1,[0])], run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), ok. scenario6(_Config) -> E = c:pid(0,465,9), - Commands = [make_enqueue(E,1,<<>>), %% 1 msg on queue - snap: prefix 1 - make_enqueue(E,2,<<>>) %% 1. msg on queue - snap: prefix 1 + Commands = [make_enqueue(E,1,msg(<<>>)), %% 1 msg on queue - snap: prefix 1 + make_enqueue(E,2,msg(<<>>)) %% 1. msg on queue - snap: prefix 1 ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), @@ -183,10 +178,10 @@ scenario7(_Config) -> C1 = {<<>>, c:pid(0,208,0)}, E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), + make_enqueue(E,1,msg(<<>>)), make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), make_settle(C1,[0])], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), @@ -196,8 +191,8 @@ scenario8(_Config) -> C1 = {<<>>, c:pid(0,208,0)}, E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<>>), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), make_checkout(C1, {auto,1,simple_prefetch}), % make_checkout(C1, cancel), {down, E, noconnection}, @@ -209,9 +204,9 @@ scenario8(_Config) -> scenario9(_Config) -> E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>)], + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), ok. @@ -221,7 +216,7 @@ scenario10(_Config) -> E = c:pid(0,188,0), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<>>), + make_enqueue(E,1,msg(<<>>)), make_settle(C1, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -232,10 +227,10 @@ scenario11(_Config) -> C1 = {<<>>, c:pid(0,215,0)}, E = c:pid(0,217,0), Commands = [ - make_enqueue(E,1,<<"1">>), % 1 + make_enqueue(E,1,msg(<<"1">>)), % 1 make_checkout(C1, {auto,1,simple_prefetch}), % 2 make_checkout(C1, cancel), % 3 - make_enqueue(E,2,<<"22">>), % 4 + make_enqueue(E,2,msg(<<"22">>)), % 4 make_checkout(C1, {auto,1,simple_prefetch}), % 5 make_settle(C1, [0]), % 6 make_checkout(C1, cancel) % 7 @@ -246,19 +241,19 @@ scenario11(_Config) -> scenario12(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<0>>), - make_enqueue(E,3,<<0>>)], + Commands = [make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<0>>)), + make_enqueue(E,3,msg(<<0>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 2}, Commands), ok. scenario13(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>), - make_enqueue(E,4,<<>>) + Commands = [make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), + make_enqueue(E,4,msg(<<>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 2}, Commands), @@ -266,7 +261,7 @@ scenario13(_Config) -> scenario14(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0,0>>)], + Commands = [make_enqueue(E,1,msg(<<0,0>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 1}, Commands), ok. @@ -274,8 +269,8 @@ scenario14(_Config) -> scenario14b(_Config) -> E = c:pid(0,217,0), Commands = [ - make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<0>>) + make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<0>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 1}, Commands), @@ -285,8 +280,8 @@ scenario15(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, msg1), - make_enqueue(E, 2, msg2), + make_enqueue(E, 1, msg(<<"msg1">>)), + make_enqueue(E, 2, msg(<<"msg2">>)), make_return(C1, [0]), make_return(C1, [2]), make_settle(C1, [1]) @@ -302,11 +297,11 @@ scenario16(_Config) -> E = c:pid(0,176,1), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E, 1, msg1), + make_enqueue(E, 1, msg(<<"msg1">>)), make_checkout(C2, {auto,1,simple_prefetch}), {down, C1Pid, noproc}, %% msg1 allocated to C2 make_return(C2, [0]), %% msg1 returned - make_enqueue(E, 2, <<>>), + make_enqueue(E, 2, msg(<<>>)), make_settle(C2, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -321,11 +316,11 @@ scenario17(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), make_checkout(C2, {auto,1,simple_prefetch}), {down, C1Pid, noconnection}, make_checkout(C2, cancel), - make_enqueue(E,2,<<"two">>), + make_enqueue(E,2,msg(<<"two">>)), {nodeup,rabbit@fake_node1}, %% this has no effect as was returned make_settle(C1, [0]), @@ -339,11 +334,11 @@ scenario17(_Config) -> scenario18(_Config) -> E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), - make_enqueue(E,4,<<"4">>), - make_enqueue(E,5,<<"5">>) + Commands = [make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), + make_enqueue(E,4,msg(<<"4">>)), + make_enqueue(E,5,msg(<<"5">>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, %% max_length => 3, @@ -354,10 +349,10 @@ scenario19(_Config) -> C1Pid = c:pid(0,883,1), C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), + Commands = [make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,3,<<"3">>), + make_enqueue(E,3,msg(<<"3">>)), make_settle(C1, [0, 1]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -369,15 +364,15 @@ scenario20(_Config) -> C1Pid = c:pid(0,883,1), C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<1>>), + Commands = [make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<1>>)), make_checkout(C1, {auto,2,simple_prefetch}), {down, C1Pid, noconnection}, - make_enqueue(E,3,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,4,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,5,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,6,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,7,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0>>) + make_enqueue(E,3,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,4,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,5,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,6,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,7,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 4, @@ -391,15 +386,15 @@ scenario21(_Config) -> E = c:pid(0,176,1), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), rabbit_fifo:make_discard(C1, [0]), rabbit_fifo:make_settle(C1, [1]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 1, - dead_letter_handler => {?MODULE, banana, []}}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}}, Commands), ok. @@ -408,16 +403,16 @@ scenario22(_Config) -> % C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), Commands = [ - make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), - make_enqueue(E,4,<<"4">>), - make_enqueue(E,5,<<"5">>) + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), + make_enqueue(E,4,msg(<<"4">>)), + make_enqueue(E,5,msg(<<"5">>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 1, max_length => 3, - dead_letter_handler => {?MODULE, banana, []}}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}}, Commands), ok. @@ -429,10 +424,10 @@ scenario24(_Config) -> Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), %% 1 make_checkout(C2, {auto,1,simple_prefetch}), %% 2 - make_enqueue(E,1,<<"1">>), %% 3 - make_enqueue(E,2,<<"2b">>), %% 4 - make_enqueue(E,3,<<"3">>), %% 5 - make_enqueue(E,4,<<"4">>), %% 6 + make_enqueue(E,1,msg(<<"1">>)), %% 3 + make_enqueue(E,2,msg(<<"2b">>)), %% 4 + make_enqueue(E,3,msg(<<"3">>)), %% 5 + make_enqueue(E,4,msg(<<"4">>)), %% 6 {down, E, noconnection} %% 7 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -440,7 +435,7 @@ scenario24(_Config) -> deliver_limit => undefined, max_length => 3, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -453,12 +448,12 @@ scenario25(_Config) -> E = c:pid(0,280,0), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), %% 1 - make_enqueue(E,1,<<0>>), %% 2 + make_enqueue(E,1,msg(<<0>>)), %% 2 make_checkout(C2, {auto,1,simple_prefetch}), %% 3 - make_enqueue(E,2,<<>>), %% 4 - make_enqueue(E,3,<<>>), %% 5 + make_enqueue(E,2,msg(<<>>)), %% 4 + make_enqueue(E,3,msg(<<>>)), %% 5 {down, C1Pid, noproc}, %% 6 - make_enqueue(E,4,<<>>), %% 7 + make_enqueue(E,4,msg(<<>>)), %% 7 rabbit_fifo:make_purge() %% 8 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -466,7 +461,7 @@ scenario25(_Config) -> release_cursor_interval => 0, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -477,19 +472,19 @@ scenario26(_Config) -> E1 = c:pid(0,436,0), E2 = c:pid(0,435,0), Commands = [ - make_enqueue(E1,2,<<>>), %% 1 - make_enqueue(E1,3,<<>>), %% 2 - make_enqueue(E2,1,<<>>), %% 3 - make_enqueue(E2,2,<<>>), %% 4 - make_enqueue(E1,4,<<>>), %% 5 - make_enqueue(E1,5,<<>>), %% 6 - make_enqueue(E1,6,<<>>), %% 7 - make_enqueue(E1,7,<<>>), %% 8 - make_enqueue(E1,1,<<>>), %% 9 + make_enqueue(E1,2,msg(<<>>)), %% 1 + make_enqueue(E1,3,msg(<<>>)), %% 2 + make_enqueue(E2,1,msg(<<>>)), %% 3 + make_enqueue(E2,2,msg(<<>>)), %% 4 + make_enqueue(E1,4,msg(<<>>)), %% 5 + make_enqueue(E1,5,msg(<<>>)), %% 6 + make_enqueue(E1,6,msg(<<>>)), %% 7 + make_enqueue(E1,7,msg(<<>>)), %% 8 + make_enqueue(E1,1,msg(<<>>)), %% 9 make_checkout(C1, {auto,5,simple_prefetch}), %% 1 - make_enqueue(E1,8,<<>>), %% 2 - make_enqueue(E1,9,<<>>), %% 2 - make_enqueue(E1,10,<<>>), %% 2 + make_enqueue(E1,8,msg(<<>>)), %% 2 + make_enqueue(E1,9,msg(<<>>)), %% 2 + make_enqueue(E1,10,msg(<<>>)), %% 2 {down, C1Pid, noconnection} ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -497,22 +492,22 @@ scenario26(_Config) -> deliver_limit => undefined, max_length => 8, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. scenario28(_Config) -> E = c:pid(0,151,0), - Conf = #{dead_letter_handler => {rabbit_fifo_prop_SUITE,banana,[]}, + Conf = #{dead_letter_handler => {at_most_once, {rabbit_fifo_prop_SUITE,banana,[]}}, delivery_limit => undefined, max_in_memory_bytes => undefined, max_length => 1,name => ?FUNCTION_NAME,overflow_strategy => drop_head, release_cursor_interval => 100,single_active_consumer_on => false}, Commands = [ - make_enqueue(E,2, <<>>), - make_enqueue(E,3, <<>>), - make_enqueue(E,1, <<>>) + make_enqueue(E,2,msg( <<>>)), + make_enqueue(E,3,msg( <<>>)), + make_enqueue(E,1,msg( <<>>)) ], ?assert(single_active_prop(Conf, Commands, false)), ok. @@ -525,39 +520,39 @@ scenario27(_Config) -> E = c:pid(0,151,0), E2 = c:pid(0,152,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E2,1,<<28,202>>), - make_enqueue(E,2,<<"Î2">>), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E2,1,msg(<<28,202>>)), + make_enqueue(E,2,msg(<<"Î2">>)), {down, E, noproc}, - make_enqueue(E2,2,<<"ê">>), + make_enqueue(E2,2,msg(<<"ê">>)), {nodeup,fakenode@fake}, - make_enqueue(E2,3,<<>>), - make_enqueue(E2,4,<<>>), - make_enqueue(E2,5,<<>>), - make_enqueue(E2,6,<<>>), - make_enqueue(E2,7,<<>>), - make_enqueue(E2,8,<<>>), - make_enqueue(E2,9,<<>>), + make_enqueue(E2,3,msg(<<>>)), + make_enqueue(E2,4,msg(<<>>)), + make_enqueue(E2,5,msg(<<>>)), + make_enqueue(E2,6,msg(<<>>)), + make_enqueue(E2,7,msg(<<>>)), + make_enqueue(E2,8,msg(<<>>)), + make_enqueue(E2,9,msg(<<>>)), {purge}, - make_enqueue(E2,10,<<>>), - make_enqueue(E2,11,<<>>), - make_enqueue(E2,12,<<>>), - make_enqueue(E2,13,<<>>), - make_enqueue(E2,14,<<>>), - make_enqueue(E2,15,<<>>), - make_enqueue(E2,16,<<>>), - make_enqueue(E2,17,<<>>), - make_enqueue(E2,18,<<>>), + make_enqueue(E2,10,msg(<<>>)), + make_enqueue(E2,11,msg(<<>>)), + make_enqueue(E2,12,msg(<<>>)), + make_enqueue(E2,13,msg(<<>>)), + make_enqueue(E2,14,msg(<<>>)), + make_enqueue(E2,15,msg(<<>>)), + make_enqueue(E2,16,msg(<<>>)), + make_enqueue(E2,17,msg(<<>>)), + make_enqueue(E2,18,msg(<<>>)), {nodeup,fakenode@fake}, - make_enqueue(E2,19,<<>>), + make_enqueue(E2,19,msg(<<>>)), make_checkout(C1, {auto,77,simple_prefetch}), - make_enqueue(E2,20,<<>>), - make_enqueue(E2,21,<<>>), - make_enqueue(E2,22,<<>>), - make_enqueue(E2,23,<<"Ýý">>), + make_enqueue(E2,20,msg(<<>>)), + make_enqueue(E2,21,msg(<<>>)), + make_enqueue(E2,22,msg(<<>>)), + make_enqueue(E2,23,msg(<<"Ýý">>)), make_checkout(C2, {auto,66,simple_prefetch}), {purge}, - make_enqueue(E2,24,<<>>) + make_enqueue(E2,24,msg(<<>>)) ], ?assert( single_active_prop(#{name => ?FUNCTION_NAME, @@ -569,7 +564,7 @@ scenario27(_Config) -> max_in_memory_bytes => 691, overflow_strategy => drop_head, single_active_consumer_on => true, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands, false)), ok. @@ -578,11 +573,11 @@ scenario30(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 - make_enqueue(E,2,<<1>>), %% 2 + make_enqueue(E,1,msg(<<>>)), %% 1 + make_enqueue(E,2,msg(<<1>>)), %% 2 make_checkout(C1, {auto,1,simple_prefetch}), %% 3 {down, C1Pid, noconnection}, %% 4 - make_enqueue(E,3,<<>>) %% 5 + make_enqueue(E,3,msg(<<>>)) %% 5 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, @@ -590,7 +585,7 @@ scenario30(_Config) -> max_length => 1, max_in_memory_length => 1, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}, single_active_consumer_on => true }, Commands), @@ -609,8 +604,8 @@ scenario31(_Config) -> % {auto,1,simple_prefetch}, % #{ack => true,args => [],prefetch => 1,username => <<"user">>}}}, % {4,{purge}}] - make_enqueue(E1,1,<<>>), %% 1 - make_enqueue(E2,2,<<1>>), %% 2 + make_enqueue(E1,1,msg(<<>>)), %% 1 + make_enqueue(E2,2,msg(<<1>>)), %% 2 make_checkout(C1, {auto,1,simple_prefetch}), %% 3 {purge} %% 4 ], @@ -618,7 +613,7 @@ scenario31(_Config) -> release_cursor_interval => 0, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -626,17 +621,17 @@ scenario31(_Config) -> scenario32(_Config) -> E1 = c:pid(0,314,0), Commands = [ - make_enqueue(E1,1,<<0>>), %% 1 - make_enqueue(E1,2,<<0,0>>), %% 2 - make_enqueue(E1,4,<<0,0,0,0>>), %% 3 - make_enqueue(E1,3,<<0,0,0>>) %% 4 + make_enqueue(E1,1,msg(<<0>>)), %% 1 + make_enqueue(E1,2,msg(<<0,0>>)), %% 2 + make_enqueue(E1,4,msg(<<0,0,0,0>>)), %% 3 + make_enqueue(E1,3,msg(<<0,0,0>>)) %% 4 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, max_length => 3, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -646,14 +641,14 @@ scenario29(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 - make_enqueue(E,2,<<>>), %% 2 + make_enqueue(E,1,msg(<<>>)), %% 1 + make_enqueue(E,2,msg(<<>>)), %% 2 make_checkout(C1, {auto,2,simple_prefetch}), %% 2 - make_enqueue(E,3,<<>>), %% 3 - make_enqueue(E,4,<<>>), %% 4 - make_enqueue(E,5,<<>>), %% 5 - make_enqueue(E,6,<<>>), %% 6 - make_enqueue(E,7,<<>>), %% 7 + make_enqueue(E,3,msg(<<>>)), %% 3 + make_enqueue(E,4,msg(<<>>)), %% 4 + make_enqueue(E,5,msg(<<>>)), %% 5 + make_enqueue(E,6,msg(<<>>)), %% 6 + make_enqueue(E,7,msg(<<>>)), %% 7 {down, E, noconnection} %% 8 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -662,7 +657,7 @@ scenario29(_Config) -> max_length => 5, max_in_memory_length => 1, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}, single_active_consumer_on => true }, Commands), @@ -672,19 +667,19 @@ scenario23(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 + make_enqueue(E,1,msg(<<>>)), %% 1 make_checkout(C1, {auto,2,simple_prefetch}), %% 2 - make_enqueue(E,2,<<>>), %% 3 - make_enqueue(E,3,<<>>), %% 4 + make_enqueue(E,2,msg(<<>>)), %% 3 + make_enqueue(E,3,msg(<<>>)), %% 4 {down, E, noconnection}, %% 5 - make_enqueue(E,4,<<>>) %% 6 + make_enqueue(E,4,msg(<<>>)) %% 6 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, deliver_limit => undefined, max_length => 2, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -697,7 +692,7 @@ single_active_01(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), make_checkout(C2, {auto,1,simple_prefetch}), make_checkout(C1, cancel), {nodeup,rabbit@fake_node1} @@ -716,7 +711,7 @@ single_active_02(_Config) -> E = test_util:fake_pid(node()), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), {down,E,noconnection}, make_checkout(C2, {auto,1,simple_prefetch}), make_checkout(C2, cancel), @@ -735,8 +730,8 @@ single_active_03(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), + make_enqueue(E, 1, msg(<<0>>)), + make_enqueue(E, 2, msg(<<1>>)), {down, Pid, noconnection}, {nodeup, node()} ], @@ -754,10 +749,10 @@ single_active_04(_Config) -> Commands = [ % make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, <<>>), - make_enqueue(E, 2, <<>>), - make_enqueue(E, 3, <<>>), - make_enqueue(E, 4, <<>>) + make_enqueue(E, 1, msg(<<>>)), + make_enqueue(E, 2, msg(<<>>)), + make_enqueue(E, 3, msg(<<>>)), + make_enqueue(E, 4, msg(<<>>)) % {down, Pid, noconnection}, % {nodeup, node()} ], @@ -796,15 +791,16 @@ snapshots(_Config) -> fun () -> ?FORALL({Length, Bytes, SingleActiveConsumer, DeliveryLimit, InMemoryLength, InMemoryBytes, - Overflow}, - frequency([{10, {0, 0, false, 0, 0, 0, drop_head}}, + Overflow, DeadLetterHandler}, + frequency([{10, {0, 0, false, 0, 0, 0, drop_head, undefined}}, {5, {oneof([range(1, 10), undefined]), oneof([range(1, 1000), undefined]), boolean(), oneof([range(1, 3), undefined]), oneof([range(1, 10), undefined]), oneof([range(1, 1000), undefined]), - oneof([drop_head, reject_publish]) + oneof([drop_head, reject_publish]), + oneof([undefined, {at_most_once, {?MODULE, banana, []}}]) }}]), begin Config = config(?FUNCTION_NAME, @@ -814,13 +810,43 @@ snapshots(_Config) -> DeliveryLimit, InMemoryLength, InMemoryBytes, - Overflow), + Overflow, + DeadLetterHandler), ?FORALL(O, ?LET(Ops, log_gen(256), expand(Ops, Config)), collect({log_size, length(O)}, snapshots_prop(Config, O))) end) end, [], 1000). +snapshots_dlx(_Config) -> + run_proper( + fun () -> + ?FORALL({Length, Bytes, SingleActiveConsumer, + DeliveryLimit, InMemoryLength, InMemoryBytes}, + frequency([{10, {0, 0, false, 0, 0, 0}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + boolean(), + oneof([range(1, 3), undefined]), + oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]) + }}]), + begin + Config = config(?FUNCTION_NAME, + Length, + Bytes, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + InMemoryBytes, + reject_publish, + at_least_once), + ?FORALL(O, ?LET(Ops, log_gen_dlx(256), expand(Ops, Config)), + collect({log_size, length(O)}, + snapshots_prop(Config, O))) + end) + end, [], 1000). + single_active(_Config) -> Size = 300, run_proper( @@ -867,7 +893,10 @@ upgrade(_Config) -> SingleActive, DeliveryLimit, InMemoryLength, - undefined), + undefined, + drop_head, + {?MODULE, banana, []} + ), ?FORALL(O, ?LET(Ops, log_gen(Size), expand(Ops, Config)), collect({log_size, length(O)}, upgrade_prop(Config, O))) @@ -923,10 +952,10 @@ single_active_ordering_01(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), E2 = test_util:fake_pid(rabbit@fake_node2), Commands = [ - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), + make_enqueue(E, 1, msg(<<"0">>)), + make_enqueue(E, 2, msg(<<"1">>)), make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E2, 1, 2), + make_enqueue(E2, 1, msg(<<"2">>)), make_settle(C1, [0]) ], Conf = config(?FUNCTION_NAME, 0, 0, true, 0, 0, 0), @@ -945,7 +974,7 @@ single_active_ordering_02(_Config) -> E = test_util:fake_pid(node()), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E, 2, 1), + make_enqueue(E, 2, msg(<<"1">>)), %% CANNOT HAPPEN {down,E,noproc}, make_settle(C1, [0]) @@ -961,9 +990,9 @@ single_active_ordering_03(_Config) -> C2 = {<<2>>, C2Pid}, E = test_util:fake_pid(rabbit@fake_node2), Commands = [ - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), - make_enqueue(E, 3, 2), + make_enqueue(E, 1, msg(<<"0">>)), + make_enqueue(E, 2, msg(<<"1">>)), + make_enqueue(E, 3, msg(<<"2">>)), make_checkout(C1, {auto,1,simple_prefetch}), make_checkout(C2, {auto,1,simple_prefetch}), make_settle(C1, [0]), @@ -1045,17 +1074,232 @@ max_length(_Config) -> end) end, [], Size). -config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes) -> -config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes, drop_head). +%% Test that rabbit_fifo_dlx can check out a prefix message. +dlx_01(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + rabbit_fifo:make_discard(C1, [0]), + rabbit_fifo_dlx:make_settle([0]), + rabbit_fifo:make_discard(C1, [1]), + rabbit_fifo_dlx:make_settle([1]) + ], + Config = config(?FUNCTION_NAME, 8, undefined, false, 2, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that dehydrating dlx_consumer works. +dlx_02(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"1">>)), + %% State contains release cursor A. + rabbit_fifo:make_discard(C1, [0]), + make_enqueue(E,2,msg(<<"2">>)), + %% State contains release cursor B + %% with the 1st msg being checked out to dlx_consumer and + %% being dehydrated. + rabbit_fifo_dlx:make_settle([0]) + %% Release cursor A got emitted. + ], + Config = config(?FUNCTION_NAME, 10, undefined, false, 5, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that dehydrating discards queue works. +dlx_03(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<"1">>)), + %% State contains release cursor A. + make_checkout(C1, {auto,1,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0]), + make_enqueue(E,2,msg(<<"2">>)), + %% State contains release cursor B. + %% 1st message sitting in discards queue got dehydrated. + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + rabbit_fifo_dlx:make_settle([0]) + %% Release cursor A got emitted. + ], + Config = config(?FUNCTION_NAME, 10, undefined, false, 5, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +dlx_04(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 3), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), + make_enqueue(E,4,msg(<<>>)), + make_enqueue(E,5,msg(<<>>)), + make_enqueue(E,6,msg(<<>>)), + make_checkout(C1, {auto,6,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0,1,2,3,4,5]), + rabbit_fifo_dlx:make_settle([0,1,2]) + ], + Config = config(?FUNCTION_NAME, undefined, undefined, true, 1, 5, 136, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that discards queue gets dehydrated with 1 message that has empty message body. +dlx_05(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<"msg2">>)), + %% 0,1 in messages + make_checkout(C1, {auto,1,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0]), + %% 0 in discards, 1 in checkout + make_enqueue(E,3,msg(<<"msg3">>)), + %% 0 in discards (rabbit_fifo_dlx msg_bytes is still 0 because body of msg 0 is empty), + %% 1 in checkout, 2 in messages + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + %% 0 in dlx_checkout, 1 in checkout, 2 in messages + make_settle(C1, [1]), + %% 0 in dlx_checkout, 2 in checkout + rabbit_fifo_dlx:make_settle([0]) + %% 2 in checkout + ], + Config = config(?FUNCTION_NAME, 0, 0, false, 0, 0, 0, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +% Test that after recovery we can differentiate between index messge and (prefix) disk message +dlx_06(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + %% The following message has 3 bytes. + %% If we cannot differentiate between disk message and prefix disk message, + %% rabbit_fifo:delete_indexes/2 will not know whether it's a disk message or + %% prefix disk message and it will therefore falsely think that 3 is an index + %% instead of a size header resulting in message 3 being deleted from the index + %% after recovery. + make_enqueue(E,2,msg(<<"111">>)), + make_enqueue(E,3,msg(<<>>)), + %% 0,1,2 in messages + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 2), + make_checkout(C1, {auto,3,simple_prefetch}), + %% 0,1,2 in checkout + rabbit_fifo:make_discard(C1, [0,1,2]), + %% 0,1 in dlx_checkout, 3 in discards + rabbit_fifo_dlx:make_settle([0,1]) + %% 3 in dlx_checkout + ], + Config = config(?FUNCTION_NAME, undefined, 749, false, 1, 1, 131, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +dlx_07(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"12">>)), + %% 0 in checkout + rabbit_fifo:make_discard(C1, [0]), + %% 0 in discard + make_enqueue(E,2,msg(<<"1234567">>)), + %% 0 in discard, 1 in checkout + rabbit_fifo:make_discard(C1, [1]), + %% 0, 1 in discard + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + %% 0 in dlx_checkout, 1 in discard + make_enqueue(E,3,msg(<<"123">>)), + %% 0 in dlx_checkout, 1 in discard, 2 in checkout + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 2), + %% 0,1 in dlx_checkout, 2 in checkout + rabbit_fifo_dlx:make_settle([0]), + %% 1 in dlx_checkout, 2 in checkout + make_settle(C1, [2]), + %% 1 in dlx_checkout + make_enqueue(E,4,msg(<<>>)), + %% 1 in dlx_checkout, 3 in checkout + rabbit_fifo_dlx:make_settle([0,1]) + %% 3 in checkout + ], + Config = config(?FUNCTION_NAME, undefined, undefined, false, undefined, undefined, undefined, + reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% This test fails if discards queue is not normalized for comparison. +dlx_08(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + %% 0 in messages + make_checkout(C1, {auto,1,simple_prefetch}), + %% 0 in checkout + make_enqueue(E,2,msg(<<>>)), + %% 1 in messages, 0 in checkout + rabbit_fifo:make_discard(C1, [0]), + %% 1 in checkout, 0 in discards + make_enqueue(E,3,msg(<<>>)), + %% 2 in messages, 1 in checkout, 0 in discards + rabbit_fifo:make_discard(C1, [1]), + %% 2 in checkout, 0,1 in discards + rabbit_fifo:make_discard(C1, [2]), + %% 0,1,2 in discards + make_enqueue(E,4,msg(<<>>)), + %% 3 in checkout, 0,1,2 in discards + %% last command emitted this release cursor + make_settle(C1, [3]), + make_enqueue(E,5,msg(<<>>)), + make_enqueue(E,6,msg(<<>>)), + rabbit_fifo:make_discard(C1, [4]), + rabbit_fifo:make_discard(C1, [5]), + make_enqueue(E,7,msg(<<>>)), + make_enqueue(E,8,msg(<<>>)), + make_enqueue(E,9,msg(<<>>)), + rabbit_fifo:make_discard(C1, [6]), + rabbit_fifo:make_discard(C1, [7]), + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_enqueue(E,10,msg(<<>>)), + rabbit_fifo:make_discard(C1, [8]), + rabbit_fifo_dlx:make_settle([0]), + rabbit_fifo:make_discard(C1, [9]), + rabbit_fifo_dlx:make_settle([1]), + rabbit_fifo_dlx:make_settle([2]) + ], + Config = config(?FUNCTION_NAME, undefined, undefined, false, undefined, undefined, undefined, + reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +config(Name, Length, Bytes, SingleActive, DeliveryLimit, InMemoryLength, InMemoryBytes) -> +config(Name, Length, Bytes, SingleActive, DeliveryLimit, InMemoryLength, InMemoryBytes, + drop_head, {at_most_once, {?MODULE, banana, []}}). config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes, Overflow) -> + InMemoryLength, InMemoryBytes, Overflow, DeadLetterHandler) -> #{name => Name, max_length => map_max(Length), max_bytes => map_max(Bytes), - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => DeadLetterHandler, single_active_consumer_on => SingleActive, delivery_limit => map_max(DeliveryLimit), max_in_memory_length => map_max(InMemoryLength), @@ -1121,6 +1365,16 @@ validate_idx_order(Idxs, ReleaseCursorIdx) -> ok end. +%%TODO write separate generator for dlx using single_active_prop() or +%% messages_total_prop() as base template. +%% +%% E.g. enqueue few messages and have a consumer rejecting those. +%% The invariant could be: Delivery effects to dlx_worker must match the number of dead-lettered messages. +%% +%% Other invariants could be: +%% * if new consumer subscribes, messages are checked out to new consumer +%% * if dlx_worker fails receiving DOWN, messages are still in state. + single_active_prop(Conf0, Commands, ValidateOrder) -> Conf = Conf0#{release_cursor_interval => 100}, Indexes = lists:seq(1, length(Commands)), @@ -1166,14 +1420,23 @@ messages_total_invariant() -> consumers = C, enqueuers = E, prefix_msgs = {PTot, _, RTot, _}, - returns = R} = S) -> + returns = R, + dlx = #rabbit_fifo_dlx{discards = D, + consumer = DlxCon}} = S) -> Base = lqueue:len(M) + lqueue:len(R) + PTot + RTot, CTot = maps:fold(fun (_, #consumer{checked_out = Ch}, Acc) -> Acc + map_size(Ch) end, Base, C), - Tot = maps:fold(fun (_, #enqueuer{pending = P}, Acc) -> + Tot0 = maps:fold(fun (_, #enqueuer{pending = P}, Acc) -> Acc + length(P) end, CTot, E), + Tot1 = Tot0 + lqueue:len(D), + Tot = case DlxCon of + undefined -> + Tot1; + #dlx_consumer{checked_out = DlxChecked} -> + Tot1 + map_size(DlxChecked) + end, QTot = rabbit_fifo:query_messages_total(S), case Tot == QTot of true -> true; @@ -1262,9 +1525,6 @@ snapshots_prop(Conf, Commands) -> end. log_gen(Size) -> - log_gen(Size, binary()). - -log_gen(Size, _Body) -> Nodes = [node(), fakenode@fake, fakenode@fake2 @@ -1287,6 +1547,35 @@ log_gen(Size, _Body) -> {1, purge} ]))))). +log_gen_dlx(Size) -> + Nodes = [node(), + fakenode@fake, + fakenode@fake2 + ], + ?LET(EPids, vector(2, pid_gen(Nodes)), + ?LET(CPids, vector(2, pid_gen(Nodes)), + resize(Size, + list( + frequency( + [{20, enqueue_gen(oneof(EPids))}, + {40, {input_event, + frequency([{1, settle}, + {1, return}, + %% dead-letter many messages + {5, discard}, + {1, requeue}])}}, + {2, checkout_gen(oneof(CPids))}, + {1, checkout_cancel_gen(oneof(CPids))}, + {1, down_gen(oneof(EPids ++ CPids))}, + {1, nodeup_gen(Nodes)}, + {1, purge}, + %% same dlx_worker can subscribe multiple times, + %% e.g. after it dlx_worker crashed + %% "last subscriber wins" + {2, {checkout_dlx, choose(1,10)}} + ]))))). + + log_gen_config(Size) -> Nodes = [node(), fakenode@fake, @@ -1359,7 +1648,18 @@ enqueue_gen(Pid, Enq, Del) -> ?LET(E, {enqueue, Pid, frequency([{Enq, enqueue}, {Del, delay}]), - binary()}, E). + msg_gen()}, E). + +%% It's fair to assume that every message enqueued is a #basic_message. +%% That's what the channel expects and what rabbit_quorum_queue invokes rabbit_fifo_client with. +msg_gen() -> + ?LET(Bin, binary(), + #basic_message{content = #content{payload_fragments_rev = [Bin], + properties = none}}). + +msg(Bin) when is_binary(Bin) -> + #basic_message{content = #content{payload_fragments_rev = [Bin], + properties = none}}. checkout_cancel_gen(Pid) -> {checkout, Pid, cancel}. @@ -1368,11 +1668,7 @@ checkout_gen(Pid) -> %% pid, tag, prefetch ?LET(C, {checkout, {binary(), Pid}, choose(1, 100)}, C). - --record(t, {state = rabbit_fifo:init(#{name => proper, - queue_resource => blah, - release_cursor_interval => 1}) - :: rabbit_fifo:state(), +-record(t, {state :: rabbit_fifo:state(), index = 1 :: non_neg_integer(), %% raft index enqueuers = #{} :: #{pid() => term()}, consumers = #{} :: #{{binary(), pid()} => term()}, @@ -1387,20 +1683,34 @@ checkout_gen(Pid) -> expand(Ops, Config) -> expand(Ops, Config, {undefined, fun ra_lib:id/1}). +%% generates a sequence of Raft commands expand(Ops, Config, EnqFun) -> %% execute each command against a rabbit_fifo state and capture all relevant %% effects - T = #t{enq_body_fun = EnqFun, + InitConfig0 = #{name => proper, + queue_resource => blah, + release_cursor_interval => 1}, + InitConfig = case Config of + #{dead_letter_handler := at_least_once} -> + %% Configure rabbit_fifo config with at_least_once so that + %% rabbit_fifo_dlx outputs dlx_delivery effects + %% which we are going to settle immediately in enq_effs/2. + %% Therefore the final generated Raft commands will include + %% {dlx, {checkout, ...}} and {dlx, {settle, ...}} Raft commands. + maps:put(dead_letter_handler, at_least_once, InitConfig0); + _ -> + InitConfig0 + end, + T = #t{state = rabbit_fifo:init(InitConfig), + enq_body_fun = EnqFun, config = Config}, #t{effects = Effs} = T1 = lists:foldl(fun handle_op/2, T, Ops), %% process the remaining effect #t{log = Log} = lists:foldl(fun do_apply/2, T1#t{effects = queue:new()}, queue:to_list(Effs)), - lists:reverse(Log). - handle_op({enqueue, Pid, When, Data}, #t{enqueuers = Enqs0, enq_body_fun = {EnqSt0, Fun}, @@ -1493,6 +1803,9 @@ handle_op({input_event, Settlement}, #t{effects = Effs, false -> do_apply(Cmd, T#t{effects = Q}) end; + {{value, {dlx, {settle, MsgIds}}}, Q} -> + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + do_apply(Cmd, T#t{effects = Q}); _ -> T end; @@ -1500,7 +1813,10 @@ handle_op(purge, T) -> do_apply(rabbit_fifo:make_purge(), T); handle_op({update_config, Changes}, #t{config = Conf} = T) -> Config = maps:merge(Conf, Changes), - do_apply(rabbit_fifo:make_update_config(Config), T). + do_apply(rabbit_fifo:make_update_config(Config), T); +handle_op({checkout_dlx, Prefetch}, #t{config = #{dead_letter_handler := at_least_once}} = T) -> + Cmd = rabbit_fifo_dlx:make_checkout(proper_dlx_worker, Prefetch), + do_apply(Cmd, T). do_apply(Cmd, #t{effects = Effs, @@ -1534,14 +1850,17 @@ enq_effs([{send_msg, P, {delivery, CTag, Msgs}, _Opts} | Rem], Q) -> %% they can be changed depending on the input event later Cmd = rabbit_fifo:make_settle({CTag, P}, MsgIds), enq_effs(Rem, queue:in(Cmd, Q)); +enq_effs([{send_msg, _, {dlx_delivery, Msgs}, _Opts} | Rem], Q) -> + MsgIds = [I || {I, _} <- Msgs], + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + enq_effs(Rem, queue:in(Cmd, Q)); enq_effs([_ | Rem], Q) -> enq_effs(Rem, Q). %% Utility run_proper(Fun, Args, NumTests) -> - ?assertEqual( - true, + ?assert( proper:counterexample( erlang:apply(Fun, Args), [{numtests, NumTests}, @@ -1585,10 +1904,12 @@ run_snapshot_test0(Conf, Commands, Invariant) -> State -> ok; _ -> ct:pal("Snapshot tests failed run log:~n" - "~p~n from ~n~p~n Entries~n~p~n" + "~p~n from snapshot index ~b " + "with snapshot state~n~p~n Entries~n~p~n" "Config: ~p~n", - [Filtered, SnapState, Entries, Conf]), - ct:pal("Expected~n~p~nGot:~n~p", [State, S]), + [Filtered, SnapIdx, SnapState, Entries, Conf]), + ct:pal("Expected~n~p~nGot:~n~p~n", [?record_info(rabbit_fifo, State), + ?record_info(rabbit_fifo, S)]), ?assertEqual(State, S) end end || {release_cursor, SnapIdx, SnapState} <- Cursors], diff --git a/deps/rabbit_common/include/rabbit.hrl b/deps/rabbit_common/include/rabbit.hrl index f40b92a24b..41a5f30458 100644 --- a/deps/rabbit_common/include/rabbit.hrl +++ b/deps/rabbit_common/include/rabbit.hrl @@ -112,7 +112,7 @@ -record(basic_message, {exchange_name, %% The exchange where the message was received routing_keys = [], %% Routing keys used during publish - content, %% The message content + content, %% The message #content record id, %% A `rabbit_guid:gen()` generated id is_persistent}). %% Whether the message was published as persistent diff --git a/deps/rabbitmq_management/priv/www/js/global.js b/deps/rabbitmq_management/priv/www/js/global.js index 1981d9439c..dbdedc2ab7 100644 --- a/deps/rabbitmq_management/priv/www/js/global.js +++ b/deps/rabbitmq_management/priv/www/js/global.js @@ -174,7 +174,7 @@ const QUEUE_EXTRA_CONTENT_REQUESTS = []; // All help ? popups var HELP = { 'delivery-limit': - 'The number of allowed unsuccessful delivery attempts. Once a message has been delivered unsuccessfully this many times it will be dropped or dead-lettered, depending on the queue configuration.', + 'The number of allowed unsuccessful delivery attempts. Once a message has been delivered unsuccessfully more than this many times it will be dropped or dead-lettered, depending on the queue configuration.', 'exchange-auto-delete': 'If yes, the exchange will delete itself after at least one queue or exchange has been bound to this one, and then all queues or exchanges have been unbound.', @@ -218,6 +218,9 @@ var HELP = { 'queue-dead-letter-routing-key': 'Optional replacement routing key to use when a message is dead-lettered. If this is not set, the message\'s original routing key will be used.
(Sets the "x-dead-letter-routing-key" argument.)', + 'queue-dead-letter-strategy': + 'Valid values are at-most-once or at-least-once. It defaults to at-most-once. This setting is understood only by quorum queues. If at-least-once is set, Overflow behaviour must be set to reject-publish. Otherwise, dead letter strategy will fall back to at-most-once.', + 'queue-single-active-consumer': 'If set, makes sure only one consumer at a time consumes from the queue and fails over to another registered consumer in case the active one is cancelled or dies.
(Sets the "x-single-active-consumer" argument.)', @@ -246,11 +249,14 @@ var HELP = { 'Set the queue initial cluster size.', 'queue-type': - 'Set the queue type, determining the type of queue to use: raft-based high availability or classic queue. Valid values are quorum or classic. It defaults to classic.
', + 'Set the queue type, determining the type of queue to use: raft-based high availability or classic queue. Valid values are quorum or classic. It defaults to classic.
', 'queue-messages': '

Message counts.

Note that "in memory" and "persistent" are not mutually exclusive; persistent messages can be in memory as well as on disc, and transient messages can be paged out if memory is tight. Non-durable queues will consider all messages to be transient.

', + 'queue-dead-lettered': + 'Applies to messages dead-lettered with dead-letter-strategy at-least-once.', + 'queue-message-body-bytes': '

The sum total of the sizes of the message bodies in this queue. This only counts message bodies; it does not include message properties (including headers) or metadata used by the queue.

Note that "in memory" and "persistent" are not mutually exclusive; persistent messages can be in memory as well as on disc, and transient messages can be paged out if memory is tight. Non-durable queues will consider all messages to be transient.

If a message is routed to multiple queues on publication, its body will be stored only once (in memory and on disk) and shared between queues. The value shown here does not take account of this effect.

', diff --git a/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs b/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs index 3b93c210ac..716e7bc83f 100644 --- a/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs +++ b/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs @@ -103,7 +103,8 @@ Overflow behaviour | Auto expire
Dead letter exchange | - Dead letter routing key
+ Dead letter routing key
+ Message TTL
Queues [Classic] @@ -114,7 +115,6 @@ HA mirror promotion on shutdown | HA mirror promotion on failure
- Message TTL | Lazy mode | Version | Master Locator
@@ -128,7 +128,9 @@ Max in memory bytes | Delivery limit - +
+ Dead letter strategy + @@ -271,13 +273,14 @@ Max length | Max length bytes | Overflow behaviour - +
+ Message TTL + Queues [Classic] - Message TTL | Auto expire diff --git a/deps/rabbitmq_management/priv/www/js/tmpl/queue.ejs b/deps/rabbitmq_management/priv/www/js/tmpl/queue.ejs index c2022f73bb..c9d7319bb4 100644 --- a/deps/rabbitmq_management/priv/www/js/tmpl/queue.ejs +++ b/deps/rabbitmq_management/priv/www/js/tmpl/queue.ejs @@ -167,6 +167,9 @@ Unacked <% if (is_quorum(queue)) { %> In memory ready + Dead-lettered + + <% } %> <% if (is_classic(queue)) { %> In memory @@ -192,6 +195,9 @@ <%= fmt_num_thousands(queue.messages_ram) %> + + <%= fmt_num_thousands(queue.messages_dlx) %> + <% } %> <% if (is_classic(queue)) { %> @@ -224,6 +230,11 @@ <%= fmt_bytes(queue.message_bytes_ram) %> <% } %> + <% if (is_quorum(queue)) { %> + + <%= fmt_bytes(queue.message_bytes_dlx) %> + + <% } %> <% if (is_classic(queue)) { %> <%= fmt_bytes(queue.message_bytes_persistent) %> diff --git a/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs b/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs index 8a14fd2dd9..0e4f2779a2 100644 --- a/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs +++ b/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs @@ -312,13 +312,11 @@ Add - <% if (queue_type == "classic") { %> - Message TTL | - <% } %> <% if (queue_type != "stream") { %> Auto expire | - Overflow behaviour | - Single active consumer
+ Message TTL | + Overflow behaviour
+ Single active consumer | Dead letter exchange | Dead letter routing key
Max length | @@ -334,7 +332,8 @@ Delivery limit | Max in memory length | Max in memory bytes - | Initial cluster size
+ | Initial cluster size
+ Dead letter strategy
<% } %> <% if (queue_type == "stream") { %> Max time retention