Merge branch 'master' into rabbitmq-server-550

docs/rabbitmq-server.service.example

@@ -0,0 +1,17 @@
+# systemd unit example
+[Unit]
+Description=RabbitMQ broker
+After=syslog.target network.target
+
+[Service]
+Type=notify
+User=rabbitmq
+Group=rabbitmq
+NotifyAccess=all
+TimeoutStartSec=3600
+WorkingDirectory=/var/lib/rabbitmq
+ExecStart=/usr/lib/rabbitmq/bin/rabbitmq-server
+ExecStop=/usr/lib/rabbitmq/bin/rabbitmqctl stop
+
+[Install]
+WantedBy=multi-user.target

packaging/RPMS/Fedora/rabbitmq-server.spec

@@ -11,7 +11,7 @@ Source2: rabbitmq-server.logrotate
 URL: http://www.rabbitmq.com/
 BuildArch: noarch
 BuildRequires: erlang >= R16B-03, python-simplejson, xmlto, libxslt, gzip, sed, zip, rsync
-Requires: erlang >= R16B-03, logrotate
+Requires: erlang >= R16B-03, logrotate, socat
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-%{_arch}-root
 Summary: The RabbitMQ server
 Requires(post): %%REQUIRES%%

packaging/debs/Debian/debian/control

@@ -18,7 +18,7 @@ Standards-Version: 3.9.4
 
 Package: rabbitmq-server
 Architecture: all
-Depends: erlang-nox (>= 1:16.b.3) | esl-erlang, adduser, logrotate, ${misc:Depends}
+Depends: erlang-nox (>= 1:16.b.3) | esl-erlang, adduser, logrotate, socat, ${misc:Depends}
 Description: Multi-protocol messaging broker
  RabbitMQ is an open source multi-protocol messaging broker.
 Homepage: http://www.rabbitmq.com/

scripts/rabbitmq-server

@@ -47,7 +47,7 @@ case "$(uname -s)" in
                    exit $EX_CANTCREAT
                fi
                if ! echo $$ > ${RABBITMQ_PID_FILE}; then
-                   # Bettern diagnostics - otherwise the only report in logs is about failed 'echo'
+                   # Better diagnostics - otherwise the only report in logs is about failed 'echo'
                    # command, but without any other details: neither what script has failed nor what
                    # file output was redirected to.
                    echo "Failed to write pid file: ${RABBITMQ_PID_FILE}"
@@ -58,8 +58,13 @@ esac
 
 RABBITMQ_EBIN_ROOT="${RABBITMQ_HOME}/ebin"
 
+[ "$NOTIFY_SOCKET" ] && RUNNING_UNDER_SYSTEMD=true
+
 set +e
 
+# NOTIFY_SOCKET is needed here to prevent epmd from impersonating the
+# success of our startup sequence to systemd.
+NOTIFY_SOCKET= \
 RABBITMQ_CONFIG_FILE=$RABBITMQ_CONFIG_FILE \
 RABBITMQ_DIST_PORT=$RABBITMQ_DIST_PORT \
     ${ERL_DIR}erl -pa "$RABBITMQ_EBIN_ROOT" \
@@ -183,7 +188,20 @@ check_not_empty() {
     fi
 }
 
-if [ 'x' = "x$RABBITMQ_ALLOW_INPUT" -a -z "$detached" ]; then
+if [ "$RABBITMQ_ALLOW_INPUT" -o "$RUNNING_UNDER_SYSTEMD" -o "$detached" ]; then
+    # Run erlang VM directly, completely replacing current shell
+    # process - so the pid file written in the code above will be
+    # valid (unless detached, which is also handled in the code
+    # above).
+    #
+    # And also this is the correct mode to run the broker under
+    # systemd - there is no need in a proxy process that converts
+    # signals to graceful shutdown command, the unit file should already
+    # contain instructions for graceful shutdown. Also by removing
+    # this additional process we could simply use value returned by
+    # `os:getpid/0` for a systemd ready notification.
+    start_rabbitmq_server "$@"
+else
     # When RabbitMQ runs in the foreground but the Erlang shell is
     # disabled, we setup signal handlers to stop RabbitMQ properly. This
     # is at least useful in the case of Docker.
@@ -192,7 +210,7 @@ if [ 'x' = "x$RABBITMQ_ALLOW_INPUT" -a -z "$detached" ]; then
     RABBITMQ_SERVER_START_ARGS="${RABBITMQ_SERVER_START_ARGS} +B i"
 
     # Signal handlers. They all stop RabbitMQ properly (using
-    # rabbitmqctl stop). Depending on the signal, this script will exwit
+    # rabbitmqctl stop). Depending on the signal, this script will exit
     # with a non-zero error code:
     #   SIGHUP SIGTERM SIGTSTP
     #     They are considered a normal process termination, so the script
@@ -208,6 +226,4 @@ if [ 'x' = "x$RABBITMQ_ALLOW_INPUT" -a -z "$detached" ]; then
     # Block until RabbitMQ exits or a signal is caught.
     # Waits for last command (which is start_rabbitmq_server)
     wait $!
-else
-    start_rabbitmq_server "$@"
 fi

src/rabbit.erl

@@ -304,16 +304,120 @@ broker_start() ->
     Plugins = rabbit_plugins:setup(),
     ToBeLoaded = Plugins ++ ?APPS,
     start_apps(ToBeLoaded),
-    case os:type() of
+    maybe_sd_notify(),
-        {win32, _} -> ok;
-        _ ->
-            %% Only for systemd unit with Type=notify. Errors are intentionally
-            %% ignored: either you have working systemd-notify(1) or you don't
-            %% care about systemd at all.
-            os:cmd("systemd-notify --ready")
-    end,
     ok = log_broker_started(rabbit_plugins:active()).
 
+%% Try to send systemd ready notification if it makes sense in the
+%% current environment. standard_error is used intentionally in all
+%% logging statements, so all this messages will end in systemd
+%% journal.
+maybe_sd_notify() ->
+    case sd_notify_ready() of
+        false ->
+            io:format(standard_error, "systemd READY notification failed, beware of timeouts~n", []);
+        _ ->
+            ok
+    end.
+
+sd_notify_ready() ->
+    case {os:type(), os:getenv("NOTIFY_SOCKET")} of
+        {{win32, _}, _} ->
+            true;
+        {_, [_|_]} -> %% Non-empty NOTIFY_SOCKET, give it a try
+            sd_notify_legacy() orelse sd_notify_socat();
+        _ ->
+            true
+    end.
+
+sd_notify_data() ->
+    "READY=1\nSTATUS=Initialized\nMAINPID=" ++ os:getpid() ++ "\n".
+
+sd_notify_legacy() ->
+    case code:load_file(sd_notify) of
+        {module, sd_notify} ->
+            SDNotify = sd_notify,
+            SDNotify:sd_notify(0, sd_notify_data()),
+            true;
+        {error, _} ->
+            false
+    end.
+
+%% socat(1) is the most portable way the sd_notify could be
+%% implemented in erlang, without introducing some NIF. Currently the
+%% following issues prevent us from implementing it in a more
+%% reasonable way:
+%% - systemd-notify(1) is unstable for non-root users
+%% - erlang doesn't support unix domain sockets.
+%%
+%% Some details on how we ended with such a solution:
+%%   https://github.com/rabbitmq/rabbitmq-server/issues/664
+sd_notify_socat() ->
+    case sd_current_unit() of
+        {ok, Unit} ->
+            io:format(standard_error, "systemd unit for activation check: \"~s\"~n", [Unit]),
+            sd_notify_socat(Unit);
+        _ ->
+            false
+    end.
+
+socat_socket_arg("@" ++ AbstractUnixSocket) ->
+    "abstract-sendto:" ++ AbstractUnixSocket;
+socat_socket_arg(UnixSocket) ->
+    "unix-sendto:" ++ UnixSocket.
+
+sd_open_port() ->
+    open_port(
+      {spawn_executable, os:find_executable("socat")},
+      [{args, [socat_socket_arg(os:getenv("NOTIFY_SOCKET")), "STDIO"]},
+       use_stdio, out]).
+
+sd_notify_socat(Unit) ->
+    case sd_open_port() of
+        {'EXIT', Exit} ->
+            io:format(standard_error, "Failed to start socat ~p~n", [Exit]),
+            false;
+        Port ->
+            Port ! {self(), {command, sd_notify_data()}},
+            Result = sd_wait_activation(Port, Unit),
+            port_close(Port),
+            Result
+    end.
+
+sd_current_unit() ->
+    case catch re:run(os:cmd("systemctl status " ++ os:getpid()), "([-.@0-9a-zA-Z]+)", [unicode, {capture, all_but_first, list}]) of
+        {'EXIT', _} ->
+            error;
+        {match, [Unit]} ->
+            {ok, Unit};
+        _ ->
+            error
+    end.
+
+sd_wait_activation(Port, Unit) ->
+    case os:find_executable("systemctl") of
+        false ->
+            io:format(standard_error, "'systemctl' unavailable, falling back to sleep~n", []),
+            timer:sleep(5000),
+            true;
+        _ ->
+            sd_wait_activation(Port, Unit, 10)
+    end.
+
+sd_wait_activation(_, _, 0) ->
+    io:format(standard_error, "Service still in 'activating' state, bailing out~n", []),
+    false;
+sd_wait_activation(Port, Unit, AttemptsLeft) ->
+    case os:cmd("systemctl show --property=ActiveState " ++ Unit) of
+        "ActiveState=activating\n" ->
+            timer:sleep(1000),
+            sd_wait_activation(Port, Unit, AttemptsLeft - 1);
+        "ActiveState=" ++ _ ->
+            true;
+        _ = Err->
+            io:format(standard_error, "Unexpected status from systemd ~p~n", [Err]),
+            false
+    end.
+
 start_it(StartFun) ->
     Marker = spawn_link(fun() -> receive stop -> ok end end),
     case catch register(rabbit_boot, Marker) of
@@ -350,6 +454,10 @@ stop_and_halt() ->
         stop()
     after
         rabbit_log:info("Halting Erlang VM~n", []),
+        %% Also duplicate this information to stderr, so console where
+        %% foreground broker was running (or systemd journal) will
+        %% contain information about graceful termination.
+        io:format(standard_error, "Gracefully halting Erlang VM~n", []),
         init:stop()
     end,
     ok.
@@ -658,7 +766,8 @@ print_banner() ->
               "~n  ######  ##"
               "~n  ##########  Logs: ~s" ++
               LogFmt ++
-              "~n~n              Starting broker...",
+              "~n~n              Starting broker..."
+              "~n",
               [Product, Version, ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE] ++
               LogLocations).