File detect_missing_user_process.patch of Package rabbitmq-server
From b149da724210dba4021dbe49936ff7cabc28c3c6 Mon Sep 17 00:00:00 2001
From: Alexey Lebedeff <alebedev@mirantis.com>
Date: Wed, 13 Apr 2016 18:37:29 +0300
Subject: [PATCH] Detect missing 'user' process
'user' process can be unrecoverably crashed with good timing. As a
result some rabbitmqctl commands will stop working, e.g. add_user:
rabbitmqctl -n rabbit@localhost add_user ley ley
Creating user "ley" ...
Error: {badarg,
[{erlang,group_leader,[undefined,<5428.28745.44>],[]},
{rabbit_log,with_local_io,1,
[{file,"src/rabbit_log.erl"},{line,99}]},
{rabbit_auth_backend_internal,add_user,2,
[{file,"src/rabbit_auth_backend_internal.erl"},{line,149}]},
{rpc,'-handle_call_call/6-fun-0-',5,
[{file,"rpc.erl"},{line,206}]}]}
Exact sequence events that will crash 'user' is the following:
- Move startup_log file to a separate partition
- Start server
- Fill partition to the fullest
- Do stop_app/start_app several times, so all the remaning slack will be
used by log records.
- At some point 'user' will crash.
- Free some space on partition with startup_log
- Observe that any rabbit action that requires logging is now broken
---
src/rabbit_log.erl | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/rabbit_log.erl b/src/rabbit_log.erl
index c6081fad0d..e3e1d7042f 100644
--- a/src/rabbit_log.erl
+++ b/src/rabbit_log.erl
@@ -96,10 +96,18 @@ with_local_io(Fun) ->
Node = node(),
case node(GL) of
Node -> Fun();
- _ -> group_leader(whereis(user), self()),
+ _ -> set_group_leader_to_user(),
try
Fun()
after
group_leader(GL, self())
end
end.
+
+set_group_leader_to_user() ->
+ case whereis(user) of
+ undefined ->
+ warning("'user' IO process has died, you'd better restart erlang VM");
+ User ->
+ group_leader(User, self())
+ end.