File bug-970733_pacemaker-fencing-random-delay.patch of Package pacemaker.9287
commit e965dde0027517cf2c634dadb380daa0b5b2ffd5
Author: Gao,Yan <ygao@suse.com>
Date: Wed Feb 11 15:48:44 2015 +0100
Feature: fencing: cl#5134 - Support random fencing delay to avoid double fencing
This prevents double fencing when using slow fencing devices such as sbd.
A "pcmk_delay_max" parameter can be configured for a fencing resource to
enable random delay for stonith actions and specify the maximum of
random delay.
diff --git a/fencing/commands.c b/fencing/commands.c
index c193a9d..742b81e 100644
--- a/fencing/commands.c
+++ b/fencing/commands.c
@@ -81,8 +81,11 @@ typedef struct async_command_s {
int pid;
int fd_stdout;
int options;
- int default_timeout;
- int timeout;
+ int default_timeout; /* seconds */
+ int timeout; /* seconds */
+
+ int start_delay; /* milliseconds */
+ int delay_id;
char *op;
char *origin;
@@ -128,6 +131,24 @@ is_action_required(const char *action, stonith_device_t *device)
}
static int
+get_action_delay_max(stonith_device_t * device, const char * action)
+{
+ const char *value = NULL;
+ int delay_max_ms = 0;
+
+ if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) {
+ return 0;
+ }
+
+ value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX);
+ if (value) {
+ delay_max_ms = crm_get_msec(value);
+ }
+
+ return delay_max_ms;
+}
+
+static int
get_action_timeout(stonith_device_t * device, const char *action, int default_timeout)
{
char buffer[512] = { 0, };
@@ -155,6 +176,11 @@ free_async_command(async_command_t * cmd)
if (!cmd) {
return;
}
+
+ if (cmd->delay_id) {
+ g_source_remove(cmd->delay_id);
+ }
+
cmd_list = g_list_remove(cmd_list, cmd);
g_list_free_full(cmd->device_list, free);
@@ -222,8 +248,16 @@ stonith_device_execute(stonith_device_t * device)
if (device->pending_ops) {
GList *first = device->pending_ops;
- device->pending_ops = g_list_remove_link(device->pending_ops, first);
cmd = first->data;
+ if (cmd && cmd->delay_id) {
+ crm_trace
+ ("Operation %s%s%s on %s was asked to run too early, waiting for start_delay timeout of %dms",
+ cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "",
+ device->id, cmd->start_delay);
+ return TRUE;
+ }
+
+ device->pending_ops = g_list_remove_link(device->pending_ops, first);
g_list_free_1(first);
}
@@ -301,9 +335,27 @@ stonith_device_dispatch(gpointer user_data)
return stonith_device_execute(user_data);
}
+static gboolean
+start_delay_helper(gpointer data)
+{
+ async_command_t *cmd = data;
+ stonith_device_t *device = NULL;
+
+ cmd->delay_id = 0;
+ device = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
+
+ if (device) {
+ mainloop_set_trigger(device->work);
+ }
+
+ return FALSE;
+}
+
static void
schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
{
+ int delay_max = 0;
+
CRM_CHECK(cmd != NULL, return);
CRM_CHECK(device != NULL, return);
@@ -330,6 +382,14 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
device->pending_ops = g_list_append(device->pending_ops, cmd);
mainloop_set_trigger(device->work);
+
+ delay_max = get_action_delay_max(device, cmd->action);
+ if (delay_max > 0) {
+ cmd->start_delay = rand() % delay_max;
+ crm_notice("Delaying %s on %s for %lldms (timeout=%ds)",
+ cmd->action, device->id, cmd->start_delay, cmd->timeout);
+ cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
+ }
}
void
@@ -1333,6 +1393,7 @@ stonith_query_capable_device_cb(GList * devices, void *user_data)
for (lpc = devices; lpc != NULL; lpc = lpc->next) {
stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
int action_specific_timeout;
+ int delay_max;
if (!device) {
/* It is possible the device got unregistered while
@@ -1354,6 +1415,12 @@ stonith_query_capable_device_cb(GList * devices, void *user_data)
if (action_specific_timeout) {
crm_xml_add_int(dev, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
}
+
+ delay_max = get_action_delay_max(device, query->action);
+ if (delay_max > 0) {
+ crm_xml_add_int(dev, F_STONITH_DELAY_MAX, delay_max / 1000);
+ }
+
if (query->target == NULL) {
xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
diff --git a/fencing/main.c b/fencing/main.c
index 70b5bde..5249664 100644
--- a/fencing/main.c
+++ b/fencing/main.c
@@ -1294,6 +1294,15 @@ main(int argc, char **argv)
printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
printf(" </parameter>\n");
+ printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_DELAY_MAX);
+ printf
+ (" <shortdesc lang=\"en\">Enable random delay for stonith actions and specify the maximum of random delay</shortdesc>\n");
+ printf
+ (" <longdesc lang=\"en\">This prevents double fencing when using slow devices such as sbd.\n"
+ "Use this to enable random delay for stonith actions and specify the maximum of random delay.</longdesc>\n");
+ printf(" <content type=\"time\" default=\"0s\"/>\n");
+ printf(" </parameter>\n");
+
for (lpc = 0; lpc < DIMOF(actions); lpc++) {
printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
printf
diff --git a/fencing/remote.c b/fencing/remote.c
index 63c0274..8331cec 100644
--- a/fencing/remote.c
+++ b/fencing/remote.c
@@ -53,6 +53,7 @@ typedef struct st_query_result_s {
gboolean tried;
GListPtr device_list;
GHashTable *custom_action_timeouts;
+ GHashTable *delay_maxes;
/* Subset of devices that peer has verified connectivity on */
GHashTable *verified_devices;
@@ -84,6 +85,7 @@ free_remote_query(gpointer data)
free(query->host);
g_list_free_full(query->device_list, free);
g_hash_table_destroy(query->custom_action_timeouts);
+ g_hash_table_destroy(query->delay_maxes);
g_hash_table_destroy(query->verified_devices);
free(query);
}
@@ -879,14 +881,20 @@ static int
get_device_timeout(st_query_result_t * peer, const char *device, int default_timeout)
{
gpointer res;
+ int delay_max = 0;
if (!peer || !device) {
return default_timeout;
}
+ res = g_hash_table_lookup(peer->delay_maxes, device);
+ if (res && GPOINTER_TO_INT(res) > 0) {
+ delay_max = GPOINTER_TO_INT(res);
+ }
+
res = g_hash_table_lookup(peer->custom_action_timeouts, device);
- return res ? GPOINTER_TO_INT(res) : default_timeout;
+ return res ? GPOINTER_TO_INT(res) + delay_max : default_timeout + delay_max;
}
static int
@@ -1244,17 +1252,20 @@ process_remote_stonith_query(xmlNode * msg)
result->host = strdup(host);
result->devices = devices;
result->custom_action_timeouts = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
+ result->delay_maxes = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
result->verified_devices = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
for (child = __xml_first_child(dev); child != NULL; child = __xml_next(child)) {
const char *device = ID(child);
int action_timeout = 0;
+ int delay_max = 0;
int verified = 0;
int required = 0;
if (device) {
result->device_list = g_list_prepend(result->device_list, strdup(device));
crm_element_value_int(child, F_STONITH_ACTION_TIMEOUT, &action_timeout);
+ crm_element_value_int(child, F_STONITH_DELAY_MAX, &delay_max);
crm_element_value_int(child, F_STONITH_DEVICE_VERIFIED, &verified);
crm_element_value_int(child, F_STONITH_DEVICE_REQUIRED, &required);
if (action_timeout) {
@@ -1263,6 +1274,12 @@ process_remote_stonith_query(xmlNode * msg)
g_hash_table_insert(result->custom_action_timeouts,
strdup(device), GINT_TO_POINTER(action_timeout));
}
+ if (delay_max > 0) {
+ crm_trace("Peer %s with device %s returned maximum of random delay %d",
+ result->host, device, delay_max);
+ g_hash_table_insert(result->delay_maxes,
+ strdup(device), GINT_TO_POINTER(delay_max));
+ }
if (verified) {
crm_trace("Peer %s has confirmed a verified device %s", result->host, device);
g_hash_table_insert(result->verified_devices,
diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h
index 6d9eb12..7f00cfd 100644
--- a/include/crm/fencing/internal.h
+++ b/include/crm/fencing/internal.h
@@ -63,6 +63,8 @@ xmlNode *create_device_registration_xml(const char *id, const char *namespace, c
# define F_STONITH_TOLERANCE "st_tolerance"
/*! Action specific timeout period returned in query of fencing devices. */
# define F_STONITH_ACTION_TIMEOUT "st_action_timeout"
+/*! Maximum of random fencing delay for a device */
+# define F_STONITH_DELAY_MAX "st_delay_max"
/*! Has this device been verified using a monitor type
* operation (monitor, list, status) */
# define F_STONITH_DEVICE_VERIFIED "st_monitor_verified"
@@ -103,6 +105,7 @@ xmlNode *create_device_registration_xml(const char *id, const char *namespace, c
# define STONITH_ATTR_HOSTMAP "pcmk_host_map"
# define STONITH_ATTR_HOSTLIST "pcmk_host_list"
# define STONITH_ATTR_HOSTCHECK "pcmk_host_check"
+# define STONITH_ATTR_DELAY_MAX "pcmk_delay_max"
# define STONITH_ATTR_ACTION_OP "action"
