File pacemaker-pengine-resources-on-remote-nodes-not... of Package pacemaker

Overview Repositories Revisions Requests Users Attributes Meta

File pacemaker-pengine-resources-on-remote-nodes-not-restarted-until-necessary.patch of Package pacemaker

commit 3f72a028f9a12a6623a7b88733a6fe7ecc879230
Author: Andrew Beekhof <andrew@beekhof.net>
Date:   Wed May 24 11:21:44 2017 +1000

PE: Assume resources on remote nodes do not need to be restarted until absolutely necessary

diff --git a/pengine/allocate.c b/pengine/allocate.c
index 6b0025bdf..90d25fc72 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -1745,6 +1745,244 @@ rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_
 extern gboolean update_action(action_t * action);
 extern void update_colo_start_chain(action_t * action);
 
+enum remote_connection_state 
+{
+    remote_state_unknown = 0,
+    remote_state_alive = 1,
+    remote_state_resting = 2,
+    remote_state_dead= 3
+};
+
+static int
+is_recurring_action(action_t *action) 
+{
+    const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
+    int interval = crm_parse_int(interval_s, "0");
+    if(interval > 0) {
+        return TRUE;
+    }
+    return FALSE;
+}
+
+static void
+apply_container_ordering(action_t *action, pe_working_set_t *data_set)
+{
+    /* VMs are also classified as containers for these purposes... in
+     * that they both involve a 'thing' running on a real or remote
+     * cluster node.
+     *
+     * This allows us to be smarter about the type and extent of
+     * recovery actions required in various scenarios
+     */
+    resource_t *remote_rsc = NULL;
+    resource_t *container = NULL;
+    enum action_tasks task = text2task(action->task);
+
+    if (action->rsc == NULL) {
+        return;
+    }
+
+    CRM_ASSERT(action->node);
+    CRM_ASSERT(is_remote_node(action->node));
+    CRM_ASSERT(action->node->details->remote_rsc);
+
+    remote_rsc = action->node->details->remote_rsc;
+    CRM_ASSERT(remote_rsc);
+
+    container = remote_rsc->container;
+    CRM_ASSERT(container);
+
+    crm_trace("%s %s %s %s %d", action->uuid, action->task, remote_rsc->id, container->id, is_set(container->flags, pe_rsc_failed));
+    switch (task) {
+        case start_rsc:
+        case action_promote:
+            /* Force resource recovery if the container is recovered */
+            custom_action_order(container, generate_op_key(container->id, RSC_START, 0), NULL,
+                                action->rsc, NULL, action,
+                                pe_order_preserve | pe_order_implies_then | pe_order_runnable_left, data_set);
+
+            /* Wait for the connection resource to be up too */
+            custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                action->rsc, NULL, action,
+                                pe_order_preserve | pe_order_runnable_left, data_set);
+            break;
+        case stop_rsc:
+            if(is_set(container->flags, pe_rsc_failed)) {
+                /* When the container representing a guest node fails,
+                 * the stop action for all the resources living in
+                 * that container is implied by the container
+                 * stopping. This is similar to how fencing operations
+                 * work for cluster nodes.
+                 */
+                custom_action_order(container, generate_op_key(container->id, RSC_STOP, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left, data_set);
+                pe_set_action_bit(action, pe_action_pseudo);
+            } else {
+                /* Otherwise, ensure the operation happens before the connection is brought down */
+                custom_action_order(action->rsc, NULL, action,
+                                    remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL,
+                                    pe_order_preserve, data_set);
+            }
+            break;
+        case action_demote:
+            if(is_set(container->flags, pe_rsc_failed)) {
+                /* Just like a stop, the demote is implied by the
+                 * container having failed/stopped
+                 *
+                 * If we really wanted to we would order the demote
+                 * after the stop, IFF the containers current role was
+                 * stopped (otherwise we re-introduce an ordering
+                 * loop)
+                 */
+                pe_set_action_bit(action, pe_action_pseudo);
+            } else {
+                /* Otherwise, ensure the operation happens before the connection is brought down */
+                custom_action_order(action->rsc, NULL, action,
+                                    remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL,
+                                    pe_order_preserve, data_set);
+            }
+            break;
+        default:
+            /* Wait for the connection resource to be up */
+            if (is_recurring_action(action)) {
+                /* In case we ever get the recovery logic wrong, force
+                 * recurring monitors to be restarted, even if just
+                 * the connection was re-established
+                 */
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set);
+            } else {
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left, data_set);
+            }
+            break;
+    }
+}
+
+static void
+apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
+{
+    resource_t *remote_rsc = NULL;
+    node_t *cluster_node = NULL;
+    enum action_tasks task = text2task(action->task);
+    enum remote_connection_state state = remote_state_unknown;
+
+    if (action->rsc == NULL) {
+        return;
+    }
+
+    CRM_ASSERT(action->node);
+    CRM_ASSERT(is_remote_node(action->node));
+    CRM_ASSERT(action->node->details->remote_rsc);
+
+    remote_rsc = action->node->details->remote_rsc;
+    CRM_ASSERT(remote_rsc);
+
+    if(remote_rsc->running_on) {
+        cluster_node = remote_rsc->running_on->data;
+    }
+
+    /* If the cluster node the remote connection resource resides on
+     * is unclean or went offline, we can't process any operations
+     * on that remote node until after it starts elsewhere.
+     */
+    if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
+        /* There is no-where left to run the connection resource
+         * We must assume the target has failed
+         */
+        state = remote_state_dead;
+
+    } else if (cluster_node == NULL) {
+        /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
+        state = remote_state_unknown;
+
+    } else if(cluster_node->details->unclean == TRUE
+              || cluster_node->details->online == FALSE) {
+        /* Connection is running on a dead node, see if we can recover it first */
+        state = remote_state_resting;
+
+    } else if (g_list_length(remote_rsc->running_on) > 1
+               && remote_rsc->partial_migration_source
+               && remote_rsc->partial_migration_target) {
+        /* We're in the middle of migrating a connection resource,
+         * wait until after the resource migrates before performing
+         * any actions.
+         */
+        state = remote_state_resting;
+
+    } else {
+        state = remote_state_alive;
+    }
+
+    crm_trace("%s %s %d", action->uuid, action->task, state);
+    switch (task) {
+        case start_rsc:
+        case action_promote:
+            if(state == remote_state_dead) {
+                /* Wait for the connection resource to be up and force recovery */
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left, data_set);
+            } else {
+                /* Ensure the connection resource is up and assume everything is as we left it */
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left, data_set);
+            }
+            break;
+        case stop_rsc:
+            /* Handle special case with remote node where stop actions need to be
+             * ordered after the connection resource starts somewhere else.
+             */
+            if(state == remote_state_resting) {
+                /* Wait for the connection resource to be up and assume everything is as we left it */
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left, data_set);
+            } else {
+                custom_action_order(action->rsc, NULL, action,
+                                    remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL,
+                                    pe_order_preserve | pe_order_implies_first, data_set);
+            }
+            break;
+        case action_demote:
+
+            /* If the connection is being torn down, we don't want
+             * to build a constraint between a resource's demotion and
+             * the connection resource starting... because the connection
+             * resource can not start. The connection might already be up,
+             * but the "start" action would not be allowed, which in turn would
+             * block the demotion of any resources living in the node.
+             */
+
+            if(state == remote_state_resting || state == remote_state_unknown) {
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve, data_set);
+            } /* Otherwise we can rely on the stop ordering */
+            break;
+        default:
+            /* Wait for the connection resource to be up */
+            if (is_recurring_action(action)) {
+                /* In case we ever get the recovery logic wrong, force
+                 * recurring monitors to be restarted, even if just
+                 * the connection was re-established
+                 */
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set);
+            } else {
+                custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+                                    action->rsc, NULL, action,
+                                    pe_order_preserve | pe_order_runnable_left, data_set);
+            }
+            break;
+    }
+}
+
 static void
 apply_remote_node_ordering(pe_working_set_t *data_set)
 {
@@ -1753,10 +1991,9 @@ apply_remote_node_ordering(pe_working_set_t *data_set)
     if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
         return;
     }
+
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
-        resource_t *remote_rsc = NULL;
-        resource_t *container = NULL;
 
         if (action->rsc == NULL) {
             continue;
@@ -1781,7 +2018,7 @@ apply_remote_node_ordering(pe_working_set_t *data_set)
                 pe_order_optional,
                 data_set);
 
-                continue;
+            continue;
         }
 
         /* If the action occurs on a Pacemaker Remote node, create
@@ -1792,138 +2029,13 @@ apply_remote_node_ordering(pe_working_set_t *data_set)
             is_remote_node(action->node) == FALSE ||
             action->node->details->remote_rsc == NULL ||
             is_set(action->flags, pe_action_pseudo)) {
-            continue;
-        }
+            crm_trace("Nothing required for %s", action->uuid);
 
-        remote_rsc = action->node->details->remote_rsc;
-        container = remote_rsc->container;
+        } else if(action->node->details->remote_rsc->container) {
+            apply_container_ordering(action, data_set);
 
-        if (safe_str_eq(action->task, "monitor") ||
-            safe_str_eq(action->task, "start") ||
-            safe_str_eq(action->task, "promote") ||
-            safe_str_eq(action->task, "notify") ||
-            safe_str_eq(action->task, CRM_OP_LRM_REFRESH) ||
-            safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT) ||
-            safe_str_eq(action->task, "delete")) {
-
-            custom_action_order(remote_rsc,
-                generate_op_key(remote_rsc->id, RSC_START, 0),
-                NULL,
-                action->rsc,
-                NULL,
-                action,
-                pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
-                data_set);
-
-        } else if (safe_str_eq(action->task, "demote")) {
-
-            /* If the connection is being torn down, we don't want
-             * to build a constraint between a resource's demotion and
-             * the connection resource starting... because the connection
-             * resource can not start. The connection might already be up,
-             * but the "start" action would not be allowed, which in turn would
-             * block the demotion of any resources living in the node.
-             *
-             * In this case, only build the constraint between the demotion and
-             * the connection's "stop" action. This allows the connection and
-             * all the resources within the node to be torn down properly.
-             */
-            if (remote_rsc->next_role == RSC_ROLE_STOPPED) {
-                custom_action_order(action->rsc,
-                    NULL,
-                    action,
-                    remote_rsc,
-                    generate_op_key(remote_rsc->id, RSC_STOP, 0),
-                    NULL,
-                    pe_order_preserve | pe_order_implies_first,
-                    data_set);
-            } else if(container == NULL) {
-                custom_action_order(remote_rsc,
-                    generate_op_key(remote_rsc->id, RSC_START, 0),
-                    NULL,
-                    action->rsc,
-                    NULL,
-                    action,
-                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
-                    data_set);
-            }
-
-            if(container && is_set(container->flags, pe_rsc_failed)) {
-                /* Just like a stop, the demote is implied by the
-                 * container having failed/stopped
-                 *
-                 * If we really wanted to we would order the demote
-                 * after the stop, IFF the containers current role was
-                 * stopped (otherwise we re-introduce an ordering
-                 * loop)
-                 */
-                pe_set_action_bit(action, pe_action_pseudo);
-            }
-
-        } else if (safe_str_eq(action->task, "stop") &&
-                   container &&
-                   is_set(container->flags, pe_rsc_failed)) {
-
-            /* When the container representing a guest node fails, the stop
-             * action for all the resources living in that container is implied
-             * by the container stopping. This is similar to how fencing
-             * operations work for cluster nodes.
-             */
-            pe_set_action_bit(action, pe_action_pseudo);
-            custom_action_order(container,
-                generate_op_key(container->id, RSC_STOP, 0),
-                NULL,
-                action->rsc,
-                NULL,
-                action,
-                pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
-                data_set);
-        } else if (safe_str_eq(action->task, "stop")) {
-            gboolean after_start = FALSE;
-
-            /* Handle special case with remote node where stop actions need to be
-             * ordered after the connection resource starts somewhere else.
-             */
-            if (is_baremetal_remote_node(action->node)) {
-                node_t *cluster_node = remote_rsc->running_on ? remote_rsc->running_on->data : NULL;
-
-                /* If the cluster node the remote connection resource resides on
-                 * is unclean or went offline, we can't process any operations
-                 * on that remote node until after it starts elsewhere.
-                 */
-                if (cluster_node == NULL ||
-                    cluster_node->details->unclean == TRUE ||
-                    cluster_node->details->online == FALSE) {
-                    after_start = TRUE;
-                } else if (g_list_length(remote_rsc->running_on) > 1 &&
-                           remote_rsc->partial_migration_source &&
-                            remote_rsc->partial_migration_target) {
-                    /* if we're caught in the middle of migrating a connection resource,
-                     * then we have to wait until after the resource migrates before performing
-                     * any actions. */
-                    after_start = TRUE;
-                }
-            }
-
-            if (after_start) {
-                custom_action_order(remote_rsc,
-                    generate_op_key(remote_rsc->id, RSC_START, 0),
-                    NULL,
-                    action->rsc,
-                    NULL,
-                    action,
-                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
-                    data_set);
-            } else {
-                custom_action_order(action->rsc,
-                    NULL,
-                    action,
-                    remote_rsc,
-                    generate_op_key(remote_rsc->id, RSC_STOP, 0),
-                    NULL,
-                    pe_order_preserve | pe_order_implies_first,
-                    data_set);
-            }
+        } else {
+            apply_remote_ordering(action, data_set);
         }
     }
 }

Places

File pacemaker-pengine-resources-on-remote-nodes-not-restarted-until-necessary.patch of Package pacemaker

Places