File bsc#1206263-0001-Test-cts-scheduler-add-test-for-preventing-a-leftove.patch of Package pacemaker.29727
From 8306ac714ab58786f678dd78a6485b998d5828ea Mon Sep 17 00:00:00 2001
From: "Gao,Yan" <ygao@suse.com>
Date: Fri, 2 Dec 2022 18:37:09 +0100
Subject: [PATCH 1/6] Test: cts-scheduler: add test for preventing a leftover
pending monitor from causing unexpected stop of other instances
- Monitor of stateful-1 failed on node-1 so that the pending monitor
didn't get overridden.
- While stateful-1 was stopping on node-1, node-2 was rejoining and
pending.
- stateful-1 was stopped on node-1 and scheduler would unexpectedly
decide to stop the promoted instance on node-3.
---
cts/cts-scheduler.in | 1 +
.../dot/leftover-pending-monitor.dot | 36 ++++++++
.../exp/leftover-pending-monitor.exp | 81 ++++++++++++++++++
.../scores/leftover-pending-monitor.scores | 28 +++++++
.../summary/leftover-pending-monitor.summary | 35 ++++++++
.../xml/leftover-pending-monitor.xml | 82 +++++++++++++++++++
6 files changed, 263 insertions(+)
create mode 100644 cts/scheduler/dot/leftover-pending-monitor.dot
create mode 100644 cts/scheduler/exp/leftover-pending-monitor.exp
create mode 100644 cts/scheduler/scores/leftover-pending-monitor.scores
create mode 100644 cts/scheduler/summary/leftover-pending-monitor.summary
create mode 100644 cts/scheduler/xml/leftover-pending-monitor.xml
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/cts-scheduler.in
===================================================================
--- pacemaker-2.1.2+20211124.ada5c3b36.orig/cts/cts-scheduler.in
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/cts-scheduler.in
@@ -525,6 +525,7 @@ TESTS = [
[ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
[ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],
[ "no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run" ],
+ [ "leftover-pending-monitor", "Prevent a leftover pending monitor from causing unexpected stop of other instances" ],
],
[
[ "history-1", "Correctly parse stateful-1 resource state" ],
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/dot/leftover-pending-monitor.dot
===================================================================
--- /dev/null
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/dot/leftover-pending-monitor.dot
@@ -0,0 +1,36 @@
+ digraph "g" {
+"promotable-1_demote_0" -> "promotable-1_demoted_0" [ style = bold]
+"promotable-1_demote_0" -> "stateful-1_demote_0 node-3" [ style = bold]
+"promotable-1_demote_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_demoted_0" -> "promotable-1_start_0" [ style = dashed]
+"promotable-1_demoted_0" -> "promotable-1_stop_0" [ style = bold]
+"promotable-1_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_running_0" [ style=dashed color="red" fontcolor="orange"]
+"promotable-1_start_0" -> "promotable-1_running_0" [ style = dashed]
+"promotable-1_start_0" -> "stateful-1:2_start_0 node-1" [ style = dashed]
+"promotable-1_start_0" -> "stateful-1_start_0 node-3" [ style = dashed]
+"promotable-1_start_0" [ style=dashed color="red" fontcolor="orange"]
+"promotable-1_stop_0" -> "promotable-1_stopped_0" [ style = bold]
+"promotable-1_stop_0" -> "stateful-1_stop_0 node-3" [ style = bold]
+"promotable-1_stop_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_stopped_0" -> "promotable-1_start_0" [ style = dashed]
+"promotable-1_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"st-sbd_monitor_0 node-2" [ style=dashed color="red" fontcolor="black"]
+"stateful-1:2_monitor_5000 node-1" [ style=dashed color="red" fontcolor="black"]
+"stateful-1:2_start_0 node-1" -> "promotable-1_running_0" [ style = dashed]
+"stateful-1:2_start_0 node-1" -> "stateful-1:2_monitor_5000 node-1" [ style = dashed]
+"stateful-1:2_start_0 node-1" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_demote_0 node-3" -> "promotable-1_demoted_0" [ style = bold]
+"stateful-1_demote_0 node-3" -> "stateful-1_stop_0 node-3" [ style = bold]
+"stateful-1_demote_0 node-3" [ style=bold color="green" fontcolor="black"]
+"stateful-1_monitor_0 node-2" -> "promotable-1_start_0" [ style = dashed]
+"stateful-1_monitor_0 node-2" -> "promotable-1_stopped_0" [ style = dashed]
+"stateful-1_monitor_0 node-2" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_monitor_10000 node-3" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_start_0 node-3" -> "promotable-1_running_0" [ style = dashed]
+"stateful-1_start_0 node-3" -> "stateful-1_monitor_10000 node-3" [ style = dashed]
+"stateful-1_start_0 node-3" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_stop_0 node-3" -> "promotable-1_stopped_0" [ style = bold]
+"stateful-1_stop_0 node-3" -> "stateful-1_start_0 node-3" [ style = dashed]
+"stateful-1_stop_0 node-3" [ style=bold color="green" fontcolor="black"]
+}
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/exp/leftover-pending-monitor.exp
===================================================================
--- /dev/null
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/exp/leftover-pending-monitor.exp
@@ -0,0 +1,81 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="8" operation="stop" operation_key="stateful-1_stop_0" internal_operation_key="stateful-1:1_stop_0" on_node="node-3" on_node_uuid="3">
+ <primitive id="stateful-1" long-id="stateful-1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="node-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1">
+ <action_set>
+ <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3">
+ <primitive id="stateful-1" long-id="stateful-1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="node-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2" priority="1000000">
+ <action_set>
+ <pseudo_event id="19" operation="demoted" operation_key="promotable-1_demoted_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3">
+ <action_set>
+ <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="4" priority="1000000">
+ <action_set>
+ <pseudo_event id="15" operation="stopped" operation_key="promotable-1_stopped_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="8" operation="stop" operation_key="stateful-1_stop_0" internal_operation_key="stateful-1:1_stop_0" on_node="node-3" on_node_uuid="3"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
+ <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0">
+ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="19" operation="demoted" operation_key="promotable-1_demoted_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+</transition_graph>
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/scores/leftover-pending-monitor.scores
===================================================================
--- /dev/null
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/scores/leftover-pending-monitor.scores
@@ -0,0 +1,28 @@
+
+pcmk__clone_allocate: promotable-1 allocation score on node-1: 0
+pcmk__clone_allocate: promotable-1 allocation score on node-2: 0
+pcmk__clone_allocate: promotable-1 allocation score on node-3: 0
+pcmk__clone_allocate: stateful-1:0 allocation score on node-1: 0
+pcmk__clone_allocate: stateful-1:0 allocation score on node-2: 0
+pcmk__clone_allocate: stateful-1:0 allocation score on node-3: 0
+pcmk__clone_allocate: stateful-1:1 allocation score on node-1: 0
+pcmk__clone_allocate: stateful-1:1 allocation score on node-2: 0
+pcmk__clone_allocate: stateful-1:1 allocation score on node-3: 11
+pcmk__clone_allocate: stateful-1:2 allocation score on node-1: 0
+pcmk__clone_allocate: stateful-1:2 allocation score on node-2: 0
+pcmk__clone_allocate: stateful-1:2 allocation score on node-3: 10
+pcmk__native_allocate: st-sbd allocation score on node-1: 1
+pcmk__native_allocate: st-sbd allocation score on node-2: 0
+pcmk__native_allocate: st-sbd allocation score on node-3: 0
+pcmk__native_allocate: stateful-1:0 allocation score on node-1: -INFINITY
+pcmk__native_allocate: stateful-1:0 allocation score on node-2: -INFINITY
+pcmk__native_allocate: stateful-1:0 allocation score on node-3: -INFINITY
+pcmk__native_allocate: stateful-1:1 allocation score on node-1: 0
+pcmk__native_allocate: stateful-1:1 allocation score on node-2: -INFINITY
+pcmk__native_allocate: stateful-1:1 allocation score on node-3: 11
+pcmk__native_allocate: stateful-1:2 allocation score on node-1: 0
+pcmk__native_allocate: stateful-1:2 allocation score on node-2: -INFINITY
+pcmk__native_allocate: stateful-1:2 allocation score on node-3: -INFINITY
+stateful-1:0 promotion score on none: 0
+stateful-1:1 promotion score on node-3: INFINITY
+stateful-1:2 promotion score on node-1: -1
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/summary/leftover-pending-monitor.summary
===================================================================
--- /dev/null
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/summary/leftover-pending-monitor.summary
@@ -0,0 +1,35 @@
+Using the original execution date of: 2022-12-02 17:04:52Z
+Current cluster status:
+ * Node List:
+ * Node node-2: pending
+ * Online: [ node-1 node-3 ]
+
+ * Full List of Resources:
+ * st-sbd (stonith:external/sbd): Started node-1
+ * Clone Set: promotable-1 [stateful-1] (promotable):
+ * stateful-1 (ocf:pacemaker:Stateful): Stopped node-1 (Monitoring)
+ * Promoted: [ node-3 ]
+ * Stopped: [ node-1 node-2 ]
+
+Transition Summary:
+ * Stop stateful-1:1 ( Promoted node-3 ) due to unrunnable stateful-1:0 monitor
+ * Start stateful-1:2 ( node-1 ) due to unrunnable stateful-1:0 monitor (blocked)
+
+Executing Cluster Transition:
+ * Pseudo action: promotable-1_demote_0
+ * Resource action: stateful-1 demote on node-3
+ * Pseudo action: promotable-1_demoted_0
+ * Pseudo action: promotable-1_stop_0
+ * Resource action: stateful-1 stop on node-3
+ * Pseudo action: promotable-1_stopped_0
+Using the original execution date of: 2022-12-02 17:04:52Z
+
+Revised Cluster Status:
+ * Node List:
+ * Node node-2: pending
+ * Online: [ node-1 node-3 ]
+
+ * Full List of Resources:
+ * st-sbd (stonith:external/sbd): Started node-1
+ * Clone Set: promotable-1 [stateful-1] (promotable):
+ * Stopped: [ node-1 node-2 node-3 ]
Index: pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/xml/leftover-pending-monitor.xml
===================================================================
--- /dev/null
+++ pacemaker-2.1.2+20211124.ada5c3b36/cts/scheduler/xml/leftover-pending-monitor.xml
@@ -0,0 +1,82 @@
+<cib crm_feature_set="3.11.0" validate-with="pacemaker-3.7" epoch="8" num_updates="53" admin_epoch="0" cib-last-written="Fri Dec 2 18:00:35 2022" update-origin="node-1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1670000692">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <nvpair name="have-watchdog" value="true" id="cib-bootstrap-options-have-watchdog"/>
+ <nvpair name="dc-version" value="2.1.2+20211124.ada5c3b36-150400.29.16-2.1.2+20211124.ada5c3b36" id="cib-bootstrap-options-dc-version"/>
+ <nvpair name="cluster-infrastructure" value="corosync" id="cib-bootstrap-options-cluster-infrastructure"/>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="1" uname="node-1"/>
+ <node id="2" uname="node-2"/>
+ <node id="3" uname="node-3"/>
+ </nodes>
+ <resources>
+ <primitive id="st-sbd" class="stonith" type="external/sbd"/>
+ <clone id="promotable-1">
+ <meta_attributes id="promotable-1-meta_attributes">
+ <nvpair name="promotable" value="true" id="promotable-1-meta_attributes-promotable"/>
+ <nvpair name="interleave" value="true" id="promotable-1-meta_attributes-interleave"/>
+ </meta_attributes>
+ <primitive id="stateful-1" class="ocf" provider="pacemaker" type="Stateful">
+ <operations>
+ <op name="monitor" interval="5" id="stateful-1-monitor-5"/>
+ <op name="monitor" interval="10" role="Promoted" id="stateful-1-monitor-10"/>
+ </operations>
+ </primitive>
+ </clone>
+ </resources>
+ <constraints>
+ <rsc_location id="cli-prefer-promotable-1" rsc="promotable-1" role="Master" node="node-3" score="INFINITY"/>
+ </constraints>
+ <rsc_defaults>
+ <meta_attributes id="build-resource-defaults">
+ <nvpair id="build-resource-stickiness" name="resource-stickiness" value="1"/>
+ </meta_attributes>
+ </rsc_defaults>
+ </configuration>
+ <status>
+ <node_state id="1" uname="node-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+ <transient_attributes id="1">
+ <instance_attributes id="status-1">
+ <nvpair id="status-1-.feature-set" name="#feature-set" value="3.11.0"/>
+ <nvpair id="status-1-fail-count-stateful-1.monitor_5000" name="fail-count-stateful-1#monitor_5000" value="2"/>
+ <nvpair id="status-1-last-failure-stateful-1.monitor_5000" name="last-failure-stateful-1#monitor_5000" value="1670000676"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="1">
+ <lrm_resources>
+ <lrm_resource id="st-sbd" type="external/sbd" class="stonith">
+ <lrm_rsc_op id="st-sbd_last_0" operation_key="st-sbd_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="3:1:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;3:1:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="11" rc-code="0" op-status="0" interval="0" last-rc-change="1670000509" exec-time="1418" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="2:7:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;2:7:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="19" rc-code="0" op-status="0" interval="0" last-rc-change="1670000676" exec-time="15102" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-1_monitor_5000" operation_key="stateful-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="-1:193;6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="-1" rc-code="193" op-status="-1" interval="5000" last-rc-change="1670000661" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ <lrm_rsc_op id="stateful-1_last_failure_0" operation_key="stateful-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:7;6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="17" rc-code="7" op-status="0" interval="5000" last-rc-change="1670000676" exec-time="15023" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="3" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" uname="node-3" join="member" expected="member">
+ <transient_attributes id="3">
+ <instance_attributes id="status-3">
+ <nvpair id="status-3-.feature-set" name="#feature-set" value="3.11.0"/>
+ <nvpair id="status-3-master-stateful-1" name="master-stateful-1" value="10"/>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="3">
+ <lrm_resources>
+ <lrm_resource id="st-sbd" type="external/sbd" class="stonith">
+ <lrm_rsc_op id="st-sbd_last_0" operation_key="st-sbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="1:1:7:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:7;1:1:7:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1670000497" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ </lrm_resource>
+ <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
+ <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="8:2:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;8:2:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="12" rc-code="0" op-status="0" interval="0" last-rc-change="1670000510" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+ <lrm_rsc_op id="stateful-1_monitor_10000" operation_key="stateful-1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="10:3:8:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:8;10:3:8:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="13" rc-code="8" op-status="0" interval="10000" last-rc-change="1670000510" exec-time="17" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state id="2" uname="node-2" crmd="offline" crm-debug-origin="post_cache_update" in_ccm="true" join="down" expected="down"/>
+ </status>
+</cib>
