File vmtoolsd_bailout_on_rpc_errors.patch of Package open-vm-tools.10446

commit 0c9174716ba828899418ba07efc3aab0bff004cc
Author: Oliver Kurth <okurth@vmware.com>
Date:   Tue Jan 29 14:03:19 2019 -0800

    Bail out vmtoolsd early when there are RPC errors.
    
    VMX state machine could give up quiescing operation for various
    reasons when vmtoolsd is busy performing necessary state transitions.
    Once VMX gives up quiescing operation, there is no point in
    vmtoolsd continuing with it. vmtoolsd should also give up the
    operation asap. vmtoolsd can detect VMX state machine change
    when it gets errors sending VMBACKUP_PROTOCOL_EVENT_SET RPC.
    
    RPC errors are only used as a trigger to abort the operation.
    We ignore the RPC errors that might occur after aborting the
    operation.

diff --git a/open-vm-tools/services/plugins/vmbackup/stateMachine.c b/open-vm-tools/services/plugins/vmbackup/stateMachine.c
index 14d08a77..28118172 100644
--- a/open-vm-tools/services/plugins/vmbackup/stateMachine.c
+++ b/open-vm-tools/services/plugins/vmbackup/stateMachine.c
@@ -224,6 +224,7 @@ VmBackup_SendEvent(const char *event,
    if (gBackupState->keepAlive != NULL) {
       g_source_destroy(gBackupState->keepAlive);
       g_source_unref(gBackupState->keepAlive);
+      gBackupState->keepAlive = NULL;
    }
 
    msg = g_strdup_printf(VMBACKUP_PROTOCOL_EVENT_SET" %s %u %s",
@@ -267,19 +268,27 @@ VmBackup_SendEvent(const char *event,
                              &result, &resultLen);
 #endif
 
-   if (!success) {
+   if (success) {
+      ASSERT(gBackupState->keepAlive == NULL);
+      gBackupState->keepAlive =
+         g_timeout_source_new(VMBACKUP_KEEP_ALIVE_PERIOD / 2);
+      VMTOOLSAPP_ATTACH_SOURCE(gBackupState->ctx,
+                               gBackupState->keepAlive,
+                               VmBackupKeepAliveCallback,
+                               NULL,
+                               NULL);
+   } else {
       g_warning("Failed to send vmbackup event: %s, result: %s.\n",
                 msg, result);
+      if (gBackupState->rpcState != VMBACKUP_RPC_STATE_IGNORE) {
+         g_debug("Changing rpcState from %d to %d\n",
+                 gBackupState->rpcState, VMBACKUP_RPC_STATE_ERROR);
+         gBackupState->rpcState = VMBACKUP_RPC_STATE_ERROR;
+      }
    }
    vm_free(result);
    g_free(msg);
 
-   gBackupState->keepAlive = g_timeout_source_new(VMBACKUP_KEEP_ALIVE_PERIOD / 2);
-   VMTOOLSAPP_ATTACH_SOURCE(gBackupState->ctx,
-                            gBackupState->keepAlive,
-                            VmBackupKeepAliveCallback,
-                            NULL,
-                            NULL);
    return success;
 }
 
@@ -440,6 +449,12 @@ VmBackupDoAbort(void)
 {
    g_debug("*** %s\n", __FUNCTION__);
    ASSERT(gBackupState != NULL);
+
+   /*
+    * Once we abort the operation, we don't care about RPC state.
+    */
+   gBackupState->rpcState = VMBACKUP_RPC_STATE_IGNORE;
+
    if (gBackupState->machineState != VMBACKUP_MSTATE_SCRIPT_ERROR &&
        gBackupState->machineState != VMBACKUP_MSTATE_SYNC_ERROR) {
       const char *eventMsg = "Quiesce aborted.";
@@ -623,6 +638,17 @@ VmBackupAsyncCallback(void *clientData)
       if (opPending) {
          goto exit;
       }
+
+      /*
+       * VMX state might have changed when we were processing
+       * currentOp. This is usually detected by failures in
+       * sending backup event to the host.
+       */
+      if (gBackupState->rpcState == VMBACKUP_RPC_STATE_ERROR) {
+         g_warning("Aborting backup operation due to RPC errors.");
+         VmBackupDoAbort();
+         goto exit;
+      }
    }
 
    switch (gBackupState->machineState) {
@@ -958,6 +984,7 @@ VmBackupStartCommon(RpcInData *data,
    gBackupState->enableNullDriver = VMBACKUP_CONFIG_GET_BOOL(ctx->config,
                                                              "enableNullDriver",
                                                              TRUE);
+   gBackupState->rpcState = VMBACKUP_RPC_STATE_NORMAL;
 
    g_debug("Using quiesceApps = %d, quiesceFS = %d, allowHWProvider = %d,"
            " execScripts = %d, scriptArg = %s, timeout = %u,"
diff --git a/open-vm-tools/services/plugins/vmbackup/vmBackupInt.h b/open-vm-tools/services/plugins/vmbackup/vmBackupInt.h
index ad3f2d7c..4258ee0a 100644
--- a/open-vm-tools/services/plugins/vmbackup/vmBackupInt.h
+++ b/open-vm-tools/services/plugins/vmbackup/vmBackupInt.h
@@ -72,6 +72,12 @@ typedef enum {
    VMBACKUP_MSTATE_SYNC_ERROR
 } VmBackupMState;
 
+typedef enum {
+   VMBACKUP_RPC_STATE_NORMAL,
+   VMBACKUP_RPC_STATE_ERROR,
+   VMBACKUP_RPC_STATE_IGNORE
+} VmBackupRpcState;
+
 /**
  * This is a "base struct" for asynchronous operations monitored by the
  * state machine. Each implementation should provide these three functions
@@ -138,6 +144,7 @@ typedef struct VmBackupState {
    Bool           vssBootableSystemState;
    Bool           vssPartialFileSupport;
    Bool           vssUseDefault;
+   VmBackupRpcState rpcState;
 } VmBackupState;
 
 typedef Bool (*VmBackupCallback)(VmBackupState *);