File SAPHana of Package SAPHanaSR.882
#!/bin/bash
#
# SAPHana
#
# Description: Manages two single SAP HANA Instance in System Replication
# Planned: do also manage scale-up scenarios
# currently the SAPHana is dependent of the analysis of
# SAPHanaTopology
# For supported scenarios please read the README file provided
# in the same software package (rpm)
#
##############################################################################
#
# SAPHana
# Author: Fabian Herschel, November 2013
# Support: linux@sap.com
# License: GNU General Public License (GPL)
# Copyright: (c) 2013,2014 SUSE Linux Products GmbH
# Copyright: (c) 2015 SUSE Linux GmbH
#
# An example usage:
# See usage() function below for more details...
#
# OCF instance parameters:
# OCF_RESKEY_SID
# OCF_RESKEY_InstanceNumber
# OCF_RESKEY_DIR_EXECUTABLE (optional, well known directories will be searched by default)
# OCF_RESKEY_DIR_PROFILE (optional, well known directories will be searched by default)
# OCF_RESKEY_INSTANCE_PROFILE (optional, well known directories will be searched by default)
# OCF_RESKEY_PREFER_SITE_TAKEOVER (optional, default is no)
# OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT (optional, time difference needed between two last-primary-tiemstampe (lpt))
# OCF_RESKEY_SAPHanaFilter (optional, should only be set if been told by support or for debugging purposes)
#
#
#######################################################################
#
# Initialization:
timeB=$(date '+%s')
: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
#
#######################################################################
#
HANA_STATE_PRIMARY=0
HANA_STATE_SECONDARY=1
HANA_STATE_STANDALONE=2
HANA_STATE_DEFECT=3
debug_attributes=0
SH=/bin/sh
#
# function: super_ocf_log - wrapper function for ocf log in order catch usual logging into super log
# params: LOG_MESSAGE
# globals: SAPHanaFilter
function super_ocf_log() {
local level="$1"
local message="$2"
local skip=1
local mtype=""
local search=0
local shf="${SAPHanaFilter:-all}"
# message levels: (dbg)|info|warn|err|error
# message types: (ACT|RA|FLOW|DBG|LPA|DEC|DBG2...
case "$level" in
debug | dbg | warn | err | error ) skip=0
;;
info )
case "$shf" in
all) skip=0
;;
none )
skip=1
;;
* ) mtype=${message%% *}
mtype=${mtype%:}
mtype=${mtype#fh}
echo "$shf"| grep -iq ${mtype}; search=$?
if [ $search -eq 0 ]; then
skip=0
else
skip=1
fi
;;
esac
;;
esac
if [ $skip -eq 0 ]; then
ocf_log "$level" "$message"
fi
}
#
# function: saphana_usage - short usage info
# params: -
# globals: $0(r)
#
function saphana_usage() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
methods=$(saphana_methods)
methods=$(echo $methods | tr ' ' '|')
cat <<-EOF
usage: $0 ($methods)
$0 manages a SAP HANA Instance as an HA resource.
The 'start' operation starts the HANA instance or bring the "clone instance" to a WAITING status
The 'stop' operation stops the HANA instance
The 'status' operation reports whether the HANA instance is running
The 'monitor' operation reports whether the HANA instance seems to be working in master/slave it also needs to check the system replication status
The 'promote' operation either runs a takeover for a secondary or a just-nothing for a primary
The 'demote' operation neary does nothing and just mark the instance as demoted
The 'notify' operation always returns SUCCESS
The 'validate-all' operation reports whether the parameters are valid
The 'methods' operation reports on the methods $0 supports
EOF
return $rc
}
function backup_global_and_nameserver() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
cp /hana/shared/LNX/global/hdb/custom/config/global.ini /hana/shared/LNX/global/hdb/custom/config/global.ini.$(date +"%s")
cp /hana/shared/LNX/global/hdb/custom/config/nameserver.ini /hana/shared/LNX/global/hdb/custom/config/nameserver.ini.$(date +"%s")
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_meta_data - print resource agent meta-data for cluster
# params: -
# globals: -
#
function saphana_meta_data() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
#
cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="SAPHana">
<version>0.151.3</version>
<shortdesc lang="en">Manages two SAP HANA instances in system replication (SR).</shortdesc>
<longdesc lang="en">
The SAPHanaSR resource agent manages two SAP Hana instances (databases) which are configured
in system replication. This first version is limited to the scale-up scenario. Scale-Out is
not supported in this version.
Managing the two SAP HANA instances means that the resource agent controls the start/stop of the
instances. In addition the resource agent is able to monitor the SAP HANA databases to check their
availability on landscape host configuration level. For this monitoring the resource agent relies on interfaces
provided by SAP. A third task of the resource agent is to also check the synchronisation status
of the two SAP HANA databases. If the synchronisation is not "SOK", then the cluster avoids to
failover to the secondary side, if the primary fails. This is to improve the data consistency.
The resource agent uses the following four interfaces provided by SAP:
1. sapcontrol/sapstartsrv
The interface sapcontrol/sapstartsrv is used to start/stop a HANA database instance/system
2. landscapeHostConfiguration
The interface is used to monitor a HANA system. The python script is named landscapeHostConfiguration.py.
landscapeHostConfiguration.py has some detailed output about HANA system status
and node roles. For our monitor the overall status is relevant. This overall
status is reported by the returncode of the script:
0: Internal Fatal, 1: ERROR, 2: WARNING, 3: INFO, 4: OK
The SAPHana resource agent will interpret returncodes 0 as FATAL, 1 as not-running or ERROR and and returncodes 2+3+4 as RUNNING.
3. hdbnsutil
The interface hdbnsutil is used to check the "topology" of the system replication as well as the current configuration
(primary/secondary) of a SAP HANA database instance. A second task of the interface is the possibility to run a
system replication takeover (sr_takeover) or to register a former primary to a newer one (sr_register).
4. hdbsql / systemReplicationStatus
Interface is SQL query into HANA (system replication table). The hdbsql query will be replaced by a python script
"systemReplicationStatus.py" in SAP HANA SPS8 or 9.
As long as we need to use hdbsql you need to setup secure store users for linux user root to be able to
access the SAP HANA database. You need to configure a secure store user key "SAPHANA${SID}SR" which can connect the SAP
HANA database:
5. saphostctrl
The interface saphostctrl uses the function ListInstances to figure out the virtual host name of the
SAP HANA instance. This is the hostname used during the HANA installation.
</longdesc>
<parameters>
<parameter name="SID" unique="0" required="1">
<longdesc lang="en">SAP System Identifier (SID) like "SLE" or "HAE"</longdesc>
<shortdesc lang="en">SAP System Identifier (SID)</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="InstanceNumber" unique="0" required="1">
<longdesc lang="en">SAP instance number like "00" or "07"</longdesc>
<shortdesc lang="en">SAP instance number</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="PREFER_SITE_TAKEOVER" unique="0" required="0">
<longdesc lang="en">Should cluster/RA prefer to switchover to slave instance instead of restarting master locally? Default="yes"
no: Do prefer restart locally
yes: Do prefer takever to remote site
</longdesc>
<shortdesc lang="en">Local or site recover preferred?</shortdesc>
<content type="boolean" default="yes" />
</parameter>
<parameter name="AUTOMATED_REGISTER" unique="0" required="0">
<shortdesc lang="en">Define, if a former primary should automatically be registered.</shortdesc>
<longdesc lang="en">The parameter AUTOMATED_REGISTER defines, whether a former primary instance should
be registered automatically by the resource agent during cluster/resource start, if the DUPLICATE_PRIMARY_TIMEOUT is expired... TDB
</longdesc>
<content type="boolean" default="false" />
</parameter>
<parameter name="DUPLICATE_PRIMARY_TIMEOUT" unique="0" required="0">
<shortdesc lang="en">Time difference needed between to primary time stamps, if a dual-primary situation occurs</shortdesc>
<longdesc lang="en">Time difference needed between to primary time stamps,
if a dual-primary situation occurs. If the time difference is
less than the time gap, then the cluster hold one or both instances in a "WAITING" status. This is to give an admin
a chance to react on a failover. A failed former primary will be registered after the time difference is passed. After
this registration to the new primary all data will be overwritten by the system replication.
</longdesc>
<content type="string" default="7200" />
</parameter>
<parameter name="DIR_EXECUTABLE" unique="0" required="0">
<longdesc lang="en">The full qualified path where to find sapstartsrv and sapcontrol. Specify this parameter, if you have changed the SAP kernel directory location after the default SAP installation.</longdesc>
<shortdesc lang="en">Path of sapstartsrv and sapcontrol</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="DIR_PROFILE" unique="0" required="0">
<longdesc lang="en">The full qualified path where to find the SAP START profile. Specify this parameter, if you have changed the SAP profile directory location after the default SAP installation.</longdesc>
<shortdesc lang="en">Path of start profile</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="INSTANCE_PROFILE" unique="1" required="0">
<longdesc lang="en">The name of the SAP HANA instance profile. Specify this parameter, if you have changed the name of the SAP HANA instance profile after the default SAP installation. Normally you do not need to set this parameter.</longdesc>
<shortdesc lang="en">HANA instance profile name</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="SAPHanaFilter" unique="0" required="0">
<shortdesc lang="en">OUTDATED PARAMETER</shortdesc>
<longdesc lang="en">OUTDATED PARAMETER</longdesc>
<content type="string" default="" />
</parameter>
</parameters>
<actions>
<action name="start" timeout="180" />
<action name="stop" timeout="240" />
<action name="status" timeout="60" />
<action name="monitor" depth="0" timeout="60" interval="120" />
<action name="monitor" depth="0" timeout="60" interval="121" role="Slave" />
<action name="monitor" depth="0" timeout="60" interval="119" role="Master" />
<action name="promote" timeout="320" />
<action name="demote" timeout="320" />
<action name="validate-all" timeout="5" />
<action name="meta-data" timeout="5" />
<action name="methods" timeout="5" />
</actions>
</resource-agent>
END
return $rc
}
#
# function: saphana_methods - report supported cluster methods
# params: -
# globals: -
# methods: What methods/operations do we support?
#
function saphana_methods() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0 m
for m in start stop status monitor promote demote notify validate-all methods meta-data usage; do
echo "$m"
done
return $rc
}
#
# function: dequote - filter: remove quotes (") from stdin
# params: -
# globals: -
function dequote()
{
local rc=0; tr -d '"'; return $rc
}
#
# function: remoteHost2remoteNode - convert a SAP remoteHost to the cluster node name
# params: remoteHost
# globals: ATTR_NAME_HANA_VHOST[*]
#
function remoteHost2remoteNode()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local -a clusterNodes=()
local cl=""
local vHost=""
local remoteHost="$1"
local remoteNode=""
local rc=1
for cl in ${otherNodes[@]}; do
vHost=$(get_hana_attribute $cl ${ATTR_NAME_HANA_VHOST[@]} "$cl")
if [ "$vHost" = "$remoteHost" ]; then # we found the correct node
remoteNode=$cl
rc=0
fi
done
if [ -n "$remoteNode" ]; then
echo "$remoteNode"
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: is_clone - report, if resource is configured as a clone (also master/slave)
# params: -
# globals: OCF_*(r)
# descript: is_clone : find out if we are configured to run in a Master/Slave configuration
# rc: 0: it is a clone, 1: it is not a clone
#
# DONE: PRIO2: For the first shipment (scale-out) we need to limit the clones to 2
#
function is_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
#
# is a clone config?
#
if [ -n "$OCF_RESKEY_CRM_meta_clone_max" ] \
&& [ "$OCF_RESKEY_CRM_meta_clone_max" -gt 0 ]; then
#
# yes it is a clone config - check, if its configured well
#
if [ "$OCF_RESKEY_CRM_meta_clone_node_max" -ne 1 ] || \
[ "$OCF_RESKEY_CRM_meta_clone_max" -ne 2 ] || \
[ "$OCF_RESKEY_CRM_meta_master_node_max" -ne 1 ] || \
[ "$OCF_RESKEY_CRM_meta_master_max" -ne 1 ]; then
super_ocf_log err "ACT: Clone options misconfigured. (expect: clone_max=2,clone_node_max=1,master_node_max=1,master_max=1)"
exit $OCF_ERR_CONFIGURED
fi
rc=0;
else
rc=1;
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: get_hana_attribute
# params: NODE ATTR [STORE]
# globals: -
# output: attribute value
#
function get_hana_attribute()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
local attr_node=$1
local attr_name=$2
local attr_store=${3:-reboot} # DONE: PRIO5 get this (optional) from parameter
local attr_default=${4:-}
local dstr
local attr_val=""
dstr=$(date)
case "$attr_store" in
reboot | forever )
echo "$dstr: SAPHana: crm_attribute -N ${attr_node} -G -n \"$attr_name\" -l $attr_store -q" >> /var/log/fhATTRIBUTE
crm_attribute -N ${attr_node} -G -n "$attr_name" -l $attr_store -q -d "$attr_default" 2>>/var/log/fhATTRIBUTE; rc=$?
;;
props )
echo "$dstr: SAPHana: crm_attribute -G -n \"$attr_name\" -t crm_config -q" >> /var/log/fhATTRIBUTE
crm_attribute -G -n "$attr_name" -t crm_config -q -d "$attr_default" 2>>/var/log/fhATTRIBUTE; rc=$?
;;
esac
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: set_hana_attribute - set the multi-state status of a node
# params: NODE VALUE ATTR [STORE]
# globals: -
#
function set_hana_attribute()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local attr_node=$1
local attr_value=$2
local attr_name=$3
local attr_store=${4:-reboot} # DONE: PRIO5 get this (optional) from parameter
local attr_default=${5:-}
local rc=1
local attr_old=""
attr_old=$(get_hana_attribute $attr_node $attr_name $attr_store $attr_default); get_rc=$?
if [ "$attr_old" != "$attr_value" ]; then
super_ocf_log debug "DBG: SET attribute $attr_name for node ${attr_node} to ${attr_value} former ($attr_old) get_rc=$get_rc "
dstr=$(date)
case "$attr_store" in
reboot | forever )
echo "$dstr: SAPHana: crm_attribute -N $attr_node -v $attr_value -n \"$attr_name\" -l $attr_store" >> /var/log/fhATTRIBUTE
crm_attribute -N $attr_node -v $attr_value -n "$attr_name" -l $attr_store 2>>/var/log/fhATTRIBUTE; rc=$?
;;
props )
echo "$dstr: SAPHana: crm_attribute -v $attr_value -n \"$attr_name\" -t crm_config -s SAPHanaSR" >> /var/log/fhATTRIBUTE
crm_attribute -v $attr_value -n "$attr_name" -t crm_config -s SAPHanaSR 2>>/var/log/fhATTRIBUTE; rc=$?
;;
esac
else
super_ocf_log debug "DBG: LET attribute $attr_name for node ${attr_node} still be ${attr_value}"
rc=0
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: assert - quickly go out of here with minimal error/return code handling and log
# params: MESSAGE
# globals: OCF_*(r)
#
function assert() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local err_msg=$1
local default_rc=$OCF_NOT_RUNNING
# DONE: Check, if we need to destinguish between probe and others
if ocf_is_probe; then
default_exit=$OCF_NOT_RUNNING
else
default_exit=$OCF_ERR_CONFIGURED
fi
if [ "$ACTION" = "stop" ]; then
cleanup_instance
exit $OCF_SUCCESS
fi
super_ocf_log err "ACT: $err_msg"
exit $OCF_NOT_RUNNING
}
#
# function: set_crm_master - set the crm master score of the local node
# params: SCORE
# globals: HA_SBIN_DIR(r), OCF_RESOURCE_INSTANCE(r)
#
function set_crm_master()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
local score=0
if [ -n "$1" ]; then
score=$1
fi
# DONE: PRIO2: Only adjust master if value is really different (try to check that)
oldscore=$(${HA_SBIN_DIR}/crm_master -G -q -l reboot)
if [ "$oldscore" != "$score" ]; then
super_ocf_log debug "DBG: SET crm master: $score (old: $oldscore)"
${HA_SBIN_DIR}/crm_master -v $score -l reboot; rc=$?
else
super_ocf_log debug "DBG: LET crm master: $score"
rc=0
fi
#logger -t fhLOG "crm_master with: $OCF_RESOURCE_INSTANCE -v $score -l reboot"
return $rc
}
#
# function: scoring_crm_master - score instance due to role ans sync match (table SCORING_TABLE_PREFERRED_SITE_TAKEOVER)
# params: NODE_ROLES NODE_SYNC_STATUS
# globals: SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@],
#
scoring_crm_master()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local roles="$1"
local sync="$2"
local skip=0
local myScore=""
for scan in "${SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@]}"; do
if [ $skip -eq 0 ]; then
read rolePatt syncPatt score <<< $scan
if grep "$rolePatt" <<< "$roles"; then
if grep "$syncPatt" <<< "$sync"; then
skip=1
myScore=$score
fi
fi
fi
done
super_ocf_log debug "DBG: scoring_crm_master adjust score $myScore"
# TODO: PRIO1: DO Not Score, If we did not found our role/sync at this moment - bsc#919925
if [ -n "$myScore" ]; then
set_crm_master $myScore
fi
}
#
# function: get_crm_master - get the crm master score of the local node
# params: -
# globals: HA_SBIN_DIR(r)
#
function get_crm_master()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
${HA_SBIN_DIR}/crm_master -G -q -l reboot; rc=$?
return $rc
}
#
# function: saphana_init - initialize variables for the resource agent
# params: InstanceName
# globals: OCF_*(r), SID(w), sid(rw), sidadm(w), InstanceName(w), InstanceNr(w), SAPVIRHOST(w), PreferSiteTakeover(w),
# globals: sr_name(w), remoteHost(w), otherNodes(w), rem_SR_name(w)
# globals: ATTR_NAME_HANA_SYNC_STATUS(w), ATTR_NAME_HANA_CLONE_STATE(w)
# globals: DIR_EXECUTABLE(w), SAPSTARTSRV(w), SAPCONTROL(w), DIR_PROFILE(w), SAPSTARTPROFILE(w), LD_LIBRARY_PATH(w), PATH(w)
# globals: LPA_DIRECTORY(w), SIDInstanceName(w), remoteNode(w), hdbSrQueryTimeout(w)
# saphana_init : Define global variables with default values, if optional parameters are not set
#
function saphana_init() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_SUCCESS
local vName
local clN
# local site
# two parameter models (for transition only)
# OLD: InstanceName
# NEW: SID InstanceNumber
SID=$OCF_RESKEY_SID
InstanceNr=$OCF_RESKEY_InstanceNumber
SIDInstanceName="${SID}_HDB${InstanceNr}"
InstanceName="HDB${InstanceNr}"
super_ocf_log debug "DBG: Used new method to get SID ($SID) and InstanceNr ($InstanceNr)"
sid=$(echo "$SID" | tr [:upper:] [:lower:])
sidadm="${sid}adm"
# TODO PRIO3: Do we need a parameter for the RA to be able to adjust hdbSrQueryTimeout?
hdbSrQueryTimeout=180
# DONE: PRIO4: SAPVIRHOST might be different to NODENAME
# DONE: PRIO1: ASK: Is the output format of ListInstances fix? Could we take that as an API? Answer: Yes
# try to catch: Inst Info : LNX - 42 - lv9041 - 740, patch 36, changelist 1444691
# We rely on the following format: SID is word#4, NR is work#6, vHost is word#8
vName=$(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances \
| awk '$4 == SID && $6=NR { print $8 }' SID=$SID NR=$InstanceNr)
if [ -z "$vName" ]; then
#
# if saphostctrl does not know the answer, try to fallback to attribute provided by SAPHanaTopology
#
vName=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_VHOST[@]} "$NODENAME");
fi
SAPVIRHOST=${vName}
PreferSiteTakeover="$OCF_RESKEY_PREFER_SITE_TAKEOVER"
AUTOMATED_REGISTER="${OCF_RESKEY_AUTOMATED_REGISTER:-false}"
LPA_DIRECTORY=/var/lib/SAPHanaRA
LPA_ATTR=("lpa_${sid}_lpt" "forever")
super_ocf_log debug "DBG: SID=$SID, sid=$sid, SIDInstanceName=$SIDInstanceName, InstanceName=$InstanceName, InstanceNr=$InstanceNr, SAPVIRHOST=$SAPVIRHOST"
ocf_env=$(env | grep 'OCF_RESKEY_CRM')
super_ocf_log debug "DBG: OCF: $ocf_env"
#
ATTR_NAME_HANA_SYNC_STATUS=("hana_${sid}_sync_state" "reboot") # SOK, SFAIL, UNKNOWN?
ATTR_NAME_HANA_PRIMARY_AT=("hana_${sid}_primary_at" "reboot") # Not used so far
ATTR_NAME_HANA_CLONE_STATE=("hana_${sid}_clone_state" "reboot") # UKNOWN?, DEMOTED, PROMOTED
ATTR_NAME_HANA_REMOTEHOST=("hana_${sid}_remoteHost" "forever")
ATTR_NAME_HANA_SITE=("hana_${sid}_site" "forever")
ATTR_NAME_HANA_ROLES=("hana_${sid}_roles" "reboot")
ATTR_NAME_HANA_SRMODE=("hana_${sid}_srmode" "forever")
ATTR_NAME_HANA_VHOST=("hana_${sid}_vhost" "forever")
ATTR_NAME_HANA_STATUS=("hana_${sid}_status" "reboot")
#
# TODO: PRIO4: Table for non-preferred-site-takeover
#
SCORING_TABLE_PREFERRED_SITE_TAKEOVER=(
"[234]*:P:[^:]*:master .* 150"
"[015-9]*:P:[^:]*:master .* 90"
"[0-9]*:P:[^:]*:slave .* 60"
"[0-9]*:P:[^:]*:\? .* 0"
"[0-9]*:P:[^:]*:- .* 0"
"[234]*:S:[^:]*:master SOK 100"
"[015-9]*:S:[^:]*:master SOK 80"
"[0-9]*:S:[^:]*:master SFAIL -INFINITY"
"[0-9]*:S:[^:]*:slave SOK 10"
"[0-9]*:S:[^:]*:slave SFAIL -INFINITY"
"[0-9]*:S:[^:]*:\? .* 0"
"[0-9]*:S:[^:]*:- .* 0"
".* .* -1"
)
SCORING_TABLE_PREFERRED_LOCAL_RESTART=(
"[0-9]*:P:[^:]*:master .* 150"
"[0-9]*:P:[^:]*:slave .* 140"
"[0-9]*:P:[^:]*:\? .* 0"
"[0-9]*:P:[^:]*:- .* 0"
"[0-9]*:S:[^:]*:master SOK 100"
"[0-9]*:S:[^:]*:master SFAIL -INFINITY"
"[0-9]*:S:[^:]*:slave SOK 10"
"[0-9]*:S:[^:]*:slave SFAIL -INFINITY"
"[0-9]*:S:[^:]*:\? .* 0"
"[0-9]*:S:[^:]*:- .* 0"
".* .* -1"
)
#
DUPLICATE_PRIMARY_TIMEOUT="${OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT:-7200}"
super_ocf_log debug "DBG: DUPLICATE_PRIMARY_TIMEOUT=$DUPLICATE_PRIMARY_TIMEOUT"
#
# Determine list of other cluster nodes and store in otherNodes variable
otherNodes=()
case $(crm_attribute --type crm_config --name cluster-infrastructure -q) in
*corosync* ) otherNodes=($(crm_node -l | awk '{ if ($2 != me) { print $2 }}' me=${NODENAME}));;
*openais* ) otherNodes=($(crm_node -l | awk '$3 == "member" { if ($2 != me) { print $2 }}' me=${NODENAME}));;
*cman* ) otherNodes=($(crm_node -l | awk '{for (i=1; i<=NF; i++) { if ($i != me) { print $i }}}' me=${NODENAME}));;
esac
#
#
remoteHost=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_REMOTEHOST[@]});
if [ -z "$remoteHost" ]; then
if [ ${#otherNodes[@]} -eq 1 ]; then # we are a 2 node cluster, lets assume the other is the remote-host
remoteNode=${otherNodes[0]}
remoteHost=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_VHOST[@]} "$remoteNode");
super_ocf_log debug "DBG: auto-guess remoteHost=$remoteHost"
else
super_ocf_log debug "DBG: Could not auto-guess remoteHost out of list (${otherNodes[@]})"
fi
else
#
# search cluster node which vhost is equal remoteHost
#
remoteNode=$(remoteHost2remoteNode $remoteHost)
# TODO: PRIO5: catch rc!=0
fi
# ATTR_NAME_HANA_SITE
sr_name=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SITE[@]});
sr_mode=$(get_hana_attribute "${NODENAME}" ${ATTR_NAME_HANA_SRMODE[@]})
if [ -z "$sr_mode" ]; then
sr_mode="sync"
fi
if [ -n "$remoteNode" ]; then
rem_SR_name=$(get_hana_attribute ${remoteNode} ${ATTR_NAME_HANA_SITE[@]});
fi
super_ocf_log debug "DBG: sr_name=$sr_name, remoteHost=$remoteHost, remoteNode=$remoteNode, sr_mode=$sr_mode"
# optional OCF parameters, we try to guess which directories are correct
if [ -z "$OCF_RESKEY_DIR_EXECUTABLE" ]
then
if have_binary /usr/sap/$SID/$InstanceName/exe/sapstartsrv && have_binary /usr/sap/$SID/$InstanceName/exe/sapcontrol
then
DIR_EXECUTABLE="/usr/sap/$SID/$InstanceName/exe"
fi
else
if have_binary "$OCF_RESKEY_DIR_EXECUTABLE/sapstartsrv" && have_binary "$OCF_RESKEY_DIR_EXECUTABLE/sapcontrol"
then
DIR_EXECUTABLE="$OCF_RESKEY_DIR_EXECUTABLE"
fi
fi
SAPSTARTSRV="$DIR_EXECUTABLE/sapstartsrv"
SAPCONTROL="$DIR_EXECUTABLE/sapcontrol"
[ -z "$DIR_EXECUTABLE" ] && assert "Cannot find sapstartsrv and sapcontrol executable, please set DIR_EXECUTABLE parameter!"
DIR_PROFILE="${OCF_RESKEY_DIR_PROFILE:-/usr/sap/$SID/SYS/profile}"
# check, if the following fall-back is ok, or if there could be multiple profiles matching this pattern
if [ -n "${SAPVIRHOST}" ]; then
SAPSTARTPROFILE="$DIR_PROFILE/${OCF_RESKEY_INSTANCE_PROFILE:-${SID}_${InstanceName}_${SAPVIRHOST}}"
else
# check, if the following fall-back is ok, or if there could be multiple profiles matching this pattern
# also take profile versions into account - they might break this fall-back
# TODO: PRIO4: Check, if it makes sense to implement an additional last fall-back: get the SAPSTARTPROFILE from /usr/sap/sapservices
#
SAPSTARTPROFILE="$(ls -1 $DIR_PROFILE/${OCF_RESKEY_INSTANCE_PROFILE:-${SID}_${InstanceName}_*})"
fi
# as root user we need the library path to the SAP kernel to be able to call sapcontrol
# check, if we already added DIR_EXECUTABLE at the beginning of LD_LIBRARY_PATH
if [ "${LD_LIBRARY_PATH%%*:}" != "$DIR_EXECUTABLE" ]
then
LD_LIBRARY_PATH=$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH
export LD_LIBRARY_PATH
fi
PATH=${PATH}:${DIR_EXECUTABLE}; export PATH
super_ocf_log info "FLOW $FUNCNAME rc=$OCF_SUCCESS"
#############################
# TODO: PRIO9: To be able to call landscapeHostConfig.py without su (so as root)
# TODO: PRIO9: Research for environment script .htacces or something like that
#export SAPSYSTEMNAME=ZLF
#export DIR_INSTANCE=/usr/sap/ZLF/HDB02
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DIR_INSTANCE/exe:$DIR_INSTANCE/exe/Python/lib
#export PYTHONPATH=$DIR_INSTANCE/$HOST:$DIR_INSTANCE/exe/python_support:$DIR_INSTANCE/exe
#export PYTHONHOME=$DIR_INSTANCE/exe/Python
#export SAP_RETRIEVAL_PATH=$DIR_INSTANCE/$HOST
#export DIR_EXECUTABLE=$DIR_INSTANCE/exe
#############################
return $OCF_SUCCESS
}
# function: check_secstore_users
# params: USER
# globals: DIR_EXECUTABLE(r)
#
# TODO: PRIO5: Might be dropped, if we get a script for fetching the sync status
function check_secstore_users()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local user=""
local rc=1
while [ $# -gt 0 ]; do
user="$1"
$DIR_EXECUTABLE/hdbuserstore list | grep -q "KEY $user" && echo "$user" && rc=0 && break
shift
done
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: check_sapstartsrv - check for sapstartsrv - optional start
# params: -
# globals: DIR_PROFILE(w), SAPSTARTPROFILE(r), SAPCONTROL(r), SID(r), InstanceName(r), InstanceNr(r), OCF_*(r)
# check_sapstartsrv : Before using sapcontrol we make sure that the sapstartsrv is running.
#
function check_sapstartsrv() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local restart=0
local runninginst=""
local rc=$OCF_SUCCESS
local output=""
if [ ! -S /tmp/.sapstream5${InstanceNr}13 ]; then
super_ocf_log warn "ACT: sapstartsrv is not running for instance $SID-$InstanceName (no UDS), it will be started now"
restart=1
else
output=$($SAPCONTROL -nr $InstanceNr -function ParameterValue INSTANCE_NAME -format script)
if [ $? -eq 0 ]
then
runninginst=$(echo "$output" | grep '^0 : ' | cut -d' ' -f3)
if [ "$runninginst" != "$InstanceName" ]
then
super_ocf_log warn "ACT: sapstartsrv is running for instance $runninginst, that service will be killed"
restart=1
else
output=$($SAPCONTROL -nr $InstanceNr -function AccessCheck Start)
if [ $? -ne 0 ]; then
super_ocf_log warn "ACT: FAILED - sapcontrol -nr $InstanceNr -function AccessCheck Start ($(ls -ld1 /tmp/.sapstream5${InstanceNr}13))"
super_ocf_log warn "ACT: sapstartsrv will be restarted to try to solve this situation, otherwise please check sapstsartsrv setup (SAP Note 927637)"
restart=1
fi
fi
else
super_ocf_log warn "ACT: sapstartsrv is not running for instance $SID-$InstanceName, it will be started now"
restart=1
fi
fi
if [ -z "$runninginst" ]; then runninginst=$InstanceName; fi
if [ $restart -eq 1 ]
then
if [ -d /usr/sap/$SID/SYS/profile/ ]
then
DIR_PROFILE="/usr/sap/$SID/SYS/profile"
else
assert "Expected /usr/sap/$SID/SYS/profile/ to be a directory, please set DIR_PROFILE parameter!"
fi
[ ! -r $SAPSTARTPROFILE ] && assert "Expected $SAPSTARTPROFILE to be the instance START profile, please set INSTANCE_PROFILE parameter!"
pkill -9 -f "sapstartsrv.*$runninginst"
# removing the unix domain socket files as they might have wrong permissions
# or ownership - they will be recreated by sapstartsrv during next start
rm -f /tmp/.sapstream5${InstanceNr}13
rm -f /tmp/.sapstream5${InstanceNr}14
$SAPSTARTSRV pf=$SAPSTARTPROFILE -D -u $sidadm
# now make sure the daemon has been started and is able to respond
local srvrc=1
while [ $srvrc -eq 1 -a $(pgrep -f "sapstartsrv.*$runninginst" | wc -l) -gt 0 ]
do
sleep 1
$SAPCONTROL -nr $InstanceNr -function GetProcessList > /dev/null 2>&1
srvrc=$?
done
if [ $srvrc -ne 1 ]
then
super_ocf_log info "ACT: sapstartsrv for instance $SID-$InstanceName was restarted!"
rc=$OCF_SUCCESS
else
super_ocf_log error "ACT: sapstartsrv for instance $SID-$InstanceName could not be started!"
rc=$OCF_ERR_GENERIC
ocf_is_probe && rc=$OCF_NOT_RUNNING
fi
fi
return $rc
}
#
# function: cleanup_instance - remove resources from a crashed instance
# params: -
# globals: -
#
function cleanup_instance() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
# TODO: PRIO5: Check, if we need HANA cleanup procedure (processes, ipc obj, pid files); Currently not needed
super_ocf_log debug "DBG: cleanup_instance currently not implemented"
rc=0
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
}
#
# function: check_for_primary - check if local SAP HANA is configured as primary
# params: -
# globals: HANA_STATE_PRIMARY(r), HANA_STATE_SECONDARY(r), HANA_STATE_DEFECT(r)
#
function check_for_primary() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$HANA_STATE_DEFECT
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh hdbnsutil -sr_state --sapcontrol=1
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of hdbnsutil and use getParameter.py as fallback
node_full_status=$(su - ${sidadm} -c "hdbnsutil -sr_state" 2>/dev/null )
node_status=$(echo "$node_full_status" | awk '$1=="mode:" {print $2}')
super_ocf_log debug "DBG: check_for_primary: node_status=$node_status"
# TODO: PRIO2: Maybe we need to use a fallback interface when hdbnsitil does not answer properly -> lookup in config files?
# This might also solve some problems when we could not figure-out the ilocal or remote site name
for i in 1 2 3 4 5 6 7 8 9; do
case "$node_status" in
primary )
super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_PRIMARY"
return $HANA_STATE_PRIMARY;;
syncmem | sync | async )
super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_SECONDARY"
return $HANA_STATE_SECONDARY;;
none ) # have seen that mode on second side BEFEORE we registered it as replica
super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_STANDALONE"
return $HANA_STATE_STANDALONE;;
* )
super_ocf_log err "ACT: check_for_primary: we didn't expect node_status to be: <$node_status>"
dump=$( echo $node_status | hexdump -C );
super_ocf_log err "ACT: check_for_primary: we didn't expect node_status to be: DUMP <$dump>"
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh hdbnsutil -sr_state --sapcontrol=1
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of hdbnsutil and use getParameter.py as fallback
node_full_status=$(su - ${sidadm} -c "hdbnsutil -sr_state" 2>/dev/null )
node_status=$(echo "$node_full_status" | awk '$1=="mode:" {print $2}')
super_ocf_log debug "DEC: check_for_primary: loop=$i: node_status=$node_status"
# TODO: PRIO1: Maybe we need to keep the old value for P/S/N, if hdbnsutil just crashes
esac;
done
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
# function: analyze_hana_sync_statusSRS
# params: -
# globals: DIR_EXECUTABLE(r), FULL_SR_STATUS(w), remoteNode
#
# systemReplicationStatus.py return-codes:
# NoHSR = 10
# Error = 11
# Unkown = 12
# Initializing = 13
# Syncing = 14
# Active = 15
function analyze_hana_sync_statusSRS()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=-1 srRc=0 all_nodes_other_side="" n="" siteParam=""
if [ -n "$rem_SR_name" ]; then
siteParam="--site=$rem_SR_name"
fi
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh systemReplicationStatus.py $siteParam
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of systemReplicationStatus.py
FULL_SR_STATUS=$(su - $sidadm -c "python $DIR_EXECUTABLE/python_support/systemReplicationStatus.py $siteParam" 2>/dev/null); srRc=$?
super_ocf_log info "DEC $FUNCNAME systemReplicationStatus.py (to site '$rem_SR_name')-> $srRc"
super_ocf_log info "FLOW $FUNCNAME systemReplicationStatus.py (to site '$rem_SR_name')-> $srRc"
#
# TODO: PRIO2: Here we might also need to filter additional sites (if multi tier should be supported)
# And is the check for return code capable for chains?
#
if [ $srRc -eq 15 ]; then
# Fix for a HANA BUG, where a non-working SR resulted in RC 15:
if grep -q "ACTIVE" <<< "$FULL_SR_STATUS"; then
super_ocf_log info "FLOW $FUNCNAME SOK"
set_hana_attribute "$remoteNode" "SOK" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
super_ocf_log info "ACT site=$sr_name, seting SOK for secondary (1)"
lpa_set_lpt 30 "$remoteNode"
rc=0;
else
# ok we should be careful and set secondary to SFAIL
super_ocf_log info "FLOW $FUNCNAME SFAIL"
set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
super_ocf_log info "ACT site=$sr_name, seting SFAIL for secondary (6) - srRc=$srRc lss=$lss No ACTIVES found in cmd output"
# TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary
lpa_set_lpt 10 "$remoteNode"
fi
elif [ $srRc -le 11 ]; then # 11 and 10
# if systemReplicationStatus is ERROR and landscapeHostConfiguration is down than do NOT set SFAIL
get_hana_landscape_status; lss=$?
if [ $lss -lt 2 ]; then
# keep everithing like it was
rc=2
else
# ok we should be careful and set secondary to SFAIL
super_ocf_log info "FLOW $FUNCNAME SFAIL"
set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
super_ocf_log info "ACT site=$sr_name, seting SFAIL for secondary (5) - srRc=$srRc lss=$lss"
# TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary
lpa_set_lpt 10 "$remoteNode"
rc1
fi
else
super_ocf_log info "FLOW $FUNCNAME SFAIL"
set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
super_ocf_log info "ACT site=$sr_name, seting SFAIL for secondary (2) - srRc=$srRc"
# TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary
lpa_set_lpt 10 "$remoteNode"
rc=1;
fi
super_ocf_log info "FLOW $FUNCNAME PRIM+LPA"
super_ocf_log info "DBG PRIM"
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
####
#### OLD HDBSQL STUFF FOR SPS6,7,8 AND SCALE-UP ONLY
####
# function: analyze_hana_sync_statusSQL - query and check hana system replication status
# params: -
# globals: DIR_EXECUTABLE(r), remoteHost(r)
# get the HANA sync status
#
function analyze_hana_sync_statusSQL()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local -a clusterNodes=()
local cl=""
local vHost=""
local n=""
local hana_sync_status="" what_does_the_chamelion_say=""
local secUser="SLEHALOC"
local chkusr;
local rc=0
local sqlrc=0
# local query_state='select distinct REPLICATION_STATUS from SYS.M_SERVICE_REPLICATION'
# select distinct REPLICATION_STATUS from SYS.M_SERVICE_REPLICATION where SITE_NAME='"SITE1"'"
local query_state="select distinct REPLICATION_STATUS from SYS.M_SERVICE_REPLICATION where SITE_NAME='"${sr_name}"'"
local query_secondaries='select distinct SECONDARY_HOST from SYS.M_SERVICE_REPLICATION'
local query_failed_secondaries="select distinct SECONDARY_HOST from SYS.M_SERVICE_REPLICATION where SECONDARY_SITE_NAME = (select distinct SECONDARY_SITE_NAME from SYS.M_SERVICE_REPLICATION WHERE REPLICATION_STATUS != 'ACTIVE')"
local all_cluster_hosts all_secondary_hosts all_broken_secondaries
#
#####################################################################################################
#
# select distinct SITE_NAME, HOST, REPLICATION_STATUS, SECONDARY_SITE_NAME, SECONDARY_HOST from SYS.M_SERVICE_REPLICATION
#
# ===> "Walldorf", "sap-app-8" "ACTIVE", "Rot", "sap-app-5"
# "Rot", "sap-app-5", "ACTIVE", "oslo", "sap-app-7"
#
#####################################################################################################
#
secUser=$(check_secstore_users SAPHANA${SID}SR SLEHALOC RHELHALOC) ; chkusr=$?
if [ $chkusr -ne 0 ]; then
super_ocf_log err "ACT: Secure store users are missing (see best practice manual how to setup the users)"
rc=$OCF_ERR_CONFIGURED
fi
hana_sync_status=$(timeout $hdbSrQueryTimeout $DIR_EXECUTABLE/hdbsql -a -x -U $secUser $query_state); sqlrc=$?
hana_sync_status=$(echo $hana_sync_status | dequote)
super_ocf_log debug "DBG: hdbsql rc=$sqlrc hana_sync_status=\"$hana_sync_status\""
if [ "$sqlrc" -eq 0 -a "$hana_sync_status" != "" ]; then
#
# UNKNOWN, ACTIVE, ERROR, INITIALIZING
#
if [ "${hana_sync_status}" == "ACTIVE" ]; then
# TODO PRIO1: REMOVE remoteNode dependency - set SOK
set_hana_attribute "$remoteNode" "SOK" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
else
super_ocf_log warn "ACT: HANA SYNC STATUS is: ${hana_sync_status}"
# TODO PRIO1: REMOVE remoteNode dependency - set SFAIL
set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
fi
else
case "$sqlrc" in
19 )
# return codes 19: license error -> set SFAIL!
# DONE: PRIO1: We should NOT set SFAIL, if HDB is exactly broken now
# When HDB breaks during monitor this could prevent a prositive remote failover
super_ocf_log warn "ACT: Was not able to fetch HANA SYNC STATUS - set sync status to SFAIL for ALL OTHER cluster hosts"
for n in $otherNodes; do
set_hana_attribute "$n" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
done
;;
esac
fi
return $rc
}
#
# function: get_hana_landscape_status - figure out hana ladscape status
# params: -
# globals: sidadm(r), DIR_EXECUTABLE(r)
#
function get_hana_landscape_status()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
#
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh landscapeHostConfiguration.py
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of landscapeHostConfiguration.py
su - $sidadm -c "python $DIR_EXECUTABLE/python_support/landscapeHostConfiguration.py" 1>/dev/null 2>/dev/null; rc=$?
return $rc;
}
#
# function: register_hana_secondary - register local hana as secondary to the other site
# params: -
# globals: sidadm(r), remoteHost(r), InstanceNr(r), sr_mode(r), sr_name(r)
# register_hana_secondary
#
function register_hana_secondary()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=2;
local remoteInstance="";
remoteInstance=$InstanceNr
if ocf_is_true ${AUTOMATED_REGISTER}; then
#
#
#
#
#
super_ocf_log info "ACT: REGISTER: hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"
#
#
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh hdbnsutil -sr_register ...
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of hdbnsutil -sr_register ????
su - $sidadm -c "hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"; rc=$?
# backup_global_and_nameserver
else
super_ocf_log info "ACT: SAPHANA DROP REGISTER because AUTOMATED_REGISTER is set to FALSE"
rc=1
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc;
}
#
# function: saphana_status - pure status check
# params: -
# globals: SIDInstanceName, OCF_*,
function saphana_status() {
local binDeam="hdb.sap${SIDInstanceName}" rc=0
binDeam=${binDeam:0:15} # Process name is limited to the first 15 characters
if pgrep $binDeam 1>/dev/null; then rc=$OCF_SUCCESS; else rc=$OCF_NOT_RUNNING; fi
return $rc
}
#
# function: saphana_start - start a hana instance
# params: -
# globals: OCF_*, SAPCONTROL, InstanceNr, SID, InstanceName,
#
function saphana_start() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_NOT_RUNNING
local output=""
local loopcount=0
check_sapstartsrv
rc=$?
#
# TODO: ASK: PRIO5: For SCALE-OUT - do we need to use an other call like StartSystem? Or better to use the HDB command?
#
if [ $rc -eq $OCF_SUCCESS ]; then
output=$($SAPCONTROL -nr $InstanceNr -function Start)
rc=$?
super_ocf_log info "ACT: Starting SAPHANA Instance $SID-$InstanceName: $output"
fi
if [ $rc -eq 0 ]
then
# TODO: PRIO9: something more dynamic than 3600 seconds in WaitforStarted
output=$($SAPCONTROL -nr $InstanceNr -function WaitforStarted 3600 1)
if [ $? -eq 0 ]
then
super_ocf_log info "ACT: SAPHANA Instance $SID-$InstanceName started: $output"
rc=$OCF_SUCCESS
else
super_ocf_log err "ACT: SAPHANA Instance $SID-$InstanceName start failed: $output"
rc=$OCF_ERR_GENERIC
fi
else
super_ocf_log err "ACT: SAPHANA Instance $SID-$InstanceName start failed: $output"
rc=$OCF_ERR_GENERIC
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_stop - stop a hana instance
# params: -
# globals: OCF_*(r), SAPCONTROL(r), SID(r), InstanceName(r)
# saphana_stop: Stop the SAP instance
#
function saphana_stop() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local output=""
local rc=0
check_sapstartsrv; rc=$?
if [ $rc -eq $OCF_SUCCESS ]; then
output=$($SAPCONTROL -nr $InstanceNr -function Stop)
rc=$?
super_ocf_log info "ACT: Stopping SAP Instance $SID-$InstanceName: $output"
fi
if [ $rc -eq 0 ]
then
output=$($SAPCONTROL -nr $InstanceNr -function WaitforStopped 3600 1)
if [ $? -eq 0 ]
then
super_ocf_log info "ACT: SAP Instance $SID-$InstanceName stopped: $output"
rc=$OCF_SUCCESS
else
super_ocf_log err "ACT: SAP Instance $SID-$InstanceName stop failed: $output"
rc=$OCF_ERR_GENERIC
fi
else
super_ocf_log err "ACT: SAP Instance $SID-$InstanceName stop failed: $output"
rc=$OCF_ERR_GENERIC
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_validate - validation of (some) variables/parameters
# params: -
# globals: OCF_*(r), SID(r), InstanceName(r), InstanceNr(r), SAPVIRHOST(r)
# saphana_validate: Check the symantic of the input parameters
#
function saphana_validate() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_SUCCESS
#
# SID is Alpha-AlphaNumeric-Alphanumeric?
#
if [ $(echo "$SID" | grep -c '^[A-Z][A-Z0-9][A-Z0-9]$') -ne 1 ]
then
super_ocf_log err "ACT: Parsing instance profile name: '$SID' is not a valid SID!"
rc=$OCF_ERR_ARGS
fi
#
# InstanceNr is a two-Digit?
#
if [ $(echo "$InstanceNr" | grep -c '^[0-9][0-9]$') -ne 1 ]
then
super_ocf_log err "ACT: Parsing instance profile name: '$InstanceNr' is not a valid instance number!"
rc=$OCF_ERR_ARGS
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_start_primary - handle startup of PRIMARY in M/S
# params:
# globals: OCF_*(r), NODENAME, ATTR_NAME_*, HANA_STATE_*,
#
function saphana_start_primary()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
local lss sqlrc;
local rc=0
local lpa_dec=4
local lpa_advice=""
#
# we will be a master (PRIMARY) so checking, if the is an OTHER master
#
super_ocf_log debug "DBG: saphana_primary - check_for_primary reports HANA_STATE_PRIMARY"
#
lpa_init_lpt $HANA_STATE_PRIMARY
lpa_check_lpt_status; lpa_dec=$?
get_hana_landscape_status; lss=$?
my_role=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_ROLES[@]})
my_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
case "$lpa_dec" in
0 ) # LPA says start-up
lpa_advice="start"
# TODO: PRIO1: We need to do a special handling for remote being a 234-Secondary in SR Status SOK
# if ( remote_role like [234]:S ) && ( remote_sync_status is SOK|PRIM ) && ( PreferSiteTakeover )
# then lpa_advice="wait"
remoteRole=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_ROLES[@]})
remoteSync=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_SYNC_STATUS[@]})
super_ocf_log info "DEC: saphana_primary - checking remoteStatus"
if ocf_is_true "${PreferSiteTakeover}"; then
remoteStatus="$remoteRole:$remoteSync"
case "$remoteStatus" in
[234]:S:*:SOK | [234]:S:*:PRIM )
lpa_advice="wait"
# TODO: PRIO3: Split WAIT into WAIT4TAKEOVER
super_ocf_log info "DEC: saphana_primary - waiting for secondary to takeover (SOK, PreferSiteTakover)"
;;
* )
super_ocf_log info "DEC: saphana_primary - remoteStatus is: $remoteStatus"
;;
esac
else
super_ocf_log info "DEC: saphana_primary - PreferSiteTakeover set to false"
fi
;;
1) # LPA says register!
lpa_advice="register"
;;
2) # LPA says wait for second LPT
lpa_advice="wait"
;;
3 | 4 ) # LPA says something is completely wrong - FAIL resource # TODO: PRIO1: RC3 for waiting remote side to report lss
lpa_advice="fail"
;;
* ) # LPA failed with an unkonown status - FAIL resource
lpa_advice="fail"
;;
esac
# DONE: PRIO2: Do we need to differ 0 and 1 here? While 0 is a fatal SAP error, 1 for down/error
if [ $lss -eq 0 ]; then
super_ocf_log err "ACT: get_hana_landscape_status reports FATAL"
# DONE: PRIO1: what to do for lss=0?
# TODO: PRIO3: Check, if OCF_ERR_GENERIC is best reaction
lpa_advice="skip"
rc=$OCF_ERR_GENERIC
fi
case "$lpa_advice" in
start ) # process a normal START
case "$lss" in
2 | 3 | 4 ) # as landcape says we are up - just set the scores and return code
super_ocf_log info "LPA: landcape: UP, LPA: start ==> keep running"
LPTloc=$(date '+%s')
lpa_set_lpt $LPTloc $NODENAME
rc=$OCF_SUCCESS
;;
1 ) # landcape says we are down, lets start and adjust scores and return code
super_ocf_log info "LPA: landcape: DOWN, LPA: start ==> start instance"
saphana_start
rc=$?
LPTloc=$(date '+%s')
lpa_set_lpt $LPTloc $NODENAME
;;
esac
scoring_crm_master "$my_role" "$my_sync"
;;
register ) # process a REGISTER
case "$lss" in
2 | 3 | 4 ) # upps we are up - but shoudn't? - we could not register with started HDB
# DONE: PRIO3: check if this reaction is correct - tell cluster about failed start
super_ocf_log info "LPA: landcape: UP, LPA: register ==> take down"
set_crm_master -inf
rc=$OCF_NOT_RUNNING
;;
1 ) # lets try to register
# DONE: PRIO2: Like Action in start_secondary
super_ocf_log info "LPA: landcape: DOWN, LPA: register ==> try to register"
super_ocf_log info "DEC: AN OTHER HANA IS AVAILABLE ==> LETS REGISTER"
set_crm_master 0
if wait_for_primary_master 1; then
register_hana_secondary
check_for_primary; primary_status=$?
if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
super_ocf_log info "ACT: Register successful"
lpa_push_lpt 10
lpa_set_lpt 10 $NODENAME
set_crm_master 0
saphana_start_secondary
rc=$?
lpa_set_lpt 10 $NODENAME
else
super_ocf_log err "ACT: Register failed"
rc=$OCF_NOT_RUNNING
fi
else
# lets check next monitor, if we can register
rc=$OCF_SUCCESS
fi
;;
esac
;;
wait ) # process a WAIT
case "$lss" in
2 | 3 | 4 ) # as we ARE up we just keep it up
# TODO: PRIO3: I now change from "just keep it up to take that down"
# TODO: PRIO1 differ lpt_advice!!
# 2 => DOWN
# 3 => KEEP
# TODO: PRIO3: OCF_SUCCESS, OCF_NOT_RUNNING or OCF_ERR_xxxx ?
set_crm_master -9000
#scoring_crm_master "$my_role" "$my_sync"
rc=$OCF_ERR_GENERIC
;;
1 ) # we are down, so we should wait --> followup in next monitor
super_ocf_log info "LPA: landcape: DOWN, LPA: wait ==> keep waiting"
# TODO: PRIO3: Check, if WAITING is correct here
set_hana_attribute ${NODENAME} "WAITING4LPA" ${ATTR_NAME_HANA_CLONE_STATE[@]}
set_crm_master -9000
rc=$OCF_SUCCESS
;;
esac
;;
fail ) # process a lpa FAIL
super_ocf_log info "LPA: LPA reports FAIL"
set_crm_master -inf
rc=$OCF_NOT_RUNNING
;;
esac
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# check_for_primary_master
# params: -
# globals: ATTR_NAME_HANA_ROLES[@], NODENAME
#
check_for_primary_master()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=1
local ch ch_role
#
# get actual list of cluster members
#
if [ -n "$otherNodes" ]; then
for ch in ${otherNodes[@]}; do
if [ $rc -eq 1 ]; then
ch_role=$(get_hana_attribute ${ch} ${ATTR_NAME_HANA_ROLES[@]})
# TODO: PRIO3: check if [0-9], [234] or [34] is correct
# TODO: PRIO4: Do we need different checks like "any-primary-master" or "running-primary-master" ?
# grep '[0-9]*:P:[^:]*:master:' <<< $ch_role && rc=0
# grep '[34]:P:[^:]*:master:' <<< $ch_role && rc=0
# Match "Running+Available Primary" Master -> Match field 1: 3/4, 2: P, 4: master
awk -F: 'BEGIN { rc=1 }
$1 ~ "[34]" && $2 ="P" && $4="master" { rc=0 }
END { exit rc }' <<< $ch_role ; rc=$?
fi
done
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# wait_for_primary_master: wait some time till a running primary master is shown in attributes
# params: optional: loop count - currently time in 10s waiting loop
# globals: -
#
wait_for_primary_master()
{
local wait=1
local rc=1
local loops=${1:-0}
local count=0
super_ocf_log info "FLOW $FUNCNAME ($*)"
#
# hana_ndb_roles=primary:master1:master:worker:master
#
while [ "$wait" -eq 1 ]; do
if check_for_primary_master; then
wait=0
rc=0
else
if [ $loops -gt 0 ]; then
(( count++ ))
if [ $count -gt $loops ]; then
wait=0
rc=1
fi
fi
sleep 10
fi
done
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_start_secondary - handle startup of PRIMARY in M/S
# params:
# globals: OCF_*(r), NODENAME, ATTR_NAME_*,
#
function saphana_start_secondary()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
local sqlrc;
set_crm_master 0
#
####### LPA - begin
#
lpa_push_lpt 10
lpa_set_lpt 10 $NODENAME
#
####### LPA - end
#
#
# we would be slave (secondary)
# we first need to check, if there are Master Nodes, because the Scecondary only starts
# successfuly, if the Primary is available. Thatfore we mark the Secondary as "WAITING"
# DONE: PRIO3: wait_for_primary_master 10 is just a test value: 10 loops x10 seconds than go to WAITING
# DONE: PRIO3: rename 'wait_for_primary_master' to match better the use case ("wait_some_time")
#
super_ocf_log debug "DBG: wait for promoted side"
# TODO: PRIO3: Check if setting SFAIL during secondary start is ok
set_hana_attribute "${NODENAME}" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
if wait_for_primary_master 10; then
saphana_start; rc=$?
if [ $rc -ne $OCF_SUCCESS ]; then
if ! wait_for_primary_master 1; then
# It seams the stating secondary could not start because of stopping primary
# so this is a WAITING situation
super_ocf_log info "ACT: PRIMARY seams to be down now ==> WAITING"
set_hana_attribute ${NODENAME} "WAITING4PRIM" ${ATTR_NAME_HANA_CLONE_STATE[@]}
set_crm_master -INFINITY
rc=$OCF_SUCCESS
fi
else
lpa_set_lpt 10 $NODENAME
fi
else
super_ocf_log info "ACT: wait_for_primary_master ==> WAITING"
set_hana_attribute ${NODENAME} "WAITING4PRIM" ${ATTR_NAME_HANA_CLONE_STATE[@]}
set_crm_master -INFINITY
rc=$OCF_SUCCESS
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_check_local_instance
# params:
# output:
# rc: rc=0 (UP) rc=1 (DOWN)
# globals:
#
function saphana_check_local_instance()
{
local rc=1
local count=0
local SERVNO
local output
local MONITOR_SERVICES="hdbnameserver|hdbdaemon" # TODO: PRIO1: exact list of Services
super_ocf_log info "FLOW $FUNCNAME ($*)"
check_sapstartsrv
rc=$?
if [ $rc -eq $OCF_SUCCESS ]
then
output=$($SAPCONTROL -nr $InstanceNr -function GetProcessList -format script)
# we have to parse the output, because the returncode doesn't tell anything about the instance status
for SERVNO in `echo "$output" | grep '^[0-9] ' | cut -d' ' -f1 | sort -u`
do
local COLOR=`echo "$output" | grep "^$SERVNO dispstatus: " | cut -d' ' -f3`
local SERVICE=`echo "$output" | grep "^$SERVNO name: " | cut -d' ' -f3`
local STATE=0
local SEARCH
case $COLOR in
GREEN|YELLOW) STATE=$OCF_SUCCESS;;
*) STATE=$OCF_NOT_RUNNING;;
esac
SEARCH=`echo "$MONITOR_SERVICES" | sed 's/\+/\\\+/g' | sed 's/\./\\\./g'`
if [ `echo "$SERVICE" | egrep -c "$SEARCH"` -eq 1 ]
then
if [ $STATE -eq $OCF_NOT_RUNNING ]
then
[ "$MONLOG" != "NOLOG" ] && ocf_log err "SAP instance service $SERVICE is not running with status $COLOR !"
rc=$STATE
fi
count=1
fi
done
if [ $count -eq 0 -a $rc -eq $OCF_SUCCESS ]
then
if ocf_is_probe
then
rc=1
else
[ "$MONLOG" != "NOLOG" ] && ocf_log err "The SAP instance does not run any services which this RA could monitor!"
rc=1
fi
fi
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_get_lpt - get lpt from cluster
# params: NODE
# output: LPT
# rc: rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
# globals: LPA_ATTR_*,
#
function lpa_get_lpt() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=1
local node=$1
local lpt=""
lpt=$(get_hana_attribute ${node} ${LPA_ATTR[@]})
if [ -n "$lpt" ]; then
rc=0
echo $lpt
else
rc=2
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_set_lpt - set lpt in cluster
# params: LPT [node]
# globals: LPA_ATTR(r), NODENAME(r),
# rc: rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_set_lpt() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=1
local crm_rc=1
local lpt=$1
local clpt=-1
local node=$2
set_hana_attribute ${node} "$lpt" ${LPA_ATTR[@]}; crm_rc=$?
if [ -n "$node" ]; then
clpt=$(lpa_get_lpt $NODENAME)
if [ "$lpt" != "$clpt" ]; then
rc=2
else
rc=0
fi
else
super_ocf_log info "DEC: lpa_set_lpt ignore to change value for empty node name"
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_pull_lpt - fetch lpt from file
# params: -
# globals: LPA_DIRECTORY(r), sid, NODENAME
# output: LPT
# rc: rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_pull_lpt() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=1
local lpt=""
local readrest=0
local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
if [ -f $lpa_file ]; then
read lpt readrest <<<$(cat $lpa_file) # exactly load first word from file to lpt
fi
if [ -n "$lpt" ]; then
rc=0
echo $lpt
else
rc=2
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_push_lpt - put lpt to file
# params: LPT
# globals: LPA_DIRECTORY(r), sid, NODENAME
# output: --
# rc: rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_push_lpt() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local lpt=$1
local clpt=-1
local rc=1
local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
#
mkdir -p $LPA_DIRECTORY
echo "$lpt" > $lpa_file
clpt=$(lpa_pull_lpt); lpt_rc=$?
if [ "$clpt" != "$lpt" -o "$lpt_rc" -ne 0 ]; then
rc=2
else
rc=0
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_init_lpt - initialize local lpt, if needed
# params: HANA_STATE
# globals: HANA_STATE_*(r), LPA_DIRECTORY(r), sid(r), NODENAME(r),
# lpa_init_lpt
#
# Returncodes:
# rc=0: OK, rc=1 InternalERROR, rc=2: ERROR
#
# Initializing (if NO local LPT-file):
# SECONDARY sets to 0
# PRIMARY sets to 1
#
function lpa_init_lpt() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=1
local LPTloc=-1
local LPTrem=-1
local hana_state=$1
local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
mkdir -p $LPA_DIRECTORY
LPTloc=$(lpa_get_lpt ${NODENAME}) || LPTloc=$(lpa_pull_lpt) || \
if [ "$hana_state" -eq "$HANA_STATE_PRIMARY" ]; then # Initialize for Primary
# init primary
LPTloc=20
lpa_push_lpt "20"; rc=$?
elif [ "$hana_state" -eq "$HANA_STATE_SECONDARY" ]; then # Initialize for Secondary
# init secondary
LPTloc=10
lpa_push_lpt "10"; rc=$?
else
rc=2
fi
lpa_set_lpt $LPTloc $NODENAME
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: lpa_check_lpt_status - start a hana clone instance
# params: -
# globals: DUPLICATE_PRIMARY_TIMEOUT, NODENAME, remoteNode
# lpa_check_lpt_status
#
# Returncodes:
# 0: start
# 1: register than start
# 2: wait4gab
# 3: wait4other
#
# Initializing (if NO local LPT-file):
# SECONDARY sets to 10
# PRIMARY sets to 20
#
# LPRlocal OR LPTremore ARE real lpt (>1000)
# THEN:
# Bigger LPR wins, if delta-gab is OK
# LPTlocal >> LPTremore ===> rc=0 (start)
# LPTRemote >> LPTlocal ===> rc=1 (register)
# Stalemate in all other cases ==> STALEMATE-HANDLING ===> rc=2 (wait4gab)
# LPRlocal AND LPTremore ARE NOT real lpt (<=1000)
# THEN:
# Bigger LPT wins
# LPTlocal > LPTremore ===> rc=0 (start)
# LPTRemote > LPTlocal ===> rc=1 (register)
# Stalemate in all other cases ==> STALEMATE-HANDLING ===> rc=2 (wait4gab)
# LPTRemote is not initialized or node not kown in cluster (crm_mon -l) (0)
# TODO: PRIO1: Need to introduce a return-code 3 for remote sides lpa not ready
# THEN:
# WAIT ==> like STALEMATE-HANDLING ===> rc=2 (wait)
#
function lpa_check_lpt_status() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
local LPTloc=-1
local LPTrem=-1
local LPTMark=1000
local delta=0
local remSn_name=""
local remHost=""
#
# First GET LPT from ATTR-FILE-DEFAULT
#
LPTloc=$(lpa_get_lpt $NODENAME); lparc=$? # ATTR
if [ "$lparc" -ne 0 ]; then
# as a fallback try to fetch the value from external status file
LPTloc=$(lpa_pull_lpt); # FILE
lparc=$?
if [ -z "$LPTloc" -o "$LPTloc" -eq -1 -o "$lparc" -ne 0 ]; then
# last option - try to initialize as PRIMARY
lpa_push_lpt 20
lpa_set_lpt 20 $NODENAME
LPTloc=20 # DEFAULT
fi
fi
# TODO PRIO1: REMOVE remoteNode dependency - lpa_get_lpt
LPTrem=$(lpa_get_lpt $remoteNode); lparc=$?
if [ $lparc -ne 0 ]; then
# LPT of the other node could not be evaluated - LPA says WAIT
super_ocf_log debug "DBG: LPA: LPTloc=$LPTloc, LPTrem undefined ==> WAIT"
rc=2
else
super_ocf_log debug "DBG: LPA: LPTloc ($LPTloc) LPTrem ($LPTrem) delta ($delta)"
if [ $LPTloc -lt $LPTMark -a $LPTrem -lt $LPTMark ]; then
delta=0 # both lpts are not a real timestamp so just take the greater one
else
delta=$DUPLICATE_PRIMARY_TIMEOUT # at least one of the lpts is a real timestamp so include delta-gap
fi
if (( delta < LPTloc - LPTrem )); then
# We are the winner - LPA says STARTUP
super_ocf_log debug "DBG: LPA: LPTloc wins $LPTloc > $LPTrem + $delta ==> START"
rc=0
elif (( delta < LPTrem - LPTloc )); then
if ocf_is_true "$AUTOMATED_REGISTER" ; then
# The other one has won - LPA says REGISTER
super_ocf_log debug "DBG: LPA: LPTrem wins $LPTrem > $LPTloc + $delta ==> REGISTER"
rc=1
else
super_ocf_log debug "DBG: LPA: LPTrem wins $LPTrem > $LPTloc + $delta BUT AUTOMATED_REGISTER='false' ==> WAIT"
rc=2
fi
else
super_ocf_log debug "DBG: LPA: Difference between LPTloc and LPTrem is less than delta ($delta) ==> WAIT"
# TODO: PRIO3: ADD STALEMATE-HANDLING HERE; currently admin should set one of the lpa to 20
rc=2
fi
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
# function: is_the_master_nameserver
# params: -
# rc: 0: yes, local node is THE master nameserver
# 1: else
# globals:
function is_the_master_nameserver()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_start_clone - start a hana clone instance
# params: -
# globals: OCF_*, ATTR_NAME_*, HANA_STATE_*, NODENAME
# saphana_start_clone
#
function saphana_start_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
local sqlrc;
local chkusr;
# TODO: PRIO4: remove check_secstore_users later
secUser=$(check_secstore_users SAPHANA${SID}SR SLEHALOC RHELHALOC) ; chkusr=$?
if [ $chkusr -ne 0 ]; then
super_ocf_log err "ACT: Secure store users are missing (see best practice manual how to setup the users)"
rc=$OCF_ERR_CONFIGURED
else
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
check_for_primary; primary_status=$?
if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
saphana_start_primary; rc=$?
else
lpa_set_lpt 10 $NODENAME
saphana_start_secondary; rc=$?
fi
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_stop_clone - stop a hana clone instance
# params: -
# globals: NODENAME(r), HANA_STATE_*(r)
# saphana_stop_clone
#
function saphana_stop_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=0
local primary_status="x"
set_hana_attribute ${NODENAME} "UNDEFINED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
check_for_primary; primary_status=$?
if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
lpa_set_lpt 10 $NODENAME
fi
saphana_stop; rc=$?
return $rc
}
#
# function: saphana_monitor_primary - monitor a hana clone instance
# params: -
# globals: HANA_STATE_*(r), remoteHost, NODENAME, ATTR_NAME_*, OCF_*, PreferSiteTakeover
#
function saphana_monitor_primary()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_ERR_GENERIC
local promoted=0
local init_attribute=0
local LPTloc=-1
local lparc=4
local lss
local remoreSync=""
local my_role=""
#
# OK, we are running/are configured as HANA PRIMARY
#
super_ocf_log debug "DBG: saphana_monitor_clone: HANA_STATE_PRIMARY"
#
##### CHECK, IF WE ARE DEMOTED (CLUSTER NODE ATTRIBUTE)
#
promote_attr=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_CLONE_STATE[@]})
super_ocf_log debug "DBG: saphana_monitor_clone: $ATTR_NAME_HANA_CLONE_STATE=$promote_attr"
if [ -z "$promote_attr" ]; then
init_attribute=1
promoted=0;
else
case "$promote_attr" in
PROMOTED )
promoted=1;
;;
DEMOTED )
promoted=0;
;;
WAITING* )
# DONE: lpa_check_lpt_status to come out of here :)
# DONE: PRIO2: CHECK IF THE FIX FOR COMING OUT OF WAITING IS CORRECT
get_hana_landscape_status; lss=$?
if [ $lss -ge 2 ]; then
# seems admin already decided that for us? -> we are running - set DEMOTED
promoted=0;
LPTloc=$(date '+%s')
lpa_set_lpt $LPTloc $NODENAME
fi
lpa_check_lpt_status; lparc=$?
# TODO: PRIO1: Need to differ lpa_check_lpt_status return codes
if [ $lparc -lt 2 ]; then
# lpa - no need to wait any longer - lets try a new start
saphana_start_clone
rc=$?
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
else
lpa_init_lpt $HANA_STATE_PRIMARY
# still waiting for second site to report lpa-lpt
if ocf_is_true "$AUTOMATED_REGISTER" ; then
super_ocf_log info "LPA: Still waiting for remote site to report LPA status"
else
super_ocf_log info "LPA: Dual primary detected and AUTOMATED_REGISTER='false' ==> WAITING"
fi
return $OCF_SUCCESS
fi
promoted=0;
;;
UNDEFINED )
if ocf_is_probe; then
promoted=0;
else
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
promoted=0;
fi
;;
* )
promoted=0;
;;
esac
fi
get_hana_landscape_status; lss=$?
super_ocf_log debug "DBG: saphana_monitor_clone: get_hana_landscape_status=$lss"
case "$lss" in
0 ) # FATAL or ERROR
rc=$OCF_ERR_GENERIC
;;
1 ) # DOWN or ERROR
# DONE: PRIO2: Maybe we need to differ between 0 and 1. While 0 is a fatal sap error, 1 is down/error
if ocf_is_probe; then
#
# leave master score untouched, only set return code
#
rc=$OCF_NOT_RUNNING
else
if [ "$promoted" -eq 1 ]; then
# INSTANCE IS FAILED PRIMARY IN PROMOTED STATE
# DONE: PRIO2: Adjust with set_crm_master?
# For Migration it would be good to decrease master score
# For Reload locally we should NOT adjust the master score
# ===> Should we rely on the migration threshold?
# set_crm_master
if ocf_is_true "${PreferSiteTakeover}" ; then
#
# DONE: PRIO1: first check, if remote site is already (and still) in sync
# TODO: PRIO4: Decide if penality (-9000) or weak (5) is better here to cover situations where other clone is gone
#
# TODO PRIO1: REMOVE remoteNode dependency - get_sync_status
remoteSync=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_SYNC_STATUS[@]})
case "$remoteSync" in
SOK | PRIM )
super_ocf_log info "DEC: PreferSiteTakeover selected so decrease promotion score here (and reset lpa)"
set_crm_master 5
if check_for_primary_master; then
lpa_set_lpt 20 $NODENAME
fi
;;
SFAIL )
super_ocf_log info "DEC: PreferSiteTakeover selected BUT remoteHost is not in sync (SFAIL) ==> local restart preferred"
;;
* )
super_ocf_log info "DEC: PreferSiteTakeover selected BUT remoteHost is not in sync ($remoteSync) ==> local restart preferred"
;;
esac
else
# TODO: PRIO5: SCALE-OUT ONLY? Implement for local restart
# It maybe that for the local restart we only need to decrease the secondaries promotion score
#super_ocf_log info "DEC: PreferSiteTakeover selected so decrease promotion score here"
my_role=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_ROLES[@]})
my_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
scoring_crm_master "$my_role" "$my_sync"
rc=$OCF_FAILED_MASTER
fi
rc=$OCF_FAILED_MASTER
else
# INSTANCE IS FAILED PRIMARY IN DEMOTED STATE
# TODO: PRIO3: Adjust with set_crm_master?
# Current decission: Do NOT adjust master score now as other
# steps should already have done that
#
rc=$OCF_NOT_RUNNING
fi
fi
;;
2 | 3 | 4 ) # WARN, INFO or OK
if ocf_is_probe; then
rc=$OCF_SUCCESS
else
LPTloc=$(date '+%s')
lpa_set_lpt $LPTloc $NODENAME
lpa_push_lpt $LPTloc
if [ "$promoted" -eq 1 ]; then
set_hana_attribute "$NODENAME" "PRIM" ${ATTR_NAME_HANA_SYNC_STATUS[@]}
rc=$OCF_RUNNING_MASTER
else
if [ "$init_attribute" -eq 1 ]; then
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
rc=$OCF_RUNNING_MASTER
else
rc=$OCF_SUCCESS
fi
fi
my_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
my_role=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_ROLES[@]})
case "$my_role" in
[12]:P:*:master:* ) # primary is down or may not anser hdbsql query so drop analyze_hana_sync_status
;;
[34]:P:*:*:* ) # primary is up and should now be able to anser hdbsql query
if [ -f $DIR_EXECUTABLE/python_support/systemReplicationStatus.py ]; then
analyze_hana_sync_statusSRS
else
analyze_hana_sync_statusSQL
fi
;;
esac
rem_role=$(get_hana_attribute ${remoteNode} ${ATTR_NAME_HANA_ROLES[@]})
rem_clone_status=$(get_hana_attribute ${remoteNode} ${ATTR_NAME_HANA_CLONE_STATE[@]})
if [ "$promote_attr" = "DEMOTED" -a "$rem_clone_status" = "PROMOTED" ]; then
case "$rem_role" in
[234]:P:* ) # dual primary, but other instance marked as PROMOTED by the cluster
lpa_check_lpt_status; again_lpa_rc=$?
if [ $again_lpa_rc -eq 2 ]; then
super_ocf_log info "DEC: Dual primary detected, other instance is PROMOTED and lpa stalemate ==> local restart"
lpa_set_lpt 10 $NODENAME
lpa_push_lpt 10
rc=$OCF_NOT_RUNNING
fi
;;
esac
fi
scoring_crm_master "$my_role" "$my_sync"
fi
;;
* ) # UNDEFINED STATUS
if ocf_is_probe; then
rc=$OCF_NOT_RUNNING
else
if [ "$promoted" -eq 1 ]; then
rc=$OCF_FAILED_MASTER
else
rc=$OCF_NOT_RUNNING
fi
fi
;;
esac
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_monitor_secondary - monitor a hana clone instance
# params: -
# globals: OCF_*, ATTR_NAME_*, NODENAME
# saphana_monitor_secondary
#
function saphana_monitor_secondary()
{
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_ERR_GENERIC
local promoted=0
local init_attribute=0
local lss
#
# OK, we are running as HANA SECONDARY
#
if ! lpa_get_lpt ${NODENAME}; then
lpa_set_lpt 10 $NODENAME
lpa_push_lpt 10
fi
promote_attr=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_CLONE_STATE[@]})
super_ocf_log debug "DBG: saphana_monitor_clone: $ATTR_NAME_HANA_CLONE_STATE=$promote_attr"
if [ -z "$promote_attr" ]; then
init_attribute=1
# DONE: PRIO3: do we need to inizialize also the DEMOTED attribute value?
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
promoted=0;
else
case "$promote_attr" in
PROMOTED ) # However - PROMOTED should never happen for a SECONDARY
promoted=1;
;;
DEMOTED ) # This is the status we expect
promoted=0;
;;
WAITING* ) # We are WAITING for PRIMARY so not testing the HANA engine now but check for a new start
if check_for_primary_master; then
super_ocf_log info "ACT: SECONDARY still in status WAITING - Primary now available - try a new start"
saphana_start_clone
rc=$?
else
super_ocf_log info "ACT: saphana_monitor_clone: SECONDARY still in status WAITING - Primary is still missing"
return $OCF_SUCCESS
fi
promoted=0;
;;
UNDEFINED | * )
if ocf_is_probe; then
promoted=0;
else
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
promoted=0;
fi
;;
esac
fi
#
super_ocf_log debug "DBG: saphana_monitor_clone: HANA_STATE_SECONDARY"
#
# old method was: saphana_monitor - new method is get_hana_landscape_status
get_hana_landscape_status; lss=$?
super_ocf_log debug "DBG: saphana_monitor_clone: get_hana_landscape_status=$lss"
case "$lss" in
0 ) # FATAL
# DONE: PRIO1: Maybe we need to differ between 0 and 1. While 0 is a fatal sap error, 1 is down/error
# TODO: PRIO3: is OCF_ERR_GENERIC best option?
lpa_set_lpt 10 $NODENAME
rc=$OCF_ERR_GENERIC
;;
1 ) # ERROR
lpa_set_lpt 10 $NODENAME
rc=$OCF_NOT_RUNNING
;;
2 | 3 | 4 ) # WARN INFO OK
rc=$OCF_SUCCESS
lpa_set_lpt 30 $NODENAME
sync_attr=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
super_ocf_log debug "DBG: sync_attr=$sync_attr"
case "$sync_attr" in
"SOK" ) # This is a possible node to promote, when primary is missing
super_ocf_log info "DEC: secondary with sync status SOK ==> possible takeover node"
my_role=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_ROLES[@]})
my_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
scoring_crm_master "$my_role" "$my_sync"
;;
"SFAIL" ) # This is currently NOT a possible node to promote
super_ocf_log info "DEC: secondary with sync status FAILED ==> EXCLUDE as possible takeover node"
set_crm_master -INFINITY
;;
"*" ) # Unknown sync status
super_ocf_log info "DEC: secondary with sync status UKNOWN/UNDEFINED ==> EXCLUDE as possible takeover node"
set_crm_master -INFINITY
;;
esac
;;
* ) # UNDEFINED STATUS
rc=$OCF_NOT_RUNNING
;;
esac
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_monitor_clone - monitor a hana clone instance
# params: -
# globals: OCF_*, ATTR_NAME_*, HOSTNANE, HANA_STATE_*
# saphana_monitor_clone
#
function saphana_monitor_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
#
# TODO: PRIO3: For the secondary, which is missing the primary (so in status WAITING) what is better:
# a) returning 7 here and force cluster a restart of the slave
# b) starting the instance here inside the monitor -> may result in longer runtime, timeouts
#
# first check with the status function (OS tools) if there could be something like a SAP instance running
# as we do not know here, if we are in master or slave state we do not want to start our monitoring
# agents (sapstartsrv) on the wrong host
local rc=$OCF_ERR_GENERIC
local promoted=0
local init_attribute=0
local lpaRc=0
local mRc=0
local myMaster=-1
my_role=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_ROLES[@]})
my_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
if ocf_is_probe; then
super_ocf_log debug "DBG: PROBE ONLY"
else
super_ocf_log debug "DBG: REGULAR MONITOR"
fi
#
# First check, if we are PRIMARY or SECONDARY
#
check_for_primary; primary_status=$?
if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
# FIX: bsc#919925 Leaving Node Maintenance stops HANA Resource Agent
# TODO: PRIO1: Maybe we need a lpa-check here to
if ocf_is_probe; then
myMaster=$(get_crm_master); mRc=$?
if [ $mRc -ne 0 ]; then
set_crm_master 5
elif [ $myMaster -eq -1 ]; then
set_crm_master 5
fi
fi
saphana_monitor_primary; rc=$?
else
if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
saphana_monitor_secondary; rc=$?
else
#
# OK, we are neither HANA PRIMARY nor HANA SECONDARY
#
super_ocf_log warn "ACT: saphana_monitor_clone: HANA_STATE_DEFECT"
# TODO: PRIO2: Or only set_crm_master -INFINITY ?
rc=$OCF_ERR_GENERIC
fi
fi
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_promote_clone - promote a hana clone
# params: -
# globals: OCF_*(r), NODENAME(r), HANA_STATE_*, SID(r), InstanceName(r),
# saphana_promote_clone:
# In a Master/Slave configuration get Master being the primary OR by running hana takeover
#
function saphana_promote_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_ERR_GENERIC;
local hana_sync;
local primary_status;
#
# first check, if we WILL be PRIMARY (checking HANA status)
#
set_hana_attribute ${NODENAME} "PROMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
check_for_primary; primary_status=$?
#
if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
#
# as we are already planned to be PRIMARY we only mark the node as PROMOTED
#
super_ocf_log info "ACT: Promoted $SID-$InstanceName as master (no hdbnsutil action needed)."
rc=$OCF_SUCCESS;
else
if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
#
# we are SECONDARY/SLAVE and need to takepover ...
# promote on the replica side...
#
hana_sync=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_SYNC_STATUS[@]})
case "$hana_sync" in
SOK )
super_ocf_log info "ACT: !!!!!!! Promote REPLICA $SID-$InstanceName to be primary. !!!!!!"
LPTloc=$(date '+%s')
# lpa_set_lpt 20 $remoteNode
lpa_set_lpt $LPTloc $NODENAME
lpa_push_lpt $LPTloc
# TODO: Get rid of the su by using a new interface:
# SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh hdbnsutil -sr_takeover ...
# TODO: Check beginning from which SPS does SAP support HDBSettings.sh?
# TODO: Limit the runtime of hdbnsutil -sr_takeover ????
su - $sidadm -c "hdbnsutil -sr_takeover"
#
# now gain check, if we are primary NOW
#
# TODO: PRIO3: check, if we need to destinguish between HANA_STATE_PRIMARY, HANA_STATE_SECONDARY, HANA_STATE_DEFECT
#
if check_for_primary; then
rc=$OCF_SUCCESS;
else
rc=$OCF_FAILED_MASTER
fi
;;
* )
super_ocf_log err "ACT: HANA SYNC STATUS IS NOT 'SOK' SO THIS HANA SITE COULD NOT BE PROMOTED"
rc=$OCF_ERR_GENERIC
;;
esac
else
#
# neither MASTER nor SLAVE - This clone instance seams to be broken!!
#
rc=$OCF_ERR_GENERIC
fi
fi
rc=$?
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: saphana_demote_clone - demote a hana clone instance
# params: -
# globals: OCF_*(r), NODENAME(r),
# saphana_demote_clone
# the HANA System Replication (SR) runs in a Master/Slave
# While we could not change a HANA instance to be really demoted, we only mark the status for
# correct monitor return codes
#
function saphana_demote_clone() {
super_ocf_log info "FLOW $FUNCNAME ($*)"
local rc=$OCF_ERR_GENERIC;
set_hana_attribute ${NODENAME} "DEMOTED" ${ATTR_NAME_HANA_CLONE_STATE[@]}
rc=$OCF_SUCCESS;
super_ocf_log info "ACT: Demoted $SID-$InstanceName."
super_ocf_log info "FLOW $FUNCNAME rc=$rc"
return $rc
}
#
# function: main - main function to operate
# params: ACTION
# globals: OCF_*(r), SID(w), sidadm(w), InstanceName(w), SAPVIRHOST(w), DIR_EXECUTABLE(w),
# globals: SAPSTARTSRV(w), SAPCONTROL(w), DIR_PROFILE(w), SAPSTARTPROFILE(w), ACTION(w), CLACT(w), ra_rc(rw), $0(r), %ENV(r)
#
## GLOBALS
SID=""
sidadm=""
InstanceName=""
InstanceNr=""
SAPVIRHOST=""
DIR_EXECUTABLE=""
SAPSTARTSRV=""
SAPCONTROL=""
DIR_PROFILE=""
SAPSTARTPROFILE=""
SAPHanaFilter="ra-act-dec-lpa"
NODENAME=$(crm_node -n)
if [ $# -ne 1 ]
then
saphana_usage
exit $OCF_ERR_ARGS
fi
ACTION=$1
if [ "$ACTION" = "status" ]; then
ACTION=monitor
fi
# These operations don't require OCF parameters to be set
# TODO: PRIO5: check, if notify is still not needing OCF parameters
case "$ACTION" in
usage|methods) saphana_$ACTION
exit $OCF_SUCCESS;;
meta-data) saphana_meta_data
exit $OCF_SUCCESS;;
notify) #saphana_notify
exit $OCF_SUCCESS;;
*);;
esac
saphana_init
if ! ocf_is_root
then
super_ocf_log err "ACT: $0 must be run as root"
exit $OCF_ERR_PERM
fi
# parameter check
if [ -z "$OCF_RESKEY_SID" ]
then
super_ocf_log err "ACT: Please set parameter SID!"
exit $OCF_ERR_ARGS
fi
if [ -z "$OCF_RESKEY_InstanceNumber" ]
then
super_ocf_log err "ACT: Please set parameter InstanceNumber!"
exit $OCF_ERR_ARGS
fi
if is_clone
then
CLACT=_clone
else
if [ "$ACTION" = "promote" -o "$ACTION" = "demote" ]
then
super_ocf_log err "ACT: $ACTION called in a non master/slave environment"
exit $OCF_ERR_ARGS
fi
fi
# What kind of method was invoked?
THE_VERSION=$(saphana_meta_data | grep '<version')
super_ocf_log info "RA ==== begin action $ACTION$CLACT ($THE_VERSION) ===="
ra_rc=$OCF_ERR_UNIMPLEMENTED
case "$ACTION" in
start|stop|monitor|promote|demote) # Standard controling actions
saphana_$ACTION$CLACT
ra_rc=$?
;;
validate-all)
saphana_validate
ra_rc=$?
;;
lpa_check)
lpa_check_lpt_status
ra_rc=$?
;;
*) # seams to be a unknown request
saphana_methods
ra_rc=$OCF_ERR_UNIMPLEMENTED
;;
esac
timeE=$(date '+%s')
(( timeR = timeE - timeB ))
#super_ocf_log info "RA ==== SAPHanaFilter=$SAPHanaFilter"
super_ocf_log info "RA ==== end action $ACTION$CLACT with rc=${ra_rc} ($THE_VERSION) (${timeR}s)===="
exit ${ra_rc}
