#!/bin/sh
########################################################################
# Begin $/opt/midas/bin/shutdown-monitor
#
# Description : Runs on MC and GUI waiting to instigate host shutdown
#
# Authors     : MLA
#
# Version     : 00.00
#
# Notes       : none.
#
########################################################################


########################################################################
# Include generic functions.
########################################################################
. /opt/midas/bin/logging
. /etc/rc.d/init.d/functions


########################################################################
# Constants
########################################################################

########################################################################
# NOTE: All files created in the /var directory do not persist over a
#       power cycle.
########################################################################

########################################################################
#
#   RUNNING_FLAG          File used as a flag to indicate that one
#                         instance of this script is already running
#                         and thereby used to prevent multiple instances
#                         running concurrently.
#
#   SHUTDOWN_FLAG         File used as a flag to signal to this script
#                         to start a controlled shutdown of the system.
#
#   RESTART_FLAG          File used as a flag to signal to this script
#                         to start a controlled restart of the system.
#
#   NO_RESPAWN_FLAG       File used as a flag to signal to the
#                         application launch scripts (launch_script,
#                         launch-script1 and launch-script2) not to
#                         respawn.
#
########################################################################
RUNNING_FLAG=/var/shutdown-monitor
SHUTDOWN_FLAG=/var/shutdown-system
RESTART_FLAG=/var/restart-system
NO_RESPAWN_FLAG=/var/norespawn


########################################################################
#
#   SHUTDOWN_AS_ACTIVE    File (empty) written to the active
#                         mastercontroller once the standby
#                         mastercontroller application has determined
#                         its role from active/standby messages from
#                         the router.
#                         NOTE: This is a fault tolerant mechanism to
#                               overcome link failure between the active
#                               mastercontroller and the router.
#
#   SHUTDOWN_AS_STANDBY   File (empty) written to the standby
#                         mastercontroller once the active
#                         mastercontroller application has determined
#                         its role from active/standby messages from
#                         the router.
#                         NOTE: This is a fault tolerant mechanism to
#                               overcome link failure between the standby
#                               mastercontroller and the router.
#
########################################################################
SHUTDOWN_AS_ACTIVE=/var/shutdown-as-active
SHUTDOWN_AS_STANDBY=/var/shutdown-as-standby


########################################################################
#
#   ACTIVE_MC_DOWN        File (empty) written to the standby
#                         mastercontroller once the active
#                         mastercontroller application has stopped.
#                         NOTE: Active host writes to standby host
#
#   STANDBY_MC_DOWN       File (empty) written to the active
#                         mastercontroller once the standby
#                         mastercontroller application has stopped.
#                         NOTE: Standby host writes to active host.
#
########################################################################
ACTIVE_MC_DOWN=/var/active-mc-down
STANDBY_MC_DOWN=/var/standby-mc-down


########################################################################
#
#   ACTIVE_CACHE_FLUSHED  File (empty) written to the standby
#                         mastercontroller once the active
#                         mastercontroller cache has been written to
#                         persistent storage.
#                         NOTE: Active host writes to standby host
#
#   STANDBY_CACHE_FLUSHED File (empty) written to the active
#                         mastercontroller once the standby
#                         mastercontroller cache has been written to
#                         persistent storage.
#                         NOTE: Standby host writes to active host.
#
########################################################################
ACTIVE_CACHE_FLUSHED=/var/active-cache-flushed
STANDBY_CACHE_FLUSHED=/var/standby-cache-flushed


########################################################################
#
#  SHUTDOWN_OS_NOW        File (empty) written to the standby
#                         mastercontroller once the active
#                         mastercontroller cache has been written to
#                         persistent storage.
#                         NOTE: Active host writes to standby host
#
########################################################################
SHUTDOWN_OS_NOW=/var/shutdown-os-now


########################################################################
#
#   MC_SHUTDOWN_TIMEOUT_SECONDS
#                         Timeout (in seconds) for the mastercontroller
#                         to shutdown. If the timer expires the
#                         application will be forcably terminated.
#
########################################################################
MC_SHUTDOWN_TIMEOUT_SECONDS=120


########################################################################
#
#   EVENT_TIMEOUT_SECONDS Timeout (in seconds) for event detection.
#
########################################################################
EVENT_TIMEOUT_SECONDS=120


########################################################################
#
#   EVENT_POLL_RESOLUTION_LOW
#   EVENT_POLL_RESOLUTION_HIGH
#     Constants used for invocation of function 'WaitForEvent'.
#
########################################################################
EVENT_POLL_RESOLUTION_LOW=0
EVENT_POLL_RESOLUTION_HIGH=1



########################################################################
# Variables
########################################################################

########################################################################
#
#   OTHER_MC_HOST         Address of the other mastercontroller if
#                         present (or accessible).
#
########################################################################
OTHER_MC_HOST=""


########################################################################
#
#    THIS_IS_MC_HOST     0 if the script is running on an MC host
#
#    THIS_IS_ACTIVE_MC_HOST
#                        0 if the script is running on an MC host that
#                        has been assign active role status
#
#    THIS_IS_GUI_HOST    0 if the script is running on a GUI host
#
########################################################################
THIS_IS_MC_HOST=1
THIS_IS_ACTIVE_MC_HOST=1
THIS_IS_GUI_HOST=1


########################################################################
# Functions
########################################################################

########################################################################
#
# InitiateGUITermination
#   If the GUI application runs on 'this' host then simply kill the
#   application, otherwise initiate a shutdown or restart (as
#   appropriate) on all remote hosts.
#
#   Arguments
#     $1   gui host is local flag (0 == gui runs locally else gui runs
#          remotely)
#     $2   fully qualified log file path
#     $3   log message prefix
#
########################################################################
function InitiateGUITermination ( )
{
	#
	# First kill the GUI if any.
	#
	if [ $1 == 0 ]; then
		#
		# Kill the GUI. It doesn't require graceful shutdown so
		# kill it with a -9.
		#
		killall --signal 9 marmaset
		LogInformation "$2" "$3 gui terminated (kill -9)"
		
	else
		#
		# The GUI isn't running locally so assume it is running remotely.
		#
		if [ -e $SHUTDOWN_FLAG ]; then
			#
			# Signal remote GUIs to start shutdown process.
			#
			LogInformation "$2" "$3 start signalling remote gui shutdown"
			
			ssh GUI-1 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-1 retcode ($?)"
			
			ssh GUI-2 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-2 retcode ($?)"
			
			ssh GUI-3 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-3 retcode ($?)"
			
			ssh GUI-4 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-4 retcode ($?)"
			
			ssh GUI-5 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-5 retcode ($?)"
			
			ssh GUI-6 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-6 retcode ($?)"
			
			ssh GUI-7 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-7 retcode ($?)"
			
			ssh GUI-8 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-8 retcode ($?)"
			
			ssh GUI-9 /opt/midas/bin/initiate-shutdown &> /dev/null
			LogInformation "$2" "$3 GUI-9 retcode ($?)"
			
			LogInformation "$2" "$3 completed signalling remote gui shutdown"
			
		elif [ -e $RESTART_FLAG ]; then
			#
			# Signal remote GUIs to start restart process.
			#
			LogInformation "$2" "$3 start signalling remote gui restart"
			
			ssh GUI-1 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-1 retcode ($?)"
			
			ssh GUI-2 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-2 retcode ($?)"
			
			ssh GUI-3 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-3 retcode ($?)"
			
			ssh GUI-4 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-4 retcode ($?)"
			
			ssh GUI-5 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-5 retcode ($?)"
			
			ssh GUI-6 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-6 retcode ($?)"
			
			ssh GUI-7 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-7 retcode ($?)"
			
			ssh GUI-8 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-8 retcode ($?)"
			
			ssh GUI-9 /opt/midas/bin/initiate-restart &> /dev/null
			LogInformation "$2" "$3 GUI-9 retcode ($?)"
			
			LogInformation "$2" "$3 completed signalling remote gui restart"
			
		fi
		
	fi
}


########################################################################
#
# SignalEvent
#   Attempt to signal the specified host with the supplied event.
#
#   Arguments
#     $1   IP address or alias of the other mastercontroller
#     $2   the event to signal
#     $3   fully qualified log file path
#     $4   log message prefix
#
#   Return
#     Follows the scripting convention of 0 for success and not 0 for
#     failure.
#
########################################################################
function SignalEvent ( )
{
	local RETURN_VALUE=0
	
	
	if [ "$1" != "" ]; then
		
		#
		# Address specified for the 'other mastercontroller'.
		# Attempt to signal the specified event.
		#
		ssh $1 touch $2
		RETURN_VALUE=$?
		LogInformation "$3" "$4 signalled $1 $2; return code $RETURN_VALUE (0 == success)"
		
	else
		
		#
		# No address specified for the 'other mastercontroller.
		#
		LogWarning "$3" "$4 failed to signal event $2; no host address specified"
		RETURN_VALUE=1
		
	fi
	
	
	return $RETURN_VALUE
}


########################################################################
#
# WaitForEvent
#   Wait for the specified event.
#
#   Arguments
#     $1   event to wait for
#     $2   timeout in seconds
#     $3   high resolution mode; 0 one second poll, 100 millisecond poll
#     $4   fully qualified log file path
#     $5   log message prefix
#
#   Return
#     Follows the scripting convention of 0 for success and not 0 for
#     failure.
#
########################################################################
function WaitForEvent ( )
{
	local RETURN_VALUE=0
	local ELAPSED_SECONDS=0
	local ELAPSED_MILLISECONDS=0
	
	
	if [ $3 == $EVENT_POLL_RESOLUTION_LOW ]; then
		
		#
		# Wait for the event to be signalled, with one second poll.
		#
		while [ ! -e $1 ] && [ $ELAPSED_SECONDS -lt $2 ]; do
			
			ELAPSED_SECONDS=$(($ELAPSED_SECONDS + 1))
			sleep 1
			
			LogInformation "$4" "$5 waiting for $1; elapsed $ELAPSED_SECONDS, timeout $2"
			
		done
		
	else
		
		#
		# Wait for the event to be signalled, with one hundred
		# millisecond poll.
		#
		while [ ! -e $1 ] && [ $ELAPSED_SECONDS -lt $2 ]; do
			
			ELAPSED_MILLISECONDS=$((ELAPSED_MILLISECONDS + 100))
			sleep 0.1
			
			# If we've waited one second, update the elapsed seconds
			# and log current status.
			if [ $ELAPSED_MILLISECONDS == 1000 ]; then
				
				ELAPSED_SECONDS=$(($ELAPSED_SECONDS + 1))
				ELAPSED_MILLISECONDS=0
				
				LogInformation "$4" "$5 waiting for $1; elapsed $ELAPSED_SECONDS, timeout $2"
				
			fi
			
		done
		
	fi
	
	
	# Determine the return code, 0 if the event was signalled, else 1.
	if [ -e $1 ]; then
		RETURN_VALUE=0
	else
		RETURN_VALUE=1
	fi
	
	
	return $RETURN_VALUE
}


########################################################################
#
# WaitForMCStopped
#   Wait until the mastercontroller has stopped or the specified timeout
#   expires.
#
#   Arguments
#     $1   timeout in seconds
#     $2   fully qualified log file path.
#     $3   log message prefix
#
#   Return
#     Follows the scripting convention of 0 for success and not 0 for
#     failure.
#
########################################################################
function WaitForMCStopped ( )
{
	local RETURN_VALUE=0
	local ELAPSED_SECONDS=0
	local MC_IS_RUNNING=0
	
	
	#
	# Wait to see if the mastercontroller shuts down.
	#
	while [ $ELAPSED_SECONDS -lt $1 ] && [ $MC_IS_RUNNING == 0 ]; do
		
		ELAPSED_SECONDS=$(($ELAPSED_SECONDS + 1))
		sleep 1
		ps -AH | grep --silent masterco
		MC_IS_RUNNING=$?
		
		LogInformation "$2" "$3 waiting for mastercontroller to stop; elapsed $ELAPSED_SECONDS, timeout $1, mc is running $MC_IS_RUNNING ( 0 is still running)"
		
	done
	
	# Determine the return code; 0 indicates MC stopped, else 1.
	if [ $MC_IS_RUNNING != 0 ]; then
		RETURN_VALUE=0
	else
		RETURN_VALUE=1
	fi
	
	
	return $RETURN_VALUE
}



########################################################################
# MAIN #################################################################
########################################################################

#
# First check that this is not already running.
#
if [ ! -e $RUNNING_FLAG ]; then
	
	echo "STARTING SHUTDOWN MONITOR"
	
	
	#
	# Clear up any previous clutter.
	#
	rm -f $SHUTDOWN_FLAG
	rm -f $RESTART_FLAG
	
	#
	# Create the singleton flag.
	#
	touch $RUNNING_FLAG
	
	#
	# Loop until signalled to shutdown in some way.
	#
	while [ ! -e $SHUTDOWN_FLAG ] && [ ! -e $RESTART_FLAG ]; do
		sleep 1
	done
	
	
	#
	# Generate a log file name at this point into which we append shutdown
	# information.
	#
	LOG_FILE=$( GenerateLogFileName "/opt/midas/var/logs/boot" "shutdown-monitor-log" )
	
	
	#
	# Determine if the host can run a mastercontroller.
	#
	ip addr | grep --silent 192.168.20
	THIS_IS_MC_HOST=$?
	
	
	#
	# If this is an MC host,
	#  - determine the IP address of the 'other' MC, if present.
	#  - then replicate the shutdown or restart event to the
	#    other mastercontroller (if present) just in case that
	#    host's link to the router is down.
	#
	if [ $THIS_IS_MC_HOST == 0 ]; then
		
		#
		# Determine the OTHER_MC_HOST IP address or alias.
		#
		#	-w to search for whole word only, i.e 192.168.20.2 is NOT the same as 192.168.20.255 !!
		ip addr | grep -w --silent 192.168.20.1
		if [ $? == 0 ]; then
			
			#
			# This is MC-1. If the system has dual mastercontrollers
			# it will be accessible via MC-2 or root@192.168.19.2.
			#
			ssh MC-2 echo "hello"
			ROUTER_LINK_UP=$?
			
			ssh root@192.168.19.2 echo "hello"
			INTERLINK_UP=$?
			
			
			if [ $ROUTER_LINK_UP == 0 ]; then
				
				# Signal via the router.
				OTHER_MC_HOST=MC-2
				
			elif [ $INTERLINK_UP == 0 ]; then
				
				# Signal via the interlink.
				OTHER_MC_HOST=root@192.168.19.2
				
			fi
			
		else
			
			#
			# This is MC-2. If the system has dual mastercontrollers
			# it will be accessible via MC-1 or root@192.168.19.1.
			#
			ssh MC-1 echo "hello"
			ROUTER_LINK_UP=$?
			
			ssh root@192.168.19.1 echo "hello"
			INTERLINK_UP=$?
			
			
			if [ $ROUTER_LINK_UP == 0 ]; then
				
				# Signal via the router.
				OTHER_MC_HOST=MC-1
				
			elif [ $INTERLINK_UP == 0 ]; then
				
				# Signal via the interlink.
				OTHER_MC_HOST=root@192.168.19.1
				
			fi
			
		fi
		
		LogInformation "$LOG_FILE" "[shutdown-monitor] OTHER_MC_HOST is ($OTHER_MC_HOST); ROUTER_LINK_UP $ROUTER_LINK_UP, INTERLINK_UP $INTERLINK_UP (0 == up)"
		
		
		#
		# Attempt to replicate the 'shutdown' event to the other
		# mastercontroller host.
		#
		if [ -e $SHUTDOWN_FLAG ]; then
			
			SignalEvent "$OTHER_MC_HOST" "$SHUTDOWN_FLAG" "$LOG_FILE" "[shutdown-monitor]" 
			
		fi
		
		if [ -e $RESTART_FLAG ]; then
			
			SignalEvent "$OTHER_MC_HOST" "$RESTART_FLAG" "$LOG_FILE" "[shutdown-monitor]" 
			
		fi
		
		
		#
		# Determine if this is the active mc
		#
		# Restrict to first 12 chars of IP as spurious chars are occasionally
		# returned e.g. "192.168.20.1Ð"
		ACTIVE_ROLE_IP=$(/opt/midas/bin/mastercontroller query-active-host-ip-address | cut -b 1-12)
		if [ $? == 0 ] && [ "$ACTIVE_ROLE_IP" != "" ]; then
			LogInformation "$LOG_FILE" "[shutdown-monitor] got from MC query... ACTIVE_ROLE_IP $ACTIVE_ROLE_IP"
	
			#
			# Command succeeded, determine if this host is the active mastercontroller.
			# NOTE: In order to overcome the loss of the router link to the other
			#       mastercontroller (whereby it will not receive the shutdown event)
			#       we also attempt to notify the other mastercontroller of its active/
			#       standby status if we can determine ours.
			#
			#	-w to search for whole word only, i.e 192.168.20.2 is NOT the same as 192.168.20.255 !!
			ip addr | grep --silent -w $ACTIVE_ROLE_IP
			THIS_IS_ACTIVE_MC_HOST=$?
			
			if [ $THIS_IS_ACTIVE_MC_HOST == 0 ]; then
				
				LogInformation "$LOG_FILE" "[shutdown-monitor] this is the active mastercontroller host; THIS_IS_ACTIVE_MC_HOST $THIS_IS_ACTIVE_MC_HOST"
				SignalEvent "$OTHER_MC_HOST" "$SHUTDOWN_AS_STANDBY" "$LOG_FILE" "[shutdown-monitor]"
				
			else
				
				LogInformation "$LOG_FILE" "[shutdown-monitor] this is the standby mastercontroller host; THIS_IS_ACTIVE_MC_HOST $THIS_IS_ACTIVE_MC_HOST"
				SignalEvent "$OTHER_MC_HOST" "$SHUTDOWN_AS_ACTIVE" "$LOG_FILE" "[shutdown-monitor]"
				
			fi
			
		elif [ "$OTHER_MC_HOST" != "" ]; then
		
			#
			# Failed to get the ip address of the active mastercontroller but
			# we know that this is a mastercontroller host and we have a link to
			# to another mastercontroller host.
			# This shouldn't happen, but if it has we can't ignore the request to
			# shutdown, so we wait to see if the other mastercontroller
			# signals us.
			#
			LogWarning "$LOG_FILE" "[shutdown-monitor] failed to determine active/standby role status; waiting for notification from other mastercontroller"
			
			#
			# Wait for the event to be signalled, with one second poll.
			#
			ELAPSED_SECONDS=0
			
			while [ ! -e $SHUTDOWN_AS_ACTIVE ] && [ ! -e $SHUTDOWN_AS_STANDBY ] && [ $ELAPSED_SECONDS -lt $EVENT_TIMEOUT_SECONDS ]; do
				
				ELAPSED_SECONDS=$(($ELAPSED_SECONDS + 1))
				sleep 1
				
				LogInformation "$LOG_FILE" "[shutdown-monitor] waiting for $SHUTDOWN_AS_ACTIVE or $SHUTDOWN_AS_STANDBY; elapsed $ELAPSED_SECONDS, timeout $EVENT_TIMEOUT_SECONDS"
				
			done
			
			
			#
			# Determine how to shutdown.
			#
			if [ -e $SHUTDOWN_AS_ACTIVE ]; then
				#
				# The other mastercontroller signalled this host to shutdown
				# as active.
				#
				THIS_IS_ACTIVE_MC_HOST=0
				LogInformation "$LOG_FILE" "[shutdown-monitor] received notification; shutdown as active"
				
			elif [ -e $SHUTDOWN_AS_STANDBY ]; then
				#
				# The other mastercontroller signalled this host to shutdown
				# as standby
				#
				THIS_IS_ACTIVE_MC_HOST=1
				LogInformation "$LOG_FILE" "[shutdown-monitor] received notification; shutdown as standby"
				
			else
				#
				# Timed out.
				# Assume active status
				#
				THIS_IS_ACTIVE_MC_HOST=0
				LogWarning "$LOG_FILE" "[shutdown-monitor] timed out waiting for notification; assume active status"
				
			fi
			
		else
			
			#
			# Failed to get the ip address of the active mastercontroller but
			# we know that this is a mastercontroller host but don't have a link
			# to another mastercontroller host.
			# This shouldn't happen, but if it has we can't ignore the request to
			# shutdown, so we assume this is am active mastercontroller
			#
			THIS_IS_ACTIVE_MC_HOST=0
			LogWarning "$LOG_FILE" "[shutdown-monitor] failed to determine active/standby role status and no link to another mastercontroller host; assume active status"
			
		fi
		
	fi
	
	
	#
	# Determine if this host runs a GUI
	#
	ps -AH | grep --silent marmaset
	THIS_IS_GUI_HOST=$?
	
	
	#
	# Disable automatic application respawn.
	#
	touch $NO_RESPAWN_FLAG
	
	
	#
	# Take appropriate active based on type of host.
	#
	if [ $THIS_IS_ACTIVE_MC_HOST == 0 ]; then
		
		#
		# This is the active mastercontroller.
		# First we need to attempt to perform a controlled shutdown.
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC> signal mc to shutdown gracefully"
		killall mastercontroller
		
		
		#
		# Update the surface illumination; 'Don't switch off'.
		# NOTE: The script is run in bacground so we can con-currently check
		#       the mastercontrolle status.
		#
		/opt/midas/bin/signal-system-shutting-down &
		LogInformation "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC> illuminiate surface 'shutting down'"
		
		
		#
		# It may take some time for the mastercontroller to perform a
		# controlled shutdown, so we can now start shutting the GUIs down.
		# NOTE: Both the active and standby mastercontroller hosts do this
		#       just in case.
		#
		InitiateGUITermination "$THIS_IS_GUI_HOST" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
		
		
		#
		# Wait to see if it shuts down.
		#
		WaitForMCStopped "$MC_SHUTDOWN_TIMEOUT_SECONDS" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
		
		
		#
		# If the mastercontroller is still running then forceably kill it.
		#
		if [ $? != 0 ]; then
			
			#
			# The mastercontroller is still running really kill it.
			#
			killall --signal 9 mastercontroller
			LogCritical "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC> mc is still running; forcing mc termination (kill -9)"
			
		fi
		
		
		#
		# Signal the standby, if any, to start shutting down the mastercontroller.
		#
		SignalEvent "$OTHER_MC_HOST" "$ACTIVE_MC_DOWN" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>" 
		WAIT_FOR_STANDBY=$?
		
		
		#
		# Write out the persisted caches.
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC> invoking '/opt/midas/bin/mc-bootstrap -shutdown'"
		/opt/midas/bin/mc-bootstrap -shutdown
		LogInformation "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC> completed '/opt/midas/bin/mc-bootstrap -shutdown'"
		
		
		#
		# Signal the standby, if any, that the active mastercontroller
		# caches have been flushed.
		#
		SignalEvent "$OTHER_MC_HOST" "$ACTIVE_CACHE_FLUSHED" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
		WAIT_FOR_STANDBY=$?
		
		
		#
		# Wait for the standby to signal completion.
		#
		if [ $WAIT_FOR_STANDBY == 0 ]; then
			
			#
			# First wait for standby mastercontroller application stopped.
			#
			WaitForEvent "$STANDBY_MC_DOWN" "$EVENT_TIMEOUT_SECONDS" "$EVENT_POLL_RESOLUTION_LOW" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
			
			#
			# Finally wait for standby cache flushed.
			#
			WaitForEvent "$STANDBY_CACHE_FLUSHED" "$EVENT_TIMEOUT_SECONDS" "$EVENT_POLL_RESOLUTION_LOW" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
			
		fi
		
		
		#
		# Finally signal that it's OK to shut the OS down.
		#
		SignalEvent "$OTHER_MC_HOST" "$SHUTDOWN_OS_NOW" "$LOG_FILE" "[shutdown-monitor] <ACTIVE MC>"
		
		
	elif [ $THIS_IS_MC_HOST == 0 ]; then
		
		#
		# This is mastercontroller host but not running as the active;
		# therefore it must be the standby.
		#
		
		
		#
		# Update the surface illumination; 'Don't switch off'.
		# We shouldn't need to do this as the active mc should have done it
		# in the case where that hasn't happened this will.
		# NOTE: The script is run in background so we start the timeout.
		#
		/opt/midas/bin/signal-system-shutting-down &
		LogInformation "$LOG_FILE" "[shutdown-monitor] <STANDBY MC> illuminiate surface 'shutting down'"
		
		
		#
		# Since we have to wait to be signalled by the active mastercontroller
		# (or timeout) the standby may as well signal the GUIs to shutdown.
		# NOTE: Both the active and standby mastercontroller hosts do this
		#       just in case.
		#
		InitiateGUITermination "$THIS_IS_GUI_HOST" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		
		
		#
		# First wait for a signal from the active mastercontroller indocating
		# that it has shutdown.
		#
		WaitForEvent "$ACTIVE_MC_DOWN" "$EVENT_TIMEOUT_SECONDS" "$EVENT_POLL_RESOLUTION_LOW" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		
		#
		# Whatever the outcome start shutting the mastercontroller done.
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] <STANDBY MC> signal mc to shutdown gracefully"
		killall mastercontroller
		
		
		#
		# Wait to see if it shuts down.
		#
		WaitForMCStopped "$MC_SHUTDOWN_TIMEOUT_SECONDS" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		
		
		#
		# If the mastercontroller is still running then forcably kill it.
		#
		if [ $? != 0 ]; then
			
			#
			# The mastercontroller is still running really kill it.
			#
			killall --signal 9 mastercontroller
			LogCritical "$LOG_FILE" "[shutdown-monitor] <STANDBY MC> mc is still running; forcing mc termination (kill -9)"
			
		fi
		
		
		#
		# Signal the active that the standby is shut down.
		#
		SignalEvent "$OTHER_MC_HOST" "$STANDBY_MC_DOWN" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		WAIT_FOR_ACTIVE=$?
		
		
		#
		# Write out the persisted caches.
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] <STANDBY MC> invoking '/opt/midas/bin/mc-bootstrap -shutdown'"
		/opt/midas/bin/mc-bootstrap -shutdown
		LogInformation "$LOG_FILE" "[shutdown-monitor] <STANDBY MC> completed '/opt/midas/bin/mc-bootstrap -shutdown'"
		
		
		#
		# Signal the active that the standby cache has been flushed.
		#
		SignalEvent "$OTHER_MC_HOST" "$STANDBY_CACHE_FLUSHED" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		WAIT_FOR_ACTIVE=$?
		
		
		#
		# Finally wait for a signal from the active mastercontroller indicating
		# that it is safe to shutdown the OS.
		#
		WaitForEvent "$SHUTDOWN_OS_NOW" "$EVENT_TIMEOUT_SECONDS" "$EVENT_POLL_RESOLUTION_HIGH" "$LOG_FILE" "[shutdown-monitor] <STANDBY MC>"
		
	elif [ $THIS_IS_GUI_HOST == 0 ]; then
		
		#
		# This is not a Mastercontroller host, but is a GUI host.
		# Attempt to stop the GUI.
		#
		InitiateGUITermination "$THIS_IS_GUI_HOST" "$LOG_FILE" "[shutdown-monitor] <GUI>"
		
	fi
	
	
	
	#
	# Remove control flag files.
	#
	rm -f $RUNNING_FLAG
	
	
	#
	# Remove role status notifications.
	#
	rm -f $SHUTDOWN_AS_ACTIVE
	rm -f $SHUTDOWN_AS_STANDBY
	
	
	#
	# Remove progress checkpoints.
	#
	rm -f $ACTIVE_MC_DOWN
	rm -f $STANDBY_MC_DOWN
	rm -f $ACTIVE_CACHE_FLUSHED
	rm -f $STANDBY_CACHE_FLUSHED
	rm -f $SHUTDOWN_OS_NOW
	
	
	#
	# Finally, shutdown or restart the operating system.
	#
	if [ -e $SHUTDOWN_FLAG ]; then
		#
		# Shutdown
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] executing 'shutdown -h now'"
		shutdown -h now
		
	elif [ -e $RESTART_FLAG ]; then
		#
		# Restart
		#
		LogInformation "$LOG_FILE" "[shutdown-monitor] executing 'shutdown -r now'"
		shutdown -r now
		
	fi
	
else
	# Monitor is already running
	echo "STOPPED AS INSTANCE IS ALREADY RUNNING"
fi