#!/bin/bash --
set -e -o pipefail -u

if [ "$EUID" != 0 ]; then
  id
  printenv
  echo "This script requires root privileges."
  exit 1
fi

PRIMARY=/etc/service/spectre
BACKUP=/etc/service/spectre_backup

function get_pid() {
  svstat "$1" | grep ": up" | egrep -o '\(pid [0-9]+\)' || echo "DOWN"
}

# wait until the service is up and check if it's spinning
# XXX: this could be smarter and use the service health checks
function wait_until_up() {
  SVC="$1"
  OLD_PID=$(get_pid "$SVC")

  for try in {1..2}; do
    sleep 5
    NEW_PID=$(get_pid "$SVC")
    if [ "$NEW_PID" != "$OLD_PID" ]; then
      if [ "$OLD_PID" = "DOWN" ]; then
        OLD_PID="$NEW_PID"
      else
        echo "Process is spinning!"
        return 1
      fi
    fi
  done

  NEW_PID=$(get_pid "$SVC")
  if [ "$NEW_PID" != "DOWN" ]; then
    return 0
  else
    echo "Process did not come back up (it's DOWN)"
    return 1
  fi
}

# wait until the pid of the process changes
function wait_until_restarts() {
  SVC="$1"
  OLD_PID="$2"

  for try in {1..30}; do
    sleep 1
    NEW_PID=$(get_pid "$SVC")
    if [ "$NEW_PID" != "DOWN" -a "$NEW_PID" != "$OLD_PID" ]; then
      return 0
    fi
  done

  return 1
}

# restart the service, will start it if it's down
function restart_service() {
  SVC="$1"
  PID=$(get_pid "$SVC")

  svc -ui "$SVC"

  # wait for the pid to change and kill the process if it doesn't
  wait_until_restarts "$SVC" "$PID" || ( svc -uk "$SVC" )
  sleep 5
  wait_until_up "$1"
}

# check if the service is up
PRIMARY_PID=$(get_pid "$PRIMARY")
BACKUP_PID=$(get_pid "$BACKUP")

if [ "$PRIMARY_PID" != "DOWN" -o "$BACKUP_PID" != "DOWN" ]; then

  # start the backup if primary is up
  if [ "$BACKUP_PID" = "DOWN" ]; then
    echo "WARNING: backup instance is down, bringing it up."
    svc -u "$BACKUP"
  fi

  # ensure the backup is running
  if ! wait_until_up "$BACKUP"; then
    echo "Could not bring the backup instance up."
    exit 1
  fi


  # failure of any of the following lines will stop the restart script
  echo "Restarting main instance: $PRIMARY"
  restart_service "$PRIMARY"
  echo "Restarting backup instance: $BACKUP"
  restart_service "$BACKUP"
else
  echo "Both instances of the service are down. No action to be done."
fi

echo "Done."
