#!/usr/bin/env bash

set -u -e -o pipefail

log() {
    local svc=$1
    local msg=$2
    echo "restart $svc: $msg"
}

# uptime <svc>
# Get the number of seconds that svc has been up. If it's not up, returns 0. If
# it doesn't exist or hits some other error, exits with code 1.
uptime() {
    local svc=$1
    local raw=$(sudo svstat $svc)

    # If the service isn't up, output 0
    if [ "$(echo $raw | awk '{print $2}')" != "up" ]; then
        echo 0
        return 0
    fi

    # Get the 'seconds' part of the svstat output and make sure its an integer
    local seconds=$(echo $raw | awk '{print $5}')
    if [[ ! $seconds =~ ^[0-9]+$ ]]; then
        log $svc "Unparseable time from svstat output \"$raw\""
        exit 1
    fi

    echo $seconds
    return 0
}

# service_state <svc>
# Returns "up" if the service is definitely up, "not up" for any other case.
service_state() {
    local svc=$1
    local raw=$(sudo svstat $svc)
    # If the service is up, output 0
    if [ "$(echo $raw | awk '{print $2}')" == "up" ]; then
        echo "up"
        return 0
    fi
    echo "not up"
    return 0
}


# wait_for_restart <svc>
# Blocks for at most 5 seconds, waiting for svc's uptime to be less than 30
# seconds. If this times out, it returns code 1, else code 0.
wait_for_restart() {
    # Wait until the service's uptime appears to be less than 30 seconds,
    # indicating it actually restarted. Time out if it takes more than 5 second
    # to restart.
    local svc=$1
    local i=0
    local t=$(uptime $svc)

    while [ "$t" -gt "30" ]; do
        if [ "$i" -gt "5" ]; then
            log $svc "timed out waiting for restart"
            return 1
            break
        fi
        let "i=i+1"
        t=$(uptime $svc)
        sleep 1
    done

    log $svc "terminated and restarted successfully"    
    return 0
}

# check_not_flapping <svc>
# Blocks for at most 10 seconds, waiting for svc to remain up for at least 10
# seconds. If it times out, returns code 1, else returns code 0
check_not_flapping() {
    # Wait until the service $1's uptime is at least 10 seconds.
    local svc=$1
    local i=0
    while [ $(service_state $svc) != "up" ]; do
        sleep 1
        let "i=i+1"
        if [ $i -gt 5 ]; then
            log $svc "timed out waiting for service to turn back on after restarting"
            return 1
        fi
    done

    log $svc "sleeping 10 seconds to make sure service isn't flapping"
    sleep 10

    local i=0
    while [ "$(uptime $svc)" -lt "10" ]; do
        sleep 2
        let i=i+1
        if [ $i -gt 5 ]; then
            log $svc "timed out waiting for service to reach uptime over 10 seconds!"
            return 1
        fi
    done

    return 0
}

# restart_svc <svc>
# Restarts svc, making sure it 1. actually stops and restarts and 2. stays up
# for at least 10 seconds. It first sends SIGTERM, but if that doesn't seem to
# do the trick it'll try SIGKILL too.
restart_svc() {
    local svc=$1
    if [ "$(service_state $svc)" == "not up" ]; then
        echo "skipping service $svc because it is not up"
        continue
    fi

    # Send a restart signal to svc.
    sudo svc -t $svc
    wait_for_restart $svc

    # Did the restart work?
    if [ $? -ne "0" ]; then
        # No, it didn't. Send a more aggressive signal, and wait again.
        sudo svc -k $svc
        wait_for_restart $svc

        # Did that work? If not, exit with an error.
        if [ $? -ne "0" ]; then
            exit 1
        fi
    fi

    # If we reach here, the restart worked. Now, block until the service has
    # been up for at least 10 seconds to avoid flapping.
    check_not_flapping $svc
    if [ $? -ne "0" ]; then
        exit 1
    fi
}

# Restart all Trace services.
for svc in /etc/service/trace*; do
    log $svc "begin"
    restart_svc $svc
    log $svc "success!"
done
