manage

#!/bin/bash

##H Usage: manage ACTION
##H
##H Available actions:
##H   help        show this help
##H   env         show env setup
##H   status      show current service's status
##H   restart     (re)start the service
##H   start       start the service
##H   stop        stop the service
##H
##H   start-minio start MIN.IO server
##H   stop-minio  stop MIN.IO server
##H
##H For more details please refer to documentation page:
##H   https://github.com/CHESSComputing/FOXDEN/blob/main/docs/main.md
#
# determine location of our script and logs area
ME=$(basename $(dirname $0))
ROOT=$(cd $(dirname $0)/.. && pwd)
LOGDIR=$ROOT/logs
HOST=`hostname -s`
services="Authz MetaData UserMetaData DataDiscovery DataManagement DataBookkeeping Frontend SyncService SpecScansService MLHub DataHub DOIService ClasseInfoService"

if [ -d $ROOT/etc/profile.d/init.sh ]; then
    . $ROOT/etc/profile.d/init.sh
fi

export FOXDEN_INSTANCE="local"
export FOXDEN_ROOT=$ROOT
if [ "`whoami`" == "chessdata_svc" ]; then
    export PATH=/home/chessdata_svc/bin:/nfs/chess/sw/chessdata_svc/go/go/bin:$PATH
    export MINIO_DIR=/home/chessdata_svc/s3
else
    export MINIO_DIR=/tmp
fi
# foxden dev should reside in /path/FOXDENDEV
if [ -n "`echo $FOXDEN_ROOT | grep '/FOXDENDEV$'`" ] && [ "`hostname -s`" == "foxden-dev" ]; then
    export FOXDEN_CONFIG=$HOME/.foxden-dev.yaml
    export FOXDEN_INSTANCE="dev"
    unset FOXDEN_RELEASES
    services+=" BeamPass"
fi
# foxden prod should reside in /path/FOXDEN
if [ "`hostname -s`" == "lnx15" ]; then
    unset FOXDEN_INSTANCE
    if [ -n "`echo $FOXDEN_ROOT | grep '/FOXDEN$'`" ]; then
        export FOXDEN_CONFIG=$HOME/.foxden.yaml
        export FOXDEN_RELEASES=/home/chessdata_svc/FOXDEN/RELEASES.md
        export FOXDEN_INSTANCE="prod"
    fi
    services+=" BeamPass"
fi
# foxden demo should reside in /path/FOXDENDEMO
if [ -n "`echo $FOXDEN_ROOT | grep '/FOXDENDEMO$'`" ] && [ "`hostname -s`" == "foxden-dev" ]; then
    export FOXDEN_CONFIG=$HOME/.foxden-demo.yaml
    export FOXDEN_INSTANCE="demo"
    unset FOXDEN_RELEASES
fi

# check if local foxden.yaml exists and use it
if [ -z "$FOXDEN_CONFIG" ]; then
    if [ -f $HOME/.foxden.yaml ]; then
        export FOXDEN_CONFIG=$HOME/.foxden.yaml
    fi
    if [ -f $PWD/foxden.yaml ]; then
        export FOXDEN_CONFIG=$PWD/foxden.yaml
    fi
fi
if [ -z "$FOXDEN_RELEASES" ]; then
    if [ -f $PWD/RELEASES.md ]; then
        export FOXDEN_RELEASES=$PWD/RELEASES.md
    fi
fi

# if FOXDEN_CONFIG not determined exit
if [ -z "$FOXDEN_CONFIG" ]; then
   echo "Unable to determine FOXDEN_CONFIG..."
   exit 1
fi
if [ -z "$FOXDEN_INSTANCE" ]; then
   echo "Unable to determine FOXDEN_INSTANCE..."
   exit 1
fi


# show all used environment variables in this script
show_env()
{
    echo "ROOT=$ROOT"
    echo "LOGDIR=$LOGDIR"
    echo "MINIO_DIR=$MINIO_DIR"
    echo "FOXDEN_INSTANCE=$FOXDEN_INSTANCE"
    echo "FOXDEN_RELEASES=$FOXDEN_RELEASES"
    echo "FOXDEN_CONFIG=${FOXDEN_CONFIG:-$HOME/.foxden.yaml}"
    echo "FOXDEN_CERTS=$FOXDEN_CERTS"
    echo "FOXDEN_ROOT=$FOXDEN_ROOT"
    echo "TFAAS_DIR=$TFAAS_DIR"
    echo "GOFAKE_DIR=$GOFAKE_DIR"
    echo "minio command: `command -v minio`"
}

# checks performs checks over used directories and env variables
checks()
{
    if [ -z "$MINIO_DIR" ]; then
        echo "WARNING: MINIO_DIR environment is not set"
    fi
    if [ ! -d $MINIO_DIR ]; then
        echo "WARNING: $MINIO_DIR area does not exist"
    fi
    if [ ! -d $LOGDIR ]; then
        echo "$LOGDIR area does not exist"
        exit 1
    fi
    if ! command -v minio &> /dev/null
    then
        echo "WARNING: minio could not be found, please install MIN.IO and setup PATH to find minio command"
    fi
}

# start ML services
start_ml_services()
{
    if [ -f $TFAAS_DIR/tfaas ]; then
        echo "### start TFaaS service..."
        cd $TFAAS_DIR
        nohup ./tfaas -config config.json 2>&1 1>& $LOGDIR/tfaas.log_$HOST < /dev/null &
        cd -
    fi
    if [ -f $GOFAKE_DIR/fake ]; then
        echo "### start GoFake service..."
        cd $GOFAKE_DIR
        nohup ./fake 2>&1 1>& $LOGDIR/gofake.log_$HOST < /dev/null &
        cd -
    fi
}

# releases function performs the following actions:
# - maintain a temporary list of successfully built services
# - abort immediately if a build fails
# - start services after the build loop completes
releases() {
    tmpfile=$(mktemp)
    echo "Compile new set of executables for FOXDEN services..."

    # -------- BUILD PHASE --------
    while read -r srv tag _; do
        echo
        echo "check releases/$srv/$tag/srv"
        mkdir -p releases/$srv/$tag

        if [ -e releases/$srv/$tag/srv ]; then
            echo "executable releases/$srv/$tag/srv already exists, continue..."
        else
            echo "Compile releases/$srv/$tag/srv executable..."
            cd $srv
            git fetch --all --tags
            BRANCH=${tag}-branch

            if git rev-parse $tag >/dev/null 2>&1; then
                echo "tag $tag exist..."
            else
                echo "tag $tag does not exist..."
                rm -f $tmpfile
                exit 1
            fi

            bhash=`git show-ref --verify refs/heads/$BRANCH`
            if [ -n "$bhash" ]; then
                echo "Branch $BRANCH already exists. Checking it out..."
                git checkout $BRANCH
            else
                echo "Branch $BRANCH does not exists. Create $BRANCH..."
                git checkout tags/$tag -b $BRANCH
            fi

            if ! make; then
                echo "ERROR: build failed for $srv"
                git checkout main
                cd -
                rm -f $tmpfile
                exit 1
            fi

            cp srv ../releases/$srv/$tag || {
                echo "ERROR: failed to copy binary for $srv"
                git checkout main
                cd -
                rm -f $tmpfile
                exit 1
            }

            echo "Comeback to main branch..."
            git checkout main
            cd -
        fi

        echo "$srv $tag" >> $tmpfile

    done < "$FOXDEN_RELEASES"

    echo
    echo "successfully compiled list of FOXDEN services"
    cat $tmpfile
    echo "restart FOXDEN services with new executables..."
    echo

    # -------- START PHASE --------
    while read -r srv tag; do
        relsrv=$FOXDEN_ROOT/releases/$srv/$tag/srv
        echo "start $relsrv ..."

        if [ -f $srv/server.json ]; then
            echo "$relsrv -config $srv/server.json"
            nohup $relsrv -config $srv/server.json \
                2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        else
            echo "$relsrv"
            nohup $relsrv \
                2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        fi

        echo "$relsrv started with PID=`cat $LOGDIR/$srv.pid_$HOST`"
        echo "log file: $LOGDIR/$srv.log_$HOST"

    done < $tmpfile

    rm -f $tmpfile
}

# original version of releases function which performs sequentially steps for
# every service: check if executable exists for specific tag, if not checkout
# tag branch, recompile executable and restart it
releases_orig() {
    while read -r srv tag _; do
        echo
        echo "check releases/$srv/$tag/srv"
        mkdir -p releases/$srv/$tag
        if [ -e releases/$srv/$tag/srv ]; then
            echo "executable releases/$srv/$tag/srv already exists, continue..."
        else
            echo "Compile releases/$srv/$tag/srv executable..."
            cd $srv
            git fetch --all --tags
            BRANCH=${tag}-branch
            if git rev-parse $TAG >/dev/null 2>&1; then
                echo "tag $tag exist..."
            else
                echo "tag $tag does not exist..."
                continue
            fi
            # Check if the branch already exists
            bhash=`git show-ref --verify refs/heads/$BRANCH`
            if [ -n "$bhash" ]; then
                echo "Branch $BRANCH already exists. Checking it out..."
                git checkout $BRANCH
            else
                echo "Branch $BRANCH does not exists. Create $BRANCH_NAMNE..."
                git checkout tags/$tag -b $BRANCH
            fi
            make
            cp srv ../releases/$srv/$tag
            echo "Comeback to main branch..."
            git checkout main
            cd -
        fi
        # start release executable
        relsrv=$FOXDEN_ROOT/releases/$srv/$tag/srv
        echo "start $relsrv ..."
        if [ -f $srv/server.json ]; then
            echo "$relsrv -config $srv/server.json"
            nohup $relsrv -config $srv/server.json 2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        else
            echo "$relsrv"
            nohup $relsrv 2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        fi
        echo "$relsrv started with PID=`cat $LOGDIR/$srv.pid_$HOST`"
        echo "log file: $LOGDIR/$srv.log_$HOST"
    done < "$FOXDEN_RELEASES"
}

# start FOXDEN/CHESS services
start_services(){
    if [ -n "$FOXDEN_RELEASES" ]; then
        echo ""
        echo "### Switch to FOXDEN_RELEASES=$FOXDEN_RELEASES"
        releases
        exit 0
    fi
    # start new processes
    for srv in $services
    do
        echo "### $srv service..."
        if [ ! -f $FOXDEN_ROOT/$srv/srv ]; then
            echo "ERROR: no $FOXDEN_ROOT/$srv/srv executable found..."
            continue
        fi
        cd $FOXDEN_ROOT/$srv
        if [ -f $FOXDEN_ROOT/$srv/server.json ]; then
            echo "Start: $FOXDEN_ROOT/$srv/srv -config $FOXDEN_ROOT/$srv/server.json"
            nohup $FOXDEN_ROOT/$srv/srv -config $FOXDEN_ROOT/$srv/server.json 2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        else
            echo "Start: $FOXDEN_ROOT/$srv/srv"
            nohup $PWD/srv 2>&1 1>& $LOGDIR/$srv.log_$HOST < /dev/null & \
                echo $! > $LOGDIR/$srv.pid_$HOST
        fi
        cd - 2>&1 1>& /dev/null
        echo "$srv started with PID=`cat $LOGDIR/$srv.pid_$HOST`"
        echo "log file: $LOGDIR/$srv.log_$HOST"
    done
    start_ml_services
}

# stop ML services
stop_ml_services()
{
    if [ -f $TFAAS_DIR/tfaas ]; then
        pid=`ps auxww | grep $FOXDEN_ROOT | grep "tfaas -config" | grep -v grep | awk '{print $2}' | awk '{print "kill -9 "$1""}'`
        if [ -n "$pid" ]; then
            echo "### stop TFaaS service..."
            echo "$pid" | sed "s,\n,,g"
            echo "$pid" | sed "s,\n,,g" | /bin/sh
        fi
    fi
    if [ -f $GOFAKE_DIR/fake ]; then
        pid=`ps auxww | grep $FOXDEN_ROOT | grep "fake" | grep -v grep | awk '{print $2}' | awk '{print "kill -9 "$1""}'`
        if [ -n "$pid" ]; then
            echo "### stop GoFake service..."
            echo "$pid" | sed "s,\n,,g"
            echo "$pid" | sed "s,\n,,g" | /bin/sh
        fi
    fi
}

# stop FOXDEN/CHESS services
stop_services()
{
    # delete existing srv processes
    pid=`ps auxww | grep $FOXDEN_ROOT | grep ".*/srv$" | grep -v grep | awk '{print $2}' | awk '{print "kill -9 "$1""}'`
    if [ -n "$pid" ]; then
        echo "$pid" | sed "s,\n,,g"
        echo "$pid" | sed "s,\n,,g" | /bin/sh
    fi
    for srv in $services
    do
        if [ -f $LOGDIR/$srv.log_$HOST ]; then
            rm  $LOGDIR/$srv.log_$HOST
        fi
        if [ -f $LOGDIR/$srv.pid_$HOST ]; then
            rm  $LOGDIR/$srv.pid_$HOST
        fi
    done
    stop_ml_services
}

# status of ML services
status_ml_services()
{
    if [ -f $TFAAS_DIR/tfaas ]; then
        pid=`ps auxww | grep "tfaas -config" | grep -v grep | awk '{print $2}'`
        if [ -n "$pid" ]; then
            echo "service TFaaS is running, PID=$pid, $LOGDIR/tfaas.log_$HOST"
        fi
    fi
    if [ -f $GOFAKE_DIR/fake ]; then
        pid=`ps auxww | grep "fake" | grep -v grep | awk '{print $2}'`
        if [ -n "$pid" ]; then
            echo "service GoFake is running, PID=$pid, $LOGDIR/gofake.log_$HOST"
        fi
    fi
    echo ""
    echo "List of running srv processes:"
    ps auxwww | grep $FOXDEN_ROOT | grep srv | grep -v grep
}

# run status over FOXDEN/CHESS services
status_services()
{
    # start new processes
    for srv in $services
    do
        pid=`ps auxww | grep $FOXDEN_ROOT | grep ".*/${srv}/.*srv$" | grep -v grep | awk '{print $2}'`
        if [ -n "$pid" ]; then
            echo "service $srv is running, PID=$pid, $LOGDIR/$srv.log_$HOST"
        else
            echo "service $srv is not running"
        fi
    done
}

# stop minio server
stop_minio()
{
    if ! command -v minio &> /dev/null
    then
        echo "minio command not found. Exiting function."
        return  # Exit the function if 'minio' is not found
    fi
    # proceed only if we have minio installed
    local pid=`ps auxwww | egrep "minio server" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'`
    if [ -n "${pid}" ]; then
        echo "killing MIN.IO runnig on $pid"
        kill -9 $pid
    fi
    if [ -f $LOGDIR/minio.log_$HOST ]; then
        rm  $LOGDIR/minio.log_$HOST
    fi
}

# start minio server
start_minio()
{
    if ! command -v minio &> /dev/null
    then
        echo "minio command not found. Exiting function."
        return  # Exit the function if 'minio' is not found
    fi
    # proceed only if we have minio installed
    if [ -n "$FOXDEN_CERTS" ]; then
        cdir=$FOXDEN_CERTS
    else
        cdir=/tmp
    fi
    local pid=`ps auxwww | egrep "minio server" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'`
    if [ ! -z "${pid}" ]; then
        echo "MIN.IO server is already running PID: $pid, $LOGDIR/minio.log_$HOST"
    else
        minio server $MINIO_DIR --address :8330 --certs-dir $cdir 2>&1 1>& $LOGDIR/minio.log_$HOST < /dev/null &
        echo "MIN.IO server is started, $LOGDIR/minio.log_$HOST"
    fi

    echo ""
    echo "Setup minio alias using mc client (replace host:port as desired):"
    echo "mc alias set myminio http://192.0.2.10:9000 <username> <password>"
    echo "Use MIN.IO console to create new access key/access secret for your users"
}

# status of minio server
status_minio()
{
    if ! command -v minio &> /dev/null
    then
        return
    fi
    local pid=`ps auxwww | egrep "minio server" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'`
    if [ ! -z "${pid}" ]; then
        echo "MIN.IO server is running PID: $pid, $LOGDIR/minio.log_$HOST"
    else
        echo "MIN.IO server is not running"
    fi
}

case ${1:-status} in
 restart)
    checks
    stop_services
    start_services 
    ;;
 start-minio)
    stop_minio
    start_minio
    ;;
 stop-minio)
    stop_minio
    ;;
 stop)
    checks
    stop_services
    stop_minio
    status_minio
    status_services
    ;;
 stop-services)
    stop_services
    ;;
 releases)
    releases
    ;;
 status)
    checks
    status_minio
    status_services
    status_ml_services
    ;;
 status-services)
    status_services
    ;;
 start)
    checks
    stop_minio
    start_minio
    stop_services
    start_services
    ;;
 start-services)
    start_services
    ;;
  env )
    show_env
    checks
    ;;
  help )
    cat $0 | grep "^##H"
    ;;
  * )
    echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
    exit 1
    ;;
esac