#!/bin/bash
#
# $Id: run_prestart 20280 2016-02-17 19:14:42Z davidl $
# $HeadURL: https://halldsvn.jlab.org/repos/trunk/online/daq/scripts/run_prestart $
#
# RUN_PERIOD and RAWDATA_DIR environment variables are set in the file
# /gluex/etc/hdonline.cshrc 
# 
# CODA will write files to:
#
#    $RAWDATA_DIR/active/$RUN_PERIOD/rawdata/RunXXXXXX
#
# where "XXXXXX" is the zero padded run number. Other files
# related to the run are also placed in this directory. 
# Some details on the strategy here are given in:
#
# https://halldweb1/wiki/index.php/Raid-to-Silo_Transfer_Strategy
#
#---- jcedit --------
#  %(DAQ_HOME)/scripts
#  run_prestart %(rn)  %(udl)  %(config) %(rt)
#  run_go       %(rn)  %(udl)  %(config) %(rt)
#  run_end      %(rn)  %(udl)  %(config) %(rt)
#===========================================================================================
#
#HDOPS gluon100:/tmp> ssh gluon41
#shell request failed on channel 0
#
#Jan 09 15:10:55 gluon41.jlab.org systemd-logind[6151]: New session 30859 of user hdops.
#Jan 09 15:10:55 gluon41.jlab.org systemd[1]: Started Session 30859 of user hdops.
#Jan 09 15:10:55 gluon41.jlab.org sshd[16828]: pam_unix(sshd:session): session opened for user hdops by (uid=0)
#Jan 09 15:10:55 gluon41.jlab.org sshd[16831]: error: do_exec_pty: fork: Resource temporarily unavailable
#Jan 09 15:10:55 gluon41.jlab.org sshd[16828]: pam_unix(sshd:session): session closed for user hdops
#Jan 09 15:10:55 gluon41.jlab.org systemd-logind[6151]: Removed session 30859.
# !!!!!!!!  STD in/out is not allowed in this script !!!!!!
#STDO=/tmp/${USER}_PRESYNC.$1
#exec >$STDO 2>&1
SCRIPT=`basename $0 `

# run_prestart_sync.local 0 'cMsg://gluon41.jlab.org:45000/cMsg/hdops?cmsgpassword=hdops'  /gluondaqfs/hdops/CDAQ/daq_dev_v0.31/daq/config/hd_all/TRG_CDC_BUSY.conf
RUNNUM=${1-0}
UDL=${2-"cMsg://gluon41.jlab.org:45000/cMsg/hdops?cmsgpassword=hdops"}
CONFIG_FILE=${3-/gluondaqfs/hdops/CDAQ/daq_dev_v0.31/daq/config/hd_all/TRG_CDC_BUSY.conf}


date                               
echo "----- > $SCRIPT ENTER <---- "

#source /gluex/etc/hdonline.cshrc
ENV_RAWDATA_DIR=`awk '/RAWDATA_DIR/ {printf "%s %s=%s\n",$1,$2,$3 }' /gluex/etc/hdonline.cshrc | grep setenv | sed s/setenv/export/g | grep -v "^#" `
ENV_RUN_PERIOD=`awk '/RUN_PERIOD/ {printf "%s %s=%s\n",$1,$2,$3 }' /gluex/etc/hdonline.cshrc | grep setenv | sed s/setenv/export/g  | sed s/\"//g`
$ENV_RAWDATA_DIR
$ENV_RUN_PERIOD
echo "ENV_RAWDATA_DIR=$ENV_RAWDATA_DIR  : $RAWDATA_DIR " 
echo "ENV_RUN_PERIOD=$ENV_RUN_PERIOD    : $RUN_PERIOD"  


source run_lib.sh
CONFIG_DIR=`dirname $CONFIG_FILE `

ER_name=ERsoftROC
get_host $ER_name ER_host

USE_RAMDISK=no
get_host USE_RAMDISK USE_RAMDISK
echo "USE_RAMDISK = $USE_RAMDISK"
USE_4PART=no
get_host USE_4PART USE_4PART
echo "USE_4PART = $USE_4PART"

get_host MONITOR  MONITOR_START
echo "start MONITOR = $MONITOR_START"

#===========================================================================================
RCM_HOST=gluon25
RCM_PORT=32767
RCM_LOG=/dev/tcp/${RCM_HOST}/${RCM_PORT}
RCM_LOG_FILE=$DAQ_HOME/../work/rcm_rc.log
#-------------------------------------------------------------------------------------------
cnt=0
function rcm_log() {
    cnt=$(($cnt+1))
    echo -e " $1 $2 "
    sleep 0.2
#   cMsgCommand -u cMsg://gluon100:45000/cMsg/$EXPID  -name $0 -subject Prestart -type DAQ -text "$1"     -string severity=$2
    [ -n "$RCM_LOG" ] && echo -e "msg:GUI_runconf presync:${cnt}: $1 " > $RCM_LOG
    [[ -n "$UDL" &&  $RUNNUM -gt 0  ]] && cMsgCommand -u $UDL  -name run_presync_loc${cnt} -subject Prestart -type DAQ -text "$1"  -string severity=$2  2>&1 > /tmp/${USER}_cMsgCommand
    echo -e "`date` :run_presync_loc:${cnt}:  $2: $1 " >> $RCM_LOG_FILE
}


rcm_log "Run=$1 user=$USER `date` UDL=$2 config=$3  4=$4 5=$5 " "INFO"


if [ "$#" -lt 1 ]; then
  echo "You must provide a run number!"   
  echo ""                               
  echo "Usage:"                         
  echo "   $0 RUN"             
  echo ""                                
  exit -1
fi

export RUN=$(printf '%06d' $1) 

#===========================================================================================
#      DAQ TEST MODE
#
if [ x$DAQMODE == "xtest" ] ; then
    rcm_log "WARN: DAQMODE=$DAQMODE " "WARN"
    DATA_HOME=`dirname  $DAQ_HOME`
    DATA_DIR=`dirname  $DAQ_HOME`/DATA
    DATA_DIR1=`dirname  $DAQ_HOME`/DATA1
#    ln -s ${DATA_HOME}/DATA_TEST $DATA_DIR
#    ln -s ${DATA_HOME}/DATA_TEST1 $DATA_DIR1
    ln -s /media/ramdisk/active/rawdata/active/RunPeriod-2021-08/rawdata/Run080927 $DATA_DIR
    ln -s /media/ramdisk/active/rawdata/active/RunPeriod-2021-08/rawdata/Run080927 $DATA_DIR1
    exit 0
fi

#===========================================================================================
#      any other accounts ..
#
if [ $USER != "hdops" ] ; then
    which run_log.sh
    run_log.sh "PRESTART" $RUN $CONFIG_FILE
    exit 0
fi

#===========================================================================================
#            HDOPS account only ...
#-------------------------------------------------------------------------------------------


#-------------------------------------------------------------------------------------------
#      select gluonraid disk
#-------------------------------------------------------------------------------------------


#---------------------- NEW style for gluonraid3/4 -------------------
rcm_log "ER_host=$ER_host select gluonraid3/4" "INFO"

# Rotate links for partitions on gluonraid3
# rcm_log " NO hd_rotate_raid_links  !" "ERROR"
tmn=`date`
rcm_log "rotating RAID partition links $tmn" "INFO"

# For some reason the LD_LIBRARY_PATH seems to not have the
# /apps/gcc/4.9.2/lib64 directory that is in the normal
# hdops environment. This causes hdlog to fail to run which
# means hd_rotate_raid_links.py never runs. This gums up
# the whole system, preventing files from making it to tape.
# This seems to have suddenly become a problem on 4/25/2018
# with the full extent of the problem not being noticed until
# 4/28/2018. To resolve it, the missing path is added to
# LD_LIBRARY_PATH temporarily to run hdlog and then removed
# in case that would screw up later items in this script.
# 4/28/2018 DL
TMPLDPATH=$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/apps/gcc/4.9.2/lib64:${LD_LIBRARY_PATH}
hdlog -c hd_rotate_raid_links.py DAQ
#hd_rotate_raid_links.py DAQ
export LD_LIBRARY_PATH==$TMPLDPATH

RAID_PATH=none

if [[ $USE_RAMDISK != "yes" ]]; then  
    GLUEX_RAID=/gluex/data/rawdata/curr
    #----  check /gluex/raid link ----
    if [ -L $GLUEX_RAID ] ; then
        rcm_log "link $GLUEX_RAID exists" "INFO"
    elif [ -e $GLUEX_RAID ] ; then
        rcm_log "file $GLUEX_RAID exists, not a link" "ERROR"
    else
        rcm_log "link $GLUEX_RAID does not exists!" "ERROR"
    fi
    #---- set DATA path ---
    RAID_PATH=`readlink $GLUEX_RAID `
else 
    RAID_PATH=/media/ramdisk/active
    rcm_log "use RAM disk = $RAID_PATH " "INFO"
fi

export RAWDATA_DIR=${RAID_PATH}/rawdata
tmn=`date`
rcm_log "new RAWDATA_DIR=$RAWDATA_DIR $tmn" "INFO"
RAID_LINK=`echo $RAWDATA_DIR | cut -f2 -d/ `
[[ $RAID_LINK != "media" && $RAID_LINK != $ER_host ]] && rcm_log "Error host/path combination:  $RAID_LINK  $ER_host " "ERROR"


#-------------------------------------------------------------------------------------------
#         create directory for new run
#-------------------------------------------------------------------------------------------

if [ -z "$RUN_PERIOD" ] ; then
  export RUN_PERIOD=RunPeriod-`date "+%F_%T"`
  rcm_log "WARNING:: RUN_PERIOD is not defined, use=$RUN_PERIOD " "WARN"
fi
if [ -z "$RAWDATA_DIR" ] ; then
    export RAWDATA_DIR=/gluex/data/rawdata/curr/rawdata
    rcm_log "WARNING:: RAWDATA_DIR is not defined, use=$RAWDATA_DIR " "WARN"
fi

export RAIDDIR=${RAWDATA_DIR}/active/${RUN_PERIOD}/rawdata/Run${RUN}


RUNPERIODDIR=`dirname $RAIDDIR`
if [ ! -w "$RUNPERIODDIR" ] ; then
  rcm_log "WARNING:: RAW DIR is write protected: $RUNPERIODDIR " "ERROR"
fi

echo 'RUN_PERIOD: '$RUN_PERIOD   
echo '       RUN: '$RUN          
echo '    OUTDIR: '$RAIDDIR      

if [ -d $RAIDDIR ]; then
    rcm_log " WARNING: Directory  $RAIDDIR already exists !! " "WARN"
    echo    "          Moving to ${RAIDDIR}_Backup_"`date "+%F_%T"`  "WARN"
    mv $RAIDDIR ${RAIDDIR}_Backup_`date "+%F_%T"`
fi

#Create new directory for all output files for this run on RAID disk 
rcm_log "Create new directory for output $RAIDDIR " "INFO"
mkdir -p $RAIDDIR  

ERR=$?
if [ $ERR != 0 ]; then 
    # Couldn't make desired output directory. Warn user and try an alternative
    rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR " "ERROR"
    export RAIDDIR=/tmp/${RAIDDIR}
    rcm_log "       trying $RAIDDIR instead ..."

    mkdir -p $RAIDDIR 
    ERR=$?
    if [ $ERR != 0 ]; then
        rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR "   "ERROR"
	rcm_log "ERROR: NO place to write data!"
    fi
fi
touch ${RAIDDIR}/lock
chmod 775 $RAIDDIR

#----------------------------------------------------------------
#--       create directory RAIDDIR1 for Second Event recorder          ---
#----------------------------------------------------------------

RAIDDIR1=$RAIDDIR
RAMDISK=`echo ${RAWDATA_DIR} | cut -f3 -d/ | sed s/data//g `


get_host ERraid0 ER0_host
get_host ERraid1 ER1_host
rcm_log "ER0=$ER0_host   ER1=$ER1_host RAMDISK=$RAMDISK" "INFO"
if  [[ "$ER1_host" != "localhost" ]] ; then

    #--- assumed format: /gluonraid3/data3   or   /media/ramdisk/active/

    #   create path on the same raid, another partition

    DATA_NUM=`echo ${RAWDATA_DIR} | cut -f3 -d/ | sed s/data//g `
    RAMDISK=$DATA_NUM
    if [[ $RAMDISK != "ramdisk" ]] ; then
        DATA_NUM1=$(($DATA_NUM%4+1))
        DATA_NUM2=$(($DATA_NUM1%4+1))
        DATA_NUM3=$(($DATA_NUM2%4+1))
        export RAIDDIR1=`echo $RAIDDIR | sed s/data$DATA_NUM/data$DATA_NUM1/g`
        export RAIDDIR2=`echo $RAIDDIR | sed s/data$DATA_NUM/data$DATA_NUM2/g`
        export RAIDDIR3=`echo $RAIDDIR | sed s/data$DATA_NUM/data$DATA_NUM3/g`
        rcm_log "File0=data$DATA_NUM  File1=data$DATA_NUM1  File2=data$DATA_NUM2  File3=data$DATA_NUM3 " "INFO"
    else
        rcm_log "second data path on RAM disk " "INFO"
        # same dir for RAM disk
        RAIDDIR1=$RAIDDIR
        RAIDDIR2=$RAIDDIR
        RAIDDIR3=$RAIDDIR
    fi
    rcm_log "second data path =$RAIDDIR1" "INFO"

    #   create path on the second raid 

    if [[ $RAMDISK != "ramdisk" ]] ; then
        if [[  "$ER1_host" != "localhost" && $ER1_host != $ER0_host ]] ; then
            export RAIDDIR2=`echo $RAIDDIR | sed s/$ER0_host/$ER1_host/g`
            rcm_log "second RAID path =$RAIDDIR2" "INFO"            
            rcm_log " use another raid: $RAIDDIR2 instead !!! " "INFO"
            [[ $ER0_host == localhost || $ER1_host == localhost ]] && rcm_log " localhost ::  $ER0_host  $ER1_host  " "INFO"
            RAIDDIR1=$RAIDDIR2
        fi
    fi

    #   
    RAID_LINK1=`echo $RAIDDIR1 | cut -f2 -d/ `
    [[ $RAID_LINK1 != "media" && $RAID_LINK1 != $ER1_host ]] && rcm_log "Error host/path combination:  dir=$RAIDDIR1  ER1_host=$ER1_host " "ERROR"

    if [[ -d $RAIDDIR1 && $RAID_LINK1 != "media" ]]; then
        rcm_log " WARNING: Directory  $RAIDDIR1 already exists !! " "WARN"
        echo    "          Moving to ${RAIDDIR1}_Backup_"`date "+%F_%T"`  "WARN"
        mv $RAIDDIR1 ${RAIDDIR1}_Backup_`date "+%F_%T"`    
    fi

    # Create new directory for all output files for this run on RAID disk
    mkdir -p $RAIDDIR1  
    ERR=$?
    if [ $ERR != 0 ]; then 
        # Couldn't make desired output directory. Warn user and try an alternative
        rcm_log "ERROR: mkdir $RAIDDIR1: error code=$ERR " "ERROR"
        export RAIDDIR1=/tmp/${RAIDDIR1}
        rcm_log "       trying $RAIDDIR1 instead ..."
        
        mkdir -p $RAIDDIR1 
        ERR=$?
        if [ $ERR != 0 ]; then
            rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR "   "ERROR"
	    rcm_log "ERROR: NO place to write data!"  "INFO"
        fi
    fi
    touch ${RAIDDIR1}/lock
    chmod 775 $RAIDDIR1
    rcm_log "Created: $RAIDDIR1"   "INFO"
    [[ "$USE_4PART" = "yes" ]] && mkdir -p $RAIDDIR2  && rcm_log "Created: $RAIDDIR2"   "INFO" || rcm_log "can not create: $RAIDDIR2"   "ERROR"
    [[ "$USE_4PART" = "yes" ]] && mkdir -p $RAIDDIR3  && rcm_log "Created: $RAIDDIR3"   "INFO" || rcm_log "can not create: $RAIDDIR3"   "ERROR"
fi


#-------------------------------------------------------------------------------------------
#         make DATA link 
#-------------------------------------------------------------------------------------------

if [ -z "$DAQ_HOME" ] ; then
  rcm_log "ERROR:: DAQ_HOME=$DAQ_HOME is not set, use /tmp "  "ERROR"
  #-- or  /scratch ? ..... useless, we do not know where the CODA output file in this case ...
  DAQ_HOME=/tmp/dummy
fi
DATA_HOME=`dirname  $DAQ_HOME`
DATA_DIR=`dirname  $DAQ_HOME`/DATA
DATA_lock=`dirname  $DAQ_HOME`/DATA.lock
rcm_log "Run=$1 Use DATA path=$DATA_DIR"  "INFO"
#----------------
for isec in `seq 4 -2 0` ; do    
    if   [ -L $DATA_DIR ] ; then
        ls -l $DATA_HOME > /dev/null
        rcm_log "Run=$1 DATA link exists: $DATA_DIR, wait.. $isec "  "WARN"
        sleep 2
    elif [ -f $DATA_lock ] ; then
        rcm_log "Run=$1 DATA lock exists: $DATA_lock, wait.. $isec "  "WARN"
        sleep 2        
    else
        ls -l $DATA_DIR > /dev/null
        if [ $? = 0 ] ; then 
            rcm_log "Delete link: wait for nfs/$ER_host ..   $isec "  "WARN"
            sleep 2
        else
            break
        fi
#        break
    fi
done
sleep 2
#---------------
if  [ -L $DATA_DIR ] ; then
    rcm_log "DATA link exists; delete link: $DATA_DIR"  "WARN"
    [[ $1 -gt 0 ]] && rm $DATA_DIR || echo " dry run: rm $DATA_DIR"
    [ $? != 0 ] &&  rcm_log "ERROR: rm link $DATA_DIR "   "ERROR"
fi
#---------------
if  [ -e $DATA_DIR ] ; then
    rcm_log "ERROR: $DATA_DIR exists, move ... " "WARN"
    [[ $1 -gt 0 ]] && mv $DATA_DIR   ${DATA_DIR}_Backup_`date "+%F_%T"` || echo " dry run: mv $DATA_DIR   ${DATA_DIR}_Backup_`date "+%F_%T"`"
fi
#---------------
rcm_log "Make link: $RAIDDIR to: $DATA_DIR"  "INFO"

[[ $1 -gt 0 ]] && ln -s $RAIDDIR  $DATA_DIR  || echo " dry run:  ln -s $RAIDDIR  $DATA_DIR"
if [ $? != 0 ]; then  
    rcm_log "ERROR: link $RAIDDIR $DATA_DIR "   "ERROR"
fi


#-------------------------------------------------------------------------------------------
#         make DATA1 link for Second ER 
#-------------------------------------------------------------------------------------------
DATA_HOME=`dirname  $DAQ_HOME`
DATA_DIR1=`dirname  $DAQ_HOME`/DATA1
DATA_DIR2=`dirname  $DAQ_HOME`/DATA2
DATA_DIR3=`dirname  $DAQ_HOME`/DATA3

if  [[ "$ER1_host" != "localhost" ]] ; then
    #--  better to check variable  $RAIDDIR1 
    DATA1_lock=`dirname  $DAQ_HOME`/DATA1.lock
    rcm_log "Run=$1 Use DATA1 path=$DATA_DIR1"  "INFO"
    #----------------
    for isec in `seq 4 -2 0` ; do    
        if   [ -L $DATA_DIR1 ] ; then
            ls -l $DATA_HOME > /dev/null
            rcm_log "Run=$1 DATA1 link exists: $DATA_DIR1, wait.. $isec "  "WARN"
            sleep 2
        else
            ls -l $DATA_DIR1 > /dev/null
            if [ $? = 0 ] ; then 
                rcm_log "Delete link: wait for nfs/$ER1_host ..   $isec "  "WARN"
                sleep 2
            else
                break
            fi
            #        break
        fi
    done
    sleep 2
    #---------------
    declare -a datadir
    datadir=( $DATA_DIR $DATA_DIR1 $DATA_DIR2 $DATA_DIR3 )
    declare -a raid_dir
    raid_dir=( $RAIDDIR $RAIDDIR1 $RAIDDIR2 $RAIDDIR3 )
    echo " check links for data1 2 3 ... "
    for dd in 1 2 3 ; do 
        [[ "$USE_4PART" != "yes" && $dd = "2" ]] && break
        echo "check DATA${dd} link:  ${datadir[dd]}" 
        if  [ -L ${datadir[dd]} ] ; then
            rcm_log "DATA${dd} link exists; delete link: ${datadir[dd]}"  "WARN"
            [[ $1 -gt 0 ]] && rm ${datadir[dd]} || echo " dry run: rm ${datadir[dd]}"
            [ $? != 0 ] &&  rcm_log "ERROR: rm link ${datadir[dd]} "   "ERROR"
        fi
        #---------------
        if  [ -e ${datadir[dd]} ] ; then
            rcm_log "ERROR: ${datadir[dd]} exists, move ... " "WARN"
            [[ $1 -gt 0 ]] && mv ${datadir[dd]}   ${datadir[dd]}_Backup_`date "+%F_%T"` || echo " dry run: mv ${datadir[dd]}   ${datadir[dd]}_Backup_`date "+%F_%T"`"
        fi
        #---------------
        rcm_log "Make link: ${raid_dir[dd]} to: ${datadir[dd]}"  "INFO"
        [[ $1 -gt 0 ]] && ln -s ${raid_dir[dd]}  ${datadir[dd]} || echo " dry run: ln -s ${raid_dir[dd]}  ${datadir[dd]}"
        if [ $? != 0 ]; then  
            rcm_log "ERROR: link ${RAIDDIR${dd}} ${datadir[dd]} "   "ERROR"
        fi
    done
    #---------------
else 
    echo "ln -s $RAIDDIR  $DATA_DIR1"
    [[ $1 -gt 0 ]] && ln -s $RAIDDIR  $DATA_DIR1 || echo " dry run: ln -s $RAIDDIR  $DATA_DIR1"
    [[ $1 -gt 0 ]] && ln -s $RAIDDIR  $DATA_DIR2 || echo " dry run: ln -s $RAIDDIR  $DATA_DIR2"
    [[ $1 -gt 0 ]] && ln -s $RAIDDIR  $DATA_DIR3 || echo " dry run: ln -s $RAIDDIR  $DATA_DIR3"
fi

#-------------------------------------------------------------------------------------------
#
#-------------------------------------------------------------------------------------------

rcm_log "Make link: check link on nfs/gluonraid[1,2,3] .. ER on $ER_host "  "INFO"
ks=0
while true ; do  
    ls -l $DATA_HOME > /dev/null
    ls -l $DAQ_HOME/../DATA > /dev/null
    if [ $? = 0 ] ; then 
        break 
    else
        rcm_log "Make link: wait for nfs/$ER_host , ret=$? $ks "  "WARN"
	ks=$(($ks+2))
        sleep 2
    fi
done

#sleep 15

rcm_log "Run=$1 Make link: done "  "INFO"

#-------------------------------------------------------------------------------------------
#         set DAQ run number and DAQ status EPICS variables
#-------------------------------------------------------------------------------------------
# Note: If this script is run from a computer other than the authorized
# list then the caput commands won't actually change the values
# As of 12/6/2019, Hovanes set this list to: gluonraid1-5, gluondaqbuff, gluon46-49
rcm_log "caput HD:coda:daq:run_number $1" "INFO"
rcm_log "caput HD:coda:daq:status $1" "INFO"
ssh gluonraid4 caput HD:coda:daq:run_number $1 &
ssh gluonraid4 caput HD:coda:daq:status 1 &


rcm_log "done "  "INFO"
echo "-------> $0  EXIT `date` <-------"