#!/bin/bash # # $Id: run_prestart 20280 2016-02-17 19:14:42Z davidl $ # $HeadURL: https://halldsvn.jlab.org/repos/trunk/online/daq/scripts/run_prestart $ # # RUN_PERIOD and RAWDATA_DIR environment variables are set in the file # /gluex/etc/hdonline.cshrc # # CODA will write files to: # # $RAWDATA_DIR/active/$RUN_PERIOD/rawdata/RunXXXXXX # # where "XXXXXX" is the zero padded run number. Other files # related to the run are also placed in this directory. # Some details on the strategy here are given in: # # https://halldweb1/wiki/index.php/Raid-to-Silo_Transfer_Strategy # #---- jcedit -------- # %(DAQ_HOME)/scripts # run_prestart %(rn) %(udl) %(config) %(rt) # run_go %(rn) %(udl) %(config) %(rt) # run_end %(rn) %(udl) %(config) %(rt) #=========================================================================================== # !!!!!!!! STD in/out is not allowed in this script !!!!!! STDO=/tmp/${USER}_PRESYNC.$1 exec >$STDO 2>&1 date echo "----- > run_prestart_sync ENTER <---- " #source /gluex/etc/hdonline.cshrc ENV_RAWDATA_DIR=`awk '/RAWDATA_DIR/ {printf "%s %s=%s\n",$1,$2,$3 }' /gluex/etc/hdonline.cshrc | grep setenv | sed s/setenv/export/g | grep -v "^#" ` ENV_RUN_PERIOD=`awk '/RUN_PERIOD/ {printf "%s %s=%s\n",$1,$2,$3 }' /gluex/etc/hdonline.cshrc | grep setenv | sed s/setenv/export/g | sed s/\"//g` $ENV_RAWDATA_DIR $ENV_RUN_PERIOD echo "ENV_RAWDATA_DIR=$ENV_RAWDATA_DIR : $RAWDATA_DIR " echo "ENV_RUN_PERIOD=$ENV_RUN_PERIOD : $RUN_PERIOD" UDL=$2 CONFIG_FILE=$3 source run_lib.sh CONFIG_DIR=`dirname $CONFIG_FILE ` ER_name=ERsoftROC get_host $ER_name ER_host #--------------------------------------- get_host USE_LOCAL USE_LOCAL_SCRIPT echo " USE_LOCAL = $USE_LOCAL_SCRIPT " if [ $USE_LOCAL_SCRIPT = "yes" ]; then echo "exec: ssh $ER_host run_prestart_sync.local $* " ssh $ER_host "run_prestart_sync.local '$1' '$2' '$3' '$4' '$5' " exit $? fi #-------- delete everything below this line later --------------!! get_host MONITOR MONITOR_START echo "start MONITOR = $MONITOR_START" #=========================================================================================== RCM_HOST=gluon25 RCM_PORT=32767 RCM_LOG=/dev/tcp/${RCM_HOST}/${RCM_PORT} RCM_LOG_FILE=$DAQ_HOME/../work/rcm_rc.log #------------------------------------------------------------------------------------------- cnt=0 function rcm_log() { cnt=$(($cnt+1)) echo -e " $1 $2 " sleep 0.2 # cMsgCommand -u cMsg://gluon100:45000/cMsg/$EXPID -name run_prestart -subject Prestart -type DAQ -text "$1" -string severity=$2 [ -n "$RCM_LOG" ] && echo -e "msg:GUI_runconf presync:${cnt}: $1 " > $RCM_LOG [ -n "$UDL" ] && cMsgCommand -u $UDL -name run_presync${cnt} -subject Prestart -type DAQ -text "$1" -string severity=$2 2>&1 > /tmp/${USER}_cMsgCommand echo -e "`date` :run_presync:${cnt}: $2: $1 " >> $RCM_LOG_FILE } rcm_log "Run=$1 user=$USER `date` UDL=$2 config=$3 4=$4 5=$5 " "INFO" if [ "$#" -lt 1 ]; then echo "You must provide a run number!" echo "" echo "Usage:" echo " run_prestart_sync RUN" echo "" exit -1 fi export RUN=$(printf '%06d' $1) #=========================================================================================== # DAQ TEST MODE # if [ x$DAQMODE == "xtest" ] ; then rcm_log "WARN: DAQMODE=$DAQMODE " "WARN" DATA_HOME=`dirname $DAQ_HOME` DATA_DIR=`dirname $DAQ_HOME`/DATA DATA_DIR1=`dirname $DAQ_HOME`/DATA1 ln -s ${DATA_HOME}/DATA_TEST $DATA_DIR ln -s ${DATA_HOME}/DATA_TEST1 $DATA_DIR1 exit 0 fi #=========================================================================================== # any other accounts .. # if [ $USER != "hdops" ] ; then which run_log.sh run_log.sh "PRESTART" $RUN $CONFIG_FILE exit 0 fi #=========================================================================================== # HDOPS account only ... #------------------------------------------------------------------------------------------- #------------------------------------------------------------------------------------------- # select gluonraid disk #------------------------------------------------------------------------------------------- if [[ "$ER_host" != "gluonraid1" ]] ; then #---------------------- NEW style for gluonraid3 ------------------- rcm_log "ER_host=$ER_host select gluonraid3/4" "INFO" # Rotate links for partitions on gluonraid3 # rcm_log " NO hd_rotate_raid_links !" "ERROR" tmn=`date` rcm_log "rotating RAID partition links $tmn" "INFO" # For some reason the LD_LIBRARY_PATH seems to not have the # /apps/gcc/4.9.2/lib64 directory that is in the normal # hdops environment. This causes hdlog to fail to run which # means hd_rotate_raid_links.py never runs. This gums up # the whole system, preventing files from making it to tape. # This seems to have suddenly become a problem on 4/25/2018 # with the full extent of the problem not being noticed until # 4/28/2018. To resolve it, the missing path is added to # LD_LIBRARY_PATH temporarily to run hdlog and then removed # in case that would screw up later items in this script. # 4/28/2018 DL TMPLDPATH=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/apps/gcc/4.9.2/lib64:${LD_LIBRARY_PATH} hdlog -c hd_rotate_raid_links.py DAQ #hd_rotate_raid_links.py DAQ export LD_LIBRARY_PATH==$TMPLDPATH GLUEX_RAID=/gluex/data/rawdata/curr #---- check /gluex/raid link ---- if [ -L $GLUEX_RAID ] ; then rcm_log "link $GLUEX_RAID exists" "INFO" elif [ -e $GLUEX_RAID ] ; then rcm_log "file $GLUEX_RAID exists, not a link" "ERROR" else rcm_log "link $GLUEX_RAID does not exists!" "ERROR" fi #---- set DATA path --- RAID_PATH=`readlink $GLUEX_RAID ` export RAWDATA_DIR=`readlink $GLUEX_RAID `/rawdata tmn=`date` rcm_log "new RAWDATA_DIR=$RAID_PATH $tmn" "INFO" RAID_LINK=`echo $RAWDATA_DIR | cut -f2 -d/ ` [[ $RAID_LINK != $ER_host ]] && rcm_log "Error host/path combination: $RAID_LINK $ER_host " "ERROR" else #---------------------- OLD style for gluonraid1,2 ------------------- rcm_log "ER_host=$ER_host select old style " "INFO" GLUEX_RAID=/home/hdops/gluonraid GLUONRAID=$(basename `readlink $GLUEX_RAID `) if [[ -n "$ER_host" && "$ER_host" != "localhost" ]] ; then rcm_log "ER_host=$ER_host select old style for gluonraid1,2 " "INFO" #---- set DATA path --- export RAWDATA_DIR=/${ER_host}/rawdata rcm_log "new RAWDATA_DIR=$RAWDATA_DIR " "INFO" #---- check /gluex/raid link ---- if [ -L $GLUEX_RAID ] ; then rcm_log "link $GLUEX_RAID exists" "INFO" if [ $GLUONRAID != $ER_host ] ; then rcm_log "link is wrong: $GLUONRAID != $ER_host " "WARN" #-- delete link -- rcm_log "GLUEX_RAID link exists; delete link: $GLUEX_RAID" "WARN" rm $GLUEX_RAID [ $? != 0 ] && rcm_log "can not delete link: $GLUEX_RAID " "ERROR" #-- create link -- ln -s /${ER_host} $GLUEX_RAID ERR=$? if [ $ERR != 0 ]; then rcm_log "link create $GLUEX_RAID " "ERROR" else rcm_log "link re-created: $GLUEX_RAID " "INFO" fi else rcm_log "link is OK $GLUONRAID == $ER_host " "INFO" fi elif [ -e $GLUEX_RAID ] ; then rcm_log "file $GLUEX_RAID exists, not a link" "ERROR" else rcm_log "link $GLUEX_RAID does not exists, create" "INFO" ln -s /${ER_host} $GLUEX_RAID ERR=$? if [ $ERR != 0 ]; then rcm_log "link create $GLUEX_RAID " "ERROR" else rcm_log "link created: $GLUEX_RAID " "INFO" fi fi else rcm_log "ER is not found, use default RAWDATA_DIR=$RAWDATA_DIR " "ERROR" fi #---------------- end OLD style : gluonraid1,2 --------------------------- fi #------------------------------------------------------------------------------------------- # create directory for new run #------------------------------------------------------------------------------------------- if [ -z "$RUN_PERIOD" ] ; then export RUN_PERIOD=RunPeriod-`date "+%F_%T"` rcm_log "WARNING:: RUN_PERIOD is not defined, use=$RUN_PERIOD " "WARN" fi if [ -z "$RAWDATA_DIR" ] ; then if [[ "$ER_host" != "gluonraid1" ]] ; then export RAWDATA_DIR=/gluex/data/rawdata/curr/rawdata else export RAWDATA_DIR=/gluex/raid/rawdata fi rcm_log "WARNING:: RAWDATA_DIR is not defined, use=$RAWDATA_DIR " "WARN" fi export RAIDDIR=${RAWDATA_DIR}/active/${RUN_PERIOD}/rawdata/Run${RUN} RUNPERIODDIR=`dirname $RAIDDIR` if [ ! -w "$RUNPERIODDIR" ] ; then rcm_log "WARNING:: RAW DIR is write protected: $RUNPERIODDIR " "ERROR" fi echo 'RUN_PERIOD: '$RUN_PERIOD echo ' RUN: '$RUN echo ' OUTDIR: '$RAIDDIR if [ -d $RAIDDIR ]; then rcm_log " WARNING: Directory $RAIDDIR already exists !! " "WARN" echo " Moving to ${RAIDDIR}_Backup_"`date "+%F_%T"` "WARN" mv $RAIDDIR ${RAIDDIR}_Backup_`date "+%F_%T"` fi # Create new directory for all output files for this run on RAID disk mkdir -p $RAIDDIR ERR=$? if [ $ERR != 0 ]; then # Couldn't make desired output directory. Warn user and try an alternative rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR " "ERROR" export RAIDDIR=/tmp/${RAIDDIR} rcm_log " trying $RAIDDIR instead ..." mkdir -p $RAIDDIR ERR=$? if [ $ERR != 0 ]; then rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR " "ERROR" rcm_log "ERROR: NO place to write data!" fi fi touch ${RAIDDIR}/lock chmod 775 $RAIDDIR #---------------------------------------------------------------- #-- create directory for Second Event recorder --- #---------------------------------------------------------------- get_host ERraid0 ER0_host get_host ERraid1 ER1_host rcm_log "ER0=$ER0_host ER1=$ER1_host" "INFO" if [[ "$ER1_host" != "gluonraid1" && "$ER0_host" != "localhost" && "$ER1_host" != "localhost" ]] ; then #--- assumed format: /gluonraid3/data3 # create path on the same raid, another partition DATA_NUM=`echo ${RAWDATA_DIR} | cut -f3 -d/ | sed s/data//g ` DATA_NUM1=$(($DATA_NUM%4+1)) rcm_log "File0=data$DATA_NUM File1=data$DATA_NUM1 " "INFO" export RAIDDIR1=`echo $RAIDDIR | sed s/data$DATA_NUM/data$DATA_NUM1/g` rcm_log "second data path =$RAIDDIR1" "INFO" # create path on the second raid RAID_NUM=`echo ${RAWDATA_DIR} | cut -f2 -d/ | sed s/gluonraid//g ` RAID_NUM1=$(($RAID_NUM%2+3)) [[ $RAID_NUM < 3 || $RAID_NUM > 4 ]] && rcm_log "raid error RAWDATA_DIR=$RAWDATA_DIR " "ERROR" rcm_log "RAID0=gluonraid$RAID_NUM RAID1=gluonraid$RAID_NUM1 " "INFO" export RAIDDIR2=`echo $RAIDDIR | sed s/gluonraid$RAID_NUM/gluonraid$RAID_NUM1/g` rcm_log "second RAID path =$RAIDDIR2" "INFO" # only gluonraid3 and gluonraid4 are valid !! if [[ $ER0_host != $ER1_host ]] ; then rcm_log " use another raid: $RAIDDIR2 instead !!! " "INFO" [[ $ER0_host == localhost || $ER1_host == localhost ]] && rcm_log " localhost :: $ER0_host $ER1_host " "INFO" RAIDDIR1=$RAIDDIR2 fi # RAID_LINK1=`echo $RAIDDIR1 | cut -f2 -d/ ` [[ $RAID_LINK1 != $ER1_host ]] && rcm_log "Error host/path combination: dir=$RAIDDIR1 ER1_host=$ER1_host " "ERROR" if [ -d $RAIDDIR1 ]; then rcm_log " WARNING: Directory $RAIDDIR1 already exists !! " "WARN" echo " Moving to ${RAIDDIR1}_Backup_"`date "+%F_%T"` "WARN" mv $RAIDDIR1 ${RAIDDIR1}_Backup_`date "+%F_%T"` fi # Create new directory for all output files for this run on RAID disk mkdir -p $RAIDDIR1 ERR=$? if [ $ERR != 0 ]; then # Couldn't make desired output directory. Warn user and try an alternative rcm_log "ERROR: mkdir $RAIDDIR1: error code=$ERR " "ERROR" export RAIDDIR1=/tmp/${RAIDDIR1} rcm_log " trying $RAIDDIR1 instead ..." mkdir -p $RAIDDIR1 ERR=$? if [ $ERR != 0 ]; then rcm_log "ERROR: mkdir $RAIDDIR: error code=$ERR " "ERROR" rcm_log "ERROR: NO place to write data!" "INFO" fi fi touch ${RAIDDIR1}/lock chmod 775 $RAIDDIR1 rcm_log "Created: $RAIDDIR1" "INFO" fi #------------------------------------------------------------------------------------------- # make DATA link #------------------------------------------------------------------------------------------- if [ -z "$DAQ_HOME" ] ; then rcm_log "ERROR:: DAQ_HOME=$DAQ_HOME is not set, use /tmp " "ERROR" #-- or /scratch ? ..... useless, we do not know where the CODA output file in this case ... DAQ_HOME=/tmp/dummy fi DATA_HOME=`dirname $DAQ_HOME` DATA_DIR=`dirname $DAQ_HOME`/DATA DATA_lock=`dirname $DAQ_HOME`/DATA.lock rcm_log "Run=$1 Use DATA path=$DATA_DIR" "INFO" #---------------- for isec in `seq 4 -2 0` ; do if [ -L $DATA_DIR ] ; then ls -l $DATA_HOME rcm_log "Run=$1 DATA link exists: $DATA_DIR, wait.. $isec " "WARN" sleep 2 elif [ -f $DATA_lock ] ; then rcm_log "Run=$1 DATA lock exists: $DATA_lock, wait.. $isec " "WARN" sleep 2 else ssh -x $ER_host "ls -l $DATA_DIR " if [ $? = 0 ] ; then rcm_log "Delete link: wait for nfs/$ER_host .. $isec " "WARN" sleep 2 else break fi # break fi done sleep 2 #--------------- if [ -L $DATA_DIR ] ; then rcm_log "DATA link exists; delete link: $DATA_DIR" "WARN" rm $DATA_DIR [ $? != 0 ] && rcm_log "ERROR: rm link $DATA_DIR " "ERROR" fi #--------------- if [ -e $DATA_DIR ] ; then rcm_log "ERROR: $DATA_DIR exists, move ... " "WARN" mv $DATA_DIR ${DATA_DIR}_Backup_`date "+%F_%T"` fi #--------------- rcm_log "Make link: $RAIDDIR to: $DATA_DIR" "INFO" ln -s $RAIDDIR $DATA_DIR if [ $? != 0 ]; then rcm_log "ERROR: link $RAIDDIR $DATA_DIR " "ERROR" fi #--------------- #------------------------------------------------------------------------------------------- # make DATA link for Second ER #------------------------------------------------------------------------------------------- if [[ "$ER1_host" != "gluonraid1" && "$ER0_host" != "localhost" && "$ER1_host" != "localhost" ]] ; then #-- better to check variable $RAIDDIR1 DATA_HOME=`dirname $DAQ_HOME` DATA_DIR1=`dirname $DAQ_HOME`/DATA1 DATA1_lock=`dirname $DAQ_HOME`/DATA1.lock rcm_log "Run=$1 Use DATA1 path=$DATA_DIR1" "INFO" #---------------- for isec in `seq 4 -2 0` ; do if [ -L $DATA_DIR1 ] ; then ls -l $DATA_HOME rcm_log "Run=$1 DATA1 link exists: $DATA_DIR1, wait.. $isec " "WARN" sleep 2 elif [ -f $DATA1_lock ] ; then rcm_log "Run=$1 DATA1 lock exists: $DATA_lock1, wait.. $isec " "WARN" sleep 2 else ssh -x $ER1_host "ls -l $DATA_DIR1 " if [ $? = 0 ] ; then rcm_log "Delete link: wait for nfs/$ER1_host .. $isec " "WARN" sleep 2 else break fi # break fi done sleep 2 #--------------- if [ -L $DATA_DIR1 ] ; then rcm_log "DATA1 link exists; delete link: $DATA_DIR1" "WARN" rm $DATA_DIR1 [ $? != 0 ] && rcm_log "ERROR: rm link $DATA_DIR1 " "ERROR" fi #--------------- if [ -e $DATA_DIR1 ] ; then rcm_log "ERROR: $DATA_DIR1 exists, move ... " "WARN" mv $DATA_DIR1 ${DATA_DIR1}_Backup_`date "+%F_%T"` fi #--------------- rcm_log "Make link: $RAIDDIR1 to: $DATA_DIR1" "INFO" ln -s $RAIDDIR1 $DATA_DIR1 if [ $? != 0 ]; then rcm_log "ERROR: link $RAIDDIR1 $DATA_DIR1 " "ERROR" fi #--------------- fi #------------------------------------------------------------------------------------------- # #------------------------------------------------------------------------------------------- TCSH=`which tcsh` runcmd="$TCSH -c" if [ $ER_host != "localhost" ]; then runcmd="ssh -x $ER_host" fi rcm_log "Make link: check link on nfs/gluonraid[1,2,3] .. ER on $ER_host " "INFO" ks=0 while true ; do $runcmd "ls -l $DATA_HOME " $runcmd "ls -l $DAQ_HOME/../DATA" if [ $? = 0 ] ; then break else rcm_log "Make link: wait for nfs/$ER_host , ret=$? $ks " "WARN" ks=$(($ks+2)) sleep 2 fi done #sleep 15 rcm_log "Run=$1 Make link: done " "INFO" #------------------------------------------------------------------------------------------- # set DAQ run number and DAQ status EPICS variables #------------------------------------------------------------------------------------------- rcm_log "caput HD:coda:daq:run_number $1" "INFO" caput HD:coda:daq:run_number $1 & caput HD:coda:daq:status 1 & rcm_log "done " "INFO" echo "-------> run_prestart_sync EXIT `date` <-------"