#!/bin/tcsh

# This script will regenerate a number of plots for a
# given workflow at NERSC. For it to work properly it
# must be run from an account with passwordless login
# to cori.nersc.gov and passwordless loging to
# gxproj4@ifarm.
#
# Usage:
#
#   ./regenerate_plots.csh [options]
#
#
# options:
#
#  -w,-workflow <workflow>     Specify the swif2 workflow
#
#  -s,-starttime <start time>  Specify the start time for submitted
#                              jobs to slurm. If this is ommitted
#                              then swif2 is run on gxproj4@ifarm
#                              to find the creation time of the given
#                              workflow.
#
#  -p,-post                    Upload the newly generated plots to the
#                              web. Location can be specified with 
#                              -u option.
#
#  -u,-uploaddir <upload dir>  Upload files to the specified directory.
#                              This option implies -p. The upload  dir
#                              specified will be passed to scp and must
#                              be reachable via passwordless login.
#                              (See example below.)
#
#  -d                          Try to open a window to display the
#                              generated plots. On Linux this will
#                              use eog if it is in your path. On Mac
#                              OS X it will just use "open".
#
#  -n, -njobs                  Set the number of jobs used for normalization
#                              in the iNjobs_vs_time plot. Superceeded
#                              by -N option if present.
#
#  -N                          Get the number of jobs used for the
#                              iNjobs_vs_time plot from the workflow.
#                              This will superceed any setting from -n.
#
# Example:  Regenerate plots locally for specified workflow
#
#    ./regenerate_plots.csh -w offmon_2018-01_ver17
#
#
# Example:  Regenerate plots and upload them to the web
#
#    ./regenerate_plots.csh -w offmon_2018-01_ver17 -u davidl@ifarm:/group/halld/www/halldweb/html/data_monitoring/launch_analysis/2018_01/launches/offline_monitoring_RunPeriod2018_01_ver17
#
# n.b. for the above to work the directory must already exist!
#
#
#

# A defaults
set workflow=""
set njobs=0
set nsucceeded=0
set uploaddir="none"  # Will be automatically set below unless user sets it to something else

# offmon_2019-11_ver13_batch01
#set start_date="2020-07-16T08:00:00" # n.b. California time
#set   plot_end="" # Use this for "now"

# recon_2019-11_ver01_batch01
set start_date="2020-08-17T20:35:00" # n.b. California time
set   plot_end="" # Use this for "now"

#set plot_start=$start_date 
set plot_start="2020-08-17T17:35:00" # n.b. not actually Virginia time. Make this California time - 6 hours

# Loop over command line args
foreach arg ($*)
	switch( "$arg" )
		case '-d':
			set display_plots=1
			breaksw
		case '-s':
		case '-starttime':
			shift
			set start_date=$argv[1]
			breaksw
		case '-w':
		case '-workflow':
			shift
			set workflow=$argv[1]
			breaksw
		case '-n':
		case '-njobs':
			shift
			set njobs=$argv[1]
			breaksw
		case '-N':
			set get_njobs_from_workflow='1'
			breaksw
		case '-p':
		case '-post':
			set post_to_web=1
			breaksw
		case '-u':
		case '-uploaddir':
			shift
			set uploaddir=$argv[1]
			set post_to_web=1
			breaksw
			
	endsw
	if ( $#argv > 0 ) shift
end

if ( $uploaddir == "none" ) then
	set uploaddir="gxproj5@ifarm1901:/group/halld/www/halldweb/html/data_monitoring/recon/summary_swif2_output_${workflow}"
endif

# Make sure root is setup in environment
if ( ! $?ROOTSYS ) then
	echo "Setting up ROOT ..."
	source /apps/root/PRO/bin/thisroot.csh
endif

# If uploaddir was not specified by user then set a default based on workflow name
if ( $uploaddir == "UNSET" ) then
	set uploaddir="gxproj5@ifarm1901:/group/halld/www/halldweb/html/data_monitoring/launch_analysis/../recon/summary_swif2_output_"$workflow
   echo "Setting default upload directory to: "$uploaddir
endif

# If user did not specify start date/time of the launch
# then get it from the workflow.
if ( ! $?start_date ) then
	set start_date=`ssh gxproj4@ifarm swif2 status -workflow $workflow | grep create_ts | awk '{print $3" "$4}' | sed 's/..$//'`  # sed is to chop off last ".0" that sacct doesn't like
endif

# Optionally get the number of jobs from workflow
if ( $?get_njobs_from_workflow ) then
	set njobs=`ssh gxproj4@ifarm1901 swif2 status -workflow $workflow | grep jobs | grep -v xfer_globus | awk '{print $3" "$4}'`
	set nsucceeded=`ssh gxproj4@ifarm1901 swif2 status -workflow $workflow | grep succeeded | awk '{print $3" "$4}'`
endif

# Get min and max runs in workflow
set runrange=`./extract_run_range.py $workflow`
set RunMin=`echo $runrange | awk '{print $1}'`
set RunMax=`echo $runrange | awk '{print $2}'`

echo "  workflow: "$workflow
echo "start time: "$start_date
echo "plot start: "$plot_start
echo "plot  end : "$plot_end
echo "     njobs: "$njobs
echo "    RunMin: "$RunMin
echo "    RunMax: "$RunMax

# Run sacct on cori to get info from slurm in form of text
# file and copy it back to local directory.
set SITE=""
if ( $workflow =~ *_PSC ) then
	echo "Mining slurm data from PSC ..."
	ssh bridges.psc.edu "cd work/accounting ; sacct --format=JobID%15,Submit,Start,End,NCPUS,CPUTimeRaw,ResvCPURAW,MaxRSS,JobName%30,ExitCode,MaxDiskRead -S '$start_date' > slurm.dat"
	scp bridges.psc.edu:work/accounting/slurm.dat .
	set SITE="PSC"
else
	echo "Mining slurm data from NERSC ..."
	echo ssh cori.nersc.gov "mkdir -p builds/accounting ; cd builds/accounting ; sacct -A m3120 -a --format=JobID%15,Submit,Start,End,NCPUS,CPUTimeRaw,ResvCPURAW,MaxRSS,JobName%30,ExitCode,MaxDiskRead -S '$start_date' > slurm.dat"
	ssh cori.nersc.gov "mkdir -p builds/accounting ; cd builds/accounting ; sacct -A m3120 -a --format=JobID%15,Submit,Start,End,NCPUS,CPUTimeRaw,ResvCPURAW,MaxRSS,JobName%30,ExitCode,MaxDiskRead -S '$start_date' > slurm.dat"
	scp cori.nersc.gov:builds/accounting/slurm.dat .
	set SITE="NERSC"
endif

# Convert ascii file to SQLite DB and CSV formats
#./slurm2sqlite.py $plot_start $plot_end
./slurm2sqlite.py $start_date $plot_end

# Run all macros to create plots
foreach m ( Njobs_vs_time.C latency_vs_time.C cpu_vs_time.C )
	echo "Running ROOT macro $m ..."
	root -l -q -b $m'("'$plot_start'",'$RunMin','$RunMax',"'$SITE'")'
end

echo 'iNjobs_vs_time.C("'$plot_start'",'$njobs','$nsucceeded','$RunMin','$RunMax',"'$SITE'")'
root -l -q -b 'iNjobs_vs_time.C("'$plot_start'",'$njobs','$nsucceeded','$RunMin','$RunMax',"'$SITE'")'

# Optionally open window to display plots on local machine
if ( $?display_plots ) then
	if ( $OSTYPE == 'darwin' ) then
		open *.png
	else if ( `where eog` != "" ) then
		eog *.png
	endif
endif

# Optionally upload files to web
if ( $?post_to_web ) then

	# Make sure destination directory exists
	if ( $uploaddir =~ gxproj5@ifarm* ) then
		set upart=`echo $uploaddir | cut -d: -f 1`
		set dirpart=`echo $uploaddir | cut -d: -f 2`
		echo "ssh $upart mkdir -p $dirpart"
		ssh $upart mkdir -p $dirpart
	endif

	# Upload index.html and all .png files
	echo "uploading to $uploaddir"
	scp index.html *.png $uploaddir
endif


