#!/usr/bin/env python
#
# 2014/11/10  Sean Dobbs (s-dobbs@northwestern.edu)
#
# This script should be run after offline monitoring ROOT files are generated
#

import sys,os
from os import listdir
from os.path import isfile, join
from math import fabs
import struct
from datetime import datetime
#import numpy as np

from datamon_db import datamon_db

RADIATOR_TOLERANCE = 5  # allow 5 mm tolerance in determining which radiator

VERBOSE = True

# switches to decide if we process certain types of information
#USE_EPICS = True
#USE_EVIO_FILES = True
#USE_CCDB = True

def EVIO_SWAP64(x):
    return ( (((x) >> 56) & 0x00000000000000FFL) | \
                 (((x) >> 40) & 0x000000000000FF00L) | \
                 (((x) >> 24) & 0x0000000000FF0000L) | \
                 (((x) >> 8)  & 0x00000000FF000000L) | \
                 (((x) << 8)  & 0x000000FF00000000L) | \
                 (((x) << 24) & 0x0000FF0000000000L) | \
                 (((x) << 40) & 0x00FF000000000000L) | \
                 (((x) << 56) & 0xFF00000000000000L) )


def EVIO_SWAP32(x):
    return ( (((x) >> 24) & 0x000000FF) | \
                 (((x) >> 8)  & 0x0000FF00) | \
                 (((x) << 8)  & 0x00FF0000) | \
                 (((x) << 24) & 0xFF000000) )

######################################
# CCDB

import ccdb.path_utils
import ccdb.cmd.themes
from ccdb import get_ccdb_home_path
from ccdb.cmd.console_context import ConsoleContext
from tempfile import NamedTemporaryFile

def InitCCDB():
    #create connection string:
    ccdb_path = get_ccdb_home_path()
    #sqlite_connection_str = os.getenv(CCDB_CONNECTION, "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite"))
    if os.environ['CCDB_CONNECTION']:
        sqlite_connection_str = os.environ['CCDB_CONNECTION']
    else:
        sqlite_connection_str = "test"
    #print "using CCDB connection = " + sqlite_connection_str

     #create console context
    context = ConsoleContext()                          # this is the main class
    context.silent_exceptions = False                   # now all exception is raised and you can try-except them
    context.theme = ccdb.cmd.themes.NoColorTheme()      # disable colored output
    context.connection_string = sqlite_connection_str   # set connection string
    #context.user_name = os.getenv(CCDB_USER, "gluex")  # your username
    if 'CCDB_USER' in os.environ and os.environ['CCDB_USER'] is not None:
        context.user_name = os.environ['CCDB_USER']
    else:
        context.user_name = "gluex"
    context.register_utilities()                        # Initialization. Register all commands (ls, rm, mktbl etc...)

    return context

######################################



# set default values
def init_property_mapping():
    run_properties = {
        'status'           : '-1',
        'start_time'       : '<None>',
        'end_time'         : '<None>',
        'num_files'        : '1',
        'num_events'       : '-1',
        'beam_energy'      : '10.1',       # hardcoded in GeV for now
        'beam_current'     : '-1',
        'radiator_type'    : '<None>',
        'luminosity'       : '-1',
        'solenoid_current' : '-1',
        'coherent_peak'    : '-1',
        'target_type'      : '<None>'
    }
    return run_properties

# go from timestamp to string
def format_date(timestamp):
    dt = datetime.fromtimestamp(timestamp)
    return str(datetime(dt.year,dt.month,dt.day,dt.hour,dt.minute,dt.second))

# parse files generated by run_epics_info.py
# ignore lines that begin with '#'
# look for lines that look like:   KEY = VALUE
def parse_condition_file(fname):
    run_conditions = {}
    try:
        if VERBOSE:
            print "opening condition file = "  + fname
        infile = open(fname)
        for line in infile:
            tokens = line.strip().split()
            if len(tokens)==0 or tokens[0][0] == '#':
                continue
            if len(tokens)<3:
                continue
            if tokens[1] == "=":
                # deal with special cases
                if tokens[0] == "TIME":
                    run_conditions[ tokens[0] ] = " ".join(tokens[2:])
                else:
                    run_conditions[ tokens[0] ] = tokens[2]
        infile.close()
    except IOError as e:
        print "I/O error({0}): {1}".format(e.errno, e.strerror)
        return None
        #sys.exit(0)
    except:
        print "Unexpected error:", sys.exc_info()[0]
        return None
        #sys.exit(0)
    return run_conditions


# algorithm taken from evio2db
def ParseEVIOFiles(filelist):
    BUFFER_SIZE_WORDS = 1000  
    all_properties = []
    # should double check file names
    for fnamepath in filelist:
        print "processing " + fnamepath + "..."
        try:
            # extract file number & check 
            fname = fnamepath.split('/')[-1]
            if len(fname) < 6 or fname[-5:] != ".evio":
                print "bad file " + fname + " , skipping..."
                continue
            fparts = fname[:-5].split('_')
            if len(fparts) < 4:
                print "bad file " + fname + " , skipping..."
                continue
            try:
                file_num = int(fparts[3])
            except ValueError:
                print "bad file " + fname + " , skipping..."
                continue                

            filesize = os.path.getsize(fnamepath)
            f = open(fnamepath,"rb")
            # initialize values
            properties = {}
            properties["file_num"] = file_num
            # figure out the first event
            in_words = BUFFER_SIZE_WORDS
            if filesize < BUFFER_SIZE_WORDS*4:
                in_words = filesize/4
            data_in = f.read(4*in_words)
            data = struct.unpack(str(in_words)+"I", data_in)
            for i in xrange(len(data)):
                w = data[i]
                # Look for a "Go" event that we can use for the beginning of the run
                if( (w & 0x0001D2FF) == 0x0001D2FF ): 
                    properties["start_time"] = EVIO_SWAP32(data[i+1]);
                    if VERBOSE:
                        print "Found start time = " + format_date(properties["start_time"])
                # Physics Event Header bits that should be set
                if( (w & 0x001050FF) != 0x001050FF ):
                    continue
                # Physics Event Header bits that should not be set
                if( (w & 0x000F0E00) != 0x00000000 ):
                    continue
                # Jump 2 words to Trigger bank
                w = data[i+2];
                # Built Trigger Bank bits that should be set
                if( (w & 0x002020FF) != 0x002020FF ):
                    continue
                # First bank in Trigger bank should be 64bit int type
                w = data[i+3];
                if( (w & 0x00000A00) != 0x00000A00 ):
                    continue
                        
                properties["first_event"]  = EVIO_SWAP64( (long(data[i+5])<<32) | long(data[i+4]) )
                properties["tfirst_event"] = EVIO_SWAP64( (long(data[i+7])<<32) | long(data[i+6]) )
                break

            # figure out the last event - search through the file in chunks from the back
            n = 0  # number of chunks we have looked through
            last_event = 0
            while True:
                if(last_event > 0):
                    break
                n += 1
                f.seek(-n*4*in_words, 2)
                data_in = f.read(4*in_words)
                Nwords = len(data_in)/4
                data = struct.unpack(str(in_words)+"I", data_in)                
                #for i in xrange(len(data)):
                i = Nwords - 7
                while True:
                    i -= 1
                    if i < 0:
                        break
                    w = data[i]
                    # Physics Event Header bits that should be set
                    if( (w & 0x001050FF) != 0x001050FF ):
                        continue
                    # Physics Event Header bits that should not be set
                    if( (w & 0x000F0E00) != 0x00000000 ):
                        continue
                    # Jump 2 words to Trigger bank
                    w = data[i+2];
                    # Built Trigger Bank bits that should be set
                    if( (w & 0x002020FF) != 0x002020FF ):
                        continue
                    # First bank in Trigger bank should be 64bit int type
                    w = data[i+3];
                    if( (w & 0x00000A00) != 0x00000A00 ):
                        continue

                    last_event = EVIO_SWAP64( (long(data[i+5])<<32) | long(data[i+4]) )
                    properties["last_event"]  = last_event
                    properties["tlast_event"] = EVIO_SWAP64( (long(data[i+7])<<32) | long(data[i+6]) )
                    break

            # calculate results for a file
            # do a better job handling bad files?
            N = 0
            end_time = 0
            if "first_event" in properties and "last_event" in properties:    
                N = (properties["last_event"] - properties["first_event"]) + 1
                end_time = 5.0E-9 * (float(properties["tlast_event"]) - float(properties["tfirst_event"]))
            if N == 0:
                N = 1
            properties["num_events"] = N
            properties["end_time"] = end_time

            # we're all done!
            all_properties.append(properties)

        except IOError as e:
            print "I/O error({0}): {1}".format(e.errno, e.strerror)
            return {}

        
    # build results
    file_properties = {}

    if VERBOSE:
        print "results from EVIO parsing = " + str(all_properties)
        print "processing: "

    file_properties["num_files"] = len(all_properties)
    file_properties["num_events"] = -1
    file_properties["start_time"] = -1
    file_properties["end_time"] = -1
    for props in sorted(all_properties, key=lambda prop: int(prop["file_num"])):
        if VERBOSE:
            print str(props)
        if file_properties["start_time"] < 0 :
            # if there are DAQ problems, the GO event might not be properly written out
            if "start_time" in props:
                file_properties["start_time"] = props["start_time"]
            else:
                file_properties["start_time"] = 0
            file_properties["end_time"] = file_properties["start_time"] + props["end_time"]
            file_properties["num_events"] = props["num_events"]
        else:
            file_properties["end_time"] += props["end_time"] 
            file_properties["num_events"] += props["num_events"] 
    file_properties["start_time"] = format_date(file_properties["start_time"])
    file_properties["end_time"] = format_date(file_properties["end_time"])
    return file_properties

def load_target_types(ccdb_context, run_number):
    mapping = {}

    # make temp file to store CCDB info in
    f = NamedTemporaryFile()
    ccdb_context.process_command_line("dump /TARGET/target_type_list:"+str(run_number)+" > "+f.name)

    # read in info
    f.flush()
    lines = f.readlines()

    if len(lines) < 2:
        print "Problem writing out CCDB table: /TARGET/target_type_list"
    else:
        # skip the first line, which is junk
        for x in range(1,len(lines)):
            vals = lines[x].split()
            #index = int(vals[0])
            mapping[ int(vals[0]) ] = " ".join(vals[1:])

    return mapping


def main(argv):
    # configuration vars
    RUN_PERIOD = "RunPeriod-2014-10"
    CONDITION_FILE_FORMAT = "/work/halld/online_monitoring/conditions/run_conditions%06d.dat"
    # assume that the files are loaded on the cache disk
    RAWDATA_DIR_FORMAT = "/cache/mss/halld/RunPeriod-2014-10/rawdata/Run%06d"

    # read in run number from command line
    try:
        run_number = int(argv[0])
    except:
        print "Need to pass the run number to process as a command line argument!"
        return

    run_properties = init_property_mapping()

    run_conditions = parse_condition_file(CONDITION_FILE_FORMAT % (run_number))
    if run_conditions is None:
        return

    # start extracting saved EPICS values
    #run_number = run_conditions['RUN']     ## check this?
    run_properties['beam_current'] = run_conditions['IBCAD00CRCUR6']
    run_properties['start_time'] = run_conditions['TIME']
    run_properties['solenoid_current'] = run_conditions['HallD-PXI:Data:I_Shunt']

    # figure out which radiator was used
    # save luminosity factor = current * radiator thickness
    amorphous_radiator_position = float(run_conditions['hd:radiator:motor.RBV'])
    if fabs(amorphous_radiator_position - 135.948) < RADIATOR_TOLERANCE:
        run_properties['radiator_type'] = '2x10-5 RL'
        run_properties['luminosity'] = 1.7e-5 * float(run_properties['beam_current'])
    elif fabs(amorphous_radiator_position - 166.095) < RADIATOR_TOLERANCE:
        run_properties['radiator_type'] = '1x10-4 RL'
        run_properties['luminosity'] = 11.2e-5 * float(run_properties['beam_current'])
    elif fabs(amorphous_radiator_position - 196.262) < RADIATOR_TOLERANCE:
        run_properties['radiator_type'] = '3x10-4 RL'
        run_properties['luminosity'] = 22.7e-5 * float(run_properties['beam_current'])
    else:
        run_properties['radiator_type'] = 'None'
        #run_properties['luminosity'] = run_properties['beam_current']
        run_properties['luminosity'] = 0.

    # parse EVIO files to extract useful information
    # eventually the DAQ will report this properly?
    rawdata_evio_dir = RAWDATA_DIR_FORMAT % (run_number)
    if os.path.isdir(rawdata_evio_dir) :
        filelist = [ join(rawdata_evio_dir,f) for f in listdir(rawdata_evio_dir) if ((f[:10]=="hd_rawdata" or f[:6]=="hd_raw")and(f[-5:]=='.evio')) ]
        filelist.sort()
        file_properties = ParseEVIOFiles(filelist)
        if len(file_properties) > 0:
            run_properties['num_events'] = file_properties['num_events']
            run_properties['num_files'] = file_properties['num_files']
            run_properties['start_time'] = file_properties['start_time']
            run_properties['end_time'] = file_properties['end_time']

    # pull out target information from the CCDB
    # load CCDB connection
    ccdb_context = InitCCDB()

    # read target index -> name mapping definition in from the CCDB
    target_types = load_target_types(ccdb_context, run_number)

    # make temp file to store CCDB info in
    fconst = NamedTemporaryFile()
    ccdb_context.process_command_line("dump /TARGET/target_parms:"+str(run_number)+" > "+fconst.name)

    # read in info
    fconst.flush()
    const_lines = fconst.readlines()

    if len(const_lines) < 2:
        print "Problem writing out CCDB constants to file!"
    else:
        # the first line of the output file from CCDB is junk, and our numbers are on the second line
        vals = const_lines[1].split()
        target_index = int(vals[0])
        if target_index in target_types:
            run_properties['target_type'] = target_types[target_index]
        else:
            print "Invalid target index from CCDB = " + str(target_index)
        fconst.close()

    if VERBOSE:
        print "RUN PROPERTIES FOR RUN " + str(run_number)
        print str(run_properties)

    # Add information to DB
    ## initialize DB
    db = datamon_db()
    ## add blank run to DB if it doesn't exist
    if(db.GetRunID(run_number) < 0):
        db.CreateRun(run_number)
    db.UpdateRunInfo(run_number, run_properties)

if __name__ == "__main__":
    main(sys.argv[1:])