# 2025-05-31 David L. # This was copied from hoss_skim_gen24s_d.config it disables gluonraid4 andf gluonraid7 # and instead uses only gluonraid6. It also only uses partitions 2,3,4. See logbook entry # from today for more details. # # SF use raid6 and raid7 with /data2 on both # #----------------------------------------------------------------- # # This is a configuration file for the hd_data_flow.py script. # # Syntax is as follows: # # Sections start with either "stage:" or "distribute:". # # "stage:" sections describe directories where files should be # linked to once they are no longer being written to. Each file # may appear in multiple destinations. Once a file is linked in # all of the specified directories, the original is unlinked. # Directories specified in stage sections must be local (i.e. # not prefixed with a host name.) # # "distribute:" sections describe directories that files should # be moved to using a round-robin algorithm. Each file appears # in only one of the destinations. Destinations may be either # local or remote and a mixture of both types is allowed. # # Both section types support multiple "source" lines and # multiple "destination" lines. The source lines may contain # wildcards and any syntax supported by python's glob.glob(). # The destination lines must specify a directory (no wildcards). # # For the "stage:" sections all sources and destinations must # be on the same filesystem. # # For "distribute:" sections, the destinations may indicate a # remote host by specifying it with a colon (:) in front of # the destination directory. "destination" lines that specify # a host will have the files copied using hdrdmacp. Ones that # do not specify a host will copy the files using rsync. Both # types may be mixed in the same section if you want to copy # some files to a local disk and send some to a remote host. # # # Uncomment to turn on test mode. (See comments in TESTING section below) # TESTMODE: #------------------------------------------------------------------- # Misc Config # # Set misc. config parameters. # # ALIAS: Define an alias. This allows a shorthand to be used in the rest # of this file where groups of nodes may be referenced with %ALIASNAME. # # MAX_PROCS: Set maximum number of processes allowed to run simultaneously on a host # # FILE_TIMEOUT: Sets a maximum time since last access before a file is deleted. # First argument is in seconds. Second argument is directory to start in. # Files in the specified directory and any subdirectories are subject # to deletion. Third argument are nodes on which to apply this rule. # # PROCESS_TIMEOUT: Set the maximum time in seconds for an external process to run before # it is killed. # # ALARM: Set alarm threshold for free disk space. # # DB_HOST: The DB_X: options specify connection info for a MySQL DB to insert # DB_USER: file movement information into. # DB_PASS: # DB_DATABASE: # DB_NODES: DB_NODES: lists the nodes which should make entries into the DB ALIAS: RAID_NODES gluonraid6,gluonraid7 ALIAS: MASTER_NODES gluondaqbuff,%RAID_NODES ALIAS: WORKER_NODES_RAID5 gluon152,gluon153 ALIAS: WORKER_NODES_RAID7 gluon154,gluon155 ALIAS: WORKER_NODES %WORKER_NODES_RAID5,%WORKER_NODES_RAID7 MAX_PROCS: 2 %WORKER_NODES MAX_PROCS: 4 %MASTER_NODES FILE_TIMEOUT: 200 /media/ramdisk/@TESTDIR/ %WORKER_NODES PROCESS_TIMEOUT: 240 ALARM: 30GB /media/ramdisk %MASTER_NODES ALARM: 1GB /media/ramdisk %WORKER_NODES DB_HOST: gluondb1.jlab.org DB_USER: hoss DB_PASS: DB_DATABASE: HOSS DB_NODES: %MASTER_NODES #------------------------------------------------------------------- # TESTING # The following is used for testing. To turn on test mode, uncomment # the line with TESTMODE: at the top of this file. # Note that these directories are hardwired to include the TEST # directory so that the they don't accidentally delete files # from the actual directories holding real data when TESTMODE is # turned off. By uncommenting the TESTMODE: line, all instances # of @TESTDIR/ in this file will be replaced by TEST. Otherwise, # they are replaced by an empty string. # # The REMOVETREE: line below will cause all directories and files # from the specified directory to be removed before starting the # scans. This is intended for testing as a way to clear out the # ram disks. # # Similar to above, the KILLALL: line will cause the specified # process to be killed (permissions permitting). This is intended # to clear out lingering processes at startup. # KILLALL: hd_ana %WORKER_NODES KILLALL: hdskims %WORKER_NODES KILLALL: hdmk_skims.py %WORKER_NODES REMOVETREE: /media/ramdisk/TEST %MASTER_NODES,%WORKER_NODES REMOVETREE: /data1/TEST %MASTER_NODES,%WORKER_NODES REMOVETREE: /data2/TEST %MASTER_NODES,%WORKER_NODES REMOVETREE: /data3/TEST %MASTER_NODES,%WORKER_NODES REMOVETREE: /data4/TEST %MASTER_NODES,%WORKER_NODES # This says to simply delete all files that show up in the # TEST staged_for_tape and volatile directories. stage: %RAID_NODES # Raw data files in both the staged_for_tape directory and volatile directories source /data*/TEST/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run*/*.evio source /data*/TEST/rawdata/volatile/@RUNPERIOD/rawdata/Run*/*.evio # Skim files in both the staged_for_tape directory and volatile directories source /data*/TEST/calib/staged_for_tape/@RUNPERIOD/calib/ver00/*/Run*/*.* source /data*/TEST/calib/volatile/@RUNPERIOD/calib/ver00/*/Run*/*.* # Delete files destination /dev/null #------------------------------------------------------------------- # DAQ # # Copy from buffer disk to RAM disk on RAIDs # # #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # # This section defines the fall-back configuration where the DAQ writes # directly to a RAID partition. It looks in /data*/rawdata/active/... # for evio files and copies them to both volatile and staged_for_tape # directories. Files are NOT copied to any other computer and no # processing is done. stage: gluonraid4 source /data1/@TESTDIR/rawdata/active/RunPeriod-2023-01/rawdata/Run1*/*.evio destination /data1/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data1/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: gluonraid4 source /data2/@TESTDIR/rawdata/active/RunPeriod-2023-01/rawdata/Run1*/*.evio destination /data2/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data2/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: gluonraid4 source /data3/@TESTDIR/rawdata/active/RunPeriod-2023-01/rawdata/Run1*/*.evio destination /data3/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data3/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: gluonraid4 source /data4/@TESTDIR/rawdata/active/RunPeriod-2023-01/rawdata/Run1*/*.evio destination /data4/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data4/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # # This section implements copying of files from gluondaqbuff to the # raid disks, but also to the farm nodes for processing. # # note: Two source directories are specified, but CODA currently # only writes to the long ones. # Also, CODA may be configured to use either gluondaqbuff # or a gluonraid RAM disk for writing. # stage: gluondaqbuff source /media/ramdisk/@TESTDIR/active/rawdata/active/RunPeriod-*/rawdata/Run*/*.evio source /media/ramdisk/@TESTDIR/active/*.evio destination /media/ramdisk/@TESTDIR/rawdata_in distribute: gluondaqbuff source /media/ramdisk/@TESTDIR/rawdata_in/*.evio destination gluonraid6:/media/ramdisk/@TESTDIR/active destination gluonraid7:/media/ramdisk/@TESTDIR/active stage: %RAID_NODES source /media/ramdisk/@TESTDIR/active/*.evio destination /media/ramdisk/@TESTDIR/rawdata_staged_for_disk destination /media/ramdisk/@TESTDIR/rawdata_staged_for_skim #------------------------------------------------------------------- # RAWDATA # # First copy from ramdisk to one of the RAID partitions and then # make links in staged_for_tape and volatile directories. distribute: %RAID_NODES source /media/ramdisk/@TESTDIR/rawdata_staged_for_disk/*.evio destination /data1/@TESTDIR/rawdata/staged_for_disk destination /data2/@TESTDIR/rawdata/staged_for_disk destination /data3/@TESTDIR/rawdata/staged_for_disk destination /data4/@TESTDIR/rawdata/staged_for_disk stage: %RAID_NODES source /data1/@TESTDIR/rawdata/staged_for_disk/*.evio destination /data1/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data1/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: %RAID_NODES source /data2/@TESTDIR/rawdata/staged_for_disk/*.evio destination /data2/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data2/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: %RAID_NODES source /data3/@TESTDIR/rawdata/staged_for_disk/*.evio destination /data3/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data3/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER stage: %RAID_NODES source /data4/@TESTDIR/rawdata/staged_for_disk/*.evio destination /data4/@TESTDIR/rawdata/staged_for_tape/@RUNPERIOD/rawdata/Run@RUNNUMBER destination /data4/@TESTDIR/rawdata/volatile/@RUNPERIOD/rawdata/Run@RUNNUMBER #------------------------------------------------------------------- # SKIMS # # - Copy to farm node # - Run hdmk_skims.py in trigger count only mode on farm node and enter into DB # # Clear out files from (potentially) non-test directories # WARNING: If there are files lingering on worker nodes awaiting # processing, this will delete them! REMOVETREE: /media/ramdisk/@TESTDIR/active %WORKER_NODES REMOVETREE: /media/ramdisk/@TESTDIR/rawdata_staged_for_skim %WORKER_NODES,%MASTER_NODES REMOVETREE: /media/ramdisk/@TESTDIR/skim_out %WORKER_NODES,%MASTER_NODES distribute: gluonraid6 source /media/ramdisk/@TESTDIR/rawdata_staged_for_skim/*.evio destination %WORKER_NODES_RAID5:/media/ramdisk/@TESTDIR/active distribute: gluonraid7 source /media/ramdisk/@TESTDIR/rawdata_staged_for_skim/*.evio destination %WORKER_NODES_RAID7:/media/ramdisk/@TESTDIR/active # n.b. use stage section to ensure only complete files appear in rawdata_staged_for_skim stage: %WORKER_NODES source /media/ramdisk/@TESTDIR/active/*.evio destination /media/ramdisk/@TESTDIR/rawdata_staged_for_skim # The -ignore_fp=X,Y,... argument to hdmk_skims.py below is used to ignore # certain front panel triggers when writing out 40 event blocks. These # are used by calibration triggers. If any are active for more than # about 2.5% of the events in the file, then most of the 40-event blocks # will be save in the first stage and parsed in the second which leads # to timeouts and the skim system failing. # # For GlueX running, we have traditionally disabled the DIRC_LED # trigger since it ran at 5kHz. For PrimEx-2021, we needed to disable # the CCAL_LED and random triggers as well. # # You can see a snapshot of what the assignments are in the MySQL DB # with the following command. Sasha S. would need to be asked directly # for the current assignments. # # echo "SELECT * FROM trigtypes;" | mysql -h gluondb1 -u hoss HOSS # +----------+--------------+ # | col_name | trig | # +----------+--------------+ # | GTP0 | FCAL_BCAL | # | GTP2 | BCAL | # | GTP3 | PS | # | GTP4 | FCAL_BCAL_ST | # | FP2 | FCAL_LED | # | FP4 | ECAL_LED1 | # | FP5 | ECAL_LED2 | # | FP8 | BCAL_LED_US | # | FP9 | BCAL_LED_DS | # | FP11 | random | # | FP14 | DIRC_LED | # +----------+--------------+ # distribute: %WORKER_NODES source /media/ramdisk/@TESTDIR/rawdata_staged_for_skim/*.evio process hdlog hdmk_skims.py -d -ignore_fp=11,14 @INFILE /media/ramdisk/@TESTDIR/skim_out # process hdlog hdmk_skims.py -d -ignore_fp=4,5,11,14 @INFILE /media/ramdisk/@TESTDIR/skim_out # n.b. assume only complete files appear in skim_out directory (i.e. it is not the working directory) distribute: %WORKER_NODES_RAID5 source /media/ramdisk/@TESTDIR/skim_out/*.* destination gluonraid6:/media/ramdisk/@TESTDIR/skim_active distribute: %WORKER_NODES_RAID7 source /media/ramdisk/@TESTDIR/skim_out/*.* destination gluonraid7:/media/ramdisk/@TESTDIR/skim_active stage: %RAID_NODES source /media/ramdisk/@TESTDIR/skim_active/*.* destination /media/ramdisk/@TESTDIR/skim_staged distribute: %RAID_NODES source /media/ramdisk/@TESTDIR/skim_staged/*.* destination /data1/@TESTDIR/skim/staged_for_disk destination /data2/@TESTDIR/skim/staged_for_disk destination /data3/@TESTDIR/skim/staged_for_disk destination /data4/@TESTDIR/skim/staged_for_disk stage: %RAID_NODES source /data1/@TESTDIR/skim/staged_for_disk/*.* destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER stage: %RAID_NODES source /data2/@TESTDIR/skim/staged_for_disk/*.* destination /data2/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER destination /data2/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER stage: %RAID_NODES source /data3/@TESTDIR/skim/staged_for_disk/*.* destination /data3/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER destination /data3/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER stage: %RAID_NODES source /data4/@TESTDIR/skim/staged_for_disk/*.* destination /data4/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER destination /data4/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/@SKIMDIR/Run@RUNNUMBER # Additions for online skims created separately from HOSS stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/BCAL_pi0/Run*/*.evio destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/BCAL_pi0/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/BCAL_pi0/Run@RUNNUMBER stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/FCAL_pi0/Run*/*.evio destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/FCAL_pi0/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/FCAL_pi0/Run@RUNNUMBER stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/FCAL_pi0_outer/Run*/*.evio destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/FCAL_pi0_outer/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/FCAL_pi0_outer/Run@RUNNUMBER stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/trd/Run*/*.evio destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/trd/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/trd/Run@RUNNUMBER stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/omega/Run*/*.evio destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/omega/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/omega/Run@RUNNUMBER stage: gluonraid4 source /data1/@TESTDIR/online_skimming/@RUNPERIOD/skim_hists/Run*/*.root destination /data1/@TESTDIR/skim/staged_for_tape/@RUNPERIOD/calib/ver00/skim_hists/Run@RUNNUMBER destination /data1/@TESTDIR/skim/volatile/@RUNPERIOD/calib/ver00/skim_hists/Run@RUNNUMBER