#!/usr/bin/env python

#
#
# Copy files from RAID server into /media/ramdisk/TEST/active 
# on gluondaqbuff so skims may be processed.
#
# This should be run from a RAID server so it can use hdrdmacp
# to copy from the local disk to gluondaqbuff without going
# through NFS.

import os
import sys
import glob
import time
import random
import subprocess

RUNPERIOD = 'RunPeriod-2019-11'

MAX_PROCS = 1
files = []
procs = {}
completed_files = []

#===========================================================================
# Loop over partitions
for ipart in range(1,5):

	#topdir = '/data%d/rawdata/volatile/%s/rawdata' % (ipart, RUNPERIOD)
	topdir = '/data%d/rawdata/active/%s/rawdata' % (ipart, RUNPERIOD)
	part_files = glob.glob(topdir + '/Run*/hd_rawdata_*_*.evio')
	print( ('Processing %d files from: ' + topdir) % len(part_files))
	files += part_files

# Loop until all files are processed
random.shuffle( files )  # shuffle files so they are not all read from same partition at once
while True:

	if len(completed_files) >= len(files): break

	#if (len(completed_files)%10) == 0: print('Completed %d/%d files' % (len(completed_files), len(files)))

	for sfile in files:

		if sfile in completed_files: continue # This file has already been processed
		if sfile in procs.keys(): continue    # This file is currently being processed

		if len(procs) >= MAX_PROCS:
			time.sleep(0.5)
			break

		cmd = ['hdrdmacp', '-u', 'hdops', '-m', '2', '-n', '8', sfile, 'gluondaqbuff:/media/ramdisk/active/'+os.path.basename(sfile)]
		prefix = '%s cmd: ' % time.ctime()
		print(prefix + ' '.join(cmd) )

		proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
		procs[sfile] = {'proc':proc, 'cmd':cmd}


	# Look for finished subprocesses
	for sfile,proc in procs.items() :
		if proc['proc'].poll() is not None :
			print( proc['proc'].communicate()[0] )
			print('subprocess completed: ' +  ' '.join(proc['cmd']))
			procs.pop( sfile )
			completed_files.append( sfile )
			print('Completed %d/%d files' % (len(completed_files), len(files)))

	# Limit rate through loop 
	#time.sleep(5)