/*----------------------------------------------------------------------------*
	GlueX file to ET  producer

	This program reads events from an EVIO(CODA) file(s) and then inserts
	them into an ET system.

	11/20/2015

	I did some profiling of this program to see how it might be optimized.
	In it's current state (svn revision 19684) it can write 27.5kB events
	from run 2931 into a local ET system at about 30kHz. Roughly 1/3 of
	this time is spent reading (from file cache in RAM). About 1/3 is
	spent byte swapping, and about 1/4 to 1/5 is spent in memcpy. This
	could be optimized to get better performance, but will require major
	refactoring (or possibly rewriting from scratch). At best, one could
	improve the speed by about a factor of 3. Rewriting this
	now is not the highest priority so I'm leaving this note to remind
	me what my thoughts are so I can take it up later.

	The optimized program should work with at least 4 threads:

	Thread 1: Read from file into large buffers
	Thread 2: Make list of pointers to individual EVIO events within
	          the large buffers. Events spanning more than one buffer
				 will need to have the parts copied into a separate, single
				 buffer.
	Thread 3: Byte swap the single event buffers into a separate set of
	          buffers.
	Thread 4: Copy events from the byte-swapped buffers into ET events

	A fifth thread may also be needed to print the event ticker. Since
	the data will flow through multiple layers of buffers handled by
	multiple threads, back pressure will need to be implemented. This
	can be done using pthread condition waits so upstream threads can
	sleep while waiting for downstream threads to finish consuming
	existing buffers.


 *----------------------------------------------------------------------------*/


#include <iostream>
#include <iomanip>
#include <string>
#include <vector>
#include <sstream>
using namespace std;

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>

#include <et.h>
#include <HDEVIO.h>
#include <evio.h>

//#define max_event_size 100000
uint32_t max_event_size=20000000;

// Globals
et_sys_id id;
et_att_id  attach;
string et_fname("");
useconds_t DELAY = 0;
vector<string> source_files;
bool QUIT = false;
bool USE_HDEVIO = true;
bool READ_EVIO = true;
bool WRITE_ET = true;
uint64_t WRITE_ET_PRESCALE = 0;
bool LOOP=false;
bool ADD_NETWORK_TRANSFER_HEADER = true;
bool FORCE_SWAP=false;
uint32_t BLOCK_SIZE=1;
int PORT = 0;
int VERBOSE = 0;

HDEVIO *hdevio = NULL;

#if defined(__APPLE__)
const char *HOST = "127.0.0.1";
#else
const char *HOST = "localhost";
#endif

// Routines
int InsertEventsIntoET(vector<char*> &buffs, vector<uint32_t> &buff_sizes, uint32_t &NwordsET);
void DumpBinary(const uint32_t *iptr, const uint32_t *iend=NULL, uint32_t MaxWords=0, const uint32_t *imark=NULL);
void Usage(void);

int SIGINT_RECEIVED = 0;

//-----------------------------------------------------------------
// ctrlCHandle
//-----------------------------------------------------------------
void ctrlCHandle(int x)
{
	SIGINT_RECEIVED++;
	cerr<<endl<<"SIGINT received ("<<SIGINT_RECEIVED<<")....."<<endl;
	if(SIGINT_RECEIVED == 3){
		cerr<<endl<<"Three SIGINTS received! Attempting graceful exit ..."<<endl<<endl;
	}
	if(SIGINT_RECEIVED ==6){
		cerr<<endl<<"Six SIGINTS received! Attempting forceful exit ..."<<endl<<endl;
		exit(-1);
	}

	QUIT = true;
}

//------------------------
// main
//------------------------
int main(int narg,char **argv)
{
	// Set up to catch SIGINTs for graceful exits
	signal(SIGINT,ctrlCHandle);

	// Loop over command line arguments.
	for(int i=1; i<narg; i++){
		string s(argv[i]);
		if(s=="-h" || s=="--help")Usage();
		if(s=="-loop"){
			LOOP = true;
			continue;
		}
		if(s=="-d"){
			DELAY = atoi(argv[++i]);
			continue;
		}
		if(s=="-noread" || s=="--noread"){
			READ_EVIO = false;
			continue;
		}
		if(s=="-nowrite" || s=="--nowrite"){
			WRITE_ET = false;
			continue;
		}
		if(s=="-hdevio" || s=="--hdevio"){
			USE_HDEVIO = true;
			continue;
		}
		if(s=="-coda" || s=="--coda"){
			USE_HDEVIO = false;
			continue;
		}
		if(s=="-f"){
			et_fname = argv[++i];
			continue;
		}
		if(s=="-n"){
			BLOCK_SIZE = atoi(argv[++i]);
			continue;
		}
		if(s=="-b"){
			max_event_size = atoi(argv[++i]);
			continue;
		}
		if(s=="-s"){
			WRITE_ET_PRESCALE = atoi(argv[++i]);
			continue;
		}
		if(s=="-H"){
			HOST = argv[++i];
			continue;
		}
		if(s=="-p"){
			PORT = atoi(argv[++i]);
			continue;
		}
		if(s=="-v"){
			VERBOSE++;
			continue;
		}
		source_files.push_back(s);
	}

	if(READ_EVIO && source_files.empty()) source_files.push_back("dummy");
	if(source_files.empty() || et_fname.size()==0) Usage();

	cout << "==========================================================" << endl;
	cout << "         ET host: " << HOST << endl;
	if(PORT!=0) cout << "         ET port: " << PORT << endl; 
	cout << "  ET system file: " << et_fname << endl;
	cout << "    EVIO file(s): ";
	for(uint32_t i=0; i<source_files.size(); i++) cout << source_files[i] << " ";
	cout << endl;
	cout << "           delay: " << DELAY << " us" << endl;
	cout << "            loop: " << (LOOP ? "Y":"N") << endl;
	cout << "    EVIO library: " << (USE_HDEVIO ? "HDEVIO":"CODA") << endl;
	cout << "==========================================================" << endl;

	// Open the ET system for inserting events into
	et_openconfig openconfig;
	et_open_config_init(&openconfig);
	et_open_config_setcast(openconfig, ET_DIRECT);
	et_open_config_setmode(openconfig, ET_HOST_AS_LOCAL); // ET_HOST_AS_LOCAL or ET_HOST_AS_REMOTE
	et_open_config_sethost(openconfig, HOST);  // for some reason, using ET_HOST_LOCAL doesn't work ???
	if(PORT != 0)et_open_config_setserverport(openconfig, PORT);
	if(et_open(&id, et_fname.c_str(), openconfig) != ET_OK){
		cerr<<__FILE__<<":"<<__LINE__<<" Error opening ET system "<<et_fname<<endl;
		return -1;
	}
	et_open_config_destroy(openconfig);

	// set the debug level (everything)
	et_system_setdebug(id, ET_DEBUG_INFO);

	// attach to GRANDCENTRAL station since we are producing events
	if (et_station_attach(id, ET_GRANDCENTRAL, &attach) < 0) {
		cerr<<__FILE__<<":"<<__LINE__<<" Error attaching to GRAND CENTRAL on "<<et_fname<<endl;
		return -1;
	}

	// Allocate buffers to hold BLOCK_SIZE events
	if(BLOCK_SIZE == 0){
		cout << "BLOCK_SIZE is < 1. Setting BLOCK_SIZE=1" << endl;
		BLOCK_SIZE = 1;
	}
	if(BLOCK_SIZE>1000000){
		cout << "BLOCK_SIZE > 1M ("<<BLOCK_SIZE<<") !" <<endl;
		cout << "This seems wrong. You'll have to recompile if you really want that!" << endl;
		exit(-1);
	}
   cout << "BLOCK_SIZE: " << BLOCK_SIZE << " EVIO events per ET event" << endl;
	vector<char*> buffs;
	for(uint32_t i=0; i<BLOCK_SIZE; i++) buffs.push_back(new char[max_event_size]);

	if(!WRITE_ET) cout<<"---- ET WRITING DISABLED ----"<<endl;

	// keep track of size of each event buffer as it is read in. This also
   // keeps track of how many entries in "buff" have events that have not
	// yet been written out.
	vector<uint32_t> buff_sizes;

	// Outer loop for when LOOP is true
	if(LOOP){
		cout<<"Entering infinite loop over events. Hit ctl-C to quit ..."<<endl;
	}
	do{

		// For printing rates and event sizes
		static uint32_t NeventsL1   = 0;
		static uint32_t NeventsEVIO = 0;
		static uint32_t NwordsEVIO  = 0;
		static uint32_t NeventsET   = 0;
		static uint32_t NwordsET    = 0;
		static time_t last_time = time(NULL);

		// Loop over source files
		for(unsigned int i=0; i<source_files.size(); i++){

			if(QUIT) break;

			// Open EVIO file
			int handle=0;
			hdevio = NULL;
			if(READ_EVIO){
				if(USE_HDEVIO){
					hdevio = new HDEVIO(source_files[i]);
					//hdevio->ResizeFileBuffer(1024*1024*1);
					//hdevio->disable_bank_swap = true;
					if(!hdevio->is_open){
						cerr<<"Error opening \""<<source_files[i]<<"\""<<endl;
						continue;
					}
				}else{
					if(evOpen((char*)source_files[i].c_str(), (char*)"r", &handle)!=S_SUCCESS){
						cerr<<"Error opening \""<<source_files[i]<<"\""<<endl;
						continue;
					}
				}
				cout << "Opened " << source_files[i] << endl;
			}else{
				cout<<"---- EVIO FILE READING DISABLED ----"<<endl;
			}

			// Loop over events in file
			//char buff[max_event_size];
			uint32_t *iptr = (uint32_t*)buffs[buff_sizes.size() ];
			while(true){

			if(QUIT) break;

				uint32_t event_size = 0;
				if(READ_EVIO){

					if(USE_HDEVIO){
						hdevio->readNoFileBuff(iptr, max_event_size, false);
						if(hdevio->err_code == HDEVIO::HDEVIO_EOF) break;
						if(hdevio->err_code != HDEVIO::HDEVIO_OK){
							cout << endl << "HEDVIO ERROR: " << hdevio->err_code << endl;
							if(!LOOP) QUIT = true; // only quit if not looping
							break;
						}
					}else{
						if(evRead(handle, iptr, max_event_size)!=S_SUCCESS) break;
					}

					// Event size in words should be first word in buffer
					event_size = (hdevio->swap_needed ? swap32(*iptr):*iptr) + 1;
					
					// Number of L1 events should be in 2nd word
					uint32_t header = (hdevio->swap_needed ? swap32(iptr[1]):iptr[1]);
					NeventsL1 += header&0xFF;
					
				}else{
					event_size = 4980; // dummy event size (~19.4kB)
				}

				if(event_size>max_event_size)event_size = max_event_size;
				buff_sizes.push_back( event_size );
				
				if(VERBOSE>=3) DumpBinary(iptr, &iptr[event_size], 256);

				if(buff_sizes.size() == BLOCK_SIZE){
					uint32_t nwordsET=0;
					if(WRITE_ET) {
						bool skip_et = false;
						if( (WRITE_ET_PRESCALE!=0) &&  (NeventsEVIO%WRITE_ET_PRESCALE!=0) ) skip_et = true;
						if(!skip_et) {
							InsertEventsIntoET(buffs, buff_sizes, nwordsET);
							NeventsET++;
							NwordsET += nwordsET;
						}
					}
					buff_sizes.clear();
				}

				// Print ticker indicating rate and average event sizes

				NeventsEVIO++;
				NwordsEVIO += event_size;

				time_t t = time(NULL);
				time_t delta_t = t - last_time;
				if(delta_t>=3){

					float rateL1      = (float)NeventsL1/(float)delta_t/1000.0;
					float rateEVIO    = (float)NeventsEVIO/(float)delta_t/1000.0;
					float rateET      = (float)NeventsET/(float)delta_t/1000.0;
					float avgsizeL1   = (float)NwordsEVIO*4.0/(float)NeventsL1/1024.0;
					float avgsizeEVIO = (float)NwordsEVIO*4.0/(float)NeventsEVIO/1024.0;
					float avgsizeET   = (float)NwordsET*4.0/(float)NeventsET/1024.0;
					cout << " L1/EVIO/ET rate: " << rateL1 << " kHz/ " << rateEVIO << " kHz/ " << rateET << " kHz  Avg. L1/EVIO/ET evt size: " << avgsizeL1 <<  "kB/ " << avgsizeEVIO << "kB/ " << avgsizeET << "kB      \r";
					cout.flush();

					NeventsL1   = 0;
					NeventsEVIO = 0;
					NwordsEVIO  = 0;
					NeventsET   = 0;
					NwordsET    = 0;
					last_time = time(NULL);
				}

				// Set buffer for next evio event read
				iptr = (uint32_t*)buffs[buff_sizes.size() ];
			}
			cout << endl;

			// Close EVIO file
			if(handle) evClose(handle);
			if(hdevio) {hdevio->PrintStats(); delete hdevio;}
		}

		if(QUIT) break;
	}while(LOOP);

	uint32_t nwordsET;
	if(!buff_sizes.empty()) InsertEventsIntoET(buffs, buff_sizes, nwordsET);

	// Free memory used for event buffers
	for(uint32_t i=0; i<buffs.size(); i++) delete[] buffs[i];

	cout << endl << endl;

	return 0;
}

//------------------------
// InsertEventsIntoET
//------------------------
int InsertEventsIntoET(vector<char*> &buffs, vector<uint32_t> &buff_sizes, uint32_t &NwordsET)
{
	et_event   *pe;
	int status;
	char *pdata;

	// Make sure ET system is still alive. Wait for it if not.
	if (!et_alive(id)) {
		et_wait_for_alive(id);
	}

	// Get total size of ET buffer needed for all events
	uint32_t nwords = 0;
	for(uint32_t i=0; i<buff_sizes.size(); i++) nwords += buff_sizes[i];

	/* get new/unused event */
	uint32_t nwords_tot = nwords + (ADD_NETWORK_TRANSFER_HEADER ? 8:0);
	status = et_event_new(id, attach, &pe, ET_SLEEP, NULL, nwords_tot*sizeof(int));
	if (status != ET_OK) {
		printf("et_producer: error in et_event_new\n");
		exit(0);
	}

	NwordsET = nwords_tot;

	// Get pointer to data of new event
    et_event_getdata(pe,(void**)&pdata);

	// When CODA writes events to ET, it adds a network transport header first
	// If the ADD_NETWORK_TRANSFER_HEADER is true, then add this header
	uint32_t *iptr = (uint32_t*)pdata;
	if(ADD_NETWORK_TRANSFER_HEADER){
		uint32_t bitinfo = (1<<9) + (1<<10); // (1<<9)=Last event in ET stack, (1<<10)="Physics" payload
		*iptr++ = nwords_tot;         // Block Length (inclusive)
		*iptr++ = 1;                  // Block Number
		*iptr++ = 8;                  // Header Length
		*iptr++ = buff_sizes.size();  // Event count
		*iptr++ = 0;                  // Reserved 1
//		*iptr++ = (1<<10) + 4;        // Bit Info/Version (Bit info in bits 10-13. 1=Physics event)
		*iptr++ = (bitinfo<<8) + 4;   // Bit Info/Version (Bit info in bits 10-13. 1=Physics event)
		*iptr++ = 0;                  // Reserved 2
		*iptr++ = 0xc0da0100;         // Magic number
		
//		if(FORCE_SWAP) hdevio->swap_block((uint32_t*)pdata, 8, (uint32_t*)pdata);
		hdevio->swap_block((uint32_t*)pdata, 8, (uint32_t*)pdata);
	}

	// Copy contents into new event
	for(uint32_t i=0; i<buff_sizes.size(); i++){
		if(FORCE_SWAP){
			if(VERBOSE>=3) cout << "swapping " << buff_sizes[i] << " words to " << hex << iptr << dec << endl;
			hdevio->swap_bank(iptr, (uint32_t*)buffs[i], buff_sizes[i]);
			if(VERBOSE>=3) cout << "err_mess: " << hdevio->err_mess.str() << endl;
		}else{
			if(VERBOSE>=3) cout << "copying " << buff_sizes[i] << " words to " << hex << iptr << dec << endl;
			memcpy( iptr, (char*)buffs[i], buff_sizes[i]*sizeof(int));
		}
		iptr = &iptr[buff_sizes[i]];
	}
	
	if(VERBOSE>=3){
		iptr = (uint32_t*)pdata;
		DumpBinary(iptr, &iptr[iptr[1]], 256);
	}

	// put event back into the ET system
	status = et_event_put(id, attach, pe);
	if (status != ET_OK) {
		printf("et_producer: put error\n");
			exit(0);
	}

	// Delay a little (if specified) to lower the event rate
	if(DELAY != 0)usleep(DELAY);

	return 0;
}

//----------------
// DumpBinary
//----------------
void DumpBinary(const uint32_t *iptr, const uint32_t *iend, uint32_t MaxWords, const uint32_t *imark)
{
	/// This is used for debugging. It will print to the screen the words
	/// starting at the address given by iptr and ending just before iend
	/// or for MaxWords words, whichever comes first. If iend is NULL,
	/// then MaxWords will be printed. If MaxWords is zero then it is ignored
	/// and only iend is checked. If both iend==NULL and MaxWords==0, then
	/// only the word at iptr is printed.

	cout << "Dumping binary: istart=" << hex << iptr << " iend=" << iend << " MaxWords=" << dec << MaxWords << endl;

	if(iend==NULL && MaxWords==0) MaxWords=1;
	if(MaxWords==0) MaxWords = (uint32_t)0xffffffff;

	uint32_t Nwords=0;
	while(iptr!=iend && Nwords<MaxWords){

		// line1 is hex and line2 is decimal
		stringstream line1, line2;

		// print words in columns 8 words wide. First part is
		// reserved for word number
		uint32_t Ncols = 8;
		line1 << setw(5) << Nwords;
		line2 << string(5, ' ');

		// Loop over columns
		for(uint32_t i=0; i<Ncols; i++, iptr++, Nwords++){

			if(iptr == iend) break;
			if(Nwords>=MaxWords) break;

			stringstream iptr_hex;
			iptr_hex << hex << "0x" << *iptr;

			string mark = (iptr==imark ? "*":" ");

			line1 << setw(12) << iptr_hex.str() << mark;
			line2 << setw(12) << *iptr << mark;
		}

		cout << line1.str() << endl;
		cout << line2.str() << endl;
		cout << endl;
	}
}
//------------------------
// Usage
//------------------------
void Usage(void)
{
	cout<<endl;
	cout<<"Usage:"<<endl;
	cout<<"      file2et [options] -f et_system_file evio_file [evio_file] ..."<<endl;
	cout<<endl;
	cout<<"Read events from an EVIO (CODA formatted file) and write "<<endl;
	cout<<"them into the specified ET system."<<endl;
	cout<<endl;
	cout<<"  options:"<<endl;
	cout<<"     -h,--help      Print this usage statement"<<endl;
	cout<<"     -d delay       Delay in us between ET events (def. 0)"<<endl;
	cout<<"     -loop          Repeatedly reopen the EVIO file(s)"<<endl;
	cout<<"     -n block_size  Stack block_size events in each ET event"<<endl;
	cout<<"     -b buffer_size Buffer size to allocate for reading event in"<<endl;
	cout<<"     -noread        Don't actually read the EVIO event"<<endl;
	cout<<"                    (for testing ET rates only. Data will"<<endl;
	cout<<"                    be garbage)."<<endl;
	cout<<"     -nowrite       Don't actually write the event to ET"<<endl;
	cout<<"                    (for testing EVIO read rates only.)"<<endl;
	cout<<"     -hdevio        Use HDEVIO for file reading (default)"<<endl;
	cout<<"     -coda          Use CODA evio library for file reading"<<endl;
	cout<<"     -f et_file     Open the specified et system file"<<endl;
	cout<<"     -s et_prescale Prescale events written to ET"<<endl;
	cout<<"     -H host        Open ET system on this host. Default is"<<endl;
	cout<<"                    \"localhost\" though note that on Mac OS X" << endl;
	cout<<"                    one needs to use \"127.0.0.1\"."<<endl;
	cout<<"     -p port        Set the server port to connect to" << endl;
	cout<<"     -v             Increase verbosity level" << endl;
	cout<<endl;
	cout<<"The value of et_system_file is usually something like:"<<endl;
	cout<<endl;
	cout<<"/tmp/et_sys_mysession"<<endl;
	cout<<endl;
	cout<<"If the -d delay option is given, then a delay of \"delay\""<<endl;
	cout<<"microseconds will be inserted between events so that the"<<endl;
	cout<<"rate can be controlled. Note that most operating systems"<<endl;
	cout<<"have a minimum delay so if you specify a value below it,"<<endl;
	cout<<"the minimum delay will still occur. No additional delay is"<<endl;
	cout<<"incurred if the value of delay is 0 or the -d argument is"<<endl;
	cout<<"ommitted."<<endl;
	cout<<endl;
	cout<<"If the -loop option is specified, then the source files are"<<endl;
	cout<<"repeatedly read over and over until the program is killed."<<endl;
	cout<<endl;

	exit(0);
}