#define ROL_NAME__ "mcROL" #define MAX_EVENT_LENGTH 1024*8 #define MAX_EVENT_POOL 500 /* POLLING_MODE */ #define POLLING___ #define POLLING_MODE #define TEST_MODE #define INIT_NAME mcROL__init #include #include #include #include #include #include //------------------------------------------------------------ // This section is here so that the conf_output.X files // below can compile. They are used to input DAQ configuration // into the data stream and it is useful to debug that here. // This feature can be turned on/off with the following #define #define USE_CONF_OUTPUT 1 #if USE_CONF_OUTPUT #ifndef dma_dabufp #define dma_dabufp rol->dabufp #endif #include "conf_utils.h" #include "conf_output.h" #include "conf_output.c" #endif // USE_CONF_OUTPUT //------------------------------------------------------------ // The definition of the rol object can be found in: // // $CODA/common/include/rolInt.h // The evio input file name is passed in as the user string // during prestart and so is set below. The string may // contain a "%d" which will be replaced with the value // in rol->pid (aka the ROC id). static char evioFileName[1024] = "nada"; // full path on gluonraid1 static char evioLocalFileName[1024] = "nada"; // path to file on local RAM disk // For EVIO static int handle; #define MCBUFSIZE 10000 static uint32_t mcbuffer[MCBUFSIZE]; // For cMsg char UDL[512] = "cMsg://gluondb1.jlab.org/cMsg/softROC"; char cMsgName[256] = "rocXXX"; char cMsgDescr[256] = "coda_roc running software trigger"; void *domainID = NULL; void DownloadFile(void); void MyCallback(void *msg, void *userArg); // This is used to slow down and roughly synchronize the // events between ROCs. If we are connected to the cMsg // server (i.e. if domainID is not NULL) then every // event will decrement this counter in the __done() // routine until it gets to 0. Then, it will go into // a sleep loop waiting for the counter to be non-zero. // If __reset() or __prestart() are called then // this is reset to "1". int USE_SOFTROC_CONTROLLER = 0; unsigned long Nevents_left = 1000; unsigned long Nevents_processed = 0; unsigned long MAX_EVENTS = 0; // 0 means no max int END_AT_NEXT_BOUNDARY = 0; // The value of BLOCKLEVEL is used to set the number of events // as though we're reading in multi-event block mode. This should // be set to 1 since setting it differently causes the PEB to crash. // (Existing code is left here since it may help future debugging // of this feature.) // // The value of NEVENTS_TO_STACK is used to stack multiple // EVIO events read from the file into a single event. Ideally, // we would just use BLOCKLEVEL here as they should be the same. // However, because the BLOCKLEVEL must be set for 1 for the time // being, we use this variable to allow us to increase the event // size to mimic that of a single DAQ event in multi-event block // mode. One will need to scale up the rates reported by CODA in // this case by this value. int BLOCKLEVEL = 1; int NEVENTS_TO_STACK = 1; int trigBankType = 0xff11; //------------------------- // download //------------------------- static void __download() { daLogMsg("INFO","Readout list compiled %s", DAYTIME); #ifdef POLLING___ rol->poll = 1; #endif *(rol->async_roc) = 0; /* Normal ROC */ // Turn off by default until we get a message from controller // This allows us to run without a controller USE_SOFTROC_CONTROLLER = 0; // Connect to remote cMsg server. // (This will be disconnected in __reset() ) sprintf(cMsgName, "roc%02d", rol->pid); int err = cMsgConnect(UDL, cMsgName, cMsgDescr, &domainID); if(err!=0){ printf("Could not connect to cMsg server! Remote event governor disabled.\n"); }else{ printf("Connected to cMsg server using: %s\n", UDL); printf("-- cMsg client name: %s\n", cMsgName); cMsgSubscribeConfig *config = cMsgSubscribeConfigCreate(); cMsgSubscribe(domainID, "softROC", "*", MyCallback, NULL, config, NULL); cMsgSubscribe(domainID, cMsgName, "*", MyCallback, NULL, config, NULL); cMsgReceiveStart(domainID); } // Full file name (on possibly remote system) sprintf(evioFileName, rol->usrString, rol->pid); // replace first %d with ROC id // Get full path to file on RAM disk int pos = strlen(evioFileName); for(pos--; pos>0; pos--) if(evioFileName[pos]=='/') break; sprintf(evioLocalFileName, "/dev/shm/%s", &evioFileName[++pos]); // Download the input evio file to RAM disk DownloadFile(); #if USE_CONF_OUTPUT // Initialize so no module type claims any slot uint32_t slot; for(slot=0; slotnevents) = 0; daLogMsg("INFO","Entering User Prestart"); TEST_INIT; CTRIGRSS(TEST,1,usrtrig,usrtrig_done); CRTTYPE(1,TEST,1); TESTflag = 0; TEST_prescale = 1; // sets iterations of a loop to slow down event rate daLogMsg("INFO","User Prestart Executed"); if (__the_event__) WRITE_EVENT_; *(rol->nevents) = 0; rol->recNb = 0; // Initialize counter Nevents_left = 1000; END_AT_NEXT_BOUNDARY = 0; } //------------------------- // end //------------------------- static void __end() { // Only turn off triggering at an event block // boundary if we are being regulated by a // remote softROCcontroller program if(USE_SOFTROC_CONTROLLER){ // Tell the remote softROCcontroller process that we received an // end transition void *cmsg = cMsgCreateMessage(); cMsgSetSubject(cmsg, "softROCcontroller"); cMsgSetType(cmsg, cMsgName); cMsgSetText(cmsg, "ending"); cMsgSend(domainID, cmsg); cMsgFreeMessage(&cmsg); //END_AT_NEXT_BOUNDARY = 1; printf("Ending: Nevents_processed=%ld Nevents_left=%ld\n", Nevents_processed, Nevents_left); }else{ TESTflag = 0; // close evio file evClose(handle); } daLogMsg("INFO","User End Executed"); if (__the_event__) WRITE_EVENT_; } //------------------------- // pause //------------------------- static void __pause() { TESTflag = 0; daLogMsg("INFO","User Pause Executed"); if (__the_event__) WRITE_EVENT_; } //------------------------- // go //------------------------- static void __go() { daLogMsg("INFO","Entering User Go"); TESTflag = 1; if (__the_event__) WRITE_EVENT_; } //------------------------- // usrtrig //------------------------- void usrtrig(unsigned long EVTYPE,unsigned long EVSOURCE) { static unsigned int event_number = 0; unsigned int ievent; unsigned long evtnum = *(rol->nevents); unsigned int *start = rol->dabufp; unsigned int *end=NULL; uint32_t i,j; event_number++; CEOPEN(ROCID,BT_BANK,BLOCKLEVEL); #if 1 CBOPEN(trigBankType,BT_SEG,BLOCKLEVEL); if(trigBankType == 0xff11) { *rol->dabufp++ = (EVTYPE<<24)|(0x01<<16)|(3); }else{ *rol->dabufp++ = (EVTYPE<<24)|(0x01<<16)|(1); } *rol->dabufp++ = (BLOCKLEVEL*(evtnum-1) + 1); if(trigBankType == 0xff11) { *rol->dabufp++ = 0x12345678; *rol->dabufp++ = 0; } CBCLOSE; #endif // Optionally write DAQ config to output stream #if USE_CONF_OUTPUT conf_output_write(); #endif // #if USE_CONF_OUTPUT #if 1 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // Stack BLOCKLEVEL events into single CODA event. These events will // not be properly formatted if BLOCKLEVEL is not set to 1 and will // likely cause the offline parsing to choke. It can be useful for // estimating rates in multi-event block mode though. for(ievent=0; ieventdabufp,(char*)buff,nword*sizeof(uint32_t)); rol->dabufp += nword; } } #endif CECLOSE; end = rol->dabufp; // printf("%s:%d start=%08x end=%08x diff(words)=%d\n\n",__FILE__, __LINE__, start,end,(end-start)); } //------------------------- // usrtrig_done //------------------------- void usrtrig_done() { } //------------------------- // done //------------------------- void __done() { poolEmpty = 0; /* global Done, Buffers have been freed */ Nevents_processed++; if(USE_SOFTROC_CONTROLLER!=0 && domainID!=NULL){ // We are connected to a cMsg server so decrement // the event counter and check it to see if we need // to wait here for a cMsg to tell us to continue. Nevents_left--; if(Nevents_left == 0){ if(TESTflag == 0){ printf("Finished block of triggers and ending...\n"); //TESTflag = 0; evClose(handle); // close evio file return; }else{ printf("Finished block of triggers. Waiting for softROCcontroller..\n"); void *cmsg = cMsgCreateMessage(); cMsgSetSubject(cmsg, "softROCcontroller"); cMsgSetType(cmsg, cMsgName); cMsgSetText(cmsg, "ready"); cMsgSend(domainID, cmsg); cMsgFreeMessage(&cmsg); } } while(Nevents_left == 0)usleep(100000); } } //------------------------- // reset //------------------------- void __reset () { // Disconnect from cMsg server if connected if(domainID != NULL){ printf("Disconnecting from cMsg server ... \n"); cMsgReceiveStop(domainID); cMsgDisconnect(&domainID); domainID = NULL; }else{ printf("No cMsg connection so skipping disconnect.\n"); } Nevents_left = 1; } //------------------------- // DownloadFile //------------------------- void DownloadFile(void) { // If the file is available on the local filesystem then copy it from there. // Otherwise, we'll have to scp it from the raid disk. We create a command // that will copy it to the RAM disk either via cp or scp and execute it below. char cmd[512]; FILE *f = fopen(evioFileName, "r"); if(f != NULL){ // File exists locally. Close it. fclose(f); // This is option is mainly for use with a sandbox CODA setup // (like used in a virtual machine). This seems to take a // while to do the copy so check if the file already exists // on the RAM disk and don't bother recopying if it does. f = fopen(evioLocalFileName, "r"); if(f != NULL){ // File exists on RAM disk. Do not re-copy fclose(f); sprintf(cmd, "echo file %s already exists. Skipping copy", evioLocalFileName); }else{ // File does not exist on RAM disk sprintf(cmd, "cp %s %s", evioFileName, evioLocalFileName); } }else{ // Download data file to RAM disk. We must do this via scp // since some ROCs do not seem to have /gluonraid1 mounted sprintf(cmd, "scp gluonraid1-daq:%s %s", evioFileName, evioLocalFileName); } // Run the command and print result printf("%s\n", cmd); int status = system(cmd); printf("Command result: %d\n", status); // Make the file world writable so it can be deleted and // replaced regardles of which account is running coda sprintf(cmd, "chmod 666 %s", evioLocalFileName); printf("%s\n", cmd); status = system(cmd); printf("Command result: %d\n", status); // open evio file if(evOpen(evioLocalFileName,"r",&handle)!=S_SUCCESS) { printf("?unable to open file %s\n",evioLocalFileName); } else { printf("successfully opened file %s\n",evioLocalFileName); } } //------------------------- // MyCallback //------------------------- void MyCallback(void *msg, void *userArg) { const char *subject = NULL; const char *type = NULL; const char *cmd = NULL; int err; cMsgGetSubject(msg, &subject); cMsgGetType(msg, &type); cMsgGetText(msg, &cmd); printf("cMsg received:\n"); printf(" subject: %s\n", subject ? subject:""); printf(" type: %s\n", type ? type:""); printf(" cmd: %s\n", cmd ? cmd:""); // If we get any kind of message, it must mean a // softROCcontroller is running USE_SOFTROC_CONTROLLER = 1; // prepare response message in case we need to send it void *cmsg = cMsgCreateMessage(); cMsgSetSubject(cmsg, type); cMsgSetType(cmsg, cMsgName); int sendResponse = 0; int save_TESTflag; if(!strcmp(cmd, "roll call")){ cMsgSetText(cmsg, "present"); sendResponse = 1; }else if(!strcmp(cmd, "process triggers")){ cMsgGetUserInt(msg, &Nevents_left); printf("--- set Nevents_left to: %d\n", Nevents_left); }else if(!strcmp(cmd, "stop triggers")){ TESTflag = 0; }else if(!strcmp(cmd, "start triggers")){ TESTflag = 1; }else if(!strcmp(cmd, "set prescale")){ int new_test_precale; cMsgGetUserInt(msg, &new_test_precale); printf("--- setting TEST_prescale to: %d\n", new_test_precale); TEST_prescale = new_test_precale; }else if(!strcmp(cmd, "delete data file")){ err = unlink(evioLocalFileName); printf("--- deleted local file \"%s\" (err=%d)\n", evioLocalFileName, err); }else if(!strcmp(cmd, "reload data file")){ // Pause self-triggering and wait half a second for // the current event to finish processing printf("--- reloading data file\n"); printf("--- pausing triggers\n"); save_TESTflag = TESTflag; TESTflag = 0; usleep(500000); // Close existing file and delete it evClose(handle); printf("--- closing local file\n"); err = unlink(evioLocalFileName); printf("--- deleted local file \"%s\" (err=%d)\n", evioLocalFileName, err); // Download data file DownloadFile(); // Resume triggers (if they were already going printf("--- resuming\n"); TESTflag = save_TESTflag; } // If flag is set, then send the response message if(sendResponse) cMsgSend(domainID, cmsg); // Free the response message cMsgFreeMessage(&cmsg); }