/************************************************************************* * * ti_slave_list.c - Library of routines for readout and buffering of * events using a JLAB Trigger Interface V3 (TI) with * a Linux VME controller in CODA 3.0. * * This is for a TI in Slave Mode controlled by a * Master TI or Trigger Supervisor * */ /* Event Buffer definitions */ #define MAX_EVENT_POOL 20 #define MAX_EVENT_LENGTH 1024*64 /* Size in Bytes */ /* Define TI Type (TI_MASTER or TI_SLAVE) */ #define TI_SLAVE /* TS Fiber Link trigger source (from TI Master, TD, or TS), POLL for available data */ #define TI_READOUT TI_READOUT_TS_POLL /* TI VME address, or 0 for Auto Initialize (search for TI by slot) */ #define TI_ADDR 0 /* Measured longest fiber length in system */ #define FIBER_LATENCY_OFFSET 0x4A //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // This ROL needs to do things in the __done and __reset routines // which tiprimary_list.c does not provide a callback for. Temporarily // define these as __reset_default and __done_default so we can define // our own in this file and call the ones defined in tiprimary_list.c // from there. #define __done __done_default #define __reset __reset_default //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> #include "dmaBankTools.h" /* Macros for handling CODA banks */ #include "tiprimary_list.c" /* Source required for CODA readout lists using the TI */ /* Define buffering level */ #define BUFFERLEVEL 10 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // (see not above on __reset_default) #undef __done #undef __reset #include #include #include // The evio input file name is passed in as the user string // during prestart and so is set below. The string may // contain a "%d" which will be replaced with the value // in rol->pid (aka the ROC id). static char evioFileName[1024] = "nada"; // full path on gluonraid1 static char evioLocalFileName[1024] = "nada"; // path to file on local RAM disk // For EVIO static int handle=0; #define MCBUFSIZE 10000 static uint32_t mcbuffer[MCBUFSIZE]; // For cMsg char UDL[512] = "cMsg://gluondb1.jlab.org/cMsg/softROC"; char cMsgName[256] = "rocXXX"; char cMsgDescr[256] = "coda_roc running software trigger"; void *domainID = NULL; void MyCallback(void *msg, void *userArg); // This is used to slow down and roughly synchronize the // events between ROCs. If we are connected to the cMsg // server (i.e. if domainID is not NULL) then every // event will decrement this counter in the __done() // routine until it gets to 0. Then, it will go into // a sleep loop waiting for the counter to be non-zero. // If __reset() or __prestart() are called then // this is reset to "1". int USE_SOFTROC_CONTROLLER = 0; unsigned long Nevents_left = 1000; int END_AT_NEXT_BOUNDARY = 0; int blklevel = 1; int trigBankType = 0xff11; //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> /**************************************** * DOWNLOAD ****************************************/ void rocDownload() { int stat; /* Setup Address and data modes for DMA transfers * * vmeDmaConfig(addrType, dataType, sstMode); * * addrType = 0 (A16) 1 (A24) 2 (A32) * dataType = 0 (D16) 1 (D32) 2 (BLK32) 3 (MBLK) 4 (2eVME) 5 (2eSST) * sstMode = 0 (SST160) 1 (SST267) 2 (SST320) */ vmeDmaConfig(2,5,1); /* Define BLock Level variable to a default */ blockLevel = 1; /***************** * TI SETUP *****************/ /* Set the sync delay width to 0x40*32 = 2.048us */ tiSetSyncDelayWidth(0x54, 0x40, 1); /* Set Trigger Buffer Level */ tiSetBlockBufferLevel(BUFFERLEVEL); /* Init the SD library so we can get status info */ stat = sdInit(); if(stat==0) { tiSetBusySource(TI_BUSY_SWB,1); sdSetActiveVmeSlots(0); sdStatus(0); } else { /* No SD or the TI is not in the Proper slot */ tiSetBusySource(TI_BUSY_LOOPBACK,1); } tiStatus(0); /* daLogMsg("ERROR","rocDownload: Test Error message\n"); */ //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // Turn off by default until we get a message from controller // This allows us to run without a controller USE_SOFTROC_CONTROLLER = 0; // Connect to remote cMsg server. // (This will be disconnected in __reset() ) sprintf(cMsgName, "roc%02d", rol->pid); int err = cMsgConnect(UDL, cMsgName, cMsgDescr, &domainID); if(err!=0){ printf("Could not connect to cMsg server! Remote event governor disabled.\n"); }else{ printf("Connected to cMsg server using: %s\n", UDL); printf("-- cMsg client name: %s\n", cMsgName); cMsgSubscribeConfig *config = cMsgSubscribeConfigCreate(); cMsgSubscribe(domainID, "softROC", "*", MyCallback, NULL, config, NULL); cMsgSubscribe(domainID, cMsgName, "*", MyCallback, NULL, config, NULL); cMsgReceiveStart(domainID); } // Full file name (on possibly remote system) sprintf(evioFileName, rol->usrString, rol->pid); // replace first %d with ROC id // Get full path to file on RAM disk int pos = strlen(evioFileName); for(pos--; pos>0; pos--) if(evioFileName[pos]=='/') break; sprintf(evioLocalFileName, "/dev/shm/%s", &evioFileName[++pos]); // If the file is available on the local filesystem then copy it from there. // Otherwise, we'll have to scp it from the raid disk. We create a command // that will copy it to the RAM disk either via cp or scp and execute it below. char cmd[512]; FILE *f = fopen(evioFileName, "r"); if(f != NULL){ // File exists locally. Close it. fclose(f); // This is option is mainly for use with a sandbox CODA setup // (like used in a virtual machine). This seems to take a // while to do the copy so check if the file already exists // on the RAM disk and don't bother recopying if it does. f = fopen(evioLocalFileName, "r"); if(f != NULL){ // File exists on RAM disk. Do not re-copy fclose(f); sprintf(cmd, "echo file %s already exists. Skipping copy", evioLocalFileName); }else{ // File does not exist on RAM disk sprintf(cmd, "cp %s %s", evioFileName, evioLocalFileName); } }else{ // Download data file to RAM disk. We must do this via scp // since some ROCs do not seem to have /gluonraid1 mounted sprintf(cmd, "scp gluonraid1-daq:%s %s", evioFileName, evioLocalFileName); } // Run the command and print result printf("%s\n", cmd); int status = system(cmd); printf("Command result: %d\n", status); //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> printf("rocDownload: User Download Executed\n"); } /**************************************** * PRESTART ****************************************/ void rocPrestart() { unsigned short iflag; int stat; int islot; /* Unlock the VME Mutex */ vmeBusUnlock(); tiStatus(0); //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // open evio file if(evOpen(evioLocalFileName,"r",&handle)!=S_SUCCESS) { printf("?unable to open file %s\n",evioLocalFileName); } else { printf("successfully opened file %s\n",evioLocalFileName); } // Initialize counter Nevents_left = 1000; END_AT_NEXT_BOUNDARY = 0; //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> printf("rocPrestart: User Prestart Executed\n"); } /**************************************** * GO ****************************************/ void rocGo() { int islot; /* Get the broadcasted Block Level from TS or TI Master */ blockLevel = tiGetCurrentBlockLevel(); printf("rocGo: Block Level set to %d\n",blockLevel); /* Enable/Set Block Level on modules, if needed, here */ } /**************************************** * END ****************************************/ void rocEnd() { int islot; tiStatus(0); //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // close evio file if(handle) evClose(handle); handle = 0; //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> printf("rocEnd: Ended after %d blocks\n",tiGetIntCount()); } /**************************************** * TRIGGER ****************************************/ void rocTrigger(int arg) { int ii, islot; int dCnt, len=0, idata; unsigned int val; unsigned int *start; /* Set TI output 1 high for diagnostics */ tiSetOutputPort(1,0,0,0); /* Readout the trigger block from the TI Trigger Block MUST be reaodut first */ dCnt = tiReadTriggerBlock(dma_dabufp); if(dCnt<=0) { printf("No data or error. dCnt = %d\n",dCnt); } else { /* TI Data is already in a bank structure. Bump the pointer */ dma_dabufp += dCnt; } /* EXAMPLE: How to open a bank (type=5) and add data words by hand */ BANKOPEN(5,BT_UI4,blockLevel); *dma_dabufp++ = tiGetIntCount(); *dma_dabufp++ = 0xdead; *dma_dabufp++ = 0xcebaf111; *dma_dabufp++ = 0xcebaf222; BANKCLOSE; //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> // Read in event from file int stat=evRead(handle,mcbuffer,MCBUFSIZE); if(stat!=S_SUCCESS) { evClose(handle); handle = 0; if(evOpen(evioLocalFileName,"r",&handle)!=S_SUCCESS) { printf("?unable to reopen file %s\n",evioFileName); } else { printf("successfully reopened file %s\n",evioFileName); stat=evRead(handle,mcbuffer,MCBUFSIZE); } } // Write EVIO event to ouput buffer if(stat==S_SUCCESS) { BANKOPEN(6,BT_UI4,blockLevel); // copy from local to output buffer uint32_t nword = mcbuffer[0]+1; memcpy((char*)dma_dabufp,(char*)&(mcbuffer[0]),nword*sizeof(uint32_t)); dma_dabufp += nword; BANKCLOSE; } //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> /* Set TI output 0 low */ tiSetOutputPort(0,0,0,0); } /**************************************** * rocCleanup (called when ROL is detached) ****************************************/ void rocCleanup() { int islot=0; printf("%s: Reset all Modules\n",__FUNCTION__); } //------------------------- // done //------------------------- void __done() { __done_default(); // call routine defined in tiprimary_list.c if(USE_SOFTROC_CONTROLLER!=0 && domainID!=NULL){ // We are connected to a cMsg server so decrement // the event counter and check it to see if we need // to wait here for a cMsg to tell us to continue. Nevents_left--; if(Nevents_left == 0){ if(END_AT_NEXT_BOUNDARY){ printf("Finished block of triggers and ending...\n"); return; }else{ printf("Finished block of triggers. Waiting for softROCcontroller..\n"); void *cmsg = cMsgCreateMessage(); cMsgSetSubject(cmsg, "softROCcontroller"); cMsgSetType(cmsg, cMsgName); cMsgSetText(cmsg, "ready"); cMsgSend(domainID, cmsg); cMsgFreeMessage(&cmsg); } } while(Nevents_left == 0)usleep(100000); } } //------------------------- // reset //------------------------- void __reset () { __reset_default(); // call routine defined in tiprimary_list.c // Disconnect from cMsg server if connected if(domainID != NULL){ printf("Disconnecting from cMsg server ... \n"); cMsgReceiveStop(domainID); cMsgDisconnect(&domainID); domainID = NULL; }else{ printf("No cMsg connection so skipping disconnect.\n"); } Nevents_left = 1; } //------------------------- // MyCallback //------------------------- void MyCallback(void *msg, void *userArg) { const char *subject = NULL; const char *type = NULL; const char *cmd = NULL; cMsgGetSubject(msg, &subject); cMsgGetType(msg, &type); cMsgGetText(msg, &cmd); printf("cMsg received:\n"); printf(" subject: %s\n", subject ? subject:""); printf(" type: %s\n", type ? type:""); printf(" cmd: %s\n", cmd ? cmd:""); // If we get any kind of message, it must mean a // softROCcontroller is running USE_SOFTROC_CONTROLLER = 1; // prepare response message in case we need to send it void *cmsg = cMsgCreateMessage(); cMsgSetSubject(cmsg, type); cMsgSetType(cmsg, cMsgName); int sendResponse = 0; if(!strcmp(cmd, "roll call")){ cMsgSetText(cmsg, "present"); sendResponse = 1; }else if(!strcmp(cmd, "process triggers")){ cMsgGetUserInt(msg, &Nevents_left); printf("--- set Nevents_left to: %d\n", Nevents_left); } // If flag is set, then send the response message if(sendResponse) cMsgSend(domainID, cmsg); // Free the response message cMsgFreeMessage(&cmsg); }