#!/usr/bin/python # # On the gluons, this will only work with the system-installed # python. (the python in /gapps won't work). Thus, we revert the # top line to refer to that rather than the python found via # the environment as in the following: # #!/usr/bin/env python import sys import os import glob import time import math import string import ctypes import gobject import gtk import locale import subprocess from epics import caget from threading import * class nodeInfo(ctypes.Structure): _fields_ = [('name', ctypes.c_char_p), ('node', ctypes.c_char_p), ('Nthreads', ctypes.c_uint), ('Nevents', ctypes.c_ulong), ('level', ctypes.c_char_p), ('rate', ctypes.c_double), ('cpu', ctypes.c_double), ('Nrespawns', ctypes.c_uint)] LogsWin = None HelpWin = None #<><><><><><><><><><><><><><><><> # HDHelpWindow #<><><><><><><><><><><><><><><><> class HDHelpWindow(gtk.Window): # Useful parameters defpad = 15 # default padding in pixels #--------------------- # Done #--------------------- def Done(self, widget=None, event=None, data=None): self.hide() return True #--------------------- # __init__ #--------------------- def __init__(self): super(HDHelpWindow, self).__init__() self.set_size_request(500, 900) self.set_position(gtk.WIN_POS_CENTER) self.connect("destroy", self.Done) self.connect("delete_event", self.Done) self.set_title("Hall-D Data Monitoring Help") # Top-level frame self.Fmain = gtk.VBox(False, self.defpad) self.add(self.Fmain) #------ Title frame self.Ftitle = gtk.HBox(False, 0) self.Fmain.pack_start(self.Ftitle, False, False, 5) label = gtk.Label() label.set_use_markup(True) label.set_markup('Hall-D Data Monitoring Help') self.Ftitle.pack_start(label, True,False, 0) #------ Middle content frame self.Fmid = gtk.HBox(False, self.defpad) self.Fmain.pack_start(self.Fmid, True, True, self.defpad) self.sw = gtk.ScrolledWindow() self.sw.set_shadow_type(gtk.SHADOW_ETCHED_IN) self.sw.set_policy(gtk.POLICY_ALWAYS, gtk.POLICY_ALWAYS) self.logLab = gtk.Label(""" OVERVIEW ---------------------------- The hdmongui.py program can be used to launch and/or monitor the health of the monitoring farm processes. It launches and kills processes by running the start_monitoring script (which can also be run by hand without this program). The monitoring function is done by continuosly communicating with each farm process via cMsg. cMsg Connection UDL ---------------------------- Communication with the farm processes is done via cMsg using the server specified by the value of the JANACTL_UDL environment variable which is normally set to: cMsg://gluondb1/cMsg/janactl If it is not set, the following is used: cMsg://localhost/cMsg/janactl This can be changed by changing your environment variable and relaunching hdmongui.py. Note, however, that monitoring processes that were already started using a different UDL will continue to communicate only through that UDL and must be killed and restarted in order to use the new UDL. LEVELS ---------------------------- The monitoring system is designed to support multiple "levels" of monitoring. The main application of this is to have some nodes process at high rate to fill histograms with large statistics. Other nodes can do a more full analysis of each event at a slower rate. There is no limit to the number of levels one can have. The levels are set by files matching the following naming pattern: ${DAQ_HOME}/config/monitoring/hdmon*.conf where DAQ_HOME is an environment varaible (normally set to something like "/home/hdops/CDAQ/daq_dev_v0.31/daq") and the "*" part defines the level name. Level names can be as simple as "1" and "2" or more complex strings like "testing". These would correspond to configuration file names "hdmon1.conf", "hdmon2.conf", and "hdmontesting.conf". To create a new level, simply create a new configuration file in the monitoring directory. Keep in mind though that users will usually want to start monitoring for all levels so don't leave extra configuration files lying around that may accidentally get used by unsuspecting shift workers. Also keep in mind that you will need to assign nodes to the new level for any processing to be done (see next section). NODE ASSIGNMENTS ---------------------------- The nodes assigned to run proesses for each level are specified in the file: ${DAQ_HOME}/config/monitoring/nodes.conf Each line has two items. The first specifies what type of process to run and the second is the node name. For example: mon2 gluon110 would specify that a level "2" monitoring process should be run on gluon110. Two important things to note: - A node can appear more than once in this file and so may have more than one process launched there - Other node assignments may be made in this file including L3 trigger nodes and the RootSpy archiver node EVENT SOURCE ---------------------------- The source of events used by the monitoring processes may be specified in multiple ways. If the "COOL" box is checked then the parameters for the ET system are extracted for the current CODA configuration (COOL configuration). Usually, this is what shift workers will want to do. If the "COOL" box is not checked, then the value in the "User specified" entry is used. This has the same format as all DANA (i.e. sim-recon) programs accept. Thus, you can actually put a file name there, but that would be of limited use since all monitoring processes would read from it independently. In most all cases, one should specify an ET system from which to read events. In the Hall-D syntax, an ET system is specified by: ET:filename:station:host:port where: filename is the name of the ET memory mapped file which is usually something like /tmp/et_hdops_ERsoftROC station is the name of the station on the ET system to attach to. It will be created if it does not exist. Usually, this should be something like "MON" or "MON1". host hostname of the ET system to attach to (e.g. gluonraid2) port TCP port the ET system is listening on. This may be one of 11111 or 23921 or something else altogether """) self.logLab.set_use_markup(True) self.logLab.set_line_wrap(False) self.logLab.set_selectable(False) eb = gtk.EventBox() # must use EventBox to get white background (Sheesh!) eb.add(self.logLab) eb.modify_bg(gtk.STATE_NORMAL, gtk.gdk.color_parse("white")) self.sw.add_with_viewport(eb) self.Fmid.pack_start(self.sw, True, True, 5) #------ Bottom buttons frame self.Fbottom = gtk.HBox(False, 8) self.Fmain.pack_end(self.Fbottom, False, True, self.defpad) self.Bquit = gtk.Button('Close') self.Fbottom.pack_end(self.Bquit, False, False, self.defpad) self.Bquit.connect("clicked", self.Done) self.show_all() #<><><><><><><><><><><><><><><><> # HDLogsWindow #<><><><><><><><><><><><><><><><> class HDLogsWindow(gtk.Window): # Useful parameters defpad = 15 # default padding in pixels logdir = os.getenv('HDLOG', os.getenv('HOME', '/tmp')) #--------------------- # Quit #--------------------- def Quit(self, widget=None, event=None, data=None): self.hide() return True #--------------------- # Refresh #--------------------- def Refresh(self, widget=None, event=None, data=None): (model, tree_iter) = self.tree_selection.get_selected() if tree_iter == None: self.logLab.set_markup('') else: node = model.get_value(tree_iter,0) self.ReadLogFile(node) return True #--------------------- # UpdateFileList #--------------------- def UpdateFileList(self, widget=None, event=None, data=None): # Get list of files ordered by modification time logfiles = filter(os.path.isfile, glob.glob(self.logdir + "/hdmon_*.log")) logfiles.sort(key=lambda x: os.path.getmtime(x)) # Loop over rows currently displayed and delete any for which files no longer exist displayed_files = [] treeiter = self.store.get_iter_first() while treeiter != None: node_name = self.store.get_value(treeiter, 0) fullpath = self.logdir + "/hdmon_%s.log" % node_name if fullpath not in logfiles: # Delete row. (treeiter is left pointing to next row or False is returned) if not self.store.remove(treeiter) : treeiter = None else: displayed_files.append(fullpath) treeiter = self.store.iter_next(treeiter) # Loop over file list and create new rows for any not currently displayed for file in logfiles: if file not in displayed_files: node = file[len(self.logdir)+7:-4] # cut node name out of full path name to log file vals = [node] self.store.prepend(vals) return True #--------------------- # MakeTreeView #--------------------- def MakeTreeView(self): # Create ListStore with correct number of columns self.store = gtk.ListStore(gobject.TYPE_STRING) self.treeView = gtk.TreeView(self.store) self.treeView.set_rules_hint(True) self.treeView.set_enable_tree_lines(True) self.tree_selection = self.treeView.get_selection() self.tree_selection.set_mode(gtk.SELECTION_SINGLE) self.tree_selection.connect("changed", self.Refresh) textid = 0 col_node = gtk.TreeViewColumn('node', gtk.CellRendererText(), text=textid) col_node.set_sort_column_id(textid) self.treeView.append_column(col_node) #--------------------- # HighlightLine #--------------------- def HighlightLine(self, line): if line.endswith('\n'): line = line[:-1] # Linux draws an empty label if "<" or ">" charaters are present line = line.replace('&', '&') line = line.replace('<', '<') line = line.replace('>', '>') if line.find('hdlog starting')>=0 : return '' + line + '' if line.find('hdlog ending')>=0 : return '' + line + '' line = line.replace('ERROR','ERROR') line = line.replace('Error','Error') line = line.replace('error','error') # bad things happen when tags not balanced Nopens = line.count('[1m') + line.count('[1;31m') Ncloses = line.count('[1m') + line.count('[1;31m') if Nopens == Ncloses: line = line.replace('[1m','') line = line.replace('[1;31m','') line = line.replace('[0m','') else: line = line.replace('[1m','*') line = line.replace('[1;31m','*') line = line.replace('[0m','*') return line #--------------------- # LogContentChanged # # This is used to scroll the log content window to the bottom of the # file whenever the label's size changes (which should usually happen # when a new log file is read in). Oddly enough, just setting the # vadjustment in the ReadLogFile method doesn't work as expected since # the label's size change is actually delayed. #--------------------- def LogContentChanged(self, widget, event, data=None): adj = self.sw.get_vadjustment() adj.set_value( adj.upper - adj.page_size ) #--------------------- # ReadLogFile #--------------------- def ReadLogFile(self, node): # This reads in and displays an entire log file. It modifies the # contents of the file line by line using the HighlightLine method # of this class. In order to display, it needs certain characters # altered or removed. (See HighlightLine for details). # # Reading and processing the file is very fast, but it takes some # time for pygtk to render it after the self.logLab.set_markup(str) # call at the end. Therefore, we print a message to the screen # telling the user to be patient # Form log filename given the node name fname = '%s/hdmon_%s.log' % (self.logdir, node) # Clear the screen and print a message to let user know self.logLab.set_markup(' Preparing file "%s" for display (this may take several seconds)...\n\n' % fname) while gtk.events_pending(): gtk.main_iteration_do(True) lines = [] with open (fname, "r") as myfile: lines=myfile.readlines() # Highlight certain strings str = '' iline = 0 for line in lines: if 'ET stalled ...' in line : continue iline += 1 if iline < (len(lines) - 1000): continue modified = self.HighlightLine(line) str += modified + '\n' # Strip out any non-printable characters. Without this, it seems # nothing will be displayed in the label on Linux if a non-printable # character is present str = filter(lambda x: x in string.printable, str) # Tell user how many lines we're about to display and include estimate # on how long it will take. The time appears to be non-linear with # time. A quick measurement of 3 different files gives these times # when run on some typical files on gluon03: # # lines seconds # ------ ------- # 4000 2.05 # 9015 6.83 # 11309 11.40 # # An exponential fit to this gives: # # t = exp(-0.162728+0.0002298*Nlines) t = math.exp(-0.162728+0.0002298*float(len(lines))) self.logLab.set_markup(' Preparing file "%s" for display (this may take several seconds)...\n (%d lines ~ %3.1f seconds)\n' % (fname, len(lines), t)) while gtk.events_pending(): gtk.main_iteration_do(True) #str = 'Hello World!' self.logLab.set_markup(str) #--------------------- # __init__ #--------------------- def __init__(self): super(HDLogsWindow, self).__init__() self.set_size_request(800, 700) self.set_position(gtk.WIN_POS_CENTER) self.connect("destroy", self.Quit) self.connect("delete_event", self.Quit) self.set_title("Hall-D Data Monitoring Logs") # Top-level frame self.Fmain = gtk.VBox(False, self.defpad) self.add(self.Fmain) #------ Title frame self.Ftitle = gtk.HBox(False, 0) self.Fmain.pack_start(self.Ftitle, False, False, 5) label = gtk.Label() label.set_use_markup(True) label.set_markup('Hall-D Data Monitoring Logs') self.Ftitle.pack_start(label, True,False, 0) #------ Middle content frame self.Fmid = gtk.HBox(False, self.defpad) self.Fmain.pack_start(self.Fmid, True, True, self.defpad) self.MakeTreeView() sw = gtk.ScrolledWindow() sw.set_shadow_type(gtk.SHADOW_ETCHED_IN) sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.treeView) sw.set_size_request(130, 10) self.Fmid.pack_start(sw, False, True, 5) #------ Middle-right content frame self.Fmidright = gtk.VBox(False, self.defpad) self.Fmid.pack_start(self.Fmidright, True, True, 5) # buttons self.Fmidrighttop = gtk.HBox(False, self.defpad) self.Fmidright.pack_start(self.Fmidrighttop, False, True, 5) self.Bnext = gtk.Button('next >>') self.Fmidrighttop.pack_end(self.Bnext, False, False, self.defpad) #self.Bnext.connect("clicked", self.Next) self.Bprev = gtk.Button('<< prev') self.Fmidrighttop.pack_end(self.Bprev, False, False, self.defpad) #self.Bprev.connect("clicked", self.Prev) # log file content area self.sw = gtk.ScrolledWindow() self.sw.set_shadow_type(gtk.SHADOW_ETCHED_IN) self.sw.set_policy(gtk.POLICY_ALWAYS, gtk.POLICY_ALWAYS) self.logLab = gtk.Label() self.logLab.set_use_markup(True) self.logLab.set_line_wrap(True) self.logLab.set_size_request(1000,-1) self.logLab.set_padding(10,2) self.logLab.set_selectable(True) eb = gtk.EventBox() # must use EventBox to get white background (Sheesh!) eb.add(self.logLab) eb.modify_bg(gtk.STATE_NORMAL, gtk.gdk.color_parse("white")) self.sw.add_with_viewport(eb) self.Fmidright.pack_start(self.sw, True, True, 5) # (see comment in LogContentChanged) self.logLab.connect('size-allocate', self.LogContentChanged) #------ Bottom buttons frame self.Fbottom = gtk.HBox(False, 8) self.Fmain.pack_end(self.Fbottom, False, True, self.defpad) self.Bquit = gtk.Button('Close') self.Fbottom.pack_end(self.Bquit, False, False, self.defpad) self.Bquit.connect("clicked", self.Quit) self.Bstop = gtk.Button('Refresh') self.Fbottom.pack_end(self.Bstop, False, False, self.defpad) self.Bstop.connect("clicked", self.Refresh) self.show_all() # Schedule the periodic update task self.UpdateFileList() gobject.timeout_add(3000, self.UpdateFileList) #self.ReadLogFile('harriet') #<><><><><><><><><><><><><><><><> # HDMonGUI #<><><><><><><><><><><><><><><><> class HDMonGUI(gtk.Window): # Useful parameters defpad = 15 # default padding in pixels udl = os.getenv('JANACTL_UDL', 'cMsg://localhost/cMsg/janactl') #--------------------- # LoadSharedLibrary #--------------------- def LoadSharedLibrary(self): # First, look relative to the actual script being run scriptdir = os.path.dirname(os.path.realpath(__file__)) fname = '%s/../plugins/hdmongui.so' % scriptdir if os.path.exists(fname): print 'Loading %s' % fname self.clib = ctypes.cdll.LoadLibrary(fname) return # Next, try looking in JANA_PLUGIN_PATH directories jana_plugin_path = os.getenv('JANA_PLUGIN_PATH', '') for path in jana_plugin_path.split(':'): fname = '%s/hdmongui.so' % path if os.path.exists(fname): print 'Loading %s' % fname self.clib = ctypes.cdll.LoadLibrary(fname) return print 'Unable to find hdmongui.so. Make sure JANA_PLUGIN_PATH is set!' sys.exit(-1) #--------------------- # Quit #--------------------- def Quit(self, widget=None, event=None, data=None): self.clib.Quit_cMsg() gtk.main_quit() return False #--------------------- # InfoDialog #--------------------- def InfoDialog(self, mess='what?'): dialog = gtk.MessageDialog(self, gtk.DIALOG_MODAL, gtk.MESSAGE_INFO, gtk.BUTTONS_OK, mess) response = dialog.run() dialog.destroy() #--------------------- # FormStartMonitoringCommand # # This generates the command used by both StarMonitoring and StopMonitoring. # The latter just appends a "-e". #--------------------- def FormStartMonitoringCommand(self, widget=None, event=None, data=None): cmd = ['start_monitoring'] if not self.CBautoSource.get_active(): # User specified source source = '-s%s' % self.Esource.get_text() cmd.append(source) # Try extracting run number from source in filename try: fname = str(self.Esource.get_text()) toks = fname.split('_') if len(toks)>2: RUN = int(toks[-2]) if int(RUN)>1 and int(RUN)<1E6 : cmd.append('-R%s' % RUN) except: pass else: # Use COOL (also explicitly set run number) RUN = caget("HD:coda:daq:run_number") print 'COOL source specified. Assuming online monitoring. Will start specifying run %s' % RUN if RUN != '' : if int(RUN)>1 and int(RUN)<1E6 : cmd.append('-R%s' % RUN) if self.CBRSArchiver.get_active() and self.CBautoSource.get_active(): cmd.append('-A') if self.CBRSTimeSeries.get_active(): cmd.append('-T') if self.CBRSAI.get_active(): cmd.append('-AI') if self.CBHydra.get_active(): cmd.append('-Hydra') if self.CBTestMode.get_active(): cmd.append('-testmode') level = self.CBlevel.get_active_text() if level != '< all levels >': cmd.append('-L%s' % level) if self.CBloop.get_active(): cmd.append('-loop') return cmd print(' '.join(cmd)) subprocess.call(cmd) return False #--------------------- # StartMonitoring #--------------------- def StartMonitoring(self, widget=None, event=None, data=None): cmd = self.FormStartMonitoringCommand() print(' '.join(cmd)) subprocess.call(cmd) return False #--------------------- # StopMonitoring #--------------------- def StopMonitoring(self, widget=None, event=None, data=None): cmd = self.FormStartMonitoringCommand() cmd.append('-e') print(' '.join(cmd)) subprocess.call(cmd) return False #--------------------- # StopAllNodes #--------------------- def StopAllNodes(self, widget=None, event=None, data=None): # Loop over all currently displayed nodes treeiter = self.store.get_iter_first() while treeiter != None: node_name = self.store.get_value(treeiter, 0) print 'stopping node_name=%s ...' % node_name treeiter = self.store.iter_next(treeiter) self.clib.QuitNode(node_name) gobject.timeout_add(2000, self.KillNode, node_name) return False #--------------------- # OpenLogsWindow #--------------------- def OpenLogsWindow(self, widget=None, event=None, data=None): global LogsWin if LogsWin != None: LogsWin.window.show() LogsWin.show_all() else: LogsWin = HDLogsWindow() return False #--------------------- # OpenHelpWindow #--------------------- def OpenHelpWindow(self, widget=None, event=None, data=None): global HelpWin if HelpWin != None: HelpWin.window.show() HelpWin.show_all() else: HelpWin = HDHelpWindow() return False #--------------------- # StopNode #--------------------- def StopNode(self, widget=None, event=None, data=None): node = self.detail_name.get_text() self.clib.QuitNode(node) gobject.timeout_add(2000, self.KillNode, node) return False #--------------------- # KillNode #--------------------- def KillNode(self, node): self.clib.KillNode(node) return False #--------------------- # ToggleSourceType #--------------------- def ToggleSourceType(self, widget=None, event=None, data=None): if self.CBautoSource.get_active(): self.CBloop.set_sensitive(False) self.sourceEntryLab.set_sensitive(False) self.Esource.set_sensitive(False) self.CBRSArchiver.set_sensitive(True) else: self.CBloop.set_sensitive(True) self.sourceEntryLab.set_sensitive(True) self.Esource.set_sensitive(True) self.CBRSArchiver.set_sensitive(False) self.CBTestMode.set_active(True) return False #--------------------- # UpdateNodesInfo #--------------------- def UpdateNodesInfo(self): # This will get the most recent node info from the plugin # and update the display with it. It's worth noting that this # could actually work by clearing the ListStore and just # re-adding all of the entries. The problem there is that it # loses track of any selected row(s). Hence, the more complicated # code. # Get most recent info from plugin array_size = 100 Nnodes = ctypes.c_uint(array_size) NodeInfoArray = nodeInfo*array_size nodes = NodeInfoArray() for i in range(0,array_size) : nodes[i].name = ctypes.create_string_buffer('',128).raw nodes[i].node = ctypes.create_string_buffer('',128).raw nodes[i].level = ctypes.create_string_buffer('',128).raw self.clib.GetRemoteProcesses(ctypes.c_double(5.0), ctypes.pointer(nodes), ctypes.byref(Nnodes)); # Make dictionary of node names that should be displayed so we can quickly # check if a row needs to have its values updated or be deleted. At the # same time, add up cumulative totals and update those labels node_info = {} tot_rate = 0.0 tot_nthreads = 0 tot_nevents = 0 tot_nnodes = Nnodes.value for i in range(0,Nnodes.value) : node_info[nodes[i].name] = nodes[i] tot_rate += nodes[i].rate tot_nthreads += nodes[i].Nthreads tot_nevents += nodes[i].Nevents # Update cumulative values self.total_rate.set_markup('%3.1f Hz' % tot_rate) self.total_events.set_markup('%s' % locale.format("%d", tot_nevents, grouping=True)) self.total_nodes.set_markup('%d' % tot_nnodes) self.total_threads.set_markup('%d' % tot_nthreads) # Record this for use by other methods self.node_info = node_info # Loop over rows currently displayed and either update or delete them displayed_node_names = [] treeiter = self.store.get_iter_first() while treeiter != None: node_name = self.store.get_value(treeiter, 0) if node_name in node_info: # Update existing row node = node_info[node_name] self.store.set(treeiter, 2, '%s' % node.level) self.store.set(treeiter, 3, '%d' % node.Nthreads) self.store.set(treeiter, 4, '%d' % node.Nevents) self.store.set(treeiter, 5, '%3.1f' % node.rate) self.store.set(treeiter, 6, '%3.1f%%' % (100.0-node.cpu)) self.store.set(treeiter, 7, '%4d' % node.Nrespawns) displayed_node_names.append(node_name) treeiter = self.store.iter_next(treeiter) else: # Delete row. (treeiter is left pointing to next row or False is returned) if not self.store.remove(treeiter) : treeiter = None # Loop over dictionary and create new rows for any not currently displayed for i in range(0,Nnodes.value) : if nodes[i].name in displayed_node_names : continue vals = [nodes[i].name, nodes[i].node, nodes[i].level, '%d' % nodes[i].Nthreads, '%d' % nodes[i].Nevents, '%3.1f' % nodes[i].rate, '%3.1f%%' % (100.0-float(nodes[i].cpu)), '%d' % nodes[i].Nrespawns] self.store.append(vals) #--------------------- # UpdateNodeInfo #--------------------- def UpdateNodeInfo(self, ts): # Get name of selected node (if any) (model, tree_iter) = ts.get_selected() if tree_iter == None: # nothing selected self.detail_name.set_text('---') self.detail_node.set_text('---') self.detail_threads.set_text('---') self.detail_source.set_text('---') self.detail_command.set_text('---') self.BstopNode.set_sensitive(False) return namestr = model.get_value(tree_iter,0) # Set name name = ctypes.c_char_p(namestr) self.detail_name.set_text(name.value) # Set node if name.value in self.node_info: self.detail_node.set_text(self.node_info[name.value].node) self.detail_threads.set_text('%d' % self.node_info[name.value].Nthreads) # Get source len = ctypes.c_uint(1024) src = ctypes.create_string_buffer('',len.value) self.clib.GetSource(name, ctypes.byref(src), len) self.detail_source.set_text(src.value) # Get Command line len = ctypes.c_uint(1024) cmd = ctypes.create_string_buffer('',len.value) self.clib.GetCommandLine(name, ctypes.byref(cmd), len) self.detail_command.set_text(cmd.value) # Enable stop button self.BstopNode.set_sensitive(True) #--------------------- # TestStuff #--------------------- def TestStuff(self, widget=None, event=None, data=None): # Get name of selected node (if any) (model, tree_iter) = self.tree_selection.get_selected() if tree_iter == None: # nothing selected self.detail_name.set_text('---') self.detail_node.set_text('---') self.detail_threads.set_text('---') self.detail_command.set_text('---') return namestr = model.get_value(tree_iter,0) # Set name name = ctypes.c_char_p(namestr) self.clib.GetSources(name) return False #--------------------- # MakeTreeView #--------------------- def MakeTreeView(self, cols): # Create ListStore with correct number of columns store = None if len(cols)== 1: store = gtk.ListStore(str) if len(cols)== 2: store = gtk.ListStore(str,str) if len(cols)== 3: store = gtk.ListStore(str,str,str) if len(cols)== 4: store = gtk.ListStore(str,str,str,str) if len(cols)== 5: store = gtk.ListStore(str,str,str,str,str) if len(cols)== 6: store = gtk.ListStore(str,str,str,str,str,str) if len(cols)== 7: store = gtk.ListStore(str,str,str,str,str,str,str) if len(cols)== 8: store = gtk.ListStore(str,str,str,str,str,str,str,str) if len(cols)== 9: store = gtk.ListStore(str,str,str,str,str,str,str,str,str) if len(cols)==10: store = gtk.ListStore(str,str,str,str,str,str,str,str,str,str) self.store = store self.treeView = gtk.TreeView(self.store) self.treeView.set_rules_hint(True) self.treeView.set_enable_tree_lines(True) self.tree_selection = self.treeView.get_selection() self.tree_selection.set_mode(gtk.SELECTION_SINGLE) self.tree_selection.connect("changed", self.UpdateNodeInfo) textid = 0 for col in cols: rendererText = gtk.CellRendererText() column = gtk.TreeViewColumn(col, rendererText, text=textid) column.set_sort_column_id(textid) self.treeView.append_column(column) textid += 1 #--------------------- # AddLabel # # This adds a label to the Node Detail section. The Label is # updated in the UpdateNodeInfo routine #--------------------- def AddLabel(self, tab, label, value='---', size='', button=None): lab = gtk.Label(label) val = gtk.Label(value) lab.set_alignment(1, 0.5) val.set_alignment(0, 0.5) val.set_line_wrap(True) Nrows = tab.get_property('n-rows') Ncols = tab.get_property('n-columns') if button!=None and Ncols<3: Ncols=3 tab.resize(Nrows+1, Ncols) tab.attach(lab, 0, 1, Nrows, Nrows+1, gtk.FILL, xpadding=10, ypadding=5) tab.attach(val, 1, 2, Nrows, Nrows+1, gtk.EXPAND | gtk.FILL, ypadding=5) if button!=None: tab.attach(button, 2, 3, Nrows, Nrows+1, ypadding=5) # if 'size' is specified, set label to use pango mark-up if size != '' : lab.set_use_markup(True) val.set_use_markup(True) lab.set_markup('%s' % (size, label)) val.set_markup('%s' % (size, value)) return val #----------------------- # FindConfigMonLevels # # The monitoring system supports multiple levels. # Look for files in the config directory matching the hdmonXXX.conf pattern # and return a list of all the "XXX" parts. (These should correspond to labels # in the nodes.conf file) #----------------------- def FindConfigMonLevels(self): # Get current mon levels mon_levels = ['< all levels >'] fname = '%s/config/monitoring/nodes.conf' % os.getenv('DAQ_HOME', '/gluex') if os.path.exists(fname) : file = open(fname, 'r') for line in file: vals = line.split() if line.find('#') == 0 : continue # skip comment lines if len(vals)==0 : continue # skip empty lines if vals[0]!='mon' : continue # skip non-"mon" lines if len(vals)<3 : print 'too few tokens in mon line in %s :' % fname print line else: levels = vals[2].split(',') if len(levels)>0 : mon_levels.extend(levels) mon_levels = sorted(set(mon_levels)) # remove duplicates and sort the levels alphabetically # Remove any existing entries that have disappeared old_mon_levels = [] if hasattr(self, 'mon_levels') : old_mon_levels = self.mon_levels for i in range(len(old_mon_levels)): pos = len(old_mon_levels) - 1 - i level = old_mon_levels[pos] if level not in mon_levels : self.CBlevel.remove(pos) # Add any new entries that have appeared for level in mon_levels: if level not in old_mon_levels : self.CBlevel.append_text(level) # Select first item if there is no selection if self.CBlevel.get_active() == -1 : self.CBlevel.set_active(0) # record current mon_levels self.mon_levels = mon_levels #--------------------- # UpdateAll #--------------------- def UpdateAll(self): # This method is called every 2000 msec so long as it does not return "False" # This is scheduled at the end of __init__ self.UpdateNodesInfo() self.UpdateNodeInfo(self.tree_selection) self.FindConfigMonLevels() return True #--------------------- # __init__ #--------------------- def __init__(self): super(HDMonGUI, self).__init__() self.LoadSharedLibrary() self.clib.Init_cMsg(ctypes.c_char_p(self.udl)); self.set_size_request(1300, 750) self.set_position(gtk.WIN_POS_CENTER) self.connect("destroy", gtk.main_quit) self.connect("delete_event", self.Quit) self.set_title("Hall-D Data Monitoring Controls") # Initialize some member variables self.node_info = {} # Top-level frame self.Fmain = gtk.VBox(False, self.defpad) self.add(self.Fmain) #------ Title frame self.Ftitle = gtk.HBox(False, 0) self.Fmain.pack_start(self.Ftitle, False, False, 5) label = gtk.Label() label.set_use_markup(True) label.set_markup('Hall-D Data Monitoring Farm Status') self.Ftitle.pack_start(label, True,False, 0) #------ Top content frame self.Ftop = gtk.VBox(False, self.defpad) self.Fmain.pack_start(self.Ftop, False, True, 0) # UDL self.Fudl = gtk.HBox(False, 5) self.Ftop.pack_start(self.Fudl, False, False, 0) label = gtk.Label('UDL:') self.Fudl.pack_start(label, False, False, 4) self.Ludl = gtk.Label(self.udl) self.Fudl.pack_start(self.Ludl, False, False, 4) self.Bchange_udl = gtk.Button('change') self.Fudl.pack_start(self.Bchange_udl, False, False, 10) # drawing_area = gtk.DrawingArea() # drawing_area.set_size_request(600, 100) # self.Ftop.pack_start(drawing_area, False, False, 0) # drawing_area.realize() # self.drawable = drawing_area.window # color_white = gtk.gdk.Color('white') # self.gc_white = self.drawable.new_gc(fill=1, foreground = color_white, background = color_white) # self.drawable.draw_rectangle(self.gc_white, True, 0,0, 600, 100) #------ Middle content frame self.Fmid = gtk.HBox(True, self.defpad) self.Fmain.pack_start(self.Fmid, True, True, self.defpad) #-- Middle left content frame self.Fmidleft = gtk.HBox(False, self.defpad) self.Fmid.pack_start(self.Fmidleft, True, True, self.defpad) colnames = ['name', 'node', 'level', 'Nthr', 'Nevents', 'rate (Hz)', 'idle', 'respawns'] self.MakeTreeView(colnames) sw = gtk.ScrolledWindow() sw.set_shadow_type(gtk.SHADOW_ETCHED_IN) sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.treeView) self.Fmidleft.pack_start(sw, True, True, 20) #-- Middle right content frame self.Fmidright = gtk.VBox(False, self.defpad) self.Fmid.pack_start(self.Fmidright, True, True, self.defpad) # Cumulative details self.Fcumulativedetail = gtk.Frame('Totals') self.Tcumulativedetail = gtk.Table(1,1) self.Fcumulativedetail.add(self.Tcumulativedetail) self.Fmidright.pack_start(self.Fcumulativedetail, False, True, self.defpad) self.total_rate = self.AddLabel(self.Tcumulativedetail, 'Total Rate:', '--', 'xx-large') self.total_events = self.AddLabel(self.Tcumulativedetail, 'Total Events:', '--', 'x-large') self.total_nodes = self.AddLabel(self.Tcumulativedetail, 'Total Nodes:', '--', 'x-large') self.total_threads = self.AddLabel(self.Tcumulativedetail, 'Total Threads:', '--', 'x-large') # Node details for selected node self.Fnodedetail = gtk.Frame('Node Details') self.FFnodedetail = gtk.VBox(False, self.defpad) self.Fnodedetail.add(self.FFnodedetail) self.Fmidright.pack_start(self.Fnodedetail, False, True, self.defpad) self.Tnodedetail = gtk.Table(1,1) self.FFnodedetail.pack_start(self.Tnodedetail, False, True, self.defpad) self.BstopNode = gtk.Button('stop node') #self.FFnodedetail.pack_start(self.BstopNode, False, False, 0) self.BstopNode.set_sensitive(False) self.BstopNode.connect("clicked", self.StopNode) self.detail_name = self.AddLabel(self.Tnodedetail, 'Name:', '--', button=self.BstopNode) self.detail_node = self.AddLabel(self.Tnodedetail, 'Node:', '--') self.detail_threads = self.AddLabel(self.Tnodedetail, 'Threads:', '--') self.detail_source = self.AddLabel(self.Tnodedetail, 'Event Source:', '--') self.detail_command = self.AddLabel(self.Tnodedetail, 'command line:', '--') # Extra space under details is used here so table is compact spacer = gtk.HBox(True, 0) self.Fmidright.pack_start(spacer, True, True, 0) #------ Bottom source frame self.Fsource = gtk.HBox(False,2) self.Fmain.pack_start(self.Fsource, False, True, 0) sourceLab = gtk.Label('Event Source:') self.CBautoSource = gtk.CheckButton(label='COOL') self.CBloop = gtk.CheckButton(label='loop') self.sourceEntryLab = gtk.Label(' User specified:') self.Fsource.pack_start(sourceLab, False, False, 5) self.Fsource.pack_start(self.CBautoSource, False, False, 0) self.Fsource.pack_start(self.sourceEntryLab, False, False, 5) self.Esource = gtk.Entry() self.Esource.set_text('ET:/tmp/et_sys_monitoring:MON:gluonraid1-ib:11122') self.Esource_example = gtk.Label() self.Esource_example.set_markup('( format is ET:et_sys_filename:station:host:port e.g. ET:/tmp/et_sysy_hdops:MON:gluon48:11111 or /gluex/data/rawdata/all/Run051088/hd_rawdata*.evio)') self.Esource_example.set_alignment(0.0, 0.5) self.Fsourceentry = gtk.VBox(False,0) self.Fsourceentry.pack_start(self.Esource, True, True, 0) self.Fsourceentry.pack_start(self.Esource_example, True, True, 0) self.Fsource.pack_start(self.Fsourceentry, True, True, 0) self.Fsource.pack_start(self.CBloop, False, False, 0) # set to "online" by default self.CBautoSource.set_active(True) self.CBloop.set_active(True) self.CBloop.set_sensitive(False) self.sourceEntryLab.set_sensitive(False) self.Esource.set_sensitive(False) self.CBautoSource.connect("toggled", self.ToggleSourceType) #------ Bottom level selector frame + RSArchiver, RSTimeSeries, RSAI self.Flevel = gtk.HBox(False,2) self.Fmain.pack_start(self.Flevel, False, True, 0) levelLab = gtk.Label('Level:') self.CBlevel = gtk.combo_box_new_text() self.FindConfigMonLevels() self.Flevel.pack_start(levelLab, False, False, 5) self.Flevel.pack_start(self.CBlevel, False, False, 5) self.CBRSArchiver = gtk.CheckButton(label='RSArchiver') self.CBRSTimeSeries = gtk.CheckButton(label='RSTimeSeries') self.CBRSAI = gtk.CheckButton(label='RSAI') self.CBHydra = gtk.CheckButton(label='Hydra') self.CBTestMode = gtk.CheckButton(label='test mode') self.Flevel.pack_start(self.CBRSArchiver, False, False, 5) self.Flevel.pack_start(self.CBRSTimeSeries, False, False, 5) self.Flevel.pack_start(self.CBRSAI, False, False, 5) self.Flevel.pack_start(self.CBHydra, False, False, 5) self.Flevel.pack_start(self.CBTestMode, False, False, 5) self.CBRSArchiver.set_active(True) self.CBRSTimeSeries.set_active(True) self.CBRSAI.set_active(True) self.CBHydra.set_active(True) self.CBTestMode.set_active(False) #------ Bottom buttons frame self.Fbottom = gtk.HBox(False, 8) self.Fmain.pack_end(self.Fbottom, False, True, self.defpad) self.Bquit = gtk.Button('Quit') self.Fbottom.pack_end(self.Bquit, False, False, self.defpad) self.Bquit.connect("clicked", self.Quit) self.Bstart = gtk.Button('Start Monitoring') self.Fbottom.pack_start(self.Bstart, False, False, self.defpad) self.Bstart.connect("clicked", self.StartMonitoring) self.Bstop = gtk.Button('Stop Monitoring') self.Fbottom.pack_start(self.Bstop, False, False, self.defpad) self.Bstop.connect("clicked", self.StopMonitoring) self.Bstop_all_nodes= gtk.Button('Stop All Nodes') self.Fbottom.pack_start(self.Bstop_all_nodes, False, False, self.defpad) self.Bstop_all_nodes.connect("clicked", self.StopAllNodes) self.Blogs = gtk.Button('Logs') self.Fbottom.pack_start(self.Blogs, False, False, self.defpad) self.Blogs.connect("clicked", self.OpenLogsWindow) self.Bhelp = gtk.Button('Help') self.Fbottom.pack_start(self.Bhelp, False, False, self.defpad) self.Bhelp.connect("clicked", self.OpenHelpWindow) self.Btest = gtk.Button('TestStuff') #self.Fbottom.pack_start(self.Btest, False, False, self.defpad) self.Btest.connect("clicked", self.TestStuff) self.show_all() # Schedule the periodic update task gobject.timeout_add(2000, self.UpdateAll) #--------------------- # main #--------------------- def main(self): gtk.main() # If the program is run directly or passed as an argument to the python # interpreter then create a HelloWorld instance and show it if __name__ == "__main__": hdmongui = HDMonGUI() hdmongui.main()