#!/usr/bin/env python
#
# GUI for monitoring hdmoods system used for moving data files
# through the counting house and online skim system.
#
# This can be tested with the hdmoods_fake_server.py script
# (must be run on same system)
#

import os
import zmq
import json
import threading
import subprocess
import traceback
import time
import datetime
from tkinter import *
import tkinter.messagebox as messagebox
import tkinter.simpledialog as simpledialog
import tkinter.ttk as ttk
import tkinter.font as tkFont
from collections import OrderedDict
from functools import partial
from epics import caput,caget

# -- Globals
DONE =False

publishers_hdrdmacp =[]
publishers_HOSS =[]
# For now, set up for testing
# Nservers = 10
# starting_port = 5555
# for i in range(0,Nservers):
# 	port = starting_port + i
# 	host = 'localhost:%d' % port
# 	publishers.append( host )

pubhosts  = ['gluondaqbuff', 'gluonraid3', 'gluonraid4', 'gluonraid5']
for i in range(100, 138):
	if i==118 or i==119: continue
	pubhosts.append('gluon' + str(i))
pubhosts.append('gluon200')

for h in pubhosts: publishers_hdrdmacp.append( h + ':10471')
for h in pubhosts: publishers_HOSS.append( h + ':10473')

#-----------------------------------------------------------------------------
# MultiColumnListbox
#
# This copied from https://stackoverflow.com/questions/5286093/display-listbox-with-columns-using-tkinter
# I've modified this by Adding the _upsrt method to insert and update data.
# The sortby proc was also made a method of this class.
#
class MultiColumnListbox(object):
	def __init__(self, parent, columns, title):
		self.parent = parent
		self.columns = columns
		self.item_map = {} # key=hostname val=item returned by insert
		self.tree = None
		self._setup_widgets(title)
		self._build_tree()

	def _setup_widgets(self, title):
		msg = ttk.Label(self.parent, wraplength="4i", justify="left", anchor="n", padding=(10, 2, 10, 6), text=title)
		msg.grid(row=0, sticky=W+E)
		container = ttk.Frame(self.parent)
		container.grid(row=1, sticky=N+S+W+E)
		# create a treeview with dual scrollbars
		self.tree = ttk.Treeview(columns=self.columns, height=15, show="headings")
		vsb = ttk.Scrollbar(orient="vertical", command=self.tree.yview)
		hsb = ttk.Scrollbar(orient="horizontal", command=self.tree.xview)
		self.tree.configure(yscrollcommand=vsb.set, xscrollcommand=hsb.set)
		self.tree.grid(column=0, row=0, sticky='nsew', in_=container)
		vsb.grid(column=1, row=0, sticky='ns', in_=container)
		hsb.grid(column=0, row=1, sticky='ew', in_=container)
		Grid.rowconfigure(container, 0, weight=1)
		Grid.columnconfigure(container, 0, weight=1)
		Grid.rowconfigure(self.parent, 1, weight=1)
		Grid.columnconfigure(self.parent, 0, weight=1)

	def _build_tree(self):
		self.tree_sort_user_set = False  # Set flag indicating user has not yet set a sorting column so we should use default
		for col in self.columns:
			self.tree.heading(col, text=col.title(), command=lambda c=col: self.sortby(self.tree, c, 1, True))
			# adjust the column's width to the header string
			myanchor = E
			if col == self.columns[0]: myanchor=W
			colwidth = int(tkFont.Font().measure(col.title()))
			self.tree.column(col, width=colwidth, anchor=myanchor, stretch=NO)

	# Upsert row in table. "row" should be dictionary with keys for every
	# column. If a row for the "host" already exists, its values are
	# updated. Otherwise a new row is inserted for that host.
	def _upsrt_row(self, row):
		# Create a list of the values for each entry corresponding to the column names
		row_list = []
		for col in self.columns:
			if col in row:
				row_list.append( row[col] )
			else:
				row_list.append( '' )
		item = tuple(row_list)
		
		# Use first column as key
		mykey = item[0]
		if mykey not in self.item_map.keys():
			self.item_map[mykey] = self.tree.insert('', 'end', values=item)
		# Update row with new values and adjust column's width if necessary to fit each value
		for ix, val in enumerate(item):
			self.tree.set(self.item_map[mykey], self.columns[ix], val)
			col_w = tkFont.Font().measure(val)
			if self.tree.column(self.columns[ix],width=None)<col_w:
				self.tree.column(self.columns[ix], width=col_w)

		# If user has not clicked a column to sort, then re-sort based on first column
		if not self.tree_sort_user_set:
			self.sortby( self.tree, self.columns[0], False, False)


	# sort tree contents when a column header is clicked on
	def sortby(self, tree, col, descending, user_set):
		# grab values to sort
		data = [(tree.set(child, col), child) for child in tree.get_children('')]
		# if the data to be sorted is numeric change to float
		#data =  change_numeric(data)
		# now sort the data in place
		data.sort(reverse=descending)
		for ix, item in enumerate(data): tree.move(item[1], '', ix)
		# switch the heading so it will sort in the opposite direction
		if user_set:
			self.tree_sort_user_set = user_set
			tree.heading(col, command=lambda column=col: self.sortby(tree, column, int(not descending), True))


#-----------------------------------------------------------------------------
# StripChart
#
# This copied from http://code.activestate.com/recipes/578871-simple-tkinter-strip-chart-python-3/
# (c) MIT License Copyright 2014 Ronald H Longo
#
# Some significant improvements made by David Lawrence.
#
# The "trackColors" argument to constructor should be dictionary of values with
# track name as key and color as value (both strings). The names should match
# values passed to plotValues. e.g. plotValues(A=1, B=2, C=0) if the keys in
# dictionary were "A", "B", and "C".
class StripChart( Frame ):
	def __init__( self, parent, scale, historySize, trackColors, trackLabels, *args, **opts ):
		# Initialize
		super().__init__( parent, *args, **opts )
		self._trackHist   = OrderedDict() # Map: Track Name -> list of canvas objID
		self._trackColor  = trackColors   # Map: Track Name -> color
		self._trackLabel  = trackLabels   # Map: Track Name -> label (for legend)
		self.ymax = 1.0E-8
		self.yautoscale = BooleanVar()
		self.yautoscale.set(True)
		self._clearValues = False
		self._ticks = 0

		# offsets in pixels
		self.yoff = 10
		self.xoff = 10
		self.num_hlines = 2  # does not include top and bottom lines!

		# Options
		self._cbautoscale = Checkbutton(self, text='y-autoscale', variable=self.yautoscale)
		self._cbautoscale.grid(row=0, sticky=W)
		self._bclear = Button(self, text='clear', command=self.clearHistory)
		self._bclear.grid(row=0, column=1, sticky=W)

		self._chartHeight = scale + 1
		self._chartLength = historySize * 2  # Stretch for readability

		self._canvas = Canvas( self, height=self._chartHeight + 17, width=self._chartLength, background='black' )
		self._canvas.grid( row=1, columnspan=100, sticky=N+S+E+W )

		# Draw horizontal to divide plot from tick labels
		self._hlines = []
		self._hlabels = []
		x, y  = 0, 0 # these will be overwritten in resizeCanvas pretty much immediately
		x2,y2 = 1,1  # these will be overwritten in resizeCanvas pretty much immediately
		for idx in range(0, self.num_hlines+2):
			dash = (3,4)                                # midlines
			if idx == 0 : dash = (2,4)                  # top line
			if idx == self.num_hlines+1 : dash = (1,1)  # bottom line
			self._hlines.append( self._canvas.create_line( x, y, x2, y2, fill='white' , dash=dash) )
			self._hlabels.append( self._canvas.create_text( x, y, text='---', anchor=W, fill='white' ) )

		# Draw legend
		i = 0
		self._legendlines = []
		self._legendtext = []
		for k,color in trackColors.items():
			self._legendlines.append( self._canvas.create_line( 0, 0, 0, 1 , width=4, fill=color ))
			label = k
			if k in self._trackLabel: label = self._trackLabel[k]
			self._legendtext.append( self._canvas.create_text( 0, 0, text='\n'.join(label), fill='white', font=('Clean', 9), anchor=N ))
			i += 1

		# Init track def and histories lists
		self._trackColor.update( { 'tick':'white', 'tickline':'white',
                                 'ticklabel':'white' } )
		for trackName in self._trackColor.keys(): self._trackHist[ trackName ] = [None] * historySize

		Grid.rowconfigure(self, 0, weight=0)    # Options
		Grid.rowconfigure(self, 1, weight=1)    # Canvas
		Grid.columnconfigure(self, 0, weight=1)
		self.bind('<Configure>', self.resizeCanvas)

	#---------------------------------------------------
	# resizeCanvas
	#
	# Called when window size changes which includes right when the program firt starts up.
	def resizeCanvas(self, event):
		print('Resized: width=%d  height=%d' % (event.width, event.height))
		self._chartHeight = event.height - 60
		self._chartLength = event.width - 50

		# Adjust number of entries in history list
		historySize = self._chartLength/2
		for trackName, trackHistory in self._trackHist.items():
			while len(trackHistory)>historySize: self._canvas.delete( trackHistory.pop(0) ) # Remove left-most canvas objs
			Nneeded = int(historySize-len(trackHistory))
			if Nneeded>0: trackHistory.extend( [None] * Nneeded )

		yoff = self.xoff
		x2  = self.xoff + self._chartLength
		y   = yoff + self._chartHeight + 2

		num_hlines = len(self._hlines)
		for idx in range(0, num_hlines):
			ymid = int((y-yoff)*idx/(num_hlines-1)) + yoff
			self._canvas.coords( self._hlines[idx],  [0, ymid, x2, ymid] )
			self._canvas.coords( self._hlabels[idx], [4, ymid+10] )

		# Legend
		for i in range(0, len(self._legendlines)):
			x = self.xoff + self._chartLength + 10 + i*10
			y = self.yoff
			self._canvas.coords( self._legendlines[i], [x, y, x, y+15] )
			self._canvas.coords( self._legendtext[i], [x, y+20] )

	#---------------------------------------------------
	# plotValues
	#
	# Add new time step to strip chart, scrolling it to the left and removing oldest time step.
	# This is called periodically from MyWindow::TimerUpdate
	def plotValues( self, **vals ):

		# If user hit Clear button, then clear history first
		if self._clearValues : self.clearValues()

		# Increment time counter and draw vertical tick occasionally.
		if (self._ticks % 30) == 0: self. drawTick( text=str(self._ticks), dash=(1,4) )
		self._ticks += 1

		for trackName, trackHistory in self._trackHist.items():
			# Scroll left-wards
			self._canvas.delete( trackHistory.pop(0) ) # Remove left-most canvas objs
			self._canvas.move( trackName, -2, 0 )      # Scroll canvas objs 2 pixels left
				  
			# Plot the new values
			try:
				val = vals[ trackName ]
				yoff = self.yoff
				xoff = self.xoff
				x = xoff + self._chartLength
				y = self.YtoCanvas( val )
				color = self._trackColor[ trackName ]

				width = 4
				# if trackName=='A': width = 4
				# if trackName=='B': width = 6
				# if trackName=='C': width = 2
				objId = self._canvas.create_line( x, y, x+2, y, fill=color, width=width, tags=trackName )
				trackHistory.append( objId )

			except:
				trackHistory.append( None )
	
		# Check if we need to change scale of chart
		if self.yautoscale.get():
			ymax = self.getYmax()
			if ymax < 1.0E-2: ymax = 1.0E-2
			if ymax != self.ymax:
				scale = self.ymax/ymax
				for trackName, trackHistory in self._trackHist.items():
					if trackName.startswith('tick'): continue  # skip tick marks and labels
					for objId in trackHistory:
						if objId is None : continue
						y = scale*self.YfromCanvas(objId)
						coords = self._canvas.coords(objId)
						coords[1] = coords[3] = self.YtoCanvas(y)
						self._canvas.coords(objId, coords)
				self.ymax = ymax # update only after all calls to YfromCanvas and YtoCanvas are made
		else:
			self.ymax = self.ymax_default

		# Redraw labels
		num_hlines = len(self._hlines)
		for idx in range(0, num_hlines):
			y = self.ymax*idx/(num_hlines-1)
			self._canvas.itemconfig(self._hlabels[idx], text='%3.1f GB/s' % (self.ymax - y))

	#---------------------------------------------------
	# clearHistory
	#
	# (see clearValues below)
	def clearHistory(self):
		print('Clear button pressed')
		self._clearValues = True

	#---------------------------------------------------
	# clearValues
	#
	# This is called from plotValues iff the self._clearValues flag is set. It needs to
	# be done that way since plotValues is called asynchronously to when the "Clear"
	# button is pressed.
	def clearValues(self):
		# Remove all scrolling objects
		print('Clearing history')
		for trackName, trackHistory in self._trackHist.items():
			# Scroll left-wards
			for t in trackHistory: self._canvas.delete( t )
			self._trackHist[trackName] = [None] * len(self._trackHist[trackName])
		# self.rate_stripchart_ticks = 0
		self._ticks = 0
		self._clearValues = False

	def drawTick( self, text=None, **lineOpts ):
		# draw vertical tick line
		yoff = self.yoff
		xoff = self.xoff
		x  = self._chartLength + xoff
		y  = yoff
		x2 = x
		y2 = yoff + self._chartHeight + 2
		color = self._trackColor[ 'tickline' ]
		
		objId = self._canvas.create_line( x, y, x2, y2, fill=color, tags='tick', **lineOpts )
		self._trackHist[ 'tickline' ].append( objId )
		
		# draw tick label
		if text is not None:
			x = self._chartLength + xoff
			y = self._chartHeight + yoff + 10
			color = self._trackColor[ 'ticklabel' ]
			
			objId = self._canvas.create_text( x, y, text=text, fill=color, tags='tick' )
			self._trackHist[ 'ticklabel' ].append( objId )

	def configTrackColors( self, **trackColors ):
		# Change plotted data color
		for trackName, colorName in trackColors.items( ):
			self._canvas.itemconfigure( trackName, fill=colorName )
		
		# Change settings so future data has the new color
		self._trackColor.update( trackColors )

	def SetYmax( self, ymax ):
		if ymax is None:
			self.yautoscale.set(True)
		else:
			self.yautoscale.set(False)
			self.ymax_default = ymax

	def YtoCanvas(self, y):
		h = self._chartHeight
		deltay = y*h/self.ymax
		return self.yoff + self._chartHeight - deltay

	def YfromCanvas(self, objId):
		h = self._chartHeight
		y = self._canvas.coords(objId)[1]
		return (self.yoff + self._chartHeight - y)*self.ymax/h

	def getYmax(self):
		ymax = 0
		for trackName, trackHistory in self._trackHist.items():
			if trackName.startswith('tick'): continue   # skip tick marks and labels
			for objId in trackHistory:
				if objId is not None :
					y = self.YfromCanvas( objId )
					if y > ymax: ymax = y
		return ymax

#-----------------------------------------------------------------------------
# ButtonWithMenu
#
# This class inherits from the normal tkinter Button, but adds a right-click
# menu to restart or stop the hoss worker on a node.
#-----------------------------------------------------------------------------
class ButtonWithMenu(Button):

	def __init__(self, parent, *args, **kwargs):
		Button.__init__(self, parent, *args, **kwargs)
		self.host = self['text']
		self.popup_menu = Menu( self, tearoff=0)
		self.popup_menu.add_command(label=self.host)
		self.popup_menu.add_separator()
		self.popup_menu.add_command(label='restart', command=self.restart)
		self.popup_menu.add_command(label='stop', command=self.stop)
		self.bind('<Button-3>', self.popup)

	def popup( self, event ):
		try:
			self.popup_menu.tk_popup(event.x_root, event.y_root, 0)
		except:
			print('ERROR: Problem with popup menu!')

	def restart( self ):
		print('restarting %s ...' % self.host)
		cmd = ['start_hoss','-r', '--hosts', self.host]
		threading.Thread(target=subprocess.call, args=([cmd])).start()

	def stop( self ):
		print('stopping %s ...' % self.host)
		cmd = ['start_hoss','-e', '--hosts', self.host]
		threading.Thread(target=subprocess.call, args=([cmd])).start()

#-----------------------------------------------------------------------------
# PagerDialog
class PagerDialog(simpledialog.Dialog):

	#=========================
	# __init__
	def __init__(self, parent, host, filename, splitstr, idx=0):
		self.parent = parent
		self.host = host
		self.filename = filename
		self.splitstr = splitstr
		self.logs = []
		self.curr_index = idx # Set to large number ot start on last item in logs

		#self.top = Toplevel(parent)
		simpledialog.Dialog.__init__(self, parent, host)
		#self.body( self )

		# Read in log file and set to last entry (large value of curr_index ensures this)
		# self.ReadLogFile()
		# self.ShowNext()

	#=========================
	# ReadLogFile
	def ReadLogFile( self ):
		self.logs = []
		try:
			with open( self.filename ) as f:

				log = ''
				for line in f.readlines():
					if self.splitstr in line:
						if log :
							self.logs.append( log )
							log = ''
					line = line.replace('\r','\n')
					line = line.replace('\\r','\n')
					line = line.replace('\\n','\n')
					log += line
				if log : self.logs.append( log )
		except:
			pass
		if not self.logs : self.logs = ['< NO LOG FILE "'+self.filename+'">'] # in case file doesn't exist

	#=========================
	# body
	def body( self, master):
		self.frame = Frame( master )

		# Main text area w/ scrollbar
		lbf = Frame( self.frame )
		yscroll = Scrollbar(lbf)
		self.lbl = Text( lbf, bg='white', padx=10, pady=10, yscrollcommand=yscroll.set, width=140, height=60)
		yscroll.configure(command=self.lbl.yview)
		self.lbl.grid(row=0, column=0, sticky=N+S+E+W)
		yscroll.grid(row=0,column=1, sticky=N+S+W)

		# Navigation buttons
		prevbutton = Button( self.frame, text='<<prev', command=self.ShowPrev)
		nextbutton = Button( self.frame, text='next>>', command=self.ShowNext)
		closebutton = Button( self.frame, text='Close', command=self.destroy)

		# For debugging
		self.config(bg='#333')
		self.frame.config(bg='#666')
		lbf.config(bg='#999')

		self.frame.grid(row=0, column=0, rowspan=3, columnspan=2, sticky=N+S+E+W)
		lbf.grid(row=0, column=0, columnspan=2, sticky=N+S+E+W)
		prevbutton.grid(row=1, column=0)
		nextbutton.grid(row=1, column=1)
		closebutton.grid(row=2, column=0, columnspan=2)
		Grid.columnconfigure(self, 0, weight=1)
		Grid.columnconfigure(self, 1, weight=1)
		Grid.rowconfigure(self, 0, weight=10)
		Grid.rowconfigure(self, 1, weight=1)
		Grid.rowconfigure(self, 2, weight=1)

		self.ShowNext()

		return closebutton

	#=========================
	# ShowNext
	def ShowNext(self):
		self.curr_index += 1
		if self.curr_index >= len(self.logs): self.ReadLogFile()
		if self.curr_index >= len(self.logs): self.curr_index = len(self.logs) - 1
		self.lbl.delete("1.0", END) # delete entire contents
		self.lbl.insert("1.0", self.logs[self.curr_index])
		# self.curr_log.set(self.logs[self.curr_index])

	#=========================
	# ShowPrev
	def ShowPrev(self):
		self.curr_index -= 1
		if self.curr_index < 0: self.curr_index = 0
		if self.curr_index >= len(self.logs): self.ReadLogFile()
		if self.curr_index >= len(self.logs): self.curr_index = len(self.logs) - 1
		self.lbl.delete("1.0", END) # delete entire contents
		self.lbl.insert("1.0", self.logs[self.curr_index])

	#=========================
	# buttonbox
	def buttonbox(self):
		pass

#-----------------------------------------------------------------------------
# MyWindow   (The GUI)
class MyWindow(Frame):

	#=========================
	# __init__
	def __init__(self, master=None):
		Frame.__init__(self, master)
		self.master = master
		self.hostinfo = {}
		self.hostinfo_HOSS = {}
		self.logbutton = {}
		self.filter_hdrdmacp_hosts = BooleanVar()
		self.filter_hdrdmacp_hosts.set(True)
		self.init_window()

	#=========================
	# init_window
	def init_window(self):
		self.master.title('HOSS Status')
		self.grid(row=0, column=0, sticky=N+S+E+W ) # Fill root window

		# Configuration
		self.configfile = StringVar()
		configframe = Frame(self)
		configframe.grid( row=0, column=0, sticky=N+W+E )
		self.configfilelabel = Label( configframe, textvariable=self.configfile )
		self.configfilelabel.grid( row=0, column=0, sticky=N+W )
		configbutton = Button(configframe, text='view', command=self.ShowConfig)
		configbutton.grid( row=0, column=1, sticky=N+W )

		# Run number
		self.run_number = StringVar()
		self.run_numberlabel = Label( configframe, textvariable=self.run_number, anchor='e')
		self.run_numberlabel.grid( row=0, column=2, sticky=N+E+W )
		configframe.columnconfigure( 2, weight=1)

		# Columns here must match keys that are either in the json records returned
		# or added to the dictionary in MyWindow.AddHostInfo().
		# bind call allows double click to pop up window with all fields from JSON record
		hossframe = Frame(self)
		columns = ['host', 'status','Busy', 'tbusy', 'tidle', 'Nprocs','Nerrs', 'respawns', 'command']
		self.hostslb2 = MultiColumnListbox(hossframe, columns, 'HOSS servers')
		self.hostslb2.tree.bind('<Double-1>', lambda event, t=self.hostslb2.tree: self.OnDoubleClick(event,t,self.hostinfo_HOSS))
		self.hostslb2.tree.column('command', anchor=W)
		hossframe.grid( row=1, column=0, sticky=N+S+E+W )

		# Columns here must match keys that are either in the json records returned
		# or added to the dictionary in MyWindow.AddHostInfo().
		# bind call allows double click to pop up window with all fields from JSON record
		hdrdmacpframe = Frame(self)
		columns = ['host', '10 sec. avg.', '1 min. avg.', '5 min. avg.', 'Nfiles', 'Received Total', 'RAM disk free', 'RAM free','idle']
		self.hostslb1 = MultiColumnListbox(hdrdmacpframe, columns, 'hdrdmacp servers')
		self.hostslb1.tree.bind('<Double-1>', lambda event, t=self.hostslb1.tree: self.OnDoubleClick(event,t,self.hostinfo))
		hdrdmacpframe.grid( row=2, column=0, sticky=N+S+E+W )

		# Log buttons
		logframe = Frame(self)
		logframe.grid( row=3, column=0 )
		for i in range(0, len(publishers_HOSS)):
			row = int(i/12)
			col = i%12
			host = publishers_HOSS[i].split(':')[0]
			but = ButtonWithMenu(logframe, text=host,command=partial(self.ShowLog, host), activebackground='#AAA' )
			but.grid( row=row, column=col, sticky=N+S+E+W )
			self.logbutton[host] = but

		# Strip chart
		# avg10sec, avg1min are hdrdmacp rates received by all gluonraids
		# hoss1min is hdrdmacp rate received by everything else
		colors = { 'avg10sec':'grey', 'avg1min':'red', 'hoss1min':'green' }
		labels = { 'avg10sec':'RAID10sec', 'avg1min':'RAID1min', 'hoss1min':'OSS' }
		self.rate_stripchart = StripChart( self, 150, 375, colors, labels )
		self.rate_stripchart_ticks = 0
		self.rate_stripchart.SetYmax( 3.0 )
		self.rate_stripchart.grid( row=4, column=0, sticky=N+S+E+W )

		# Restart, Stop, and Quit buttons
		buttonsframe = Frame(self)
		restartButton = Button(buttonsframe, text="Restart HOSS",command=self.RestartHOSS)
		restartButton.grid( row=0, column=0, sticky=W )
		restartButton = Button(buttonsframe, text="Stop HOSS",command=self.StopHOSS)
		restartButton.grid( row=0, column=1, sticky=W )
		quitButton = Button(buttonsframe, text="Quit",command=self.Quit)
		quitButton.grid( row=0, column=2, sticky=E )
		buttonsframe.grid( row=5, column=0, sticky=E+W)
		buttonsframe.columnconfigure( 2, weight=1)

	#self.master.rowconfigure(self,0, weight=1)
		#self.master.columnconfigure(self,0, weight=1)
		Grid.rowconfigure(self, 0, weight=0)  # Configuration
		Grid.rowconfigure(self, 1, weight=5)  # Listbox HOSS
		Grid.rowconfigure(self, 2, weight=5)  # Listbox hdrdmacp
		Grid.rowconfigure(self, 3, weight=0)  # Log buttons
		Grid.rowconfigure(self, 4, weight=2)  # stripchart
		Grid.rowconfigure(self, 5, weight=0)  # Bottom buttons
		Grid.columnconfigure(self, 0, weight=1)

	#=========================
	# RestartHOSS
	def RestartHOSS(self):
		cmd = ['start_hoss','-r']
		threading.Thread(target=subprocess.call, args=([cmd])).start()

	#=========================
	# StopHOSS
	def StopHOSS(self):
		cmd = ['start_hoss','-e']
		threading.Thread(target=subprocess.call, args=([cmd])).start()

	#=========================
	# Quit
	def Quit(self):
		global DONE, root
		DONE=True
		root.destroy()

	#=========================
	# ShowConfig
	def ShowConfig(self):
		PagerDialog(self, 'HOSS Configuration', self.configfilename, '---------------------')

	#=========================
	# OnDoubleClick
	def OnDoubleClick(self, event, tree, hostinfo):
		# Show info dialog with all values from JSON record
		item = tree.identify('item', event.x, event.y)
		vals = tree.item(item)['values']
		if len(vals) < 1: return
		host = vals[0]
		mess = ''
		for k,v in hostinfo[host].items():
			mess += k + ' = ' + str(v) + '\n'
			if k == 'received':
				dt = datetime.datetime.fromtimestamp( float(v) )
				mess += '           (' +dt.strftime('%c') + ')\n'
		messagebox.showinfo(host, mess)

	#=========================
	# Add_hdrdmacp_HostInfo
	def Add_hdrdmacp_HostInfo(self, hostinfo):
		# Keep clean copy of most recent info from server indexed by host
		host = hostinfo['host']
		self.hostinfo[host] = hostinfo

		ram_disk_percent_free = 100.0-float(hostinfo['/media/ramdisk_percent_used'])
		
		# Modify some fields for presentation in listbox
		myhostinfo = dict.copy(hostinfo)
		myhostinfo['avg5min'] = '%3.1fGB/s' % float(hostinfo['avg5min'])
		myhostinfo['10 sec. avg.'] = '%3.2fGB/s' % float(hostinfo['avg10sec'])
		myhostinfo['1 min. avg.'] = '%3.2fGB/s' % float(hostinfo['avg1min'])
		myhostinfo['5 min. avg.'] = '%3.2fGB/s' % float(hostinfo['avg5min'])
		myhostinfo['Received Total'] = '%3.3fTB' % float(hostinfo['TB_received'])
		myhostinfo['RAM disk free'] = '%3.1f GB (%3.1f%%)' % (myhostinfo['/media/ramdisk_avail'], ram_disk_percent_free)
		myhostinfo['RAM free'] = '%3.1f%%' % (100.0*float(myhostinfo['ram_avail_GB'])/float(myhostinfo['ram_tot_GB']))
		myhostinfo['idle'] = ('%3.1f' % myhostinfo['cpu_idle']) + '%'

		# Optionally filter out hosts that are not also reporting HOSS information
		if self.filter_hdrdmacp_hosts.get():
			if host not in self.hostinfo_HOSS.keys(): return

		self.hostslb1._upsrt_row(myhostinfo)

	#=========================
	# Add_HOSS_HostInfo
	def Add_HOSS_HostInfo(self, hostinfo):
		# Keep clean copy of most recent info from server indexed by host
		host = hostinfo['host']
		self.hostinfo_HOSS[host] = hostinfo

		tbusy = float(hostinfo['tbusy'])
		tidle = float(hostinfo['tidle'])
		try:
			# Replace single quotes with double quotes so json.loads is happy then count total errors
			Nerrs = sum( json.loads(hostinfo['errors'].replace("'",'"')).values() )
		except:
			traceback.print_exc()
			print("myhostinfo['errors'] = '" + hostinfo['errors'] + "'")
			Nerrs = 0

		# Modify some fields for presentation in listbox
		myhostinfo = dict.copy(hostinfo)
		myhostinfo['Busy'] = '%3.1f%%' % (100.0*tbusy/(tbusy+tidle))
		myhostinfo['tbusy'] = '%3.1fs' % tbusy
		myhostinfo['tidle'] = '%3.1fs' % tidle
		myhostinfo['Nprocs'] = myhostinfo['num_procs_total']
		myhostinfo['Nerrs'] = Nerrs
		myhostinfo['command'] = myhostinfo['last_cmd']

		if len(myhostinfo['command']) > 70:
			myhostinfo['command'] = myhostinfo['command'][:67] + '...'

		self.hostslb2._upsrt_row(myhostinfo)

		# Turn log button green to indicate it is in currently active group
		base_host = host.split('.')[0].split('-')[0]  # remove domain and any "-daq" or "-ib"
		if base_host in self.logbutton.keys():
			self.logbutton[base_host].config(bg="green")

	#=========================
	# ShowLog
	def ShowLog(self, host):
		print('Showing log for ' + host)
		filename = '/gluex/log/start_hoss_worker_' + host + '.log'
		if not os.path.exists(filename) : filename = '/gluex/log/start_hoss_worker_' + host + '-daq.log' # accommodate gluonraid3-daq.log, etc...
		PagerDialog(self, host, filename, 'hdlog starting', 1000000)

	#=========================
	# TimerUpdate
	#
	# This method is run in a separate thread and continuously loops to
	# update things outside of the mainloop.
	def TimerUpdate(self):
		while not DONE:

			# Update configuration file
			self.configfilename = '/gluondaqfs/hdops/CDAQ/daq_dev_v0.31/daq/config/HOSS/hoss_default.config'
			try:
				link = os.readlink(self.configfilename)
				s = self.configfilename + '    ->    ' + link + '    '
			except:
				s = '**********  NO CONFIG FILE: ' + self.configfilename + ' *********    '
			if self.configfile.get() != s : self.configfile.set(s)
		
			# Update hdrdmacp listbox
			now = time.time()
			hostinfos = dict.copy(self.hostinfo)
			for host, hostinfo in hostinfos.items():
				if (now - hostinfo['received']) > 6.0:
					myhostinfo = dict.copy(hostinfo)
					myhostinfo['avg5min'] = '---- MB/s'
					myhostinfo['Nfiles'] = '----'
					self.hostslb1._upsrt_row(myhostinfo)

			# Update HOSS listbox
			now = time.time()
			hostinfos = dict.copy(self.hostinfo_HOSS)
			for host, hostinfo in hostinfos.items():
				if (now - hostinfo['received']) > 6.0:
					myhostinfo = dict.copy(hostinfo)
					myhostinfo['status'] = '----'
					myhostinfo['tbusy'] = '----'
					myhostinfo['tidle'] = '----'
					myhostinfo['Busy fraction'] = '----'
					myhostinfo['respawns'] = '----'
					self.hostslb2._upsrt_row(myhostinfo)

					# turn log button red indicating communication was there, but is now gone
					base_host = host.split('.')[0].split('-')[0]  # remove domain and any "-daq" or "-ib"
					if base_host in self.logbutton.keys(): self.logbutton[base_host].config(bg="red")

			# Update strip charts
			gluonraid_tot_rate10sec = 0.0
			gluonraid_tot_rate1min = 0.0
			gluonworker_tot_rate1min = 0.0
			for host, hostinfo in self.hostinfo.items():
				if (now - hostinfo['received']) > 12.0: continue
				if 'gluonraid' in host:
					gluonraid_tot_rate10sec += hostinfo['avg10sec']
					gluonraid_tot_rate1min += hostinfo['avg1min']
				elif 'gluondaqbuff' in host:
					pass
				else:
					gluonworker_tot_rate1min += hostinfo['avg1min']

			self.rate_stripchart.plotValues( avg10sec=gluonraid_tot_rate10sec, avg1min=gluonraid_tot_rate1min, hoss1min=gluonworker_tot_rate1min )

			# Things to do only every 4 seconds
			if int(time.time())%4 == 0:
				run = GetEPICSvar('HD:coda:daq:run_number')
				s = 'Run Number: ' + str(run)
				if s != self.run_number.get():
					self.run_number.set( s )
					self.run_numberlabel.config( bg='yellow' )
				else:
					# Reset to default color for labels (assume configfilelabel is that color)
					self.run_numberlabel.config( bg=self.configfilelabel.cget('bg') )


			# Limit rate through loop
			time.sleep(1)

#-----------------------------------------------------------------------------
# MySubscriber_hdrdmacp
#
# This is run in each thread created to subscribe to a publisher host.
# It makes the connection and subscribes to all messages then loops
# continuously as long as the global DONE variable is set to True.
def MySubscriber_hdrdmacp(context, host):
	global DONE, app

	print('Connecting to ' + host + ' ...')

	# Create socket
	socket = context.socket(zmq.SUB)
	socket.connect("tcp://" + host)
	socket.setsockopt_string(zmq.SUBSCRIBE, '') # Accept all messages from publisher

	while not DONE:
		try:
			string = socket.recv_string(flags=zmq.NOBLOCK)
			#print('Received string:\n' + string + '\n')
			myinfo = json.loads( string )
			myinfo['received'] = time.time()

			app.Add_hdrdmacp_HostInfo( myinfo )
		
		except zmq.Again as e:
			time.sleep(1)

#-----------------------------------------------------------------------------
# MySubscriber_HOSS
#
# This is run in each thread created to subscribe to a publisher host.
# It makes the connection and subscribes to all messages then loops
# continuously as long as the global DONE variable is set to True.
def MySubscriber_HOSS(context, host):
	global DONE, app

	print('Connecting to ' + host + ' ...')

	# Create socket
	socket = context.socket(zmq.SUB)
	socket.connect("tcp://" + host)
	socket.setsockopt_string(zmq.SUBSCRIBE, '') # Accept all messages from publisher

	while not DONE:
		try:
			string = socket.recv_string(flags=zmq.NOBLOCK)
			#print('Received string:\n' + string + '\n')
			myinfo = json.loads( string )
			myinfo['received'] = time.time()

			app.Add_HOSS_HostInfo( myinfo )

		except zmq.Again as e:
			time.sleep(1)

#-----------------------------------------------------------------------------
# SetEPICSvar
#
# This just wraps the caput call in a try except block so if there is a problem
# with setting the EPICS variable, it does not cause the whole script to fail
def SetEPICSvar( name, val):
	try:
		caput(name, val)
	except Exception as e:
		print('Unable to set EPICS variable "' + name + '" to: ' + str(val))
		print(str(e))

#-----------------------------------------------------------------------------
# GetEPICSvar
#
# This just wraps the caget call in a try except block so if there is a problem
# with getting the EPICS variable, it does not cause the whole script to fail
def GetEPICSvar( name):
	try:
		return caget(name)
	except Exception as e:
		print('Unable to get EPICS variable "' + name + '" ')
		print(str(e))
	return ''

#=============================================================================
#------------------- main  (rest of script is in global scope) ---------------

# Create window
root = Tk()
root.geometry("1100x1050")
Grid.rowconfigure(root, 0, weight=1)
Grid.columnconfigure(root, 0, weight=1)
app = MyWindow(root)

#  Only single 0MQ context is needed
context = zmq.Context()
	
# Loop over publishers, creating a thread for each
threads = []
for host in publishers_hdrdmacp:
	t = threading.Thread(target=MySubscriber_hdrdmacp, args=(context, host))
	t.start()
	threads.append( t )
for host in publishers_HOSS:
	t = threading.Thread(target=MySubscriber_HOSS, args=(context, host))
	t.start()
	threads.append( t )

# Create a thread for periodic updates
t = threading.Thread(target=app.TimerUpdate)
t.start()

# Run main GUI loop until user closes window
root.mainloop()

#-----------------------------------------------------------------------------
print('GUI finished. Cleaning up ...')

# Close all zeroMQ threads
DONE = True
for t in threads: t.join()

print('\nFinished\n')