-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathheartbeat.py
181 lines (137 loc) · 8.2 KB
/
heartbeat.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#!/user/bin/python
from datetime import datetime, timedelta
import time
from constants import ProgramCodes, Statuses, Devices, MessageTypes, MessageCodes, DebugLevels
from programrunner import ProgramRunner
from workerthread import WorkerThread
class Heartbeat(WorkerThread):
FRIENDLY_NAME = "Heartbeat"
RUNTIME = 75
OVERRIDE_SAFE_MODE = True
DEBUG_LEVEL = DebugLevels.NONE
systemStatus = Statuses.UNDEFINED
heartbeatStatus = True
safeModeEndTime = None
information = {}
initialising = True
def dowork(self):
self.debug("dowork: {0} {1}".format(self.looprequest, self.stoprequest))
while not (self.stoprequest or self.looprequest):
self.heartbeat(self.RUNTIME)
if not self.looprequest and not self.stoprequest:
if self.systemStatus >= Statuses.CRITICAL:
self.rebootrequest("Heartbeat has not been renewed")
else:
self.systemStatus += 1
self.logalert("Incrementing status", "{0}".format(self.systemStatus))
def heartbeat(self, sleepTime):
# We're only interested in the most recent status
# This is bogus. If we do it then we'll return OK to all status requests
# This is why we need test driven development (TODO)
#while not self.inQueue.empty():
# self.sleep(0)
endtime = datetime.now() + timedelta(seconds=sleepTime)
heartbeatMode = self.program[ProgramCodes.CODE]
previousHeartbeatStatus = self.heartbeatStatus
if heartbeatMode == 'NORMAL':
if self.systemStatus == Statuses.OK or self.systemStatus == Statuses.WARNING or self.systemStatus == Statuses.ALERT \
or (self.systemStatus == Statuses.UNDEFINED and self.initialising):
if self.safeModeEndTime != None and self.safeModeEndTime > datetime.now():
self.setstatus(Statuses.ALERT, "Heartbeat Suspended: Safe Mode Engaged until {0}".format(self.safeModeEndTime.strftime("%H:%M:%S")))
self.heartbeatStatus = False
else:
# TODO: This is the start of a software solution to what is probably an electronics problem. The relays do initialise correctly if they're supplied
# with a 5V signal, but this circuit loses some voltage and only provides about 3.5V in practice. Probably it's best to use a higher source voltage,
# rather than cobbling a software patch for this.
if not self.heartbeatStatus:
# The relays don't necessarily return to their correct state if the heartbeat is restarted because there isn't quite enough power.
# Need to request that they are switched off and back on again after the heartbeat is started
#reinitialiseRelays = True
self.loginformation("Heartbeat restarting", "Restarting Heartbeat - Status {0}".format(Statuses.codes[self.systemStatus]))
self.heartbeatStatus = True
else:
self.heartbeatStatus = False
self.safeModeEndTime = datetime.now() + timedelta(seconds=300)
self.setstatus(Statuses.ALERT, "Heartbeat Suspended: Safe Mode Engaged")
self.logalert("Heartbeat suspended", "Safe mode engaged: status {0} - initialising: {1}, until {2}".format(self.systemStatus, self.initialising, self.safeModeEndTime))
elif heartbeatMode == 'SAFE_MODE':
self.setstatus(Statuses.WARNING, 'Safe Mode active by request')
self.heartbeatStatus = False
elif heartbeatMode == 'OVERRIDE_ON':
self.setstatus(Statuses.WARNING, 'Heartbeat active by request')
self.heartbeatStatus = True
heartbeatText = "On" if self.heartbeatStatus else "Off"
if previousHeartbeatStatus != self.heartbeatStatus:
self.debug("Sending SAFE_MODE message", DebugLevels.ALL)
self.outQueue.put({
MessageCodes.CODE:MessageTypes.SAFE_MODE,
MessageCodes.VALUE:not self.heartbeatStatus
})
self.showstatus()
self.information["Heartbeat"] = {
MessageCodes.NAME:"Heartbeat",
MessageCodes.VALUE:heartbeatText
}
self.information["Status"] = {
MessageCodes.NAME:"Indicated Status",
MessageCodes.VALUE:Statuses.codes[self.systemStatus]
}
if self.status == Statuses.UNDEFINED:
# We don't want to keep reefx waiting too long for a correct status
self.showstatus()
self.sleep(0) # Sleep to make sure we're not showing an out of date status
if self.looprequest or self.stoprequest:
self.debug("There's another message waiting in the queue")
return
if self.systemStatus != Statuses.UNDEFINED:
self.initialising = False
if self.systemStatus == Statuses.OK:
statusLEDs = [Devices.STATUS_LED_GREEN]
elif self.systemStatus == Statuses.WARNING:
statusLEDs = [Devices.STATUS_LED_YELLOW]
elif self.systemStatus == Statuses.ALERT:
statusLEDs = [Devices.STATUS_LED_RED]
elif self.systemStatus == Statuses.UNDEFINED:
statusLEDs = [Devices.STATUS_LED_GREEN, Devices.STATUS_LED_YELLOW, Devices.STATUS_LED_RED]
else:
statusLEDs = [Devices.STATUS_LED_RED]
self.debug("heartbeat {0} until {1}".format(self.systemStatus, endtime))
self.showleds([Devices.STATUS_LED_GREEN, Devices.STATUS_LED_YELLOW, Devices.STATUS_LED_RED], False)
self.debug("Loop: {0} {1} {2}".format(endtime, self.looprequest, self.stoprequest))
while endtime > datetime.now() and not self.looprequest and not self.stoprequest:
#self.debug("Flash LED")
# Note: we use time.sleep, rather than self.sleep as we don't want to be interrupted
# in the middle of a flash cycle (it makes the LED flashing look messy)
self.deviceoutput(Devices.HEARTBEAT_2, False, requireResponse=False)
self.deviceoutput(Devices.HEARTBEAT_1, self.heartbeatStatus, requireResponse=False)
self.showleds(statusLEDs, True)
time.sleep(0.75)
self.deviceoutput(Devices.HEARTBEAT_1, False, requireResponse=False)
self.deviceoutput(Devices.HEARTBEAT_2, self.heartbeatStatus, requireResponse=False)
time.sleep(0.25)
if self.systemStatus != Statuses.CRITICAL:
self.showleds(statusLEDs, False)
time.sleep(0.5)
self.deviceoutput(Devices.HEARTBEAT_1, False, requireResponse=False)
self.deviceoutput(Devices.HEARTBEAT_2, False, requireResponse=False)
# Now call self.sleep to catch any pending requests
self.sleep(0)
self.debug("heartbeat done")
def showleds(self, leds, output):
for led in leds:
self.deviceoutput(led, output, requireResponse=False)
def processrequest(self, request):
self.debug("Received a request {0}: {1}".format(request[MessageCodes.CODE], request[MessageCodes.VALUE]))
if request[MessageCodes.CODE] == MessageTypes.INDICATE_STATUS:
self.systemStatus = request[MessageCodes.VALUE]
self.looprequest = True
return True
else:
return super(Heartbeat, self).processrequest(request)
def teardown(self, message):
self.debug("Turning off LEDs: {0}".format(message))
self.showleds([Devices.STATUS_LED_GREEN, Devices.STATUS_LED_YELLOW, Devices.STATUS_LED_RED], False)
self.deviceoutput(Devices.HEARTBEAT_1, False, requireResponse=False)
self.deviceoutput(Devices.HEARTBEAT_2, False, requireResponse=False)
def setup(self):
pass