forked from ianmiell/shutit
-
Notifications
You must be signed in to change notification settings - Fork 0
/
shutit_background.py
154 lines (141 loc) · 7.31 KB
/
shutit_background.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
r"""Represents a ShutIt background command.
- options to:
- cancel command
- get status (running, suspended etc)
- check_timeout
"""
import time
import logging
import shutit_global
import shutit_util
class ShutItBackgroundCommand(object):
"""Background command in ShutIt
"""
def __init__(self,
sendspec):
# Stub this with a simple command for now
self.sendspec = sendspec
self.block_other_commands = sendspec.block_other_commands
self.retry = sendspec.retry
self.tries = 0
self.pid = None
self.return_value = None
self.start_time = None
self.run_state = 'N' # State as per ps man page, but 'C' == Complete, 'N' == not started, 'F' == failed, 'S' == sleeping/running, 'T' == timed out by ShutIt
self.cwd = self.sendspec.shutit_pexpect_child.send_and_get_output(' command pwd', ignore_background=True)
self.id = shutit_util.random_id()
self.output_file = '/tmp/shutit_background_' + self.id + '_output.log'
self.exit_code_file = '/tmp/shutit_background_' + self.id + '_exit_code_file.log'
self.command_file = '/tmp/shutit_background_' + self.id + '_command.log'
if self.sendspec.run_in_background:
self.sendspec.send = ' set +m && { : $(echo "' + self.sendspec.original_send + '" >' + self.command_file + ' && command cd ' + self.cwd + '>' + self.output_file + ' && ' + self.sendspec.send + ' >>' + self.output_file + ' 2>&1; echo $? >' + self.exit_code_file + ') & } 2>/dev/null'
def __str__(self):
string = str(self.sendspec)
string += '\n---- Background object BEGIN ----'
string += '\nblock_other_commands: ' + str(self.block_other_commands)
string += '\ncwd: ' + str(self.cwd)
string += '\npid: ' + str(self.pid)
string += '\nretry: ' + str(self.retry)
string += '\nreturn_value: ' + str(self.return_value)
string += '\nrun_state: ' + str(self.run_state)
string += '\nstart_time: ' + str(self.start_time)
string += '\ntries: ' + str(self.tries)
string += '\n---- END ----'
return string
def run_background_command(self):
# reset object
self.pid = None
self.return_value = None
self.run_state = 'N'
self.start_time = time.time() # record start time
# run command
self.tries += 1
if self.sendspec.run_in_background:
# Run in the background
self.sendspec.shutit_pexpect_child.quick_send(self.sendspec.send)
# Put into an 'S' state as that means 'running'
self.run_state = 'S'
# Required to reset terminal after a background send. (TODO: why?)
self.sendspec.shutit_pexpect_child.reset_terminal()
# record pid
self.pid = self.sendspec.shutit_pexpect_child.send_and_get_output(" echo ${!}",ignore_background=True)
else:
# Run synchronously and mark complete
# We need to set this to ignore background before we run it, so that
# it does not block itself and end up in an infinite loop.
self.sendspec.ignore_background = True
self.sendspec.shutit_pexpect_child.send(self.sendspec)
self.run_state = 'C'
self.sendspec.started = True
assert self.run_state in ('C','S','F')
return True
def check_background_command_state(self):
shutit_global.shutit_global_object.log('CHECKING background task: ' + self.sendspec.send + ', id: ' + self.id,level=logging.DEBUG)
assert self.start_time is not None
# Check the command has been started
if not self.sendspec.started:
assert self.run_state == 'N'
return self.run_state
if self.run_state in ('C','F'):
assert self.sendspec.started
return self.run_state
assert self.run_state in ('S',), 'State should be in S, is in fact: ' + self.run_state
# Update the run state.
updated_run_state = self.sendspec.shutit_pexpect_child.send_and_get_output(""" command ps -o stat """ + self.pid + """ | command sed '1d' """, ignore_background=True)
# Ensure we get the first character only, if one exists.
if len(updated_run_state) > 0:
# Task is unfinished.
self.run_state = updated_run_state[0]
updated_run_state = None
# state The state is given by a sequence of characters, for example, ``RWNA''. The first character indicates the run state of the process:
# I Marks a process that is idle (sleeping for longer than about 20 seconds).
# R Marks a runnable process.
# S Marks a process that is sleeping for less than about 20 seconds.
# T Marks a stopped process.
# U Marks a process in uninterruptible wait.
# Z Marks a dead process (a ``zombie'').
if self.run_state in ('I','R','T','U','Z'):
shutit_global.shutit_global_object.log('background task run state: ' + self.run_state, level=logging.DEBUG)
self.run_state = 'S'
assert self.run_state in ('S',), 'State should be in S having gleaned from ps, is in fact: ' + self.run_state
# honour sendspec.timeout
if self.sendspec.timeout is not None:
current_time = time.time()
time_taken = current_time - self.start_time
if time_taken > self.sendspec.timeout:
# We've timed out, kill with prejudice.
self.sendspec.shutit_pexpect_child.quick_send(' kill -9 ' + self.pid)
self.run_state = 'T'
assert self.run_state in ('S','T')
return self.run_state
else:
# Task is finished.
self.run_state = 'C'
shutit_global.shutit_global_object.log('background task: ' + self.sendspec.send + ', id: ' + self.id + ' complete',level=logging.DEBUG)
# Stop this from blocking other commands from here.
assert self.return_value is None, 'check_background_command_state called with self.return_value already set?' + str(self)
self.sendspec.shutit_pexpect_child.quick_send(' wait ' + self.pid)
# Get the exit code.
self.return_value = self.sendspec.shutit_pexpect_child.send_and_get_output(' cat ' + self.exit_code_file, ignore_background=True)
# If the return value is deemed a failure:
if self.return_value not in self.sendspec.exit_values:
shutit_global.shutit_global_object.log('background task: ' + self.sendspec.send + ' failed with exit code: ' + self.return_value, level=logging.DEBUG)
shutit_global.shutit_global_object.log('background task: ' + self.sendspec.send + ' failed with output: ' + self.sendspec.shutit_pexpect_child.send_and_get_output(' cat ' + self.output_file, ignore_background=True), level=logging.DEBUG)
if self.retry > 0:
shutit_global.shutit_global_object.log('background task: ' + self.sendspec.send + ' retrying',level=logging.DEBUG)
self.retry -= 1
self.run_background_command()
# recurse
return self.check_background_command_state()
else:
shutit_global.shutit_global_object.log('background task final failure: ' + self.sendspec.send + ' failed with exit code: ' + self.return_value, level=logging.DEBUG)
self.run_state = 'F'
assert self.run_state in ('C','F')
return self.run_state
else:
# Task succeeded.
shutit_global.shutit_global_object.log('background task: ' + self.sendspec.send + ' succeeded with exit code: ' + self.return_value, level=logging.DEBUG)
assert self.run_state in ('C',)
return self.run_state
# Should never get here.
assert False