1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35 import os, re, tempfile, time, signal
36
37 import Ganga.Utility.logging
38 logger = Ganga.Utility.logging.getLogger()
39 from Ganga.Utility.Config import getConfig
40
41
43
44 exceptions=getConfig('Shell')['IgnoredVars']
45
46 - def __init__(self,setup=None, setup_args=[]):
47
48 """The setup script is sourced (with possible arguments) and the
49 environment is captured. The environment variables are expanded
50 automatically (this is a fix for bug #44259: GangaLHCb tests fail due to
51 gridProxy check).
52
53 Example of variable expansion:
54
55 os.environ['BAR'] = 'rabarbar'
56 os.environ['FOO'] = '$BAR'
57 s = Shell() # with or without the setup script
58 assert s.env['FOO'] == 'rabarbar' # NOT literal string '$BAR'
59
60 NOTE: the behaviour is not 100% bash compatible: undefined variables in
61 bash are empty strings, Shell() leaves the literals unchanged,so:
62
63 os.environ['FOO'] = '$NO_BAR'
64 s = Shell()
65 if os.environ.not has_key('NO_BAR'):
66 assert s.env['FOO'] == '$NO_BAR'
67
68 """
69
70 def expand_vars(env):
71 tmp_dict = {}
72 for k,v in env.iteritems():
73 tmp_dict[k] = os.path.expandvars(v)
74 return tmp_dict
75
76 if setup:
77 logger.error('source %s %s > /dev/null 2>&1; python -c "import os; print os.environ"' % (setup," ".join(setup_args)))
78 pipe=os.popen('source %s %s > /dev/null 2>&1; python -c "import os; print os.environ"' % (setup," ".join(setup_args)))
79 output=pipe.read()
80 rc=pipe.close()
81 if rc: logger.warning('Unexpected rc %d from setup command %s',rc,setup)
82
83 env = expand_vars(eval(output))
84
85 for key in Shell.exceptions:
86 try:
87 del env[key]
88 except KeyError:
89 pass
90 self.env = env
91 else:
92 env=dict(os.environ)
93 self.env = expand_vars(env)
94
95 self.dirname=None
96
97 - def cmd(self,cmd,soutfile=None,allowed_exit=[0], capture_stderr=False,timeout=None, mention_outputfile_on_errors=True):
98 "Execute an OS command and captures the stderr and stdout which are returned in a file"
99
100 if not soutfile: soutfile=tempfile.mktemp('.out')
101
102 logger.debug('Running shell command: %s' % cmd)
103 try:
104 t0 = time.time()
105 already_killed = False
106 timeout0 = timeout
107 pid = os.spawnve(os.P_NOWAIT,'/bin/sh',['/bin/sh','-c','%s > %s 2>&1' % (cmd,soutfile)],self.env)
108 while 1:
109 wpid,sts = os.waitpid(pid,os.WNOHANG)
110 if wpid!=0:
111 if os.WIFSIGNALED(sts):
112 rc = -os.WTERMSIG(sts)
113 break
114 elif os.WIFEXITED(sts):
115 rc = os.WEXITSTATUS(sts)
116 break
117 if timeout and time.time()-t0>timeout:
118 logger.warning('Command interrupted - timeout %ss reached: %s', timeout0,cmd)
119 if already_killed:
120 sig = signal.SIGKILL
121 else:
122 sig = signal.SIGTERM
123 logger.debug('killing process %d with signal %d',pid,sig)
124 os.kill(pid,sig)
125 t0=time.time()
126 timeout = 5
127 already_killed = True
128 time.sleep(0.1)
129
130 except OSError, (num,text):
131 logger.warning( 'Problem with shell command: %s, %s', num,text)
132 rc = 255
133
134 BYTES = 4096
135 if rc not in allowed_exit:
136 logger.warning('exit status [%d] of command %s',rc,cmd)
137 if mention_outputfile_on_errors:
138 logger.warning('full output is in file: %s',soutfile)
139 logger.warning('<first %d bytes of output>\n%s',BYTES,file(soutfile).read(BYTES))
140 logger.warning('<end of first %d bytes of output>',BYTES)
141
142
143 m = None
144 if rc != 0:
145 m = re.search('command not found\n',file(soutfile).read())
146 if m: logger.warning('command %s not found',cmd)
147
148 return rc,soutfile,m is None
149
150 - def cmd1(self,cmd,allowed_exit=[0],capture_stderr=False,timeout=None):
151 "Executes an OS command and captures the stderr and stdout which are returned as a string"
152
153 rc,outfile,m = self.cmd(cmd,None,allowed_exit,capture_stderr,timeout,mention_outputfile_on_errors=False)
154 output=file(outfile).read()
155 os.unlink(outfile)
156
157 return rc,output, m
158
159 - def system(self,cmd,allowed_exit=[0], stderr_file=None):
160 """Execute on OS command. Useful for interactive commands. Stdout and Stderr are not
161 caputured and are passed on the caller.
162
163 stderr_capture may specify a name of a file to which stderr is redirected.
164 """
165
166 logger.debug('Running shell command: %s' % cmd)
167
168 if stderr_file:
169 cmd += " 2> %s"%stderr_file
170
171 try:
172 rc = os.spawnve(os.P_WAIT,'/bin/sh',['/bin/sh','-c',cmd],self.env)
173 except OSError, (num,text):
174 logger.warning( 'Problem with shell command: %s, %s', num,text)
175 rc = 255
176 return rc
177
178 - def wrapper(self,cmd,preexecute=None):
179 """Write wrapper script for command
180
181 A wrapper around cmd is written including the setting of the environment.
182 Useful for situations where it is an external Python module that is
183 calling the command. It is callers responsibility to enter
184 new location into PATH as this might have external effects. Full path of
185 wrapper script is returned. Preexecute can contain extra commands to be
186 executed before cmd
187
188 fullpath = s.wrapper('lcg-cp', 'echo lcg-cp called with arguments $*'"""
189
190 from Ganga.Utility.tempfile_compatibility import mkdtemp
191 from os.path import join
192
193 if not self.dirname:
194 self.dirname=mkdtemp()
195
196 fullpath = join(self.dirname,cmd)
197 f = open(fullpath,'w')
198 f.write("#!/bin/bash\n")
199 for k,v in self.env.iteritems():
200 f.write("export %s='%s'\n" % (k,v))
201 if preexecute:
202 f.write("%s\n" % preexecute)
203 f.write("%s $*\n" % cmd)
204 f.close()
205 import stat,os
206 os.chmod(fullpath,stat.S_IRWXU)
207
208 return fullpath
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266