Source Code for Module Ganga.Utility.feedback_report

  1   
  2 -def report(job=None): 
  3          """ Upload error reports (snapshot of configuration,job parameters, input/output files, command history etc.). Job argument is optional. """ 
  4          import mimetypes 
  5          import urllib2 
  6          import httplib 
  7          import string 
  8          import random 
  9          import sys 
 10          import os 
 11          import platform 
 12   
 13          from Ganga.GPI import config 
 14          from Ganga.GPI import full_print 
 15          from Ganga.GPI import Job 
 16           
 17          import Ganga 
 18   
 19          #global variables that will print sumamry report to the user along with the download link 
 20          global JOB_REPORT, GANGA_VERSION, BACKEND_NAME, APPLICATION_NAME, PYTHON_PATH 
 21          JOB_REPORT = False 
 22          GANGA_VERSION = '' 
 23          BACKEND_NAME = '' 
 24          APPLICATION_NAME = '' 
 25          PYTHON_PATH = '' 
 26   
 27          def random_string (length): 
 28                  return ''.join ([random.choice (string.letters) for ii in range (length + 1)]) 
 29   
 30          def encode_multipart_data (files): 
 31                  boundary = random_string (30) 
 32   
 33                  def get_content_type (filename): 
 34                          return mimetypes.guess_type (filename)[0] or 'application/octet-stream' 
 35   
 36                  def encode_file (field_name): 
 37                          filename = files [field_name] 
 38                          return ('--' + boundary, 
 39                                  'Content-Disposition: form-data; name="%s"; filename="%s"' % (field_name, filename), 
 40                                  'Content-Type: %s' % get_content_type(filename), 
 41                                  '', open (filename, 'rb').read ()) 
 42           
 43                  lines = [] 
 44                  for name in files: 
 45                          lines.extend (encode_file (name)) 
 46                  lines.extend (('--%s--' % boundary, '')) 
 47                  body = '\r\n'.join (lines) 
 48   
 49                  headers = {'content-type': 'multipart/form-data; boundary=' + boundary, 
 50                          'content-length': str (len (body))} 
 51   
 52                  return body, headers 
 53      
 54   
 55          def make_upload_file (server): 
 56   
 57                  def upload_file (path): 
 58   
 59                          #print 'Uploading %r to %r' % (path, server) 
 60   
 61                          data = {'MAX_FILE_SIZE': '3145728', 
 62                          'sub': '', 
 63                          'mode': 'regist'} 
 64                          files = {'file': path} 
 65   
 66                          send_post (server, files) 
 67   
 68                  return upload_file 
 69   
 70          def send_post (url, files): 
 71                  req = urllib2.Request (url) 
 72                  connection = httplib.HTTPConnection (req.get_host ()) 
 73                  connection.request ('POST', req.get_selector (), 
 74                          *encode_multipart_data (files)) 
 75                  response = connection.getresponse () 
 76   
 77                  responseResult = response.read() 
 78   
 79                  responseResult = responseResult[responseResult.find("<span id=\"download_path\""):] 
 80                  startIndex = responseResult.find("path:") + 5 
 81                  endIndex = responseResult.find("</span>") 
 82   
 83                  print 'Your error report was uploaded to ganga developers with the following URL. ' 
 84                  print 'You may include this URL and the following summary information in your bug report or in the support email to the developers.' 
 85                  print 
 86                  print '***',responseResult[startIndex:endIndex],'***' 
 87                  print 
 88                  global GANGA_VERSION, JOB_REPORT, APPLICATION_NAME, BACKEND_NAME, PYTHON_PATH 
 89                  print 'Ganga Version : ' + GANGA_VERSION 
 90                  print 'Python Version : ' + "%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) 
 91                  print 'Operation System Version : ' + platform.platform()  
 92   
 93                  if JOB_REPORT: 
 94                          print 'Application Name : ' + APPLICATION_NAME 
 95                          print 'Backend Name : ' + BACKEND_NAME 
 96   
 97                  print 'Python Path : ' + PYTHON_PATH 
 98                  print 
 99   
100                  JOB_REPORT = False 
101                  GANGA_VERSION = '' 
102                  BACKEND_NAME = '' 
103                  APPLICATION_NAME = '' 
104                  PYTHON_PATH = '' 
105   
106          def run_upload (server, path): 
107   
108                  upload_file = make_upload_file (server) 
109                  upload_file (path) 
110           
111   
112          def report_inner(job=None): 
113           
114                  userInfoDirName = "userreport" 
115                  tempDirName = "reportsRepository" 
116                  #job relevant info 
117                  inputDirName = "inputdir" 
118                  outputDirName = "outputdir" 
119                  jobSummaryFileName = "jobsummary.txt" 
120                  jobFullPrintFileName = "jobfullprint.txt" 
121                  #non job relevant info - user's info 
122                  environFileName = "environ.txt" 
123                  userConfigFileName = "userconfig.txt" 
124                  defaultConfigFileName = "gangarc.txt" 
125                  ipythonHistoryFileName = "ipythonhistory.txt" 
126                  gangaLogFileName = "gangalog.txt" 
127                  jobsListFileName = "jobslist.txt" 
128                  repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx" 
129                  uploadFileServer= "http://gangamon.cern.ch/django/errorreports/" 
130                  #uploadFileServer= "http://127.0.0.1:8000/errorreports" 
131   
132                  def printDictionary(dictionary): 
133                          for k,v in dictionary.iteritems(): 
134                                  print '%s: %s' % (k,v) 
135                                  print 
136                                   
137                                  if k == 'PYTHONPATH': 
138                                          global PYTHON_PATH 
139                                          PYTHON_PATH = v 
140   
141                  def extractFileObjects(fileName, targetDirectoryName): 
142                          try: 
143                                  fileToRead = open(fileName, 'r') 
144                                  try: 
145                                          fileText = fileToRead.read() 
146                                          import re 
147                                          pattern = "File\(name=\'(.+?)\'" 
148                                          matches = re.findall(pattern, fileText)  
149                                           
150                                          for fileName in matches: 
151                                                  fileName = os.path.expanduser(fileName)  
152                                                  targetFileName = os.path.join(targetDirectoryName, os.path.basename(fileName))                                   
153                                                  shutil.copyfile(fileName, targetFileName)                
154   
155                                  finally: 
156                                          fileToRead.close() 
157                          #except IOError, OSError: 
158                          except: 
159                                  writeErrorLog(str(sys.exc_value)) 
160           
161                  def writeErrorLog(errorMessage): 
162                          try: 
163                                  fileToWrite = open(errorLogPath, 'a') 
164                                  try: 
165                                          fileToWrite.write(errorMessage) 
166                                          fileToWrite.write("\n") 
167                                  finally: 
168                                          fileToWrite.close() 
169                          except: 
170                                  pass 
171                                   
172   
173                  def writeStringToFile(fileName, stringToWrite): 
174   
175                          try: 
176                                  #uncomment this to try the error logger 
177                                  #fileName = '~/' + fileName 
178                                  fileToWrite = open(fileName, 'w') 
179                                  try: 
180                                          fileToWrite.write(stringToWrite) 
181                                  finally: 
182                                          fileToWrite.close() 
183                          #except IOError: 
184                          except: 
185                                  writeErrorLog(str(sys.exc_value)) 
186   
187                  def renameDataFiles(directory): 
188   
189   
190                          for fileName in os.listdir(directory): 
191                                  fullFileName = os.path.join(directory, fileName) 
192                                  if os.path.isfile(fullFileName): 
193                                          if fileName == 'data': 
194                                                  os.rename(fullFileName, fullFileName + '.txt') 
195                                  else: 
196                                          renameDataFiles(fullFileName)    
197   
198                  import shutil 
199                  import tarfile 
200                  import tempfile 
201   
202                  userHomeDir = os.getenv("HOME") 
203                  tempDir = tempfile.mkdtemp() 
204   
205                  errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt') 
206   
207                  fullPathTempDir = os.path.join(tempDir, tempDirName) 
208                  fullLogDirName = '' 
209                  #create temp dir and specific dir for the job/user 
210   
211                  try: 
212                          if not os.path.exists(fullPathTempDir): 
213                                  os.mkdir(fullPathTempDir) 
214   
215                          import datetime 
216                          now = datetime.datetime.now() 
217                          userInfoDirName = userInfoDirName + now.strftime("%Y-%m-%d-%H:%M:%S") 
218                          fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName)  
219           
220                          #if report directory exists -> delete it's content(we would like last version of the report) 
221                          if os.path.exists(fullLogDirName): 
222                                  shutil.rmtree(fullLogDirName) 
223           
224                          os.mkdir(fullLogDirName) 
225                  #except OSError: 
226                  except: 
227                          writeErrorLog(str(sys.exc_value)) 
228   
229                  #import os.environ in a file 
230                  fullEnvironFileName = os.path.join(fullLogDirName, environFileName) 
231   
232                  try: 
233                          inputFile = open(fullEnvironFileName, 'w') 
234                          try: 
235                                  sys.stdout = inputFile 
236                                  printDictionary(os.environ) 
237   
238                                  print 'OS VERSION : ' + platform.platform() 
239   
240                          finally: 
241                                  sys.stdout = sys.__stdout__ 
242                                  inputFile.close()        
243                  #except IOError          
244                  except: 
245                          writeErrorLog(str(sys.exc_value)) 
246   
247                  #import user config in a file    
248                  userConfigFullFileName = os.path.join(fullLogDirName, userConfigFileName) 
249                   
250                  try: 
251                          inputFile = open(userConfigFullFileName, 'w') 
252                          try: 
253                           
254                                  sys.stdout = inputFile 
255   
256                                  print "#GANGA_VERSION = %s" % config.System.GANGA_VERSION  
257                                  print 
258   
259                                  global GANGA_VERSION 
260                                  GANGA_VERSION = config.System.GANGA_VERSION 
261   
262                                  #this gets the default values 
263                                  #Ganga.GPIDev.Lib.Config.Config.print_config_file() 
264                                   
265                                  #this should get the changed values 
266                                  for c in config: 
267                                          print config[c] 
268   
269                          finally: 
270                                  sys.stdout = sys.__stdout__ 
271                                  inputFile.close() 
272                  #except IOError does not catch the exception ??? 
273                  except: 
274                          writeErrorLog(str(sys.exc_value)) 
275   
276                  #write gangarc - default configuration 
277                  defaultConfigFullFileName = os.path.join(fullLogDirName, defaultConfigFileName) 
278   
279                  try: 
280                          outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r') 
281           
282                          try: 
283                                  writeStringToFile(defaultConfigFullFileName, outputFile.read())   
284                          finally: 
285                                  outputFile.close() 
286                                           
287                  #except IOError does not catch the exception ???                 
288                  except: 
289                          writeErrorLog(str(sys.exc_value)) 
290   
291                  #import ipython history in a file 
292                  try: 
293                          ipythonFile = open(os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r') 
294                           
295                          try:                     
296                                  lastIPythonCommands = ipythonFile.readlines()[-20:]              
297                                  writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands)) 
298                                  #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read()) 
299                          finally: 
300                                  ipythonFile.close() 
301                  #except IOError does not catch the exception ???                 
302                  except: 
303                          writeErrorLog(str(sys.exc_value)) 
304           
305                  #import gangalog in a file       
306                  userLogFileLocation = config["Logging"]._logfile 
307                  userLogFileLocation = os.path.expanduser(userLogFileLocation) 
308   
309                  try: 
310                          gangaLogFile = open(userLogFileLocation, 'r') 
311                          try:             
312                                  writeStringToFile(os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read())   
313                          finally: 
314                                  gangaLogFile.close() 
315                  #except IOError: 
316                  except: 
317                          writeErrorLog(str(sys.exc_value)) 
318   
319                  #import the result of jobs command in the report 
320                  jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName) 
321                   
322                  try: 
323                          outputFile = open(jobsListFullFileName, 'w') 
324                          try: 
325                           
326                                  sys.stdout = outputFile 
327                                   
328                                  from Ganga.GPI import jobs 
329                                  print jobs       
330                                           
331                          finally: 
332                                  sys.stdout = sys.__stdout__ 
333                                  outputFile.close() 
334                                           
335                  #except IOError does not catch the exception ???                 
336                  except: 
337                          writeErrorLog(str(sys.exc_value)) 
338           
339                  #save it here because we will change fullLogDirName, but we want this to be the archive and to be deleted  
340                  folderToArchive = fullLogDirName 
341   
342                  #import job relevant info 
343                  if job is not None: 
344   
345                          global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME 
346   
347                          JOB_REPORT = True 
348                          APPLICATION_NAME = job.application.__class__.__name__ 
349                          BACKEND_NAME = job.backend.__class__.__name__ 
350   
351   
352                          #create job folder                       
353                          jobFolder = 'job_%s' % str(job.fqid) 
354                          fullLogDirName = os.path.join(fullLogDirName, jobFolder) 
355                          os.mkdir(fullLogDirName)                         
356   
357                          #import job summary in a file    
358                          fullJobSummaryFileName = os.path.join(fullLogDirName, jobSummaryFileName) 
359                          writeStringToFile(fullJobSummaryFileName, str(job)) 
360   
361                          #import job full print in a file 
362                          fullJobPrintFileName = os.path.join(fullLogDirName, jobFullPrintFileName) 
363                           
364                          try: 
365                                  inputFile = open(fullJobPrintFileName, 'w') 
366                                  try: 
367                                          full_print(job, inputFile) 
368                                  finally: 
369                                          inputFile.close() 
370                          #except IOError, OSError: 
371                          except: 
372                                  writeErrorLog(str(sys.exc_value)) 
373   
374                          #extract file objects 
375                          try:                     
376                                  fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects') 
377                                  os.mkdir(fileObjectsPath) 
378                                  extractFileObjects(fullJobSummaryFileName, fileObjectsPath) 
379                          #except OSError: 
380                          except: 
381                                  writeErrorLog(str(sys.exc_value)) 
382   
383                          #copy dir of the job ->input/output and subjobs 
384                          try: 
385                                  parentDir, currentDir = os.path.split(job.inputdir[:-1]) 
386                                  workspaceDir = os.path.join(fullLogDirName, 'workspace') 
387                                  shutil.copytree(parentDir, workspaceDir) 
388                          #except IOError, OSError 
389                          except: 
390                                  writeErrorLog(str(sys.exc_value)) 
391   
392                          #copy repository job file 
393                          try: 
394                                  indexFileName = str(job.id) + '.index' 
395   
396                                  repositoryPath = repositoryPath.replace('$usr', os.getenv("USER")) 
397                                   
398                                  #check if the job is subjob -> different way of forming the path to the repository 
399                                  is_subjob = job.fqid.find('.') > -1 
400   
401                                  if is_subjob: 
402   
403                                          jobid, subjobid = job.fqid.split('.')[0], job.fqid.split('.')[1] 
404                                          repositoryPath = repositoryPath.replace('$thousandsNum', str(int(jobid)/1000)) 
405                                          repositoryPath = os.path.join(repositoryPath, jobid) 
406   
407                                  else:    
408                                          repositoryPath = repositoryPath.replace('$thousandsNum', str(job.id/1000)) 
409   
410                                  repositoryFullPath = os.path.join(config.Configuration.gangadir, repositoryPath) 
411                                  indexFileSourcePath = os.path.join(repositoryFullPath, indexFileName) 
412                                  repositoryFullPath = os.path.join(repositoryFullPath, str(job.id)) 
413   
414                                  repositoryTargetPath = os.path.join(fullLogDirName, 'repository', str(job.id)) 
415                                   
416                                  os.mkdir(os.path.join(fullLogDirName, 'repository')) 
417   
418                                  shutil.copytree(repositoryFullPath, repositoryTargetPath) 
419                                  #data files are copied but can not be opened -> add .txt to their file names 
420                                  renameDataFiles(repositoryTargetPath) 
421   
422                                  if not is_subjob: 
423                                          #copy .index file 
424                                          indexFileTargetPath = os.path.join(fullLogDirName, 'repository', indexFileName) 
425                                          shutil.copyfile(indexFileSourcePath, indexFileTargetPath) 
426                                   
427                          #except OSError, IOError: 
428                          except: 
429                                  writeErrorLog(str(sys.exc_value)) 
430   
431                  resultArchive = '%s.tar.gz' % folderToArchive 
432   
433                  try: 
434                          resultFile = tarfile.TarFile.open(resultArchive, 'w:gz') 
435                          try: 
436                                  resultFile.add(folderToArchive, arcname=os.path.basename(folderToArchive)) 
437                                  #put the error log in the archive 
438                                  if(os.path.exists(errorLogPath)): 
439                                          resultFile.add(errorLogPath, arcname=os.path.basename(errorLogPath))     
440                   
441                          finally:         
442                                  resultFile.close() 
443                  except: 
444                          raise #pass 
445                   
446                  #remove temp dir 
447                  if(os.path.exists(folderToArchive)): 
448                          shutil.rmtree(folderToArchive) 
449   
450                  #print the error if there is something 
451                  if os.path.exists(errorLogPath): 
452                          print 
453                          print 'An error occured while collecting report information : ' +  open(errorLogPath, 'r').read() 
454                          print   
455   
456                  #delete the errorfile from user's pc 
457                  if(os.path.exists(errorLogPath)): 
458                          os.remove(errorLogPath) 
459   
460                  #return the path to the archive and the path to the upload server 
461                  return (resultArchive, uploadFileServer, tempDir) 
462   
463          def removeTempFiles(tempDir): 
464                  import shutil 
465                   
466                  #remove temp dir 
467                  if os.path.exists(tempDir): 
468                          shutil.rmtree(tempDir)   
469   
470                  #remove temp files from django upload-> if the file is bigger than 2.5 mb django internally stores it in tmp file during the upload 
471                  userTempDir = '/tmp/' 
472                   
473                  for fileName in os.listdir(userTempDir): 
474                          if fileName.find('.upload') > -1: 
475                                  os.remove(os.path.join(userTempDir, fileName))  
476   
477   
478          tempDir = '' 
479   
480          #call the report function 
481          try: 
482   
483                  #make typecheck of the param passed 
484                  if job is not None: 
485                          if not isinstance(job,Job): 
486                                  print "report() function argument should be reference to a job object" 
487                                  return 
488   
489                  resultArchive, uploadFileServer, tempDir = report_inner(job) 
490   
491                  report_bytes = os.path.getsize(resultArchive) 
492                   
493                  if report_bytes > 1024*1024*100: #if bigger than 100MB 
494                          print 'The report is bigger than 100MB and can not be uploaded' 
495                  else: 
496                          run_upload(server=uploadFileServer, path=resultArchive) 
497   
498                   
499          except: 
500                  removeTempFiles(tempDir) 
501                  raise #pass 
502                  #raise   
503   
504          removeTempFiles(tempDir) 
505