Ganga.Core.Sandbox.Sandbox

1 import os 2 import sys 3 import shutil 4 import Ganga.Utility.logging 5 logger = Ganga.Utility.logging.getLogger(modulename=True) 6 7 from WNSandbox import INPUT_TARBALL_NAME, OUTPUT_TARBALL_NAME, PYTHON_DIR 8 from Ganga.Core import GangaException 9

10 -class SandboxError(GangaException):

11 - def __init__(self,message):

12 GangaException.__init__(self,message) 13 self.message = message

14

15 - def __str__(self):

16 return "SandboxError: %s "%(self.message)

17 18 19 #FIXME: os.system error handling missing in this module! 20

21 -def getDefaultModules():

22 """ Return list of ganga modules which are needed for WNSandbox. """ 23 import Ganga.Utility.files 24 import Ganga.Core.Sandbox 25 import subprocess 26 import tarfile 27 return [Ganga,Ganga.Utility,Ganga.Utility.files, Ganga.Utility.tempfile_compatibility, Ganga.Utility.ospath_fix, subprocess,tarfile]

28

29 -def getGangaModulesAsSandboxFiles(modules):

30 """ This returns a list of sandbox files corresponding to specified Ganga modules. 31 Ganga modules are placed in a well-known location in the sandbox. 32 """ 33 import inspect,sys 34 from Ganga.Utility.files import remove_prefix 35 from Ganga.GPIDev.Lib.File import File 36 37 files = [] 38 for m in modules: 39 fullpath = os.path.realpath(inspect.getsourcefile(m)) 40 dir,fn = os.path.split(remove_prefix(fullpath,sys.path)) 41 if os.path.join(dir,fn) == fullpath: 42 raise Exception('Cannot find the prefix for %s'%fullpath) 43 files.append(File(fullpath,subdir=os.path.join(PYTHON_DIR,dir))) 44 return files

45

46 -def createPackedInputSandbox(sandbox_files,inws,name):

47 """Put all sandbox_files into tarball called name and write it into to the input workspace. 48 This function is called by Ganga client at the submission time. 49 Arguments: 50 'sandbox_files': a list of File or FileBuffer objects. 51 'inws': a InputFileWorkspace object 52 Return: a list containing a path to the tarball 53 """ 54 55 # from Ganga.Core import FileWorkspace 56 # from Ganga.GPIDev.Lib.File import File 57 58 # import Ganga.Utility.tempfile_compatibility as tempfile 59 # tmpdir = tempfile.mkdtemp() 60 61 # tgzfile = os.path.join(tmpdir,name) 62 63 tgzfile = inws.getPath(name) 64 65 import tarfile 66 import stat 67 68 # 69 ## Curent release with os module 70 # 71 72 # wsdir = os.path.join(tmpdir,"ws") 73 # ws = FileWorkspace.FileWorkspace(wsdir) 74 # ws.create() 75 # for f in sandbox_files: 76 # ws.writefile(f) 77 78 # if os.system("tar -C %s -czf %s ."%(wsdir,tgzfile)) !=0: 79 # print "ERROR:: can't create tarball file with InputSandbox" 80 81 # 82 ## Future release with tarball module 83 84 tf = tarfile.open(tgzfile,"w:gz") 85 tf.dereference=True # --not needed in Windows 86 87 for f in sandbox_files: 88 try: 89 contents=f.getContents() # is it FileBuffer? 90 91 except AttributeError: # File 92 # tf.add(f.name,os.path.join(f.subdir,os.path.basename(f.name))) 93 try: 94 fileobj = file(f.name) 95 except: 96 raise SandboxError("File %s does not exist."%f.name) 97 tinfo = tf.gettarinfo(f.name,os.path.join(f.subdir,os.path.basename(f.name))) 98 99 else: # FileBuffer 100 from StringIO import StringIO 101 fileobj = StringIO(contents) 102 103 tinfo = tarfile.TarInfo() 104 tinfo.name = os.path.join(f.subdir,os.path.basename(f.name)) 105 import time 106 tinfo.mtime = time.time() 107 tinfo.size = fileobj.len 108 109 if f.isExecutable(): 110 tinfo.mode=tinfo.mode|stat.S_IXUSR 111 tf.addfile(tinfo,fileobj) 112 113 tf.close() 114 115 return [tgzfile]

116 117 # gFile = File(tgzfile) 118 # finalpath = inws.writefile(gFile) 119 # try: 120 # shutil.rmtree(tmpdir) 121 # except OSError: 122 # logger.warning( 'Cannot remove temporary directory ignored' ) 123 # return [finalpath] 124

125 -def createInputSandbox(sandbox_files,inws):

126 """Put all sandbox_files into the input workspace. 127 This function is called by Ganga client at the submission time. 128 Arguments: 129 'sandbox_files': a list of File or FileBuffer objects. 130 'inws': a InputFileWorkspace object 131 Return: a list of paths to sanbdox files in the input workspace 132 """ 133 134 # from Ganga.Core import FileWorkspace 135 136 return [inws.writefile(f,f.isExecutable()) for f in sandbox_files]

137

138 -def getPackedOutputSandbox(src_dir,dest_dir):

139 """Unpack output files from tarball in source directory and 140 write them to destination directory 141 This function is called by Ganga client at the completing of job. 142 Complementary to createPackedOutput() 143 Arguments: 144 'src_dir': source directory with tarball 145 'dest_dir': destination directory for output files 146 """ 147 148 149 tgzfile = os.path.join(src_dir,OUTPUT_TARBALL_NAME) 150 if os.access(tgzfile,os.F_OK): 151 152 # workaround for broken tarfile module (2.4) which does 153 # not open certain tarfiles 154 # see: http://bugs.python.org/issue4218 155 if sys.hexversion < 0x020500F0: 156 if os.system("tar -C %s -xzf %s"%(dest_dir,tgzfile)): 157 logger.warning("Problem with extracting sandbox file %s to %s. This is os.system() workaround for python < 2.5.",tgzfile,dest_dir) 158 return 159 160 import tarfile 161 162 try: 163 tf = tarfile.open(tgzfile,"r:gz") 164 except tarfile.ReadError: 165 logger.warning('Sandbox is empty or unreadable') 166 return 167 else: 168 [tf.extract(tarinfo,dest_dir) for tarinfo in tf] 169 tf.close()

170 171 172 173 174 175 ##################################################### 176

Source Code for Module Ganga.Core.Sandbox.Sandbox