Ganga.Core.FileWorkspace

Source Code for Module Ganga.Core.FileWorkspace

1 ################################################################################ 2 # Ganga Project. http://cern.ch/ganga 3 # 4 # $Id: FileWorkspace.py,v 1.2 2009-05-20 09:23:46 moscicki Exp $ 5 ################################################################################ 6 7 """ 8 FileWorkspace subsystem (aka LSE, SE,...) defines the interface to 9 create, copy and upload files which are part of Job definition such as 10 input and output sandboxes. This allows to handle files in the 11 FileWorkspace in a location-independent way. 12 """ 13 14 import Ganga.Utility.logging 15 logger = Ganga.Utility.logging.getLogger(modulename=1) 16 17 import os,time 18 19 from Ganga.Utility.files import expandfilename, chmod_executable 20 21

22 -class FileWorkspace:

23 """ 24 File workspace on a local file system. FileWorkspace object may 25 represent any part of the directory tree (including 'top' i.e. 26 root of the workspace). 27 28 Files may be stored on a 'per job' basis (optional jobid argument 29 to create method). A subpath may specify a specific part of the 30 file workspace (such as input/output sandbox). Note that you may 31 use FileWorkspace to store files which are not related to any 32 partical job (or are shared between jobs). 33 34 The general directory layout : 35 getPath() resolves to 'top/jobid/subpath/*' 36 37 Old 'splittree' option has been disabled and always defaults to 0. 38 It should not be modified because it will not work with sbjobs in the future. 39 40 If jobid is None then FileWorkspace represents the topmost 41 directory (with a given subpath which may be an empty string ''), 42 i.e.: getPath() resolves to 'top/subpath/*' or 'top/*' """ 43

44 - def __init__(self,top,subpath='',splittree=0):

45 self.jobid = None 46 self.top = top 47 self.subpath = subpath 48 self.splittree=0 49 50 #LEGACY: 51 if splittree: 52 logger.warning('FileWorkspace splittree option is obsolete and has no-effect')

53

54 - def create(self,jobid=None):

55 """ create a workspace, an optional jobid parameter specifies the job directory 56 you can call create() as many times as you want without any harm """ 57 58 # FIXME: make a helper method for os.makedirs 59 logger.debug('creating %s',self.getPath()) 60 self.jobid = jobid 61 try: 62 import os 63 os.makedirs(self.getPath()) 64 except OSError,x: 65 import errno 66 if x.errno == errno.EEXIST: 67 logger.debug('EEXIT: %s',self.getPath()) 68 else: 69 raise

70 71 # resolve a path to the filename in the context of the file workspace 72 # if filename is None then return the directory corresponding to this file workspace

73 - def getPath(self,filename=None):

74 subpath = self.subpath 75 if filename is None: filename = '' 76 if not self.jobid is None: 77 jobdir = str(self.jobid) 78 else: 79 jobdir = '' 80 # do not use subpath if no tree splitting applies 81 if not self.splittree: subpath = '' 82 83 84 if self.splittree: 85 return expandfilename(os.path.join(self.top,subpath,jobdir,filename)) 86 else: 87 return expandfilename(os.path.join(self.top,jobdir,subpath,filename))

88 89 # write a file (represent as file object) to the workspace 90 # file object may be: 91 # - a File instance - referes to an existing file which will be copied to workspace directory 92 # - a FileBuffer instance - refers to a file which is does not yet exist but which contents is available in a memory buffer 93 # this is a handy way of creating wrapper scripts etc. 94 # - a tuple (name,contents) - deprecated - equivalent to FileBuffer 95 # 96 # File classes are define in Ganga.GPIDev.Lib.File package 97 #

98 - def writefile(self,fileobj,executable=None):

99 100 from Ganga.GPIDev.Lib.File import File,FileBuffer 101 102 try: 103 name,contents = fileobj 104 except TypeError: 105 pass 106 else: 107 fileobj = FileBuffer(name,contents) 108 logger.warning('file "%s": usage of tuples is deprecated, use FileBuffer instead',name) 109 110 # output file name 111 # Added a subdir to files, (see Ganga/GPIDev/Lib/File/File.py) This allows 112 # to copy files into the a subdirectory of the workspace 113 114 # FIXME: make a helper method for os.makedirs 115 try: 116 import os 117 os.makedirs(self.getPath()+fileobj.subdir) 118 logger.debug('created %s',self.getPath()) 119 except OSError,x: 120 import errno 121 if x.errno == errno.EEXIST: 122 logger.debug('EEXIT: %s',self.getPath()) 123 else: 124 raise 125 126 outname = expandfilename(self.getPath(fileobj.getPathInSandbox())) 127 128 fileobj.create(outname) 129 130 if executable: 131 chmod_executable(outname) 132 133 return outname

134 135 # remove the workspace (including all files and directories) 136 # the part of the tree as resolved by getPath() is pruned recursively 137 # if preserve_top is true then the directory specified by getPath() will be preserved

138 - def remove(self,preserve_top=None):

139 try: 140 import shutil 141 logger.debug('removing %s',self.getPath()) 142 if os.path.exists(self.getPath()): 143 self.__removeTrials=0 144 145 def retryRemove(function, path, excinfo): 146 """ Address AFS/NSF problems with left-over lock files which prevents 147 the 'shutil.rmtree' to delete the directory (the idea is to wait a bit 148 for the fs to automatically remove these lock files and try again) 149 """ 150 self.__removeTrials+=1 151 if self.__removeTrials<=5: 152 logger.debug('Cannot delete %s (retry count=%s) ... Will wait a bit and try again' 153 % (self.getPath(),self.__removeTrials)) 154 time.sleep(0.5) 155 shutil.rmtree(self.getPath(),ignore_errors=False, onerror=retryRemove) 156 else: 157 exctype, value = excinfo[:2] 158 logger.warning('Cannot delete %s after %s retries due to: %s:%s (there might some AFS/NSF lock files left over)' 159 % (self.getPath(),self.__removeTrials,exctype,value))

160 161 shutil.rmtree(self.getPath(),ignore_errors=False, onerror=retryRemove) 162 logger.debug('removed %s',self.getPath()) 163 if preserve_top and not os.path.exists(self.getPath()): 164 logger.debug('preserving the topdir: mkdir %s'%self.getPath()) 165 os.mkdir(self.getPath()) 166 else: 167 logger.debug('%s : DOES NOT EXIST',self.getPath()) 168 #FIXME: error strategy 169 except OSError,x: 170 raise 171 except Exception,x: 172 raise 173

174 -def gettop():

175 c = Ganga.Utility.Config.getConfig('Configuration') 176 return os.path.join(c['gangadir'],'workspace',c['user'],c['repositorytype'])

177

178 -class InputWorkspace(FileWorkspace):

179 """ Part of the workspace for storing input sandbox. 180 """

181 - def __init__(self):

182 FileWorkspace.__init__(self,gettop(),subpath='input',splittree=False)

183 184

185 -class OutputWorkspace(FileWorkspace):

186 """ Part of the workspace for storing output sandbox. 187 """

188 - def __init__(self):

189 FileWorkspace.__init__(self,gettop(),subpath='output',splittree=False)

190

191 -class DebugWorkspace(FileWorkspace):

192 """ Part of the workspace for storing output sandbox. 193 """

194 - def __init__(self):

195 FileWorkspace.__init__(self,gettop(),subpath='debug',splittree=False)

196 197 198 # 199 # 200 # $Log: not supported by cvs2svn $ 201 # Revision 1.1 2008/07/17 16:40:49 moscicki 202 # migration of 5.0.2 to HEAD 203 # 204 # the doc and release/tools have been taken from HEAD 205 # 206 # Revision 1.16.28.7 2008/04/18 08:17:24 moscicki 207 # minor fix 208 # 209 # Revision 1.16.28.6 2008/04/02 15:25:24 moscicki 210 # bugfix: #31691 j.remove() removes workspace for all jobs (not just job in question) 211 # 212 # Revision 1.16.28.5 2008/03/11 15:20:30 moscicki 213 # merge from Ganga-5-0-restructure-config-branch 214 # 215 # Revision 1.16.28.4.2.1 2008/03/07 13:36:07 moscicki 216 # removal of [DefaultJobRepository] and [FileWorkspace] 217 # new options in [Configuration] user, gangadir, repositorytype, workspacetype 218 # 219 # Revision 1.16.28.4 2008/02/05 12:29:56 amuraru 220 # bugfix #32850 221 # 222 # Revision 1.16.28.3 2007/12/10 19:25:03 amuraru 223 # merged changes from Ganga 4.4.4 224 # 225 # Revision 1.16.28.2 2007/10/25 11:39:33 roma 226 # Config update 227 # 228 # Revision 1.16.28.1 2007/10/12 13:56:23 moscicki 229 # merged with the new configuration subsystem 230 # 231 # Revision 1.16.30.1 2007/09/25 09:45:11 moscicki 232 # merged from old config branch 233 # 234 # Revision 1.16.8.1 2007/06/18 07:44:51 moscicki 235 # config prototype 236 # 237 # Revision 1.17 2007/11/23 15:11:03 amuraru 238 # fixed bug #22428 and #29825 239 # 240 # Revision 1.19 2008/04/02 15:11:33 moscicki 241 # bugfix: #31691 j.remove() removes workspace for all jobs (not just job in question) 242 # 243 # Revision 1.18 2008/01/25 09:42:20 amuraru 244 # fixed bug #32850 245 # 246 # Revision 1.17 2007/11/23 15:11:03 amuraru 247 # fixed bug #22428 and #29825 248 # 249 # Revision 1.16 2006/07/27 20:02:01 moscicki 250 # comments 251 # 252 # Revision 1.15 2006/02/10 14:07:03 moscicki 253 # code cleanup 254 # 255 # Revision 1.14 2005/11/14 10:03:44 moscicki 256 # possibility to leave or remove the empty workspace directory 257 # 258 # Revision 1.13 2005/10/12 13:35:23 moscicki 259 # renamed _gangadir into gangadir 260 # 261 # Revision 1.12 2005/10/07 15:06:57 moscicki 262 # renamed __Ganga4__ into _gangadir 263 # 264 # Revision 1.11 2005/09/23 09:06:41 moscicki 265 # splittree option now defaults to 0 and is obsolete 266 # 267 # Revision 1.10 2005/09/06 12:02:14 moscicki 268 # minor fix 269 # 270 # Revision 1.9 2005/08/23 17:18:13 moscicki 271 # using new File methods, removed Karl's fix as obsolete 272 # 273 # Revision 1.8 2005/08/16 10:59:27 karl 274 # KH: Allow old and new style configuration 275 # 276 # Revision 1.7 2005/08/10 15:01:10 moscicki 277 # Fixed the exception handling for makedirs -- do not mask runtime errors, ignore EEXIST 278 # 279 # Revision 1.6 2005/08/10 09:45:36 andrew 280 # Added a subdir to File and FileBuffer objects. Changed the writefile method 281 # in FileWorspace to use the subdirectory 282 # 283 # 284 # 285