Ganga.Utility.external.ARDAMDClient.extendedLists

1 ################################################################################ 2 # Ganga Project. http://cern.ch/ganga 3 # 4 # $Id: extendedLists.py,v 1.1 2008-07-17 16:41:02 moscicki Exp $ 5 ################################################################################ 6 import os 7 import types 8 import time 9 10 from diskutils import * 11

12 -class Storage(object):

13 - def __init__(self, dirname, tries_limit = 200):

14 ## dirname is the name of the directory to store blocks 15 ## tries_limit is the maximum allowed limit for attempts 16 ## to move, remove or write files 17 self.dirname = dirname 18 self.tries_limit = tries_limit

19 20

21 -class Block(object):

22 - def __init__(self, entries = None, entrlen = None, name = None, storage = None):

23 if entries is None: 24 entries = [] 25 if entrlen is None: 26 entrlen = len(entries) 27 self.entries = entries 28 self.name = name 29 self.storage = storage 30 self.in_memory = True 31 self.on_disk = False #if modified: self.on_disk = False 32 self.setLen(entrlen) 33 self.updateKeys()

34

35 - def __getitem__(self, i):

36 if i < self.__len__(): 37 return self.entries[i] 38 else: 39 raise IndexError('list index out of range')

40

41 - def __setitem__(self, i, item):

42 if i < self.__len__(): 43 if self.entries[i][0] != item[0]: 44 self.delKey(self.entries[i][0]) 45 self.addKey(item[0]) 46 self.entries[i] = item 47 self.on_disk = False 48 else: 49 raise IndexError('list index out of range')

50

51 - def __delitem__(self, i):

52 entrlen = self.__len__() 53 if i < entrlen: 54 self.delKey(self.entries[i][0]) 55 del self.entries[i] 56 self.setLen(entrlen - 1) 57 self.on_disk = False 58 else: 59 raise IndexError('list index out of range')

60

61 - def __len__(self):

62 return self.entrlen

63

64 - def append(self, item):

65 entrlen = self.__len__() 66 self.addKey(item[0]) 67 if entrlen < len(self.entries): 68 self.entries[entrlen] = item 69 else: 70 self.entries.append(item) 71 self.setLen(entrlen + 1) 72 self.on_disk = False

73

74 - def insert(self, i, item):

75 entrlen = self.__len__() 76 if i < entrlen: 77 self.addKey(item[0]) 78 self.entries.insert(i, item) 79 self.setLen(entrlen + 1) 80 self.on_disk = False 81 else: 82 self.append(item)

83

84 - def genericName(self):

85 if self.name: 86 return self.name.split('_')[0]

87

88 - def setLen(self, entrlen):

89 self.entrlen = entrlen

90

91 - def addKey(self, key):

92 self.entry_keys[key] = {}

93

94 - def delKey(self, key):

95 del self.entry_keys[key]

96

97 - def hasKey(self, key):

98 return self.entry_keys.has_key(key)

99

100 - def updateKeys(self):

101 self.entry_keys = {} 102 for i in xrange(self.__len__()): 103 self.addKey(self.entries[i][0])

104

105 - def memoryRelease(self):

106 # release memory, so that self.entries list can be reused by another block 107 # this is done to fight memory leak in Python 108 if self.in_memory: 109 if not self.on_disk: 110 self.save() 111 self.in_memory = False 112 entries = self.entries 113 self.entries = [] 114 return entries

115

116 - def load(self, forced = False, entries = None):

117 if self.storage: 118 dirname = self.storage.dirname 119 generic_name = self.genericName() 120 if generic_name: 121 try: 122 last = getLast(os.path.join(dirname, generic_name)) 123 except OSError, e: 124 self.on_disk = False 125 else: 126 self.on_disk = True 127 lfn = os.path.basename(last) 128 if self.name != lfn or forced: 129 self.entries, entrlen = read(last, entries) 130 self.name = lfn 131 self.in_memory = True 132 self.setLen(entrlen) 133 self.updateKeys()

134

135 - def save(self):

136 if not self.on_disk: 137 if self.in_memory: 138 if self.storage: 139 dirname = self.storage.dirname 140 tries_limit = self.storage.tries_limit 141 generic_name = self.genericName() 142 if generic_name: 143 fn = os.path.join(dirname, generic_name) 144 entrlen = self.__len__() 145 if entrlen > 0: 146 self.name = os.path.basename(write(self.entries, fn, entrlen, tries_limit)) 147 self.on_disk = True 148 else: 149 remove(fn) 150 self.on_disk = False

151 152

153 -class EntryBlock(Block):

154

155 - def addKey(self, key):

156 # key will be slit using "." separator and saved in a tree like structure 157 # {k1:{k2:{}}, kk:{}} 158 kk = key.split('.') 159 dd = self.entry_keys 160 for k in kk: 161 if not dd.has_key(k): 162 dd[k] = {} 163 dd = dd[k]

164

165 - def delKey(self, key):

166 # will delete empty dictionaries 167 kk = key.split('.') 168 dd = self.entry_keys 169 visited = [] 170 for k in kk: 171 if dd.has_key(k): 172 visited.append((dd,k)) 173 dd = dd[k] 174 else: 175 break 176 else: 177 visited.reverse() 178 for dd, k in visited: 179 del dd[k] 180 if len(dd) > 0: 181 break

182

183 - def hasKey(self, key):

184 kk = key.split('.') 185 dd = self.entry_keys 186 for k in kk: 187 if not dd.has_key(k): 188 return False 189 else: 190 dd = dd[k] 191 return True

192 193

194 -class BlockCache(object):

195

196 - def __init__(self, maxsize = 3):

197 # maxsize is in blocks 198 # if maxsize <= 0, there is no limit 199 self.maxsize = maxsize 200 self.blocks = []

201

202 - def put(self, block):

203 if block.in_memory: 204 if block not in self.blocks: 205 self.blocks.append(block)

206

207 - def get(self):

208 if self.maxsize > 0: 209 if len(self.blocks) >= self.maxsize: 210 return self.blocks.pop(0).memoryRelease()

211 212

213 -class BlockedList(object):

214 block_prefix = 'BLOCK-' 215 block_class = Block 216

217 - def __init__(self, ll = None, 218 dirname = '', 219 blocklength = 100, 220 cache_size = 3, 221 tries_limit = 200):

222 self.storage = Storage(dirname, tries_limit) 223 self.blocklength = blocklength 224 if ll == None: 225 ll = [] 226 self.cache = BlockCache(cache_size) 227 self.blocks = [] 228 start = 0 229 stop = self.blocklength 230 while 1: 231 entries = ll[start:stop] 232 if not entries: 233 break 234 self._addNewBlock(entries) 235 start = stop 236 stop += self.blocklength

237

238 - def _addNewBlock(self, entries):

239 bni = len(self.blocks) 240 if bni > 0: 241 last_blk = self.blocks[-1] 242 gn = last_blk.genericName() 243 if gn: 244 bni = int(gn.split('-')[1])+ 1 245 generic_name = self.block_prefix + str(bni) 246 entrlen = len(entries) 247 free_entries = self.cache.get() 248 if free_entries is not None: 249 free_entries[:entrlen] = entries 250 entries = free_entries 251 blk = self.block_class(entries = entries, 252 entrlen = entrlen, 253 name = generic_name, 254 storage = self.storage) 255 self.blocks.append(blk) 256 self.cache.put(blk)

257

258 - def _loadBlock(self, blk, forced = True):

259 # if block is in memory the forced loading is suppressed 260 if not (blk.in_memory and forced): 261 free_entries = self.cache.get() 262 blk.load(forced, free_entries) 263 self.cache.put(blk)

264

265 - def _translateIndex(self, i):

266 if i < 0: 267 i += self.__len__() 268 i = max(0,i) 269 blk_i = i 270 for blk in self.blocks: 271 tt = blk_i - len(blk) 272 if tt < 0: 273 return (blk, blk_i) 274 blk_i = tt 275 raise IndexError('list index out of range')

276

277 - def __getitem__(self, i):

278 if type(i) == types.SliceType: 279 ss = i.indices(self.__len__()) 280 ii = range(*ss) 281 return map(self.__getitem__, ii) 282 else: 283 (blk, blk_i) = self._translateIndex(i) 284 self._loadBlock(blk) 285 return blk[blk_i]

286

287 - def __setitem__(self, i, item):

288 if type(i) == types.SliceType: 289 ss = i.indices(self.__len__()) 290 ii = range(*ss) 291 len_ii = len(ii) 292 len_item = len(item) 293 if i.step != None: #extended slice 294 if len_ii != len_item: 295 msg = "attempt to assign sequence of size %d to extended slice of size %d" %(len_item, len_ii) 296 raise ValueError(msg) 297 else: 298 delete = ii[len_item:] 299 delete.reverse() #items to delete 300 insert = item[len_ii:] 301 insert.reverse() #items to insert 302 for d in delete: 303 del self[d] 304 for s in insert: 305 self.insert(len_ii, s) 306 map(lambda x: self.__setitem__(*x), zip(ii,item)) 307 else: 308 (blk, blk_i) = self._translateIndex(i) 309 self._loadBlock(blk) 310 blk[blk_i] = item

311

312 - def __delitem__(self, i):

313 if type(i) == types.SliceType: 314 ss = i.indices(self.__len__()) 315 ii = range(*ss) 316 if ss[0] < ss[1]: 317 ii.reverse() 318 map(self.__delitem__, ii) 319 else: 320 (blk, blk_i) = self._translateIndex(i) 321 self._loadBlock(blk) 322 del blk[blk_i]

323

324 - def __len__(self):

325 blen = 0 326 for blk in self.blocks: 327 blen += len(blk) 328 return blen

329

330 - def append(self, item):

331 if not self.blocks: 332 self._addNewBlock([]) 333 blk = self.blocks[-1] 334 if len(blk) >= self.blocklength: 335 self._addNewBlock([]) 336 blk = self.blocks[-1] 337 self._loadBlock(blk) 338 blk.append(item)

339

340 - def extend(self, iter):

341 for item in iter: 342 self.append(item)

343

344 - def insert(self, i, item):

345 # can increase block length over self.blocklength 346 (blk, blk_i) = self._translateIndex(i) 347 self._loadBlock(blk) 348 blk.insert(blk_i, item)

349

350 - def _getBlockNames(self):

351 dd = {} 352 for blk in self.blocks: 353 generic_name = blk.genericName() 354 if generic_name != None: 355 dd[generic_name] = blk 356 return dd

357

358 - def _sorter(self, x, y):

359 x, y = map(lambda f: int(f.split('-')[1]), (x, y)) 360 if x < y: 361 return -1 362 elif x == y: 363 return 0 364 elif x > y: 365 return 1

366

367 - def _getStoredBlockNames(self):

368 dirname = self.storage.dirname 369 names = [] 370 for fn in listFiles(dirname): 371 if fn.startswith(self.block_prefix): 372 ff = fn.split('_') 373 if len(ff) == 2: 374 if ff[1] != '': 375 fn = ff[0] 376 if fn not in names: 377 names.append(fn) 378 names.sort(self._sorter) 379 return names

380

381 - def load(self, dirname = ''):

382 if dirname: 383 self.storage.dirname = dirname 384 blocks = [] 385 bns = self._getStoredBlockNames() 386 ndict = self._getBlockNames() 387 for bn in bns: 388 if bn in ndict: 389 blk = ndict[bn] 390 else: 391 blk = self.block_class(name = bn, storage = self.storage) 392 self._loadBlock(blk, forced = False) 393 blocks.append(blk) 394 self.blocks = blocks

395

396 - def save(self, dirname = ''):

397 if dirname: 398 self.storage.dirname = dirname 399 for i in xrange(len(self.blocks)-1, -1, -1): 400 blk = self.blocks[i] 401 blk.save() 402 if len(blk) == 0: 403 del self.blocks[i]

404

405 - def mark(self):

406 map(lambda x: setattr(x,'on_disk',False), self.blocks)

407

408 - def has_key(self, key):

409 for blk in self.blocks: 410 if blk.hasKey(key): 411 return True 412 return False

413 414

415 -class Attributes(BlockedList):

416 block_prefix = 'Attr-' 417

418 - def __init__(self, ll = None, 419 dirname = '', 420 blocklength = 20, 421 cache_size = 3, 422 tries_limit = 200):

423 super(Attributes, self).__init__(ll, dirname, blocklength, cache_size, tries_limit)

424 425

426 -class Entries(BlockedList):

427 block_prefix = 'Entr-' 428 block_class = EntryBlock 429

430 - def __init__(self, ll = None, 431 dirname = '', 432 blocklength = 100, 433 cache_size = 3, 434 tries_limit = 200):

435 super(Entries, self).__init__(ll, dirname, blocklength, cache_size, tries_limit)

436

Source Code for Module Ganga.Utility.external.ARDAMDClient.extendedLists