1
2
3
4
5
6
7
8 import sys, os
9 import re
10 import time
11 import tempfile
12
13 import extendedListsTest
14 _thisDir = os.path.dirname(extendedListsTest.__file__)
15 if not _thisDir:
16 _thisDir = os.getcwd()
17
18 from diskutils import *
19 from extendedLists import *
20
21
22 DEBUG = True
23
24
25
27 """Gives memory used by the calling process in kb"""
28 ss = os.popen('pmap %d | tail -1'%os.getpid(), 'r').read()
29 if ss:
30 m = re.search(r'([0-9]*)K', ss)
31 if m:
32 return int(m.group(1))
33
34 -def _startText(ff, txt):
35 ff.write(txt)
36 t1 = time.time()
37 ff.write('operation started at %s \n' % time.ctime(t1))
38 m1 = getmem()
39 return t1, m1
40
41 -def _endText(ff, t1, m1):
42 t2 = time.time()
43 m2 = getmem()
44 dt = t2 - t1
45 dm = m2 - m1
46 ff.write('operation finished at %s \n' % time.ctime(t2))
47 ff.write('time used: %f seconds \n' % dt)
48 ff.write('-s->%f<-s-\n\n\n'% dt)
49 ff.write('memory used: %f Kb \n' % dm)
50 ff.write('-s->%f<-s-\n\n\n'% dm)
51 return dt, dm
52
53
54 -def runTest(NTEST, LEN, CACHE, rootDir, output_dir):
55 def _append(jj, i, LEN):
56
57 j = []
58 for k in range(10):
59 j.append(str(i) + '-' + str(k))
60 j.append(LEN*'x'+str(i))
61 jj.append(j)
62
63 if DEBUG:
64 print 'from runTest: rootDir %s, output_dir %s'%(rootDir, output_dir)
65 if not os.path.isdir(rootDir):
66 try:
67 os.makedirs(rootDir)
68 except OSError:
69 pass
70 nn = tempfile.mktemp(suffix = '.test')
71 nn = os.path.join(output_dir, os.path.basename(nn))
72 ff = file(nn, 'w')
73 lock = RLock(os.path.join(rootDir, 'Lock'))
74 try:
75 t1, m1 = _startText(ff, 'registering %d jobs...' % NTEST)
76 if lock.acquire():
77 try:
78 m3 = getmem()
79 jj = Entries(dirname = rootDir, cache_size = CACHE)
80 for i in range(NTEST):
81 _append(jj, i, LEN)
82 m4 = getmem()
83 jj.save()
84 m5 = getmem()
85 finally:
86 lock.release()
87 dt, dm = _endText(ff, t1, m1)
88 if DEBUG:
89 print 'registering %d jobs...' % NTEST
90 print "--->command status", "OK"
91 print '--->len(jj) after registering', len(jj)
92 print "(dt, dm):", (dt, dm), "\n"
93 print "dm for (append, save)", (m4-m3, m5-m4), '\n\n'
94
95
96 t1, m1 = _startText(ff, 'retrieving info about ALL jobs')
97 if lock.acquire():
98 try:
99 jj.load()
100 finally:
101 lock.release()
102 dt, dm = _endText(ff, t1, m1)
103 if DEBUG:
104 print 'retrieving info about ALL jobs'
105
106 print '--->len(jj) after retrieval', len(jj)
107 print "(dt, dm):", (dt, dm), "\n"
108
109 t1, m1 = _startText(ff, 'commiting %d jobs...' % NTEST)
110 if lock.acquire():
111 try:
112 m3 = getmem()
113 jj.load()
114 m4 = getmem()
115 pid = os.getpid()
116 for i in range(len(jj)):
117 j = jj[i]
118 j[0] = str(pid)
119 jj[i] = j[0]
120 m5 = getmem()
121 jj.save()
122 m6 = getmem()
123 finally:
124 lock.release()
125 dt, dm = _endText(ff, t1, m1)
126 if DEBUG:
127 print 'commiting %d jobs...' % NTEST
128 print "--->command status", "OK"
129 print '--->len(jj) after commiting', len(jj)
130 print "(dt, dm):", (dt, dm), "\n"
131 print "dm for (load, change, save)", (m4-m3, m5-m4, m6-m5), '\n\n'
132
133 t1, m1 = _startText(ff, 'retrieving info about ALL jobs')
134 if lock.acquire():
135 try:
136 jj.load()
137 finally:
138 lock.release()
139 dt, dm = _endText(ff, t1, m1)
140 if DEBUG:
141 print 'retrieving info about ALL jobs'
142
143 print '--->len(jj) after second retrieval', len(jj)
144 print "(dt, dm):", (dt, dm), "\n"
145 for j in jj:
146 assert(jj[0][0] == j[0])
147
148 t1, m1 = _startText(ff, 'deleting %d jobs...' % NTEST)
149 if lock.acquire():
150 try:
151 jj.load()
152 for i in range(NTEST):
153 del jj[0]
154 jj.save()
155 finally:
156 lock.release()
157 dt, dm = _endText(ff, t1, m1)
158 if DEBUG:
159 print 'deleting %d jobs...' % NTEST
160 print "--->rest jobs", len(jj), map(lambda j: j[0], jj)
161 print "(dt, dm):", (dt, dm), "\n"
162 for j in jj:
163 assert(jj[0][0] == j[0])
164 finally:
165 ff.close()
166
167
169 if sys.platform == 'win32':
170 directory, file = os.path.split(path)
171 drive, tail = os.path.splitdrive(directory)
172 path_elem = tail.split(os.sep)
173 path = drive + os.sep
174 for i in range(len(path_elem)):
175 elem = path_elem[i]
176 if elem:
177 sub_elem = elem.split()
178 elem = sub_elem[0]
179 if len(elem) > 8 or len(sub_elem) > 1:
180 elem = elem[:6] + '~1'
181 path = os.path.join(path, elem)
182
183 path = os.path.join(path, file)
184
185 return path
186
187
188 if __name__ == '__main__':
189 NTEST = int(raw_input('Enter a number of job to test --->'))
190 NUSERS = int(raw_input('Enter a number of '"processes"' to test --->'))
191 LEN = int(raw_input('Enter job length --->'))
192 CACHE = int(raw_input('Enter max cache size --->'))
193 OUTPUT = raw_input('Enter a name of output dir --->')
194 dname = 'users_' + str(NUSERS) + '__jobs_' + str(NTEST)
195 output_dir = os.path.join(os.getcwd(), OUTPUT, dname)
196 if not os.path.exists(output_dir):
197 os.makedirs(output_dir)
198 print "output dir is ", output_dir
199 rootDir = os.path.join(os.getcwd(), 'testdir', 'GangaTest', 'user')
200
201 python_path = NormPath(sys.executable)
202 i = 0
203 while i < NUSERS:
204 cmd = "import sys\nsys.path.append(\'%s\')\nfrom extendedListsTest import runTest\nrunTest(%d, %d, %d, \'%s\',\'%s\')" % (_thisDir, NTEST, LEN, CACHE, rootDir, output_dir)
205 if DEBUG:
206 print cmd
207 pid = os.spawnl(os.P_NOWAIT, python_path, python_path, "-c", cmd)
208 if DEBUG:
209 print "new user process started %d" % pid
210 i+=1
211