runner.py - sphere - GPU-based 3D discrete element method algorithm with optional fluid coupling
HTML git clone git://src.adamsgaard.dk/sphere
DIR Log
DIR Files
DIR Refs
DIR LICENSE
---
runner.py (21313B)
---
1 import subprocess
2
3
4 def convert(graphics_format='png', folder='../img_out', remove_ppm=False):
5 '''
6 Converts all PPM images in img_out to graphics_format using ImageMagick. All
7 PPM images are subsequently removed if `remove_ppm` is `True`.
8
9 :param graphics_format: Convert the images to this format
10 :type graphics_format: str
11 :param folder: The folder containing the PPM images to convert
12 :type folder: str
13 :param remove_ppm: Remove ALL ppm files in `folder` after conversion
14 :type remove_ppm: bool
15 '''
16
17 #quiet = ' > /dev/null'
18 quiet = ''
19 # Convert images
20 subprocess.call('for F in ' + folder \
21 + '/*.ppm ; do BASE=`basename $F .ppm`; convert $F ' \
22 + folder + '/$BASE.' + graphics_format + ' ' \
23 + quiet + ' ; done', shell=True)
24
25 # Remove PPM files
26 if remove_ppm:
27 subprocess.call('rm ' + folder + '/*.ppm', shell=True)
28
29 def render(binary, method='pres', max_val=1e3, lower_cutoff=0.0,
30 graphics_format='png', verbose=True):
31 '''
32 Render target binary using the ``sphere`` raytracer. The target may be
33 a shell glob matching several binaries.
34
35 :param binary: Input file(s) for the raytracer, relative to the
36 ``sphere`` root folder. May contain shell wildcards.
37 :type binary: str
38 :param method: The color visualization method to use for the particles.
39 Possible values are: 'normal': color all particles with the same
40 color, 'pres': color by pressure, 'vel': color by translational
41 velocity, 'angvel': color by rotational velocity, 'xdisp': color by
42 total displacement along the x-axis, 'angpos': color by angular
43 position.
44 :type method: str
45 :param max_val: The maximum value of the color bar
46 :type max_val: float
47 :param lower_cutoff: Do not render particles with a value below this
48 value, of the field selected by ``method``
49 :type lower_cutoff: float
50 :param graphics_format: Convert the PPM images generated by the ray
51 tracer to this image format using Imagemagick
52 :type graphics_format: str
53 :param verbose: Show verbose information during ray tracing
54 :type verbose: bool
55 '''
56 quiet = ''
57 if not verbose:
58 quiet = '-q'
59
60 # Render images using sphere raytracer
61 if method == 'normal':
62 subprocess.call('cd ..; for F in `ls ' + binary + '`; do '
63 + './sphere ' + quiet
64 + ' --render $F; done', shell=True)
65 else:
66 subprocess.call('cd ..; for F in `ls ' + binary + '`; do '
67 + './sphere ' + quiet
68 + ' --method ' + method + ' {}'.format(max_val)
69 + ' -l {}'.format(lower_cutoff)
70 + ' --render $F; done', shell=True)
71
72 # Convert images to compressed format
73 if verbose:
74 print('converting images to ' + graphics_format)
75 convert(graphics_format)
76
77 def video(project, out_folder='./', video_format='mp4',
78 graphics_folder='../img_out/', graphics_format='png', fps=25,
79 verbose=True):
80 '''
81 Uses ffmpeg to combine images to animation. All images should be
82 rendered beforehand using :func:`render()`.
83
84 :param project: The simulation id of the project to render
85 :type project: str
86 :param out_folder: The output folder for the video file
87 :type out_folder: str
88 :param video_format: The format of the output video
89 :type video_format: str
90 :param graphics_folder: The folder containing the rendered images
91 :type graphics_folder: str
92 :param graphics_format: The format of the rendered images
93 :type graphics_format: str
94 :param fps: The number of frames per second to use in the video
95 :type fps: int
96 :param qscale: The output video quality, in ]0;1]
97 :type qscale: float
98 :param bitrate: The bitrate to use in the output video
99 :type bitrate: int
100 :param verbose: Show ffmpeg output
101 :type verbose: bool
102 '''
103 # Possible loglevels:
104 # quiet, panic, fatal, error, warning, info, verbose, debug
105 loglevel = 'info'
106 if not verbose:
107 loglevel = 'error'
108
109 outfile = out_folder + '/' + project + '.' + video_format
110 subprocess.call('ffmpeg -loglevel ' + loglevel + ' '
111 + '-i ' + graphics_folder + project + '.output%05d.'
112 + graphics_format
113 + ' -c:v libx264 -profile:v high -pix_fmt yuv420p -g 30'
114 + ' -r {} -y '.format(fps)
115 + outfile, shell=True)
116 if verbose:
117 print('saved to ' + outfile)
118
119 def thinsectionVideo(project, out_folder="./", video_format="mp4", fps=25,
120 qscale=1, bitrate=1800, verbose=False):
121 '''
122 Uses ffmpeg to combine thin section images to an animation. This function
123 will implicity render the thin section images beforehand.
124
125 :param project: The simulation id of the project to render
126 :type project: str
127 :param out_folder: The output folder for the video file
128 :type out_folder: str
129 :param video_format: The format of the output video
130 :type video_format: str
131 :param fps: The number of frames per second to use in the video
132 :type fps: int
133 :param qscale: The output video quality, in ]0;1]
134 :type qscale: float
135 :param bitrate: The bitrate to use in the output video
136 :type bitrate: int
137 :param verbose: Show ffmpeg output
138 :type verbose: bool
139 '''
140 ''' Use ffmpeg to combine thin section images to animation.
141 This function will start off by rendering the images.
142 '''
143
144 # Render thin section images (png)
145 lastfile = status(project)
146 from .core import sim
147 sb = sim(fluid=False)
148 for i in range(lastfile+1):
149 fn = "../output/{0}.output{1:0=5}.bin".format(project, i)
150 sb.sid = project + ".output{:0=5}".format(i)
151 sb.readbin(fn, verbose=False)
152 sb.thinsection_x1x3(cbmax=sb.w_sigma0[0]*4.0)
153
154 # Combine images to animation
155 # Possible loglevels:
156 # quiet, panic, fatal, error, warning, info, verbose, debug
157 loglevel = "info"
158 if not verbose:
159 loglevel = "error"
160
161 subprocess.call("ffmpeg -qscale {0} -r {1} -b {2} -y ".format(\
162 qscale, fps, bitrate)
163 + "-loglevel " + loglevel + " "
164 + "-i ../img_out/" + project + ".output%05d-ts-x1x3.png "
165 + "-vf 'crop=((in_w/2)*2):((in_h/2)*2)' " \
166 + out_folder + "/" + project + "-ts-x1x3." + video_format,
167 shell=True)
168
169 def run(binary, verbose=True, hideinputfile=False, dry=False, valgrind=False,
170 cudamemcheck=False, device=-1, fluid=False):
171 '''
172 Execute ``sphere`` with target binary file as input.
173
174 :param binary: Input file for ``sphere``, relative to the ``sphere``
175 root folder
176 :type binary: str
177 :param verbose: Show ``sphere`` output
178 :type verbose: bool
179 :param hideinputfile: Hide the input file
180 :type hideinputfile: bool
181 :param dry: Perform a dry run. Important parameter values are shown by
182 the ``sphere`` program, and it exits afterwards.
183 :type dry: bool
184 :param valgrind: Run the program with ``valgrind`` in order to check
185 memory leaks in the host code. This causes a significant increase in
186 computational time.
187 :type valgrind: bool
188 :param cudamemcheck: Run the program with ``cudamemcheck`` in order to
189 check for device memory leaks and errors. This causes a significant
190 increase in computational time.
191 :type cudamemcheck: bool
192 :param device: Specify the GPU device to execute the program on.
193 If not specified, sphere will use the device with the most CUDA cores.
194 To see a list of devices, run ``nvidia-smi`` in the system shell.
195 :type device: int
196 :param fluid: Simulate fluid between the particles
197 :type fluid: bool
198 '''
199
200 quiet = ''
201 stdout = ''
202 dryarg = ''
203 fluidarg = ''
204 devicearg = ''
205 valgrindbin = ''
206 cudamemchk = ''
207 if not verbose:
208 quiet = '-q '
209 if hideinputfile:
210 stdout = ' > /dev/null'
211 if dry:
212 dryarg = '--dry '
213 if valgrind:
214 valgrindbin = 'valgrind -q --track-origins=yes '
215 if cudamemcheck:
216 cudamemchk = 'cuda-memcheck --leak-check full '
217 if fluid:
218 fluidarg = '--fluid '
219 if device != -1:
220 devicearg = '-d ' + str(device) + ' '
221
222 status = subprocess.call('cd ..; ' + valgrindbin + cudamemchk
223 + './sphere ' + quiet + dryarg + fluidarg
224 + devicearg + binary + ' ' + stdout, shell=True)
225
226 if status != 0:
227 print('Warning: the sphere run returned with status ' + str(status))
228
229 def torqueScriptParallel3(obj1, obj2, obj3, email='adc@geo.au.dk',
230 email_alerts='ae', walltime='24:00:00',
231 queue='qfermi', cudapath='/com/cuda/4.0.17/cuda',
232 spheredir='/home/adc/code/sphere',
233 use_workdir=False,
234 workdir='/scratch'):
235 '''
236 Create job script for the Torque queue manager for three binaries,
237 executed in parallel, ideally on three GPUs.
238
239 :param email: The e-mail address that Torque messages should be sent to
240 :type email: str
241 :param email_alerts: The type of Torque messages to send to the e-mail
242 address. The character 'b' causes a mail to be sent when the
243 execution begins. The character 'e' causes a mail to be sent when
244 the execution ends normally. The character 'a' causes a mail to be
245 sent if the execution ends abnormally. The characters can be written
246 in any order.
247 :type email_alerts: str
248 :param walltime: The maximal allowed time for the job, in the format
249 'HH:MM:SS'.
250 :type walltime: str
251 :param queue: The Torque queue to schedule the job for
252 :type queue: str
253 :param cudapath: The path of the CUDA library on the cluster compute nodes
254 :type cudapath: str
255 :param spheredir: The path to the root directory of sphere on the cluster
256 :type spheredir: str
257 :param use_workdir: Use a different working directory than the sphere folder
258 :type use_workdir: bool
259 :param workdir: The working directory during the calculations, if
260 `use_workdir=True`
261 :type workdir: str
262
263 :returns: The filename of the script
264 :return type: str
265
266 See also :func:`torqueScript()`
267 '''
268
269 filename = obj1.sid + '_' + obj2.sid + '_' + obj3.sid + '.sh'
270
271 fh = None
272 try:
273 fh = open(filename, "w")
274
275 fh.write('#!/bin/sh\n')
276 fh.write('#PBS -N ' + obj1.sid + '_' + obj2.sid + '_' + obj3.sid + '\n')
277 fh.write('#PBS -l nodes=1:ppn=1\n')
278 fh.write('#PBS -l walltime=' + walltime + '\n')
279 fh.write('#PBS -q ' + queue + '\n')
280 fh.write('#PBS -M ' + email + '\n')
281 fh.write('#PBS -m ' + email_alerts + '\n')
282 fh.write('CUDAPATH=' + cudapath + '\n')
283 fh.write('export PATH=$CUDAPATH/bin:$PATH\n')
284 fh.write('export LD_LIBRARY_PATH=$CUDAPATH/lib64')
285 fh.write(':$CUDAPATH/lib:$LD_LIBRARY_PATH\n')
286 fh.write('echo "`whoami`@`hostname`"\n')
287 fh.write('echo "Start at `date`"\n')
288 if use_workdir:
289 fh.write('ORIGDIR=' + spheredir + '\n')
290 fh.write('WORKDIR=' + workdir + "/$PBS_JOBID\n")
291 fh.write('cp -r $ORIGDIR/* $WORKDIR\n')
292 fh.write('cd $WORKDIR\n')
293 else:
294 fh.write('cd ' + spheredir + '\n')
295 fh.write('cmake . && make\n')
296 fh.write('./sphere input/' + obj1.sid + '.bin > /dev/null &\n')
297 fh.write('./sphere input/' + obj2.sid + '.bin > /dev/null &\n')
298 fh.write('./sphere input/' + obj3.sid + '.bin > /dev/null &\n')
299 fh.write('wait\n')
300 if use_workdir:
301 fh.write('cp $WORKDIR/output/* $ORIGDIR/output/\n')
302 fh.write('echo "End at `date`"\n')
303 return filename
304
305 finally:
306 if fh is not None:
307 fh.close()
308
309 def status(project):
310 '''
311 Check the status.dat file for the target project, and return the last output
312 file number.
313
314 :param project: The simulation id of the target project
315 :type project: str
316
317 :returns: The last output file written in the simulation calculations
318 :return type: int
319 '''
320
321 fh = None
322 try:
323 filepath = "../output/{0}.status.dat".format(project)
324 fh = open(filepath)
325 data = fh.read()
326 return int(data.split()[2]) # Return last file number
327 finally:
328 if fh is not None:
329 fh.close()
330
331 def cleanup(sb):
332 '''
333 Removes the input/output files and images belonging to the object simulation
334 ID from the ``input/``, ``output/`` and ``img_out/`` folders.
335
336 :param sb: A sphere.sim object
337 :type sb: sim
338 '''
339 subprocess.call("rm -f ../input/" + sb.sid + ".bin", shell=True)
340 subprocess.call("rm -f ../output/" + sb.sid + ".*.bin", shell=True)
341 subprocess.call("rm -f ../img_out/" + sb.sid + ".*", shell=True)
342 subprocess.call("rm -f ../output/" + sb.sid + ".status.dat", shell=True)
343 subprocess.call("rm -f ../output/" + sb.sid + ".*.vtu", shell=True)
344 subprocess.call("rm -f ../output/fluid-" + sb.sid + ".*.vti", shell=True)
345 subprocess.call("rm -f ../output/" + sb.sid + "-conv.png", shell=True)
346 subprocess.call("rm -f ../output/" + sb.sid + "-conv.log", shell=True)
347
348
349 class SimRun:
350 'Execution of the sphere program and related job control.'
351
352 def run(self, verbose=True, hideinputfile=False, dry=False, valgrind=False,
353 cudamemcheck=False, device=-1):
354 '''
355 Start ``sphere`` calculations on the ``sim`` object
356
357 :param verbose: Show ``sphere`` output
358 :type verbose: bool
359 :param hideinputfile: Hide the file name of the ``sphere`` input file
360 :type hideinputfile: bool
361 :param dry: Perform a dry run. Important parameter values are shown by
362 the ``sphere`` program, and it exits afterwards.
363 :type dry: bool
364 :param valgrind: Run the program with ``valgrind`` in order to check
365 memory leaks in the host code. This causes a significant increase in
366 computational time.
367 :type valgrind: bool
368 :param cudamemcheck: Run the program with ``cudamemcheck`` in order to
369 check for device memory leaks and errors. This causes a significant
370 increase in computational time.
371 :type cudamemcheck: bool
372 :param device: Specify the GPU device to execute the program on.
373 If not specified, sphere will use the device with the most CUDA cores.
374 To see a list of devices, run ``nvidia-smi`` in the system shell.
375 :type device: int
376 '''
377
378 self.writebin(verbose=False)
379 run('input/' + self.sid + '.bin', verbose, hideinputfile, dry,
380 valgrind, cudamemcheck, device, self.fluid)
381
382 def cleanup(self):
383 '''
384 Removes the input/output files and images belonging to the object
385 simulation ID from the ``input/``, ``output/`` and ``img_out/`` folders.
386 '''
387 cleanup(self)
388
389 def torqueScript(self, email='adc@geo.au.dk', email_alerts='ae',
390 walltime='24:00:00', queue='qfermi',
391 cudapath='/com/cuda/4.0.17/cuda',
392 spheredir='/home/adc/code/sphere',
393 use_workdir=False, workdir='/scratch'):
394 '''
395 Creates a job script for the Torque queue manager for the simulation
396 object.
397
398 :param email: The e-mail address that Torque messages should be sent to
399 :type email: str
400 :param email_alerts: The type of Torque messages to send to the e-mail
401 address. The character 'b' causes a mail to be sent when the
402 execution begins. The character 'e' causes a mail to be sent when
403 the execution ends normally. The character 'a' causes a mail to be
404 sent if the execution ends abnormally. The characters can be written
405 in any order.
406 :type email_alerts: str
407 :param walltime: The maximal allowed time for the job, in the format
408 'HH:MM:SS'.
409 :type walltime: str
410 :param queue: The Torque queue to schedule the job for
411 :type queue: str
412 :param cudapath: The path of the CUDA library on the cluster compute
413 nodes
414 :type cudapath: str
415 :param spheredir: The path to the root directory of sphere on the
416 cluster
417 :type spheredir: str
418 :param use_workdir: Use a different working directory than the sphere
419 folder
420 :type use_workdir: bool
421 :param workdir: The working directory during the calculations, if
422 `use_workdir=True`
423 :type workdir: str
424
425 '''
426
427 filename = self.sid + ".sh"
428 fh = None
429 try:
430 fh = open(filename, "w")
431
432 fh.write('#!/bin/sh\n')
433 fh.write('#PBS -N ' + self.sid + '\n')
434 fh.write('#PBS -l nodes=1:ppn=1\n')
435 fh.write('#PBS -l walltime=' + walltime + '\n')
436 fh.write('#PBS -q ' + queue + '\n')
437 fh.write('#PBS -M ' + email + '\n')
438 fh.write('#PBS -m ' + email_alerts + '\n')
439 fh.write('CUDAPATH=' + cudapath + '\n')
440 fh.write('export PATH=$CUDAPATH/bin:$PATH\n')
441 fh.write('export LD_LIBRARY_PATH=$CUDAPATH/lib64'
442 + ':$CUDAPATH/lib:$LD_LIBRARY_PATH\n')
443 fh.write('echo "`whoami`@`hostname`"\n')
444 fh.write('echo "Start at `date`"\n')
445 fh.write('ORIGDIR=' + spheredir + '\n')
446 if use_workdir:
447 fh.write('WORKDIR=' + workdir + "/$PBS_JOBID\n")
448 fh.write('cp -r $ORIGDIR/* $WORKDIR\n')
449 fh.write('cd $WORKDIR\n')
450 else:
451 fh.write('cd ' + spheredir + '\n')
452 fh.write('cmake . && make\n')
453 fh.write('./sphere input/' + self.sid + '.bin > /dev/null &\n')
454 fh.write('wait\n')
455 if use_workdir:
456 fh.write('cp $WORKDIR/output/* $ORIGDIR/output/\n')
457 fh.write('echo "End at `date`"\n')
458
459 finally:
460 if fh is not None:
461 fh.close()
462
463 def render(self, method="pres", max_val=1e3, lower_cutoff=0.0,
464 graphics_format="png", verbose=True):
465 '''
466 Using the built-in ray tracer, render all output files that belong to
467 the simulation, determined by the simulation id (``sid``).
468
469 :param method: The color visualization method to use for the particles.
470 Possible values are: 'normal': color all particles with the same
471 color, 'pres': color by pressure, 'vel': color by translational
472 velocity, 'angvel': color by rotational velocity, 'xdisp': color by
473 total displacement along the x-axis, 'angpos': color by angular
474 position.
475 :type method: str
476 :param max_val: The maximum value of the color bar
477 :type max_val: float
478 :param lower_cutoff: Do not render particles with a value below this
479 value, of the field selected by ``method``
480 :type lower_cutoff: float
481 :param graphics_format: Convert the PPM images generated by the ray
482 tracer to this image format using Imagemagick
483 :type graphics_format: str
484 :param verbose: Show verbose information during ray tracing
485 :type verbose: bool
486 '''
487
488 print("Rendering {} images with the raytracer".format(self.sid))
489 render('output/' + self.sid + '*.bin', method, max_val, lower_cutoff,
490 graphics_format, verbose)
491
492 def video(self, out_folder="./", video_format="mp4",
493 graphics_folder="../img_out/", graphics_format="png", fps=25,
494 verbose=False):
495 '''
496 Uses ffmpeg to combine images to animation. All images should be
497 rendered beforehand using :func:`render()`.
498
499 :param out_folder: The output folder for the video file
500 :type out_folder: str
501 :param video_format: The format of the output video
502 :type video_format: str
503 :param graphics_folder: The folder containing the rendered images
504 :type graphics_folder: str
505 :param graphics_format: The format of the rendered images
506 :type graphics_format: str
507 :param fps: The number of frames per second to use in the video
508 :type fps: int
509 :param qscale: The output video quality, in ]0;1]
510 :type qscale: float
511 :param bitrate: The bitrate to use in the output video
512 :type bitrate: int
513 :param verbose: Show ffmpeg output
514 :type verbose: bool
515 '''
516
517 video(self.sid, out_folder, video_format, graphics_folder,
518 graphics_format, fps, verbose)
519
520 def status(self):
521 '''
522 Returns the current simulation status by using the simulation id
523 (``sid``) as an identifier.
524
525 :returns: The number of the last output file written
526 :return type: int
527 '''
528 return status(self.sid)