task_profiler.py
71 lines
| 2.2 KiB
| text/x-python
|
PythonLexer
|
r3670 | #!/usr/bin/env python | ||
"""Test the performance of the task farming system. | ||||
|
r3675 | This script submits a set of tasks via a LoadBalancedView. The tasks | ||
|
r3670 | are basically just a time.sleep(t), where t is a random number between | ||
|
r4872 | two limits that can be configured at the command line. To run | ||
|
r3670 | the script there must first be an IPython controller and engines running:: | ||
|
r3675 | ipclusterz start -n 16 | ||
|
r3670 | |||
A good test to run with 16 engines is:: | ||||
python task_profiler.py -n 128 -t 0.01 -T 1.0 | ||||
|
r4872 | This should show a speedup of 13-14x. The limitation here is that the | ||
|
r3670 | overhead of a single task is about 0.001-0.01 seconds. | ||
""" | ||||
import random, sys | ||||
from optparse import OptionParser | ||||
from IPython.utils.timing import time | ||||
|
r3675 | from IPython.parallel import Client | ||
|
r3670 | |||
def main(): | ||||
parser = OptionParser() | ||||
parser.set_defaults(n=100) | ||||
|
r4184 | parser.set_defaults(tmin=1e-3) | ||
parser.set_defaults(tmax=1) | ||||
|
r3675 | parser.set_defaults(profile='default') | ||
|
r4872 | |||
|
r3670 | parser.add_option("-n", type='int', dest='n', | ||
help='the number of tasks to run') | ||||
|
r4872 | parser.add_option("-t", type='float', dest='tmin', | ||
|
r3670 | help='the minimum task length in seconds') | ||
parser.add_option("-T", type='float', dest='tmax', | ||||
help='the maximum task length in seconds') | ||||
|
r3675 | parser.add_option("-p", '--profile', type='str', dest='profile', | ||
help="the cluster profile [default: 'default']") | ||||
|
r4872 | |||
|
r3670 | (opts, args) = parser.parse_args() | ||
assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" | ||||
|
r4872 | |||
|
r3675 | rc = Client() | ||
view = rc.load_balanced_view() | ||||
print view | ||||
|
r3670 | rc.block=True | ||
|
r3675 | nengines = len(rc.ids) | ||
|
r4184 | with rc[:].sync_imports(): | ||
from IPython.utils.timing import time | ||||
|
r3670 | |||
# the jobs should take a random time within a range | ||||
times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)] | ||||
stime = sum(times) | ||||
|
r4872 | |||
|
r3670 | print "executing %i tasks, totalling %.1f secs on %i engines"%(opts.n, stime, nengines) | ||
time.sleep(1) | ||||
start = time.time() | ||||
|
r3675 | amr = view.map(time.sleep, times) | ||
amr.get() | ||||
|
r3670 | stop = time.time() | ||
ptime = stop-start | ||||
scale = stime/ptime | ||||
|
r4872 | |||
|
r3670 | print "executed %.1f secs in %.1f secs"%(stime, ptime) | ||
print "%.3fx parallel performance on %i engines"%(scale, nengines) | ||||
print "%.1f%% of theoretical max"%(100*scale/nengines) | ||||
if __name__ == '__main__': | ||||
main() | ||||