|
|
#!/usr/bin/env python
|
|
|
"""
|
|
|
A simple python program of solving a 2D wave equation in parallel.
|
|
|
Domain partitioning and inter-processor communication
|
|
|
are done by an object of class ZMQRectPartitioner2D
|
|
|
(which is a subclass of RectPartitioner2D and uses 0MQ via pyzmq)
|
|
|
|
|
|
An example of running the program is (8 processors, 4x2 partition,
|
|
|
200x200 grid cells)::
|
|
|
|
|
|
$ ipclusterz start -n 8 # start 8 engines
|
|
|
$ ./parallelwave.py --grid 200 200 --partition 4 2
|
|
|
|
|
|
See also parallelwave-mpi, which runs the same program, but uses MPI
|
|
|
(via mpi4py) for the inter-engine communication.
|
|
|
|
|
|
Authors
|
|
|
-------
|
|
|
|
|
|
* Xing Cai
|
|
|
* Min Ragan-Kelley
|
|
|
|
|
|
"""
|
|
|
#
|
|
|
import sys
|
|
|
import time
|
|
|
|
|
|
from numpy import exp, zeros, newaxis, sqrt
|
|
|
|
|
|
from IPython.external import argparse
|
|
|
from IPython.parallel.client import Client, Reference
|
|
|
|
|
|
def setup_partitioner(comm, addrs, index, num_procs, gnum_cells, parts):
|
|
|
"""create a partitioner in the engine namespace"""
|
|
|
global partitioner
|
|
|
p = ZMQRectPartitioner2D(comm, addrs, my_id=index, num_procs=num_procs)
|
|
|
p.redim(global_num_cells=gnum_cells, num_parts=parts)
|
|
|
p.prepare_communication()
|
|
|
# put the partitioner into the global namespace:
|
|
|
partitioner=p
|
|
|
|
|
|
def setup_solver(*args, **kwargs):
|
|
|
"""create a WaveSolver in the engine namespace."""
|
|
|
global solver
|
|
|
solver = WaveSolver(*args, **kwargs)
|
|
|
|
|
|
def wave_saver(u, x, y, t):
|
|
|
"""save the wave state for each timestep."""
|
|
|
global u_hist
|
|
|
global t_hist
|
|
|
t_hist.append(t)
|
|
|
u_hist.append(1.0*u)
|
|
|
|
|
|
|
|
|
# main program:
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
|
paa = parser.add_argument
|
|
|
paa('--grid', '-g',
|
|
|
type=int, nargs=2, default=[100,100], dest='grid',
|
|
|
help="Cells in the grid, e.g. --grid 100 200")
|
|
|
paa('--partition', '-p',
|
|
|
type=int, nargs=2, default=None,
|
|
|
help="Process partition grid, e.g. --partition 4 2 for 4x2")
|
|
|
paa('-c',
|
|
|
type=float, default=1.,
|
|
|
help="Wave speed (I think)")
|
|
|
paa('-Ly',
|
|
|
type=float, default=1.,
|
|
|
help="system size (in y)")
|
|
|
paa('-Lx',
|
|
|
type=float, default=1.,
|
|
|
help="system size (in x)")
|
|
|
paa('-t', '--tstop',
|
|
|
type=float, default=1.,
|
|
|
help="Time units to run")
|
|
|
paa('--profile',
|
|
|
type=unicode, default=u'default',
|
|
|
help="Specify the ipcluster profile for the client to connect to.")
|
|
|
paa('--save',
|
|
|
action='store_true',
|
|
|
help="Add this flag to save the time/wave history during the run.")
|
|
|
paa('--scalar',
|
|
|
action='store_true',
|
|
|
help="Also run with scalar interior implementation, to see vector speedup.")
|
|
|
|
|
|
ns = parser.parse_args()
|
|
|
# set up arguments
|
|
|
grid = ns.grid
|
|
|
partition = ns.partition
|
|
|
Lx = ns.Lx
|
|
|
Ly = ns.Ly
|
|
|
c = ns.c
|
|
|
tstop = ns.tstop
|
|
|
if ns.save:
|
|
|
user_action = wave_saver
|
|
|
else:
|
|
|
user_action = None
|
|
|
|
|
|
num_cells = 1.0*(grid[0]-1)*(grid[1]-1)
|
|
|
final_test = True
|
|
|
|
|
|
# create the Client
|
|
|
rc = Client(profile=ns.profile)
|
|
|
num_procs = len(rc.ids)
|
|
|
|
|
|
if partition is None:
|
|
|
partition = [num_procs,1]
|
|
|
else:
|
|
|
num_procs = min(num_procs, partition[0]*partition[1])
|
|
|
|
|
|
assert partition[0]*partition[1] == num_procs, "can't map partition %s to %i engines"%(partition, num_procs)
|
|
|
|
|
|
# construct the View:
|
|
|
view = rc[:num_procs]
|
|
|
print "Running %s system on %s processes until %f"%(grid, partition, tstop)
|
|
|
|
|
|
# functions defining initial/boundary/source conditions
|
|
|
def I(x,y):
|
|
|
from numpy import exp
|
|
|
return 1.5*exp(-100*((x-0.5)**2+(y-0.5)**2))
|
|
|
def f(x,y,t):
|
|
|
return 0.0
|
|
|
# from numpy import exp,sin
|
|
|
# return 10*exp(-(x - sin(100*t))**2)
|
|
|
def bc(x,y,t):
|
|
|
return 0.0
|
|
|
|
|
|
# initialize t_hist/u_hist for saving the state at each step (optional)
|
|
|
view['t_hist'] = []
|
|
|
view['u_hist'] = []
|
|
|
|
|
|
# set vector/scalar implementation details
|
|
|
impl = {}
|
|
|
impl['ic'] = 'vectorized'
|
|
|
impl['inner'] = 'scalar'
|
|
|
impl['bc'] = 'vectorized'
|
|
|
|
|
|
# execute some files so that the classes we need will be defined on the engines:
|
|
|
view.execute('import numpy')
|
|
|
view.run('communicator.py')
|
|
|
view.run('RectPartitioner.py')
|
|
|
view.run('wavesolver.py')
|
|
|
|
|
|
# scatter engine IDs
|
|
|
view.scatter('my_id', range(num_procs), flatten=True)
|
|
|
|
|
|
# create the engine connectors
|
|
|
view.execute('com = EngineCommunicator()')
|
|
|
|
|
|
# gather the connection information into a single dict
|
|
|
ar = view.apply_async(lambda : com.info)
|
|
|
peers = ar.get_dict()
|
|
|
# print peers
|
|
|
# this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
|
|
|
|
|
|
# setup remote partitioner
|
|
|
# note that Reference means that the argument passed to setup_partitioner will be the
|
|
|
# object named 'com' in the engine's namespace
|
|
|
view.apply_sync(setup_partitioner, Reference('com'), peers, Reference('my_id'), num_procs, grid, partition)
|
|
|
time.sleep(1)
|
|
|
# convenience lambda to call solver.solve:
|
|
|
_solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
|
|
|
|
|
|
if ns.scalar:
|
|
|
impl['inner'] = 'scalar'
|
|
|
# setup remote solvers
|
|
|
view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly, partitioner=Reference('partitioner'), dt=0,implementation=impl)
|
|
|
|
|
|
# run first with element-wise Python operations for each cell
|
|
|
t0 = time.time()
|
|
|
ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test, user_action=user_action)
|
|
|
if final_test:
|
|
|
# this sum is performed element-wise as results finish
|
|
|
s = sum(ar)
|
|
|
# the L2 norm (RMS) of the result:
|
|
|
norm = sqrt(s/num_cells)
|
|
|
else:
|
|
|
norm = -1
|
|
|
t1 = time.time()
|
|
|
print 'scalar inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
|
|
|
|
|
|
# run again with faster numpy-vectorized inner implementation:
|
|
|
impl['inner'] = 'vectorized'
|
|
|
# setup remote solvers
|
|
|
view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
|
|
|
|
|
|
t0 = time.time()
|
|
|
|
|
|
ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test)#, user_action=wave_saver)
|
|
|
if final_test:
|
|
|
# this sum is performed element-wise as results finish
|
|
|
s = sum(ar)
|
|
|
# the L2 norm (RMS) of the result:
|
|
|
norm = sqrt(s/num_cells)
|
|
|
else:
|
|
|
norm = -1
|
|
|
t1 = time.time()
|
|
|
print 'vector inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
|
|
|
|
|
|
# if ns.save is True, then u_hist stores the history of u as a list
|
|
|
# If the partion scheme is Nx1, then u can be reconstructed via 'gather':
|
|
|
if ns.save and partition[-1] == 1:
|
|
|
import pylab
|
|
|
view.execute('u_last=u_hist[-1]')
|
|
|
u_last = view.gather('u_last', block=True)
|
|
|
pylab.pcolor(u_last)
|
|
|
pylab.show()
|
