organize IPython.parallel into subpackages

IPython/config/default/ipcluster_config.py

@@ -1,241 +1,241 b''
 import os
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Select which launchers to use
 #-----------------------------------------------------------------------------
 
 # This allows you to control what method is used to start the controller
 # and engines.  The following methods are currently supported:
 # - Start as a regular process on localhost.
 # - Start using mpiexec.
 # - Start using the Windows HPC Server 2008 scheduler
 # - Start using PBS/SGE
 # - Start using SSH
 
 
 # The selected launchers can be configured below.
 
 # Options are:
 # - LocalControllerLauncher
 # - MPIExecControllerLauncher
 # - PBSControllerLauncher
 # - SGEControllerLauncher
 # - WindowsHPCControllerLauncher
-# c.Global.controller_launcher = 'IPython.parallel.launcher.LocalControllerLauncher'
-# c.Global.controller_launcher = 'IPython.parallel.launcher.PBSControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.LocalControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.PBSControllerLauncher'
 
 # Options are:
 # - LocalEngineSetLauncher
 # - MPIExecEngineSetLauncher
 # - PBSEngineSetLauncher
 # - SGEEngineSetLauncher
 # - WindowsHPCEngineSetLauncher
-# c.Global.engine_launcher = 'IPython.parallel.launcher.LocalEngineSetLauncher'
+# c.Global.engine_launcher = 'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # The default number of engines that will be started. This is overridden by
 # the -n command line option: "ipcluster start -n 4"
 # c.Global.n = 2
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 # The command line arguments to call the controller with.
 # c.LocalControllerLauncher.controller_args = \
 #    ['--log-to-file','--log-level', '40']
 
 # The working directory for the controller
 # c.LocalEngineSetLauncher.work_dir = u''
 
 # Command line argument passed to the engines.
 # c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
 # The mpiexec/mpirun command to use in both the controller and engines.
 # c.MPIExecLauncher.mpi_cmd = ['mpiexec']
 
 # Additional arguments to pass to the actual mpiexec command.
 # c.MPIExecLauncher.mpi_args = []
 
 # The mpiexec/mpirun command and args can be overridden if they should be different
 # for controller and engines.
 # c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
 # c.MPIExecControllerLauncher.mpi_args = []
 # c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
 # c.MPIExecEngineSetLauncher.mpi_args = []
 
 # The command line argument to call the controller with.
 # c.MPIExecControllerLauncher.controller_args = \
 #     ['--log-to-file','--log-level', '40']
 
 # Command line argument passed to the engines.
 # c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 # The default number of engines to start if not given elsewhere.
 # c.MPIExecEngineSetLauncher.n = 1
 
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
 # ipclusterz can be used to launch controller and engines remotely via ssh.
 # Note that currently ipclusterz does not do any file distribution, so if
 # machines are not on a shared filesystem, config and json files must be
 # distributed.  For this reason, the reuse_files defaults to True on an
 # ssh-launched Controller.  This flag can be overridded by the program_args
 # attribute of c.SSHControllerLauncher.
 
 # set the ssh cmd for launching remote commands. The default is ['ssh']
 # c.SSHLauncher.ssh_cmd = ['ssh']
 
 # set the ssh cmd for launching remote commands. The default is ['ssh']
 # c.SSHLauncher.ssh_args = ['tt']
 
 # Set the user and hostname for the controller
 # c.SSHControllerLauncher.hostname = 'controller.example.com'
 # c.SSHControllerLauncher.user = os.environ.get('USER','username')
 
 # Set the arguments to be passed to ipcontrollerz
 # note that remotely launched ipcontrollerz will not get the contents of 
 # the local ipcontrollerz_config.py unless it resides on the *remote host*
 # in the location specified by the --cluster_dir argument.
 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
 
 # Set the default args passed to ipenginez for SSH launched engines
 # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
 
 # SSH engines are launched as a dict of locations/n-engines.
 # if a value is a tuple instead of an int, it is assumed to be of the form
 # (n, [args]), setting the arguments to passed to ipenginez on `host`.
 # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
 
 # In this case, there will be 3 engines at my.example.com, and
 # 2 at [e-mail] with a special json connector location.
 # c.SSHEngineSetLauncher.engines = {'my.example.com' : 3,
 #                                   '[e-mail]' : (2, ['-f', '/path/to/ipcontroller-engine.json']}
 #                                  }
 
 #-----------------------------------------------------------------------------
 # Unix batch (PBS) schedulers launchers
 #-----------------------------------------------------------------------------
 
 # SGE and PBS are very similar. All configurables in this section called 'PBS*'
 # also exist as 'SGE*'.
 
 # The command line program to use to submit a PBS job.
 # c.PBSLauncher.submit_command = ['qsub']
 
 # The command line program to use to delete a PBS job.
 # c.PBSLauncher.delete_command = ['qdel']
 
 # The PBS queue in which the job should run
 # c.PBSLauncher.queue = 'myqueue'
 
 # A regular expression that takes the output of qsub and find the job id.
 # c.PBSLauncher.job_id_regexp = r'\d+'
 
 # If for some reason the Controller and Engines have different options above, they
 # can be set as c.PBSControllerLauncher.<option> etc.
 
 # PBS and SGE have default templates, but you can specify your own, either as strings
 # or from files, as described here:
 
 # The batch submission script used to start the controller. This is where
 # environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
 # c.PBSControllerLauncher.batch_template = """
 # #PBS -N ipcontroller
 # #PBS -q $queue
 # 
 # ipcontrollerz --cluster-dir $cluster_dir
 # """
 
 # You can also load this template from a file
 # c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
 # c.PBSControllerLauncher.batch_file_name = u'pbs_controller'
 
 # The batch submission script used to start the engines. This is where
 # environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
 # c.PBSEngineSetLauncher.batch_template = """
 # #PBS -N ipcontroller
 # #PBS -l nprocs=$n
 # 
 # ipenginez --cluster-dir $cluster_dir$s
 # """
 
 # You can also load this template from a file
 # c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
 # c.PBSEngineSetLauncher.batch_file_name = u'pbs_engines'
 
 
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 launcher configuration
 #-----------------------------------------------------------------------------
 
 # c.IPControllerJob.job_name = 'IPController'
 # c.IPControllerJob.is_exclusive = False
 # c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
 # c.IPControllerJob.priority = 'Highest'
 # c.IPControllerJob.requested_nodes = ''
 # c.IPControllerJob.project = 'MyProject'
 
 # c.IPControllerTask.task_name = 'IPController'
 # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
 # c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
 # c.IPControllerTask.environment_variables = {}
 
 # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
 
 
 # c.IPEngineSetJob.job_name = 'IPEngineSet'
 # c.IPEngineSetJob.is_exclusive = False
 # c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
 # c.IPEngineSetJob.priority = 'Highest'
 # c.IPEngineSetJob.requested_nodes = ''
 # c.IPEngineSetJob.project = 'MyProject'
 
 # c.IPEngineTask.task_name = 'IPEngine'
 # c.IPEngineTask.engine_cmd = [u'ipengine.exe']
 # c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
 # c.IPEngineTask.environment_variables = {}
 
 # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
 
 
 
 
 
 
 

IPython/config/default/ipcontroller_config.py

@@ -1,180 +1,180 b''
 from IPython.config.loader import Config
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # Basic Global config attributes
 
 # Start up messages are logged to stdout using the logging module.
 # These all happen before the twisted reactor is started and are
 # useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
 # and smaller is more verbose.
 # c.Global.log_level = 20
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # A list of Python statements that will be run before starting the 
 # controller. This is provided because occasionally certain things need to 
 # be imported in the controller for pickling to work.
 # c.Global.import_statements = ['import math']
 
 # Reuse the controller's JSON files. If False, JSON files are regenerated
 # each time the controller is run. If True, they will be reused, *but*, you
 # also must set the network ports by hand. If set, this will override the
 # values set for the client and engine connections below.
 # c.Global.reuse_files = True
 
 # Enable exec_key authentication on all messages. Default is True
 # c.Global.secure = True
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
 # The log url for logging to an `iploggerz` application.  This will override
 # log-to-file.
 # c.Global.log_url = 'tcp://127.0.0.1:20202'
 
 # The specific external IP that is used to disambiguate multi-interface URLs.
 # The default behavior is to guess from external IPs gleaned from `socket`.
 # c.Global.location = '192.168.1.123'
 
 # The ssh server remote clients should use to connect to this controller.
 # It must be a machine that can see the interface specified in client_ip.
 # The default for client_ip is localhost, in which case the sshserver must
 # be an external IP of the controller machine.
 # c.Global.sshserver = 'controller.example.com'
 
 # the url to use for registration.  If set, this overrides engine-ip,
 # engine-transport client-ip,client-transport, and regport.
 # c.RegistrationFactory.url = 'tcp://*:12345'
 
 # the port to use for registration.  Clients and Engines both use this
 # port for registration.
 # c.RegistrationFactory.regport = 10101
 
 #-----------------------------------------------------------------------------
 # Configure the Task Scheduler
 #-----------------------------------------------------------------------------
 
 # The routing scheme. 'pure' will use the pure-ZMQ scheduler. Any other 
 # value will use a Python scheduler with various routing schemes.
 # python schemes are: lru, weighted, random, twobin. Default is 'weighted'.
 # Note that the pure ZMQ scheduler does not support many features, such as
 # dying engines, dependencies, or engine-subset load-balancing.
 # c.ControllerFactory.scheme = 'pure'
 
 # The pure ZMQ scheduler can limit the number of outstanding tasks per engine
 # by using the ZMQ HWM option.  This allows engines with long-running tasks
 # to not steal too many tasks from other engines. The default is 0, which
 # means agressively distribute messages, never waiting for them to finish.
 # c.ControllerFactory.hwm = 1
 
 # Whether to use Threads or Processes to start the Schedulers.  Threads will
 # use less resources, but potentially reduce throughput. Default is to 
 # use processes.  Note that the a Python scheduler will always be in a Process.
 # c.ControllerFactory.usethreads
 
 #-----------------------------------------------------------------------------
 # Configure the Hub
 #-----------------------------------------------------------------------------
 
 # Which class to use for the db backend.  Currently supported are DictDB (the
 # default), and MongoDB. Uncomment this line to enable MongoDB, which will
 # slow-down the Hub's responsiveness, but also reduce its memory footprint.
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
 
 # The heartbeat ping frequency.  This is the frequency (in ms) at which the
 # Hub pings engines for heartbeats.  This determines how quickly the Hub
 # will react to engines coming and going.  A lower number means faster response
 # time, but more network activity.  The default is 100ms
 # c.HubFactory.ping = 100
 
 # HubFactory queue port pairs, to set by name: mux, iopub, control, task.  Set
 # each as a tuple of length 2 of ints.  The default is to find random
 # available ports
 # c.HubFactory.mux = (10102,10112)
 
 #-----------------------------------------------------------------------------
 # Configure the client connections
 #-----------------------------------------------------------------------------
 
 # Basic client connection config attributes
 
 # The network interface the controller will listen on for client connections.
 # This should be an IP address or interface on the controller. An asterisk
 # means listen on all interfaces. The transport can be any transport
 # supported by zeromq (tcp,epgm,pgm,ib,ipc):
 # c.HubFactory.client_ip = '*'
 # c.HubFactory.client_transport = 'tcp'
 
 # individual client ports to configure by name: query_port, notifier_port
 # c.HubFactory.query_port = 12345
 
 #-----------------------------------------------------------------------------
 # Configure the engine connections
 #-----------------------------------------------------------------------------
 
 # Basic config attributes for the engine connections.
 
 # The network interface the controller will listen on for engine connections.
 # This should be an IP address or interface on the controller. An asterisk
 # means listen on all interfaces. The transport can be any transport
 # supported by zeromq (tcp,epgm,pgm,ib,ipc):
 # c.HubFactory.engine_ip = '*'
 # c.HubFactory.engine_transport = 'tcp'
 
 # set the engine heartbeat ports to use:
 # c.HubFactory.hb = (10303,10313)
 
 #-----------------------------------------------------------------------------
 # Configure the TaskRecord database backend
 #-----------------------------------------------------------------------------
 
 # For memory/persistance reasons, tasks can be stored out-of-memory in a database.
 # Currently, only sqlite and mongodb are supported as backends, but the interface
 # is fairly simple, so advanced developers could write their own backend.
 
 # ----- in-memory configuration --------
 # this line restores the default behavior: in-memory storage of all results.
-# c.HubFactory.db_class = 'IPython.parallel.dictdb.DictDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
 
 # ----- sqlite configuration --------
 # use this line to activate sqlite:
-# c.HubFactory.db_class = 'IPython.parallel.sqlitedb.SQLiteDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
 
 # You can specify the name of the db-file.  By default, this will be located
 # in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
 # c.SQLiteDB.filename = 'tasks.db'
 
 # You can also specify the location of the db-file, if you want it to be somewhere
 # other than the cluster_dir.
 # c.SQLiteDB.location = '/scratch/'
 
 # This will specify the name of the table for the controller to use.  The default
 # behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
 # this will result in results persisting for multiple sessions.
 # c.SQLiteDB.table = 'results'
 
 # ----- mongodb configuration --------
 # use this line to activate mongodb:
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
 
 # You can specify the args and kwargs pymongo will use when creating the Connection.
 # For more information on what these options might be, see pymongo documentation.
 # c.MongoDB.connection_kwargs = {}
 # c.MongoDB.connection_args = []
 
 # This will specify the name of the mongo database for the controller to use.  The default
 # behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
 # this will result in task results persisting through multiple sessions.
 # c.MongoDB.database = 'ipythondb'
 
 

IPython/external/ssh/tunnel.py

@@ -1,295 +1,295 b''
 """Basic ssh tunneling utilities."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import os,sys, atexit
 from multiprocessing import Process
 from getpass import getpass, getuser
 import warnings
 
 try:
     with warnings.catch_warnings():
         warnings.simplefilter('ignore', DeprecationWarning)
         import paramiko
 except ImportError:
     paramiko = None
 else:
     from forward import forward_tunnel
 
 try:
     from IPython.external import pexpect
 except ImportError:
     pexpect = None
 
-from IPython.parallel.entry_point import select_random_ports
+from IPython.parallel.util import select_random_ports
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Check for passwordless login
 #-----------------------------------------------------------------------------
 
 def try_passwordless_ssh(server, keyfile, paramiko=None):
     """Attempt to make an ssh connection without a password.
     This is mainly used for requiring password input only once
     when many tunnels may be connected to the same server.
     
     If paramiko is None, the default for the platform is chosen.
     """
     if paramiko is None:
         paramiko = sys.platform == 'win32'
     if not paramiko:
         f = _try_passwordless_openssh
     else:
         f = _try_passwordless_paramiko
     return f(server, keyfile)
 
 def _try_passwordless_openssh(server, keyfile):
     """Try passwordless login with shell ssh command."""
     if pexpect is None:
         raise ImportError("pexpect unavailable, use paramiko")
     cmd = 'ssh -f '+ server
     if keyfile:
         cmd += ' -i ' + keyfile
     cmd += ' exit'
     p = pexpect.spawn(cmd)
     while True:
         try:
             p.expect('[Ppassword]:', timeout=.1)
         except pexpect.TIMEOUT:
             continue
         except pexpect.EOF:
             return True
         else:
             return False
 
 def _try_passwordless_paramiko(server, keyfile):
     """Try passwordless login with paramiko."""
     if paramiko is None:
         raise ImportError("paramiko unavailable, use openssh")
     username, server, port = _split_server(server)
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     client.set_missing_host_key_policy(paramiko.WarningPolicy())
     try:
         client.connect(server, port, username=username, key_filename=keyfile,
                look_for_keys=True)
     except paramiko.AuthenticationException:
         return False
     else:
         client.close()
         return True
 
 
 def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None):
     """Connect a socket to an address via an ssh tunnel.
     
     This is a wrapper for socket.connect(addr), when addr is not accessible
     from the local machine.  It simply creates an ssh tunnel using the remaining args,
     and calls socket.connect('tcp://localhost:lport') where lport is the randomly
     selected local port of the tunnel.
     
     """
     lport = select_random_ports(1)[0]
     transport, addr = addr.split('://')
     ip,rport = addr.split(':')
     rport = int(rport)
     if paramiko is None:
         paramiko = sys.platform == 'win32'
     if paramiko:
         tunnelf = paramiko_tunnel
     else:
         tunnelf = openssh_tunnel
     tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password)
     socket.connect('tcp://127.0.0.1:%i'%lport)
     return tunnel
 
 def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=15):
     """Create an ssh tunnel using command-line ssh that connects port lport
     on this machine to localhost:rport on server.  The tunnel
     will automatically close when not in use, remaining open
     for a minimum of timeout seconds for an initial connection.
     
     This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,
     as seen from `server`.
     
     keyfile and password may be specified, but ssh config is checked for defaults.
     
     Parameters
     ----------
     
         lport : int
             local port for connecting to the tunnel from this machine.
         rport : int
             port on the remote machine to connect to.
         server : str
             The ssh server to connect to. The full ssh server string will be parsed.
             user@server:port
         remoteip : str [Default: 127.0.0.1]
             The remote ip, specifying the destination of the tunnel.
             Default is localhost, which means that the tunnel would redirect
             localhost:lport on this machine to localhost:rport on the *server*.
         
         keyfile : str; path to public key file
             This specifies a key to be used in ssh login, default None.
             Regular default ssh keys will be used without specifying this argument.
         password : str; 
             Your ssh password to the ssh server. Note that if this is left None,
             you will be prompted for it if passwordless key based login is unavailable.
     
     """
     if pexpect is None:
         raise ImportError("pexpect unavailable, use paramiko_tunnel")
     ssh="ssh "
     if keyfile:
         ssh += "-i " + keyfile 
     cmd = ssh + " -f -L 127.0.0.1:%i:%s:%i %s sleep %i"%(lport, remoteip, rport, server, timeout)
     tunnel = pexpect.spawn(cmd)
     failed = False
     while True:
         try:
             tunnel.expect('[Pp]assword:', timeout=.1)
         except pexpect.TIMEOUT:
             continue
         except pexpect.EOF:
             if tunnel.exitstatus:
                 print (tunnel.exitstatus)
                 print (tunnel.before)
                 print (tunnel.after)
                 raise RuntimeError("tunnel '%s' failed to start"%(cmd))
             else:
                 return tunnel.pid
         else:
             if failed:
                 print("Password rejected, try again")
                 password=None
             if password is None:
                 password = getpass("%s's password: "%(server))
             tunnel.sendline(password)
             failed = True
     
 def _split_server(server):
     if '@' in server:
         username,server = server.split('@', 1)
     else:
         username = getuser()
     if ':' in server:
         server, port = server.split(':')
         port = int(port)
     else:
         port = 22
     return username, server, port
 
 def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=15):
     """launch a tunner with paramiko in a subprocess. This should only be used
     when shell ssh is unavailable (e.g. Windows).
     
     This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,
     as seen from `server`.
     
     If you are familiar with ssh tunnels, this creates the tunnel:
     
     ssh server -L localhost:lport:remoteip:rport
     
     keyfile and password may be specified, but ssh config is checked for defaults.
     
     
     Parameters
     ----------
     
         lport : int
             local port for connecting to the tunnel from this machine.
         rport : int
             port on the remote machine to connect to.
         server : str
             The ssh server to connect to. The full ssh server string will be parsed.
             user@server:port
         remoteip : str [Default: 127.0.0.1]
             The remote ip, specifying the destination of the tunnel.
             Default is localhost, which means that the tunnel would redirect
             localhost:lport on this machine to localhost:rport on the *server*.
         
         keyfile : str; path to public key file
             This specifies a key to be used in ssh login, default None.
             Regular default ssh keys will be used without specifying this argument.
         password : str; 
             Your ssh password to the ssh server. Note that if this is left None,
             you will be prompted for it if passwordless key based login is unavailable.
     
     """
     if paramiko is None:
         raise ImportError("Paramiko not available")
     
     if password is None:
         if not _check_passwordless_paramiko(server, keyfile):
             password = getpass("%s's password: "%(server))
 
     p = Process(target=_paramiko_tunnel, 
             args=(lport, rport, server, remoteip), 
             kwargs=dict(keyfile=keyfile, password=password))
     p.daemon=False
     p.start()
     atexit.register(_shutdown_process, p)
     return p
     
 def _shutdown_process(p):
     if p.isalive():
         p.terminate()
 
 def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None):
     """Function for actually starting a paramiko tunnel, to be passed
     to multiprocessing.Process(target=this), and not called directly.
     """
     username, server, port = _split_server(server)
     client = paramiko.SSHClient()
     client.load_system_host_keys()
     client.set_missing_host_key_policy(paramiko.WarningPolicy())
 
     try:
         client.connect(server, port, username=username, key_filename=keyfile,
                        look_for_keys=True, password=password)
 #    except paramiko.AuthenticationException:
 #        if password is None:
 #            password = getpass("%s@%s's password: "%(username, server))
 #            client.connect(server, port, username=username, password=password)
 #        else:
 #            raise
     except Exception as e:
         print ('*** Failed to connect to %s:%d: %r' % (server, port, e))
         sys.exit(1)
 
     # print ('Now forwarding port %d to %s:%d ...' % (lport, server, rport))
 
     try:
         forward_tunnel(lport, remoteip, rport, client.get_transport())
     except KeyboardInterrupt:
         print ('SIGINT: Port forwarding stopped cleanly')
         sys.exit(0)
     except Exception as e:
         print ("Port forwarding stopped uncleanly: %s"%e)
         sys.exit(255)
 
 if sys.platform == 'win32':
     ssh_tunnel = paramiko_tunnel
 else:
     ssh_tunnel = openssh_tunnel
 
     
 __all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh']
 
 

IPython/parallel/__init__.py

@@ -1,25 +1,26 b''
 """The IPython ZMQ-based parallel computing interface."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import zmq
 
-if zmq.__version__ < '2.1.3':
-    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.3, you appear to have %s"%zmq.__version__)
+if zmq.__version__ < '2.1.4':
+    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)
 
-from .asyncresult import *
-from .client import Client
-from .dependency import *
-from .remotefunction import *
-from .view import *
 from IPython.utils.pickleutil import Reference
 
+from .client.asyncresult import *
+from .client.client import Client
+from .client.remotefunction import *
+from .client.view import *
+from .controller.dependency import *
+
 

IPython/parallel/apps/ipclusterapp.py

@@ -1,592 +1,592 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The ipcluster application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import errno
 import logging
 import os
 import re
 import signal
 
 import zmq
 from zmq.eventloop import ioloop
 
 from IPython.external.argparse import ArgumentParser, SUPPRESS
 from IPython.utils.importstring import import_item
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir, ClusterDirConfigLoader,
     ClusterDirError, PIDFileError
 )
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 default_config_file_name = u'ipcluster_config.py'
 
 
 _description = """\
 Start an IPython cluster for parallel computing.\n\n
 
 An IPython cluster consists of 1 controller and 1 or more engines.
 This command automates the startup of these processes using a wide
 range of startup methods (SSH, local processes, PBS, mpiexec,
 Windows HPC Server 2008). To start a cluster with 4 engines on your
 local host simply do 'ipcluster start -n 4'. For more complex usage 
 you will typically do 'ipcluster create -p mycluster', then edit
 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
 """
 
 
 # Exit codes for ipcluster
 
 # This will be the exit code if the ipcluster appears to be running because
 # a .pid file exists
 ALREADY_STARTED = 10
 
 
 # This will be the exit code if ipcluster stop is run, but there is not .pid
 # file to be found.
 ALREADY_STOPPED = 11
 
 # This will be the exit code if ipcluster engines is run, but there is not .pid
 # file to be found.
 NO_CLUSTER = 12
 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPClusterAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         # Don't call ClusterDirConfigLoader._add_arguments as we don't want
         # its defaults on self.parser. Instead, we will put those on
         # default options on our subparsers.
         
         # This has all the common options that all subcommands use
         parent_parser1 = ArgumentParser(
             add_help=False,
             argument_default=SUPPRESS
         )
         self._add_ipython_dir(parent_parser1)
         self._add_log_level(parent_parser1)
 
         # This has all the common options that other subcommands use
         parent_parser2 = ArgumentParser(
             add_help=False,
             argument_default=SUPPRESS
         )
         self._add_cluster_profile(parent_parser2)
         self._add_cluster_dir(parent_parser2)
         self._add_work_dir(parent_parser2)
         paa = parent_parser2.add_argument
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file', 
             help='Log to a file in the log directory (default is stdout)')
 
         # Create the object used to create the subparsers.
         subparsers = self.parser.add_subparsers(
             dest='Global.subcommand',
             title='ipcluster subcommands',
             description=
             """ipcluster has a variety of subcommands. The general way of 
             running ipcluster is 'ipcluster <cmd> [options]'. To get help
             on a particular subcommand do 'ipcluster <cmd> -h'."""
             # help="For more help, type 'ipcluster <cmd> -h'",
         )
 
         # The "list" subcommand parser
         parser_list = subparsers.add_parser(
             'list',
             parents=[parent_parser1],
             argument_default=SUPPRESS,
             help="List all clusters in cwd and ipython_dir.",
             description=
             """List all available clusters, by cluster directory, that can
             be found in the current working directly or in the ipython
             directory. Cluster directories are named using the convention
             'cluster_<profile>'."""
         )
 
         # The "create" subcommand parser
         parser_create = subparsers.add_parser(
             'create',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Create a new cluster directory.",
             description=
             """Create an ipython cluster directory by its profile name or 
             cluster directory path. Cluster directories contain 
             configuration, log and security related files and are named 
             using the convention 'cluster_<profile>'. By default they are
             located in your ipython directory. Once created, you will
             probably need to edit the configuration files in the cluster
             directory to configure your cluster. Most users will create a
             cluster directory by profile name, 
             'ipcluster create -p mycluster', which will put the directory
             in '<ipython_dir>/cluster_mycluster'. 
             """
         )
         paa = parser_create.add_argument
         paa('--reset-config',
             dest='Global.reset_config', action='store_true',
             help=
             """Recopy the default config files to the cluster directory.
             You will loose any modifications you have made to these files.""")
 
         # The "start" subcommand parser
         parser_start = subparsers.add_parser(
             'start',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Start a cluster.",
             description=
             """Start an ipython cluster by its profile name or cluster 
             directory. Cluster directories contain configuration, log and
             security related files and are named using the convention
             'cluster_<profile>' and should be creating using the 'start'
             subcommand of 'ipcluster'. If your cluster directory is in 
             the cwd or the ipython directory, you can simply refer to it
             using its profile name, 'ipcluster start -n 4 -p <profile>`,
             otherwise use the '--cluster-dir' option.
             """
         )
         
         paa = parser_start.add_argument
         paa('-n', '--number',
             type=int, dest='Global.n',
             help='The number of engines to start.',
             metavar='Global.n')
         paa('--clean-logs',
             dest='Global.clean_logs', action='store_true',
             help='Delete old log flies before starting.')
         paa('--no-clean-logs',
             dest='Global.clean_logs', action='store_false',
             help="Don't delete old log flies before starting.")
         paa('--daemon',
             dest='Global.daemonize', action='store_true',
             help='Daemonize the ipcluster program. This implies --log-to-file')
         paa('--no-daemon',
             dest='Global.daemonize', action='store_false',
             help="Dont't daemonize the ipcluster program.")
         paa('--delay',
             type=float, dest='Global.delay',
             help="Specify the delay (in seconds) between starting the controller and starting the engine(s).")
 
         # The "stop" subcommand parser
         parser_stop = subparsers.add_parser(
             'stop',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Stop a running cluster.",
             description=
             """Stop a running ipython cluster by its profile name or cluster 
             directory. Cluster directories are named using the convention
             'cluster_<profile>'. If your cluster directory is in 
             the cwd or the ipython directory, you can simply refer to it
             using its profile name, 'ipcluster stop -p <profile>`, otherwise
             use the '--cluster-dir' option.
             """
         )
         paa = parser_stop.add_argument
         paa('--signal',
             dest='Global.signal', type=int,
             help="The signal number to use in stopping the cluster (default=2).",
             metavar="Global.signal")
         
         # the "engines" subcommand parser
         parser_engines = subparsers.add_parser(
             'engines',
             parents=[parent_parser1, parent_parser2],
             argument_default=SUPPRESS,
             help="Attach some engines to an existing controller or cluster.",
             description=
             """Start one or more engines to connect to an existing Cluster
             by profile name or cluster directory.
             Cluster directories contain configuration, log and
             security related files and are named using the convention
             'cluster_<profile>' and should be creating using the 'start'
             subcommand of 'ipcluster'. If your cluster directory is in 
             the cwd or the ipython directory, you can simply refer to it
             using its profile name, 'ipcluster engines -n 4 -p <profile>`,
             otherwise use the '--cluster-dir' option.
             """
         )
         paa = parser_engines.add_argument
         paa('-n', '--number',
             type=int, dest='Global.n',
             help='The number of engines to start.',
             metavar='Global.n')
         paa('--daemon',
             dest='Global.daemonize', action='store_true',
             help='Daemonize the ipcluster program. This implies --log-to-file')
         paa('--no-daemon',
             dest='Global.daemonize', action='store_false',
             help="Dont't daemonize the ipcluster program.")
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPClusterApp(ApplicationWithClusterDir):
 
     name = u'ipcluster'
     description = _description
     usage = None
     command_line_loader = IPClusterAppConfigLoader
     default_config_file_name = default_config_file_name
     default_log_level = logging.INFO
     auto_create_cluster_dir = False
 
     def create_default_config(self):
         super(IPClusterApp, self).create_default_config()
         self.default_config.Global.controller_launcher = \
             'IPython.parallel.launcher.LocalControllerLauncher'
         self.default_config.Global.engine_launcher = \
             'IPython.parallel.launcher.LocalEngineSetLauncher'
         self.default_config.Global.n = 2
         self.default_config.Global.delay = 2
         self.default_config.Global.reset_config = False
         self.default_config.Global.clean_logs = True
         self.default_config.Global.signal = signal.SIGINT
         self.default_config.Global.daemonize = False
 
     def find_resources(self):
         subcommand = self.command_line_config.Global.subcommand
         if subcommand=='list':
             self.list_cluster_dirs()
             # Exit immediately because there is nothing left to do.
             self.exit()
         elif subcommand=='create':
             self.auto_create_cluster_dir = True
             super(IPClusterApp, self).find_resources()
         elif subcommand=='start' or subcommand=='stop':
             self.auto_create_cluster_dir = True
             try:
                 super(IPClusterApp, self).find_resources()
             except ClusterDirError:
                 raise ClusterDirError(
                     "Could not find a cluster directory. A cluster dir must "
                     "be created before running 'ipcluster start'.  Do "
                     "'ipcluster create -h' or 'ipcluster list -h' for more "
                     "information about creating and listing cluster dirs."
                 )
         elif subcommand=='engines':
             self.auto_create_cluster_dir = False
             try:
                 super(IPClusterApp, self).find_resources()
             except ClusterDirError:
                 raise ClusterDirError(
                     "Could not find a cluster directory. A cluster dir must "
                     "be created before running 'ipcluster start'.  Do "
                     "'ipcluster create -h' or 'ipcluster list -h' for more "
                     "information about creating and listing cluster dirs."
                 )
 
     def list_cluster_dirs(self):
         # Find the search paths
         cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
         if cluster_dir_paths:
             cluster_dir_paths = cluster_dir_paths.split(':')
         else:
             cluster_dir_paths = []
         try:
             ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             ipython_dir = self.default_config.Global.ipython_dir
         paths = [os.getcwd(), ipython_dir] + \
             cluster_dir_paths
         paths = list(set(paths))
 
         self.log.info('Searching for cluster dirs in paths: %r' % paths)
         for path in paths:
             files = os.listdir(path)
             for f in files:
                 full_path = os.path.join(path, f)
                 if os.path.isdir(full_path) and f.startswith('cluster_'):
                     profile = full_path.split('_')[-1]
                     start_cmd = 'ipcluster start -p %s -n 4' % profile
                     print start_cmd + " ==> " + full_path
 
     def pre_construct(self):
         # IPClusterApp.pre_construct() is where we cd to the working directory.
         super(IPClusterApp, self).pre_construct()
         config = self.master_config
         try:
             daemon = config.Global.daemonize
             if daemon:
                 config.Global.log_to_file = True
         except AttributeError:
             pass
 
     def construct(self):
         config = self.master_config
         subcmd = config.Global.subcommand
         reset = config.Global.reset_config
         if subcmd == 'list':
             return
         if subcmd == 'create':
             self.log.info('Copying default config files to cluster directory '
             '[overwrite=%r]' % (reset,))
             self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
         if subcmd =='start':
             self.cluster_dir_obj.copy_all_config_files(overwrite=False)
             self.start_logging()
             self.loop = ioloop.IOLoop.instance()
             # reactor.callWhenRunning(self.start_launchers)
             dc = ioloop.DelayedCallback(self.start_launchers, 0, self.loop)
             dc.start()
         if subcmd == 'engines':
             self.start_logging()
             self.loop = ioloop.IOLoop.instance()
             # reactor.callWhenRunning(self.start_launchers)
             engine_only = lambda : self.start_launchers(controller=False)
             dc = ioloop.DelayedCallback(engine_only, 0, self.loop)
             dc.start()
 
     def start_launchers(self, controller=True):
         config = self.master_config
 
         # Create the launchers. In both bases, we set the work_dir of
         # the launcher to the cluster_dir. This is where the launcher's
         # subprocesses will be launched. It is not where the controller
         # and engine will be launched.
         if controller:
             cl_class = import_item(config.Global.controller_launcher)
             self.controller_launcher = cl_class(
                 work_dir=self.cluster_dir, config=config,
                 logname=self.log.name
             )
             # Setup the observing of stopping. If the controller dies, shut
             # everything down as that will be completely fatal for the engines.
             self.controller_launcher.on_stop(self.stop_launchers)
             # But, we don't monitor the stopping of engines. An engine dying
             # is just fine and in principle a user could start a new engine.
             # Also, if we did monitor engine stopping, it is difficult to
             # know what to do when only some engines die. Currently, the
             # observing of engine stopping is inconsistent. Some launchers
             # might trigger on a single engine stopping, other wait until
             # all stop.  TODO: think more about how to handle this.
         else:
             self.controller_launcher = None
         
         el_class = import_item(config.Global.engine_launcher)
         self.engine_launcher = el_class(
             work_dir=self.cluster_dir, config=config, logname=self.log.name
         )
 
         # Setup signals
         signal.signal(signal.SIGINT, self.sigint_handler)
 
         # Start the controller and engines
         self._stopping = False  # Make sure stop_launchers is not called 2x.
         if controller:
             self.start_controller()
         dc = ioloop.DelayedCallback(self.start_engines, 1000*config.Global.delay*controller, self.loop)
         dc.start()
         self.startup_message()
 
     def startup_message(self, r=None):
         self.log.info("IPython cluster: started")
         return r
         
     def start_controller(self, r=None):
         # self.log.info("In start_controller")
         config = self.master_config
         d = self.controller_launcher.start(
             cluster_dir=config.Global.cluster_dir
         )
         return d
             
     def start_engines(self, r=None):
         # self.log.info("In start_engines")
         config = self.master_config
         
         d = self.engine_launcher.start(
             config.Global.n,
             cluster_dir=config.Global.cluster_dir
         )
         return d
 
     def stop_controller(self, r=None):
         # self.log.info("In stop_controller")
         if self.controller_launcher and self.controller_launcher.running:
             return self.controller_launcher.stop()
 
     def stop_engines(self, r=None):
         # self.log.info("In stop_engines")
         if self.engine_launcher.running:
             d = self.engine_launcher.stop()
             # d.addErrback(self.log_err)
             return d
         else:
             return None
 
     def log_err(self, f):
         self.log.error(f.getTraceback())
         return None
         
     def stop_launchers(self, r=None):
         if not self._stopping:
             self._stopping = True
             # if isinstance(r, failure.Failure):
             #     self.log.error('Unexpected error in ipcluster:')
             #     self.log.info(r.getTraceback())
             self.log.error("IPython cluster: stopping")
             # These return deferreds. We are not doing anything with them
             # but we are holding refs to them as a reminder that they 
             # do return deferreds.
             d1 = self.stop_engines()
             d2 = self.stop_controller()
             # Wait a few seconds to let things shut down.
             dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
             dc.start()
             # reactor.callLater(4.0, reactor.stop)
 
     def sigint_handler(self, signum, frame):
         self.stop_launchers()
         
     def start_logging(self):
         # Remove old log files of the controller and engine
         if self.master_config.Global.clean_logs:
             log_dir = self.master_config.Global.log_dir
             for f in os.listdir(log_dir):
                 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
                     os.remove(os.path.join(log_dir, f))
         # This will remove old log files for ipcluster itself
         super(IPClusterApp, self).start_logging()
 
     def start_app(self):
         """Start the application, depending on what subcommand is used."""
         subcmd = self.master_config.Global.subcommand
         if subcmd=='create' or subcmd=='list':
             return
         elif subcmd=='start':
             self.start_app_start()
         elif subcmd=='stop':
             self.start_app_stop()
         elif subcmd=='engines':
             self.start_app_engines()
 
     def start_app_start(self):
         """Start the app for the start subcommand."""
         config = self.master_config
         # First see if the cluster is already running
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             pass
         else:
             self.log.critical(
                 'Cluster is already running with [pid=%s]. '
                 'use "ipcluster stop" to stop the cluster.' % pid
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STARTED)
 
         # Now log and daemonize
         self.log.info(
             'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if config.Global.daemonize:
             if os.name=='posix':
                 from twisted.scripts._twistd_unix import daemonize
                 daemonize()
 
         # Now write the new pid file AFTER our new forked pid is active.
         self.write_pid_file()
         try:
             self.loop.start()
         except KeyboardInterrupt:
             pass
         except zmq.ZMQError as e:
             if e.errno == errno.EINTR:
                 pass
             else:
                 raise
         self.remove_pid_file()
 
     def start_app_engines(self):
         """Start the app for the start subcommand."""
         config = self.master_config
         # First see if the cluster is already running
         
         # Now log and daemonize
         self.log.info(
             'Starting engines with [daemon=%r]' % config.Global.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if config.Global.daemonize:
             if os.name=='posix':
                 from twisted.scripts._twistd_unix import daemonize
                 daemonize()
 
         # Now write the new pid file AFTER our new forked pid is active.
         # self.write_pid_file()
         try:
             self.loop.start()
         except KeyboardInterrupt:
             pass
         except zmq.ZMQError as e:
             if e.errno == errno.EINTR:
                 pass
             else:
                 raise
         # self.remove_pid_file()
     
     def start_app_stop(self):
         """Start the app for the stop subcommand."""
         config = self.master_config
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             self.log.critical(
                 'Problem reading pid file, cluster is probably not running.'
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STOPPED)
         else:
             if os.name=='posix':
                 sig = config.Global.signal
                 self.log.info(
                     "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
                 )
                 os.kill(pid, sig)
             elif os.name=='nt':
                 # As of right now, we don't support daemonize on Windows, so
                 # stop will not do anything. Minimally, it should clean up the
                 # old .pid files.
                 self.remove_pid_file()
 
 
 def launch_new_instance():
     """Create and run the IPython cluster."""
     app = IPClusterApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/parallel/apps/ipcontrollerapp.py

@@ -1,431 +1,432 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import copy
 import os
 import logging
 import socket
 import stat
 import sys
 import uuid
 
 import zmq
 from zmq.log.handlers import PUBHandler
 from zmq.utils import jsonapi as json
 
 from IPython.config.loader import Config
+
 from IPython.parallel import factory
-from IPython.parallel.controller import ControllerFactory
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.parallel.util import disambiguate_ip_address, split_url
 # from IPython.kernel.fcutil import FCServiceFactory, FURLError
 from IPython.utils.traitlets import Instance, Unicode
 
+from IPython.parallel.controller.controller import ControllerFactory
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 #: The default config file name for this application
 default_config_file_name = u'ipcontroller_config.py'
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your ipython directory and named as "cluster_<profile>". See the --profile
 and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Default interfaces
 #-----------------------------------------------------------------------------
 
 # The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Default.url_file = 'ipcontroller-client.url'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.url_file = u'ipcontroller-engine.url'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
 #-----------------------------------------------------------------------------
 # Service factories
 #-----------------------------------------------------------------------------
 
 # 
 # class FCClientServiceFactory(FCServiceFactory):
 #     """A Foolscap implementation of the client services."""
 # 
 #     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
 #     interfaces = Instance(klass=Config, kw=default_client_interfaces,
 #                           allow_none=False, config=True)
 # 
 # 
 # class FCEngineServiceFactory(FCServiceFactory):
 #     """A Foolscap implementation of the engine services."""
 # 
 #     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
 #     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
 #                           allow_none=False, config=True)
 # 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPControllerAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         
         ## Hub Config:
         paa('--mongodb', 
             dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.mongodb.MongoDB', 
+            const='IPython.parallel.controller.mongodb.MongoDB', 
             help='Use MongoDB for task storage [default: in-memory]')
         paa('--sqlite', 
             dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.sqlitedb.SQLiteDB', 
+            const='IPython.parallel.controller.sqlitedb.SQLiteDB',
             help='Use SQLite3 for DB task storage [default: in-memory]')
         paa('--hb',
             type=int, dest='HubFactory.hb', nargs=2,
             help='The (2) ports the Hub\'s Heartmonitor will use for the heartbeat '
             'connections [default: random]',
             metavar='Hub.hb_ports')
         paa('--ping',
             type=int, dest='HubFactory.ping',
             help='The frequency at which the Hub pings the engines for heartbeats '
             ' (in ms) [default: 100]',
             metavar='Hub.ping')
         
         # Client config
         paa('--client-ip',
             type=str, dest='HubFactory.client_ip', 
             help='The IP address or hostname the Hub will listen on for '
             'client connections. Both engine-ip and client-ip can be set simultaneously '
             'via --ip [default: loopback]',
             metavar='Hub.client_ip')
         paa('--client-transport',
             type=str, dest='HubFactory.client_transport', 
             help='The ZeroMQ transport the Hub will use for '
             'client connections. Both engine-transport and client-transport can be set simultaneously '
             'via --transport [default: tcp]',
             metavar='Hub.client_transport')
         paa('--query',
             type=int, dest='HubFactory.query_port', 
             help='The port on which the Hub XREP socket will listen for result queries from clients [default: random]',
             metavar='Hub.query_port')
         paa('--notifier',
             type=int, dest='HubFactory.notifier_port', 
             help='The port on which the Hub PUB socket will listen for notification connections [default: random]',
             metavar='Hub.notifier_port')
         
         # Engine config
         paa('--engine-ip',
             type=str, dest='HubFactory.engine_ip', 
             help='The IP address or hostname the Hub will listen on for '
             'engine connections. This applies to the Hub and its schedulers'
             'engine-ip and client-ip can be set simultaneously '
             'via --ip [default: loopback]',
             metavar='Hub.engine_ip')
         paa('--engine-transport',
             type=str, dest='HubFactory.engine_transport', 
             help='The ZeroMQ transport the Hub will use for '
             'client connections. Both engine-transport and client-transport can be set simultaneously '
             'via --transport [default: tcp]',
             metavar='Hub.engine_transport')
         
         # Scheduler config
         paa('--mux',
             type=int, dest='ControllerFactory.mux', nargs=2,
             help='The (2) ports the MUX scheduler will listen on for client,engine '
             'connections, respectively [default: random]',
             metavar='Scheduler.mux_ports')
         paa('--task',
             type=int, dest='ControllerFactory.task', nargs=2,
             help='The (2) ports the Task scheduler will listen on for client,engine '
             'connections, respectively [default: random]',
             metavar='Scheduler.task_ports')
         paa('--control',
             type=int, dest='ControllerFactory.control', nargs=2,
             help='The (2) ports the Control scheduler will listen on for client,engine '
             'connections, respectively [default: random]',
             metavar='Scheduler.control_ports')
         paa('--iopub',
             type=int, dest='ControllerFactory.iopub', nargs=2,
             help='The (2) ports the IOPub scheduler will listen on for client,engine '
             'connections, respectively [default: random]',
             metavar='Scheduler.iopub_ports')
             
         paa('--scheme',
             type=str, dest='HubFactory.scheme',
             choices = ['pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'],
             help='select the task scheduler scheme  [default: Python LRU]',
             metavar='Scheduler.scheme')
         paa('--usethreads',
             dest='ControllerFactory.usethreads', action="store_true",
             help='Use threads instead of processes for the schedulers',
             )
         paa('--hwm',
             dest='ControllerFactory.hwm', type=int,
             help='specify the High Water Mark (HWM) for the downstream '
             'socket in the pure ZMQ scheduler. This is the maximum number '
             'of allowed outstanding tasks on each engine.',
             )
         
         ## Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
         paa('--log-url',
             type=str, dest='Global.log_url',
             help='Broadcast logs to an iploggerz process [default: disabled]')
         paa('-r','--reuse-files', 
             action='store_true', dest='Global.reuse_files',
             help='Try to reuse existing json connection files.')
         paa('--no-secure',
             action='store_false', dest='Global.secure',
             help='Turn off execution keys (default).')
         paa('--secure',
             action='store_true', dest='Global.secure',
             help='Turn on execution keys.')
         paa('--execkey',
             type=str, dest='Global.exec_key',
             help='path to a file containing an execution key.',
             metavar='keyfile')
         paa('--ssh',
             type=str, dest='Global.sshserver',
             help='ssh url for clients to use when connecting to the Controller '
             'processes. It should be of the form: [user@]server[:port]. The '
             'Controller\'s listening addresses must be accessible from the ssh server',
             metavar='Global.sshserver')
         paa('--location',
             type=str, dest='Global.location',
             help="The external IP or domain name of this machine, used for disambiguating "
             "engine and client connections.",
             metavar='Global.location')
         factory.add_session_arguments(self.parser)
         factory.add_registration_arguments(self.parser)
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 
 
 class IPControllerApp(ApplicationWithClusterDir):
 
     name = u'ipcontroller'
     description = _description
     command_line_loader = IPControllerAppConfigLoader
     default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
     
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
         # Don't set defaults for Global.secure or Global.reuse_furls
         # as those are set in a component.
         self.default_config.Global.import_statements = []
         self.default_config.Global.clean_logs = True
         self.default_config.Global.secure = True
         self.default_config.Global.reuse_files = False
         self.default_config.Global.exec_key = "exec_key.key"
         self.default_config.Global.sshserver = None
         self.default_config.Global.location = None
 
     def pre_construct(self):
         super(IPControllerApp, self).pre_construct()
         c = self.master_config
         # The defaults for these are set in FCClientServiceFactory and
         # FCEngineServiceFactory, so we only set them here if the global
         # options have be set to override the class level defaults.
         
         # if hasattr(c.Global, 'reuse_furls'):
         #     c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
         #     c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
         #     del c.Global.reuse_furls
         # if hasattr(c.Global, 'secure'):
         #     c.FCClientServiceFactory.secure = c.Global.secure
         #     c.FCEngineServiceFactory.secure = c.Global.secure
         #     del c.Global.secure
     
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
         c = self.master_config
         url = cdict['url']
         location = cdict['location']
         if not location:
             try:
                 proto,ip,port = split_url(url)
             except AssertionError:
                 pass
             else:
                 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
             cdict['location'] = location
         fname = os.path.join(c.Global.security_dir, fname)
         with open(fname, 'w') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
         c = self.master_config
         # load from engine config
         with open(os.path.join(c.Global.security_dir, 'ipcontroller-engine.json')) as f:
             cfg = json.loads(f.read())
         key = c.SessionFactory.exec_key = cfg['exec_key']
         xport,addr = cfg['url'].split('://')
         c.HubFactory.engine_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.engine_ip = ip
         c.HubFactory.regport = int(ports)
         c.Global.location = cfg['location']
         
         # load client config
         with open(os.path.join(c.Global.security_dir, 'ipcontroller-client.json')) as f:
             cfg = json.loads(f.read())
         assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
         xport,addr = cfg['url'].split('://')
         c.HubFactory.client_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.client_ip = ip
         c.Global.sshserver = cfg['ssh']
         assert int(ports) == c.HubFactory.regport, "regport mismatch"
     
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
         c = self.master_config
         
         self.import_statements()
         reusing = c.Global.reuse_files
         if reusing:
             try:
                 self.load_config_from_json()
             except (AssertionError,IOError):
                 reusing=False
         # check again, because reusing may have failed:
         if reusing:
             pass
         elif c.Global.secure:
             keyfile = os.path.join(c.Global.security_dir, c.Global.exec_key)
             key = str(uuid.uuid4())
             with open(keyfile, 'w') as f:
                 f.write(key)
             os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
             c.SessionFactory.exec_key = key
         else:
             c.SessionFactory.exec_key = ''
             key = None
         
         try:
             self.factory = ControllerFactory(config=c, logname=self.log.name)
             self.start_logging()
             self.factory.construct()
         except:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
         
         if not reusing:
             # save to new json config files
             f = self.factory
             cdict = {'exec_key' : key,
                     'ssh' : c.Global.sshserver,
                     'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                     'location' : c.Global.location
                     }
             self.save_connection_dict('ipcontroller-client.json', cdict)
             edict = cdict
             edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
             self.save_connection_dict('ipcontroller-engine.json', edict)
         
     
     def save_urls(self):
         """save the registration urls to files."""
         c = self.master_config
         
         sec_dir = c.Global.security_dir
         cf = self.factory
         
         with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
         
         with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
         
     
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 self.log.msg("Error running statement: %s" % s)
 
     def start_logging(self):
         super(IPControllerApp, self).start_logging()
         if self.master_config.Global.log_url:
             context = self.factory.context
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.master_config.Global.log_url)
             handler = PUBHandler(lsock)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
     # 
     def start_app(self):
         # Start the subprocesses:
         self.factory.start()
         self.write_pid_file(overwrite=True)
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPControllerApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/parallel/apps/ipengineapp.py

@@ -1,294 +1,295 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython engine application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import json
 import os
 import sys
 
 import zmq
 from zmq.eventloop import ioloop
 
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.zmq.log import EnginePUBHandler
 
 from IPython.parallel import factory
-from IPython.parallel.engine import EngineFactory
-from IPython.parallel.streamkernel import Kernel
+from IPython.parallel.engine.engine import EngineFactory
+from IPython.parallel.engine.streamkernel import Kernel
 from IPython.parallel.util import disambiguate_url
+
 from IPython.utils.importstring import import_item
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 
 _description = """Start an IPython engine for parallel computing.\n\n
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your ipython directory and named as "cluster_<profile>".
 See the --profile and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPEngineAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Controller config
         paa('--file', '-f',
             type=unicode, dest='Global.url_file',
             help='The full location of the file containing the connection information fo '
             'controller. If this is not given, the file must be in the '
             'security directory of the cluster directory.  This location is '
             'resolved using the --profile and --app-dir options.',
             metavar='Global.url_file')
         # MPI
         paa('--mpi',
             type=str, dest='MPI.use',
             help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
             metavar='MPI.use')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
         paa('--log-url',
             dest='Global.log_url',
             help="url of ZMQ logger, as started with iploggerz")
         # paa('--execkey',
         #     type=str, dest='Global.exec_key',
         #     help='path to a file containing an execution key.',
         #     metavar='keyfile')
         # paa('--no-secure',
         #     action='store_false', dest='Global.secure',
         #     help='Turn off execution keys.')
         # paa('--secure',
         #     action='store_true', dest='Global.secure',
         #     help='Turn on execution keys (default).')
         # init command
         paa('-c',
             type=str, dest='Global.extra_exec_lines',
             help='specify a command to be run at startup')
         
         factory.add_session_arguments(self.parser)
         factory.add_registration_arguments(self.parser)
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPEngineApp(ApplicationWithClusterDir):
 
     name = u'ipengine'
     description = _description
     command_line_loader = IPEngineAppConfigLoader
     default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()
 
         # The engine should not clean logs as we don't want to remove the
         # active log files of other running engines.
         self.default_config.Global.clean_logs = False
         self.default_config.Global.secure = True
 
         # Global config attributes
         self.default_config.Global.exec_lines = []
         self.default_config.Global.extra_exec_lines = ''
 
         # Configuration related to the controller
         # This must match the filename (path not included) that the controller
         # used for the FURL file.
         self.default_config.Global.url_file = u''
         self.default_config.Global.url_file_name = u'ipcontroller-engine.json'
         # If given, this is the actual location of the controller's FURL file.
         # If not, this is computed using the profile, app_dir and furl_file_name
         # self.default_config.Global.key_file_name = u'exec_key.key'
         # self.default_config.Global.key_file = u''
 
         # MPI related config attributes
         self.default_config.MPI.use = ''
         self.default_config.MPI.mpi4py = mpi4py_init
         self.default_config.MPI.pytrilinos = pytrilinos_init
 
     def post_load_command_line_config(self):
         pass
 
     def pre_construct(self):
         super(IPEngineApp, self).pre_construct()
         # self.find_cont_url_file()
         self.find_url_file()
         if self.master_config.Global.extra_exec_lines:
             self.master_config.Global.exec_lines.append(self.master_config.Global.extra_exec_lines)
 
     # def find_key_file(self):
     #     """Set the key file.
     # 
     #     Here we don't try to actually see if it exists for is valid as that
     #     is hadled by the connection logic.
     #     """
     #     config = self.master_config
     #     # Find the actual controller key file
     #     if not config.Global.key_file:
     #         try_this = os.path.join(
     #             config.Global.cluster_dir, 
     #             config.Global.security_dir,
     #             config.Global.key_file_name
     #         )
     #         config.Global.key_file = try_this
         
     def find_url_file(self):
         """Set the key file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.master_config
         # Find the actual controller key file
         if not config.Global.url_file:
             try_this = os.path.join(
                 config.Global.cluster_dir, 
                 config.Global.security_dir,
                 config.Global.url_file_name
             )
             config.Global.url_file = try_this
         
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
         config = self.master_config
         # if os.path.exists(config.Global.key_file) and config.Global.secure:
         #     config.SessionFactory.exec_key = config.Global.key_file
         if os.path.exists(config.Global.url_file):
             with open(config.Global.url_file) as f:
                 d = json.loads(f.read())
                 for k,v in d.iteritems():
                     if isinstance(v, unicode):
                         d[k] = v.encode()
             if d['exec_key']:
                 config.SessionFactory.exec_key = d['exec_key']
             d['url'] = disambiguate_url(d['url'], d['location'])
             config.RegistrationFactory.url=d['url']
             config.EngineFactory.location = d['location']
         
         
         
         config.Kernel.exec_lines = config.Global.exec_lines
 
         self.start_mpi()
 
         # Create the underlying shell class and EngineService
         # shell_class = import_item(self.master_config.Global.shell_class)
         try:
             self.engine = EngineFactory(config=config, logname=self.log.name)
         except:
             self.log.error("Couldn't start the Engine", exc_info=True)
             self.exit(1)
         
         self.start_logging()
 
         # Create the service hierarchy
         # self.main_service = service.MultiService()
         # self.engine_service.setServiceParent(self.main_service)
         # self.tub_service = Tub()
         # self.tub_service.setServiceParent(self.main_service)
         # # This needs to be called before the connection is initiated
         # self.main_service.startService()
 
         # This initiates the connection to the controller and calls
         # register_engine to tell the controller we are ready to do work
         # self.engine_connector = EngineConnector(self.tub_service)
 
         # self.log.info("Using furl file: %s" % self.master_config.Global.furl_file)
 
         # reactor.callWhenRunning(self.call_connect)
 
 
     def start_logging(self):
         super(IPEngineApp, self).start_logging()
         if self.master_config.Global.log_url:
             context = self.engine.context
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.master_config.Global.log_url)
             handler = EnginePUBHandler(self.engine, lsock)
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
     
     def start_mpi(self):
         global mpi
         mpikey = self.master_config.MPI.use
         mpi_import_statement = self.master_config.MPI.get(mpikey, None)
         if mpi_import_statement is not None:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
 
     def start_app(self):
         self.engine.start()
         try:
             self.engine.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Engine Interrupted, shutting down...\n")
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPEngineApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/parallel/apps/iploggerapp.py

@@ -1,132 +1,132 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 A simple IPython logger application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 import zmq
 
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from .logwatcher import LogWatcher
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application
 default_config_file_name = u'iplogger_config.py'
 
 _description = """Start an IPython logger for parallel computing.\n\n
 
 IPython controllers and engines (and your own processes) can broadcast log messages
 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
 logger can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your ipython directory and named as "cluster_<profile>".
 See the --profile and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPLoggerAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPLoggerAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Controller config
         paa('--url',
             type=str, dest='LogWatcher.url',
             help='The url the LogWatcher will listen on',
             )
         # MPI
         paa('--topics',
             type=str, dest='LogWatcher.topics', nargs='+',
             help='What topics to subscribe to',
             metavar='topics')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
         
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPLoggerApp(ApplicationWithClusterDir):
 
     name = u'iploggerz'
     description = _description
     command_line_loader = IPLoggerAppConfigLoader
     default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPLoggerApp, self).create_default_config()
 
         # The engine should not clean logs as we don't want to remove the
         # active log files of other running engines.
         self.default_config.Global.clean_logs = False
 
         # If given, this is the actual location of the logger's URL file.
         # If not, this is computed using the profile, app_dir and furl_file_name
         self.default_config.Global.url_file_name = u'iplogger.url'
         self.default_config.Global.url_file = u''
 
     def post_load_command_line_config(self):
         pass
 
     def pre_construct(self):
         super(IPLoggerApp, self).pre_construct()
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_logging()
 
         try:
             self.watcher = LogWatcher(config=self.master_config, logname=self.log.name)
         except:
             self.log.error("Couldn't start the LogWatcher", exc_info=True)
             self.exit(1)
         
 
     def start_app(self):
         try:
             self.watcher.start()
             self.watcher.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Logging Interrupted, shutting down...\n")
 
 
 def launch_new_instance():
     """Create and run the IPython LogWatcher"""
     app = IPLoggerApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/parallel/apps/launcher.py

@@ -1,971 +1,971 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Facilities for launching IPython processes asynchronously.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import copy
 import logging
 import os
 import re
 import stat
 
 from signal import SIGINT, SIGTERM
 try:
     from signal import SIGKILL
 except ImportError:
     SIGKILL=SIGTERM
 
 from subprocess import Popen, PIPE, STDOUT
 try:
     from subprocess import check_output
 except ImportError:
     # pre-2.7, define check_output with Popen
     def check_output(*args, **kwargs):
         kwargs.update(dict(stdout=PIPE))
         p = Popen(*args, **kwargs)
         out,err = p.communicate()
         return out
 
 from zmq.eventloop import ioloop
 
 from IPython.external import Itpl
 # from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import Any, Str, Int, List, Unicode, Dict, Instance, CUnicode
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
 
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 # load winhpcjob only on Windows
 try:
     from .winhpcjob import (
         IPControllerTask, IPEngineTask,
         IPControllerJob, IPEngineSetJob
     )
 except ImportError:
     pass
 
 
 #-----------------------------------------------------------------------------
 # Paths to the kernel apps
 #-----------------------------------------------------------------------------
 
 
 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipclusterapp'
+    'IPython.parallel.apps.ipclusterapp'
 ))
 
 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipengineapp'
+    'IPython.parallel.apps.ipengineapp'
 ))
 
 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipcontrollerapp'
+    'IPython.parallel.apps.ipcontrollerapp'
 ))
 
 #-----------------------------------------------------------------------------
 # Base launchers and errors
 #-----------------------------------------------------------------------------
 
 
 class LauncherError(Exception):
     pass
 
 
 class ProcessStateError(LauncherError):
     pass
 
 
 class UnknownStatus(LauncherError):
     pass
 
 
 class BaseLauncher(LoggingFactory):
     """An asbtraction for starting, stopping and signaling a process."""
 
     # In all of the launchers, the work_dir is where child processes will be
     # run. This will usually be the cluster_dir, but may not be. any work_dir
     # passed into the __init__ method will override the config value.
     # This should not be used to set the work_dir for the actual engine
     # and controller. Instead, use their own config files or the
     # controller_args, engine_args attributes of the launchers to add
     # the --work-dir option.
     work_dir = Unicode(u'.')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     
     start_data = Any()
     stop_data = Any()
     
     def _loop_default(self):
         return ioloop.IOLoop.instance()
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
         self.state = 'before' # can be before, running, after
         self.stop_callbacks = []
         self.start_data = None
         self.stop_data = None
 
     @property
     def args(self):
         """A list of cmd and args that will be used to start the process.
 
         This is what is passed to :func:`spawnProcess` and the first element
         will be the process name.
         """
         return self.find_args()
 
     def find_args(self):
         """The ``.args`` property calls this to find the args list.
 
         Subcommand should implement this to construct the cmd and args.
         """
         raise NotImplementedError('find_args must be implemented in a subclass')
 
     @property
     def arg_str(self):
         """The string form of the program arguments."""
         return ' '.join(self.args)
 
     @property
     def running(self):
         """Am I running."""
         if self.state == 'running':
             return True
         else:
             return False
 
     def start(self):
         """Start the process.
 
         This must return a deferred that fires with information about the
         process starting (like a pid, job id, etc.).
         """
         raise NotImplementedError('start must be implemented in a subclass')
 
     def stop(self):
         """Stop the process and notify observers of stopping.
 
         This must return a deferred that fires with information about the
         processing stopping, like errors that occur while the process is
         attempting to be shut down. This deferred won't fire when the process
         actually stops. To observe the actual process stopping, see
         :func:`observe_stop`.
         """
         raise NotImplementedError('stop must be implemented in a subclass')
 
     def on_stop(self, f):
         """Get a deferred that will fire when the process stops.
 
         The deferred will fire with data that contains information about
         the exit status of the process.
         """
         if self.state=='after':
             return f(self.stop_data)
         else:
             self.stop_callbacks.append(f)
 
     def notify_start(self, data):
         """Call this to trigger startup actions.
 
         This logs the process startup and sets the state to 'running'.  It is
         a pass-through so it can be used as a callback.
         """
 
         self.log.info('Process %r started: %r' % (self.args[0], data))
         self.start_data = data
         self.state = 'running'
         return data
 
     def notify_stop(self, data):
         """Call this to trigger process stop actions.
 
         This logs the process stopping and sets the state to 'after'. Call
         this to trigger all the deferreds from :func:`observe_stop`."""
 
         self.log.info('Process %r stopped: %r' % (self.args[0], data))
         self.stop_data = data
         self.state = 'after'
         for i in range(len(self.stop_callbacks)):
             d = self.stop_callbacks.pop()
             d(data)
         return data
 
     def signal(self, sig):
         """Signal the process.
 
         Return a semi-meaningless deferred after signaling the process.
 
         Parameters
         ----------
         sig : str or int
             'KILL', 'INT', etc., or any signal number
         """
         raise NotImplementedError('signal must be implemented in a subclass')
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 
 class LocalProcessLauncher(BaseLauncher):
     """Start and stop an external process in an asynchronous manner.
 
     This will launch the external process with a working directory of
     ``self.work_dir``.
     """
 
     # This is used to to construct self.args, which is passed to 
     # spawnProcess.
     cmd_and_args = List([])
     poll_frequency = Int(100) # in ms
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(LocalProcessLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
         self.process = None
         self.start_deferred = None
         self.poller = None
 
     def find_args(self):
         return self.cmd_and_args
 
     def start(self):
         if self.state == 'before':
             self.process = Popen(self.args,
                 stdout=PIPE,stderr=PIPE,stdin=PIPE,
                 env=os.environ,
                 cwd=self.work_dir
             )
             
             self.loop.add_handler(self.process.stdout.fileno(), self.handle_stdout, self.loop.READ)
             self.loop.add_handler(self.process.stderr.fileno(), self.handle_stderr, self.loop.READ)
             self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
             self.poller.start()
             self.notify_start(self.process.pid)
         else:
             s = 'The process was already started and has state: %r' % self.state
             raise ProcessStateError(s)
 
     def stop(self):
         return self.interrupt_then_kill()
 
     def signal(self, sig):
         if self.state == 'running':
             self.process.send_signal(sig)
 
     def interrupt_then_kill(self, delay=2.0):
         """Send INT, wait a delay and then send KILL."""
         self.signal(SIGINT)
         self.killer  = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
         self.killer.start()
 
     # callbacks, etc:
     
     def handle_stdout(self, fd, events):
         line = self.process.stdout.readline()
         # a stopped process will be readable but return empty strings
         if line:
             self.log.info(line[:-1])
         else:
             self.poll()
     
     def handle_stderr(self, fd, events):
         line = self.process.stderr.readline()
         # a stopped process will be readable but return empty strings
         if line:
             self.log.error(line[:-1])
         else:
             self.poll()
     
     def poll(self):
         status = self.process.poll()
         if status is not None:
             self.poller.stop()
             self.loop.remove_handler(self.process.stdout.fileno())
             self.loop.remove_handler(self.process.stderr.fileno())
             self.notify_stop(dict(exit_code=status, pid=self.process.pid))
         return status
 
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
 
     def find_args(self):
         return self.controller_cmd + self.controller_args
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.log.info("Starting LocalControllerLauncher: %r" % self.args)
         return super(LocalControllerLauncher, self).start()
 
 
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
     engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
 
     def find_args(self):
         return self.engine_cmd + self.engine_args
 
     def start(self, cluster_dir):
         """Start the engine by cluster_dir."""
         self.engine_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         return super(LocalEngineLauncher, self).start()
 
 
 class LocalEngineSetLauncher(BaseLauncher):
     """Launch a set of engines as regular external processes."""
 
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     # launcher class
     launcher_class = LocalEngineLauncher
     
     launchers = Dict()
     stop_data = Dict()
     
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(LocalEngineSetLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
         self.stop_data = {}
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.cluster_dir = unicode(cluster_dir)
         dlist = []
         for i in range(n):
             el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
             # Copy the engine args over to each engine launcher.
             el.engine_args = copy.deepcopy(self.engine_args)
             el.on_stop(self._notice_engine_stopped)
             d = el.start(cluster_dir)
             if i==0:
                 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
             self.launchers[i] = el
             dlist.append(d)
         self.notify_start(dlist)
         # The consumeErrors here could be dangerous
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         # dfinal.addCallback(self.notify_start)
         return dlist
 
     def find_args(self):
         return ['engine set']
 
     def signal(self, sig):
         dlist = []
         for el in self.launchers.itervalues():
             d = el.signal(sig)
             dlist.append(d)
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         return dlist
 
     def interrupt_then_kill(self, delay=1.0):
         dlist = []
         for el in self.launchers.itervalues():
             d = el.interrupt_then_kill(delay)
             dlist.append(d)
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         return dlist
 
     def stop(self):
         return self.interrupt_then_kill()
     
     def _notice_engine_stopped(self, data):
         pid = data['pid']
         for idx,el in self.launchers.iteritems():
             if el.process.pid == pid:
                 break
         self.launchers.pop(idx)
         self.stop_data[idx] = data
         if not self.launchers:
             self.notify_stop(self.stop_data)
 
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
 
 class MPIExecLauncher(LocalProcessLauncher):
     """Launch an external process using mpiexec."""
 
     # The mpiexec command to use in starting the process.
     mpi_cmd = List(['mpiexec'], config=True)
     # The command line arguments to pass to mpiexec.
     mpi_args = List([], config=True)
     # The program to start using mpiexec.
     program = List(['date'], config=True)
     # The command line argument to the program.
     program_args = List([], config=True)
     # The number of instances of the program to start.
     n = Int(1, config=True)
 
     def find_args(self):
         """Build self.args using all the fields."""
         return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
                self.program + self.program_args
 
     def start(self, n):
         """Start n instances of the program using mpiexec."""
         self.n = n
         return super(MPIExecLauncher, self).start()
 
 
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
     n = Int(1, config=False)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
         return super(MPIExecControllerLauncher, self).start(1)
 
     def find_args(self):
         return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
                self.controller_cmd + self.controller_args
 
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
     program = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     program_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     n = Int(1, config=True)
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.program_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.n = n
         self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
         return super(MPIExecEngineSetLauncher, self).start(n)
 
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
 # TODO: Get SSH Launcher working again.
 
 class SSHLauncher(LocalProcessLauncher):
     """A minimal launcher for ssh.
 
     To be useful this will probably have to be extended to use the ``sshx``
     idea for environment variables.  There could be other things this needs
     as well.
     """
 
     ssh_cmd = List(['ssh'], config=True)
     ssh_args = List(['-tt'], config=True)
     program = List(['date'], config=True)
     program_args = List([], config=True)
     hostname = CUnicode('', config=True)
     user = CUnicode('', config=True)
     location = CUnicode('')
 
     def _hostname_changed(self, name, old, new):
         if self.user:
             self.location = u'%s@%s' % (self.user, new)
         else:
             self.location = new
 
     def _user_changed(self, name, old, new):
         self.location = u'%s@%s' % (new, self.hostname)
 
     def find_args(self):
         return self.ssh_cmd + self.ssh_args + [self.location] + \
                self.program + self.program_args
 
     def start(self, cluster_dir, hostname=None, user=None):
         self.cluster_dir = unicode(cluster_dir)
         if hostname is not None:
             self.hostname = hostname
         if user is not None:
             self.user = user
         
         return super(SSHLauncher, self).start()
     
     def signal(self, sig):
         if self.state == 'running':
             # send escaped ssh connection-closer
             self.process.stdin.write('~.')
             self.process.stdin.flush()
         
 
 
 class SSHControllerLauncher(SSHLauncher):
 
     program = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
 
 
 class SSHEngineLauncher(SSHLauncher):
     program = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     program_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     
 class SSHEngineSetLauncher(LocalEngineSetLauncher):
     launcher_class = SSHEngineLauncher
     engines = Dict(config=True)
     
     def start(self, n, cluster_dir):
         """Start engines by profile or cluster_dir.
         `n` is ignored, and the `engines` config property is used instead.
         """
         
         self.cluster_dir = unicode(cluster_dir)
         dlist = []
         for host, n in self.engines.iteritems():
             if isinstance(n, (tuple, list)):
                 n, args = n
             else:
                 args = copy.deepcopy(self.engine_args)
             
             if '@' in host:
                 user,host = host.split('@',1)
             else:
                 user=None
             for i in range(n):
                 el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
                 
                 # Copy the engine args over to each engine launcher.
                 i
                 el.program_args = args
                 el.on_stop(self._notice_engine_stopped)
                 d = el.start(cluster_dir, user=user, hostname=host)
                 if i==0:
                     self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
                 self.launchers[host+str(i)] = el
                 dlist.append(d)
         self.notify_start(dlist)
         return dlist
     
 
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 scheduler launchers
 #-----------------------------------------------------------------------------
 
 
 # This is only used on Windows.
 def find_job_cmd():
     if os.name=='nt':
         try:
             return find_cmd('job')
         except FindCmdError:
             return 'job'
     else:
         return 'job'
 
 
 class WindowsHPCLauncher(BaseLauncher):
 
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = Str(r'\d+', config=True)
     # The filename of the instantiated job script.
     job_file_name = CUnicode(u'ipython_job.xml', config=True)
     # The full path to the instantiated job script. This gets made dynamically
     # by combining the work_dir with the job_file_name.
     job_file = CUnicode(u'')
     # The hostname of the scheduler to submit the job to
     scheduler = CUnicode('', config=True)
     job_cmd = CUnicode(find_job_cmd(), config=True)
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(WindowsHPCLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
 
     @property
     def job_file(self):
         return os.path.join(self.work_dir, self.job_file_name)
 
     def write_job_file(self, n):
         raise NotImplementedError("Implement write_job_file in a subclass.")
 
     def find_args(self):
         return [u'job.exe']
         
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
         m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
         self.log.info('Job started with job id: %r' % job_id)
         return job_id
 
     def start(self, n):
         """Start n copies of the process using the Win HPC job scheduler."""
         self.write_job_file(n)
         args = [
             'submit',
             '/jobfile:%s' % self.job_file,
             '/scheduler:%s' % self.scheduler
         ]
         self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         # Twisted will raise DeprecationWarnings if we try to pass unicode to this
         output = check_output([self.job_cmd]+args,
             env=os.environ,
             cwd=self.work_dir,
             stderr=STDOUT
         )
         job_id = self.parse_job_id(output)
         self.notify_start(job_id)
         return job_id
 
     def stop(self):
         args = [
             'cancel',
             self.job_id,
             '/scheduler:%s' % self.scheduler
         ]
         self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         try:
             output = check_output([self.job_cmd]+args,
                 env=os.environ,
                 cwd=self.work_dir,
                 stderr=STDOUT
             )
         except:
             output = 'The job already appears to be stoppped: %r' % self.job_id
         self.notify_stop(dict(job_id=self.job_id, output=output))  # Pass the output of the kill cmd
         return output
 
 
 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
 
     job_file_name = CUnicode(u'ipcontroller_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPControllerJob(config=self.config)
 
         t = IPControllerTask(config=self.config)
         # The tasks work directory is *not* the actual work directory of 
         # the controller. It is used as the base path for the stdout/stderr
         # files that the scheduler redirects to.
         t.work_directory = self.cluster_dir
         # Add the --cluster-dir and from self.start().
         t.controller_args.extend(self.extra_args)
         job.add_task(t)
 
         self.log.info("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCControllerLauncher, self).start(1)
 
 
 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
 
     job_file_name = CUnicode(u'ipengineset_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPEngineSetJob(config=self.config)
 
         for i in range(n):
             t = IPEngineTask(config=self.config)
             # The tasks work directory is *not* the actual work directory of 
             # the engine. It is used as the base path for the stdout/stderr
             # files that the scheduler redirects to.
             t.work_directory = self.cluster_dir
             # Add the --cluster-dir and from self.start().
             t.engine_args.extend(self.extra_args)
             job.add_task(t)
 
         self.log.info("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, n, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCEngineSetLauncher, self).start(n)
 
 
 #-----------------------------------------------------------------------------
 # Batch (PBS) system launchers
 #-----------------------------------------------------------------------------
 
 class BatchSystemLauncher(BaseLauncher):
     """Launch an external process using a batch system.
 
     This class is designed to work with UNIX batch systems like PBS, LSF,
     GridEngine, etc.  The overall model is that there are different commands
     like qsub, qdel, etc. that handle the starting and stopping of the process.
 
     This class also has the notion of a batch script. The ``batch_template``
     attribute can be set to a string that is a template for the batch script.
     This template is instantiated using Itpl. Thus the template can use
     ${n} fot the number of instances. Subclasses can add additional variables
     to the template dict.
     """
 
     # Subclasses must fill these in.  See PBSEngineSet
     # The name of the command line program used to submit jobs.
     submit_command = List([''], config=True)
     # The name of the command line program used to delete jobs.
     delete_command = List([''], config=True)
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = CUnicode('', config=True)
     # The string that is the batch script template itself.
     batch_template = CUnicode('', config=True)
     # The file that contains the batch template
     batch_template_file = CUnicode(u'', config=True)
     # The filename of the instantiated batch script.
     batch_file_name = CUnicode(u'batch_script', config=True)
     # The PBS Queue
     queue = CUnicode(u'', config=True)
     
     # not configurable, override in subclasses
     # PBS Job Array regex
     job_array_regexp = CUnicode('')
     job_array_template = CUnicode('')
     # PBS Queue regex
     queue_regexp = CUnicode('')
     queue_template = CUnicode('')
     # The default batch template, override in subclasses
     default_template = CUnicode('')
     # The full path to the instantiated batch script.
     batch_file = CUnicode(u'')
     # the format dict used with batch_template:
     context = Dict()
 
     
     def find_args(self):
         return self.submit_command + [self.batch_file]
     
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(BatchSystemLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
         self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
 
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
         m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
         self.log.info('Job submitted with job id: %r' % job_id)
         return job_id
 
     def write_batch_script(self, n):
         """Instantiate and write the batch script to the work_dir."""
         self.context['n'] = n
         self.context['queue'] = self.queue
         print self.context
         # first priority is batch_template if set
         if self.batch_template_file and not self.batch_template:
             # second priority is batch_template_file
             with open(self.batch_template_file) as f:
                 self.batch_template = f.read()
         if not self.batch_template:
             # third (last) priority is default_template
             self.batch_template = self.default_template
         
         regex = re.compile(self.job_array_regexp)
         # print regex.search(self.batch_template)
         if not regex.search(self.batch_template):
             self.log.info("adding job array settings to batch script")
             firstline, rest = self.batch_template.split('\n',1)
             self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
         
         regex = re.compile(self.queue_regexp)
         # print regex.search(self.batch_template)
         if self.queue and not regex.search(self.batch_template):
             self.log.info("adding PBS queue settings to batch script")
             firstline, rest = self.batch_template.split('\n',1)
             self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
             
         script_as_string = Itpl.itplns(self.batch_template, self.context)
         self.log.info('Writing instantiated batch script: %s' % self.batch_file)
     
         with open(self.batch_file, 'w') as f:
             f.write(script_as_string)
         os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
 
     def start(self, n, cluster_dir):
         """Start n copies of the process using a batch system."""
         # Here we save profile and cluster_dir in the context so they
         # can be used in the batch script template as ${profile} and
         # ${cluster_dir}
         self.context['cluster_dir'] = cluster_dir
         self.cluster_dir = unicode(cluster_dir)
         self.write_batch_script(n)
         output = check_output(self.args, env=os.environ)
         
         job_id = self.parse_job_id(output)
         self.notify_start(job_id)
         return job_id
 
     def stop(self):
         output = check_output(self.delete_command+[self.job_id], env=os.environ)
         self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
         return output
 
 
 class PBSLauncher(BatchSystemLauncher):
     """A BatchSystemLauncher subclass for PBS."""
 
     submit_command = List(['qsub'], config=True)
     delete_command = List(['qdel'], config=True)
     job_id_regexp = CUnicode(r'\d+', config=True)
     
     batch_file = CUnicode(u'')
     job_array_regexp = CUnicode('#PBS\W+-t\W+[\w\d\-\$]+')
     job_array_template = CUnicode('#PBS -t 1-$n')
     queue_regexp = CUnicode('#PBS\W+-q\W+\$?\w+')
     queue_template = CUnicode('#PBS -q $queue')
 
 
 class PBSControllerLauncher(PBSLauncher):
     """Launch a controller using PBS."""
 
     batch_file_name = CUnicode(u'pbs_controller', config=True)
     default_template= CUnicode("""#!/bin/sh
 #PBS -V
 #PBS -N ipcontroller
 %s --log-to-file --cluster-dir $cluster_dir
 """%(' '.join(ipcontroller_cmd_argv)))
 
     def start(self, cluster_dir):
         """Start the controller by profile or cluster_dir."""
         self.log.info("Starting PBSControllerLauncher: %r" % self.args)
         return super(PBSControllerLauncher, self).start(1, cluster_dir)
 
 
 class PBSEngineSetLauncher(PBSLauncher):
     """Launch Engines using PBS"""
     batch_file_name = CUnicode(u'pbs_engines', config=True)
     default_template= CUnicode(u"""#!/bin/sh
 #PBS -V
 #PBS -N ipengine
 %s --cluster-dir $cluster_dir
 """%(' '.join(ipengine_cmd_argv)))
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
         return super(PBSEngineSetLauncher, self).start(n, cluster_dir)
 
 #SGE is very similar to PBS
 
 class SGELauncher(PBSLauncher):
     """Sun GridEngine is a PBS clone with slightly different syntax"""
     job_array_regexp = CUnicode('#$$\W+-t\W+[\w\d\-\$]+')
     job_array_template = CUnicode('#$$ -t 1-$n')
     queue_regexp = CUnicode('#$$\W+-q\W+\$?\w+')
     queue_template = CUnicode('#$$ -q $queue')
 
 class SGEControllerLauncher(SGELauncher):
     """Launch a controller using SGE."""
 
     batch_file_name = CUnicode(u'sge_controller', config=True)
     default_template= CUnicode(u"""#$$ -V
 #$$ -S /bin/sh
 #$$ -N ipcontroller
 %s --log-to-file --cluster-dir $cluster_dir
 """%(' '.join(ipcontroller_cmd_argv)))
 
     def start(self, cluster_dir):
         """Start the controller by profile or cluster_dir."""
         self.log.info("Starting PBSControllerLauncher: %r" % self.args)
         return super(PBSControllerLauncher, self).start(1, cluster_dir)
 
 class SGEEngineSetLauncher(SGELauncher):
     """Launch Engines with SGE"""
     batch_file_name = CUnicode(u'sge_engines', config=True)
     default_template = CUnicode("""#$$ -V
 #$$ -S /bin/sh
 #$$ -N ipengine
 %s --cluster-dir $cluster_dir
 """%(' '.join(ipengine_cmd_argv)))
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
         return super(SGEEngineSetLauncher, self).start(n, cluster_dir)
 
 
 #-----------------------------------------------------------------------------
 # A launcher for ipcluster itself!
 #-----------------------------------------------------------------------------
 
 
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
     ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
     # Command line arguments to pass to ipcluster.
     ipcluster_args = List(
         ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
     ipcluster_subcommand = Str('start')
     ipcluster_n = Int(2)
 
     def find_args(self):
         return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
             ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
 
     def start(self):
         self.log.info("Starting ipcluster: %r" % self.args)
         return super(IPClusterLauncher, self).start()
 

IPython/parallel/apps/logwatcher.py

@@ -1,98 +1,98 b''
 #!/usr/bin/env python
 """A simple logger object that consolidates messages incoming from ipcluster processes."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
 import logging
 import sys
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.traitlets import Int, Str, Instance, List
 
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 
 class LogWatcher(LoggingFactory):
     """A simple class that receives messages on a SUB socket, as published
     by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself.
     
     This can subscribe to multiple topics, but defaults to all topics.
     """
     # configurables
     topics = List([''], config=True)
     url = Str('tcp://127.0.0.1:20202', config=True)
     
     # internals
     context = Instance(zmq.Context, (), {})
     stream = Instance('zmq.eventloop.zmqstream.ZMQStream')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         return ioloop.IOLoop.instance()
     
     def __init__(self, **kwargs):
         super(LogWatcher, self).__init__(**kwargs)
         s = self.context.socket(zmq.SUB)
         s.bind(self.url)
         self.stream = zmqstream.ZMQStream(s, self.loop)
         self.subscribe()
         self.on_trait_change(self.subscribe, 'topics')
     
     def start(self):
         self.stream.on_recv(self.log_message)
     
     def stop(self):
         self.stream.stop_on_recv()
     
     def subscribe(self):
         """Update our SUB socket's subscriptions."""
         self.stream.setsockopt(zmq.UNSUBSCRIBE, '')
         for topic in self.topics:
             self.log.debug("Subscribing to: %r"%topic)
             self.stream.setsockopt(zmq.SUBSCRIBE, topic)
     
     def _extract_level(self, topic_str):
         """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')"""
         topics = topic_str.split('.')
         for idx,t in enumerate(topics):
             level = getattr(logging, t, None)
             if level is not None:
                 break
         
         if level is None:
             level = logging.INFO
         else:
             topics.pop(idx)
         
         return level, '.'.join(topics)
             
             
     def log_message(self, raw):
         """receive and parse a message, then log it."""
         if len(raw) != 2 or '.' not in raw[0]:
             self.log.error("Invalid log message: %s"%raw)
             return
         else:
             topic, msg = raw
             # don't newline, since log messages always newline:
             topic,level_name = topic.rsplit('.',1)
             level,topic = self._extract_level(topic)
             if msg[-1] == '\n':
                 msg = msg[:-1]
             logging.log(level, "[%s] %s" % (topic, msg))
 

IPython/parallel/client/asyncresult.py

@@ -1,340 +1,340 b''
 """AsyncResult objects for the client"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import time
 
 from zmq import MessageTracker
 
 from IPython.external.decorator import decorator
-from . import error
+from IPython.parallel import error
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 # global empty tracker that's always done:
 finished_tracker = MessageTracker()
 
 @decorator
 def check_ready(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.wait(0)
     if not self._ready:
         raise error.TimeoutError("result not ready")
     return f(self, *args, **kwargs)
 
 class AsyncResult(object):
     """Class for representing results of non-blocking calls.
     
     Provides the same interface as :py:class:`multiprocessing.pool.AsyncResult`.
     """
     
     msg_ids = None
     _targets = None
     _tracker = None
     _single_result = False
     
     def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None):
         if isinstance(msg_ids, basestring):
             # always a list
             msg_ids = [msg_ids]
         if tracker is None:
             # default to always done
             tracker = finished_tracker
         self._client = client
         self.msg_ids = msg_ids
         self._fname=fname
         self._targets = targets
         self._tracker = tracker
         self._ready = False
         self._success = None
         if len(msg_ids) == 1:
             self._single_result = not isinstance(targets, (list, tuple))
         else:
             self._single_result = False
     
     def __repr__(self):
         if self._ready:
             return "<%s: finished>"%(self.__class__.__name__)
         else:
             return "<%s: %s>"%(self.__class__.__name__,self._fname)
     
     
     def _reconstruct_result(self, res):
         """Reconstruct our result from actual result list (always a list)
         
         Override me in subclasses for turning a list of results
         into the expected form.
         """
         if self._single_result:
             return res[0]
         else:
             return res
         
     def get(self, timeout=-1):
         """Return the result when it arrives. 
         
         If `timeout` is not ``None`` and the result does not arrive within
         `timeout` seconds then ``TimeoutError`` is raised. If the
         remote call raised an exception then that exception will be reraised
         by get() inside a `RemoteError`.
         """
         if not self.ready():
             self.wait(timeout)
         
         if self._ready:
             if self._success:
                 return self._result
             else:
                 raise self._exception
         else:
             raise error.TimeoutError("Result not ready.")
     
     def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
         return self._ready
     
     def wait(self, timeout=-1):
         """Wait until the result is available or until `timeout` seconds pass.
         
         This method always returns None.
         """
         if self._ready:
             return
         self._ready = self._client.wait(self.msg_ids, timeout)
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
             
     
     def successful(self):
         """Return whether the call completed without raising an exception. 
         
         Will raise ``AssertionError`` if the result is not ready.
         """
         assert self.ready()
         return self._success
     
     #----------------------------------------------------------------
     # Extra methods not in mp.pool.AsyncResult
     #----------------------------------------------------------------
     
     def get_dict(self, timeout=-1):
         """Get the results as a dict, keyed by engine_id.
         
         timeout behavior is described in `get()`.
         """
         
         results = self.get(timeout)
         engine_ids = [ md['engine_id'] for md in self._metadata ]
         bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))
         maxcount = bycount.count(bycount[-1])
         if maxcount > 1:
             raise ValueError("Cannot build dict, %i jobs ran on engine #%i"%(
                     maxcount, bycount[-1]))
         
         return dict(zip(engine_ids,results))
     
     @property
     def result(self):
         """result property wrapper for `get(timeout=0)`."""
         return self.get()
     
     # abbreviated alias:
     r = result
     
     @property
     @check_ready
     def metadata(self):
         """property for accessing execution metadata."""
         if self._single_result:
             return self._metadata[0]
         else:
             return self._metadata
     
     @property
     def result_dict(self):
         """result property as a dict."""
         return self.get_dict()
     
     def __dict__(self):
         return self.get_dict(0)
     
     def abort(self):
         """abort my tasks."""
         assert not self.ready(), "Can't abort, I am already done!"
         return self.client.abort(self.msg_ids, targets=self._targets, block=True)
 
     @property
     def sent(self):
         """check whether my messages have been sent."""
         return self._tracker.done
     
     def wait_for_send(self, timeout=-1):
         """wait for pyzmq send to complete.
         
         This is necessary when sending arrays that you intend to edit in-place.
         `timeout` is in seconds, and will raise TimeoutError if it is reached
         before the send completes.
         """
         return self._tracker.wait(timeout)
 
     #-------------------------------------
     # dict-access
     #-------------------------------------
     
     @check_ready
     def __getitem__(self, key):
         """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
         """
         if isinstance(key, int):
             return error.collect_exceptions([self._result[key]], self._fname)[0]
         elif isinstance(key, slice):
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, basestring):
             values = [ md[key] for md in self._metadata ]
             if self._single_result:
                 return values[0]
             else:
                 return values
         else:
             raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
     
     @check_ready
     def __getattr__(self, key):
         """getattr maps to getitem for convenient attr access to metadata."""
         if key not in self._metadata[0].keys():
             raise AttributeError("%r object has no attribute %r"%(
                     self.__class__.__name__, key))
         return self.__getitem__(key)
     
     # asynchronous iterator:
     def __iter__(self):
         if self._single_result:
             raise TypeError("AsyncResults with a single result are not iterable.")
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 yield ar.get()
         else:
             # already done
             for r in rlist:
                 yield r
 
 
     
 class AsyncMapResult(AsyncResult):
     """Class for representing results of non-blocking gathers.
     
     This will properly reconstruct the gather.
     """
     
     def __init__(self, client, msg_ids, mapObject, fname=''):
         AsyncResult.__init__(self, client, msg_ids, fname=fname)
         self._mapObject = mapObject
         self._single_result = False
     
     def _reconstruct_result(self, res):
         """Perform the gather on the actual results."""
         return self._mapObject.joinPartitions(res)
     
     # asynchronous iterator:
     def __iter__(self):
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 rlist = ar.get()
                 try:
                     for r in rlist:
                         yield r
                 except TypeError:
                     # flattened, not a list
                     # this could get broken by flattened data that returns iterables
                     # but most calls to map do not expose the `flatten` argument
                     yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
 
 class AsyncHubResult(AsyncResult):
     """Class to wrap pending results that must be requested from the Hub.
     
     Note that waiting/polling on these objects requires polling the Hubover the network,
     so use `AsyncHubResult.wait()` sparingly.
     """
     
     def wait(self, timeout=-1):
         """wait for result to complete."""
         start = time.time()
         if self._ready:
             return
         local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)
         local_ready = self._client.wait(local_ids, timeout)
         if local_ready:
             remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)
             if not remote_ids:
                 self._ready = True
             else:
                 rdict = self._client.result_status(remote_ids, status_only=False)
                 pending = rdict['pending']
                 while pending and (timeout < 0 or time.time() < start+timeout):
                     rdict = self._client.result_status(remote_ids, status_only=False)
                     pending = rdict['pending']
                     if pending:
                         time.sleep(0.1)
                 if not pending:
                     self._ready = True
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
         
 __all__ = ['AsyncResult', 'AsyncMapResult', 'AsyncHubResult']
  No newline at end of file

IPython/parallel/client/client.py

@@ -1,1278 +1,1279 b''
 """A semi-synchronous Client for the ZMQ cluster"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import json
 import time
 import warnings
 from datetime import datetime
 from getpass import getpass
 from pprint import pprint
 
 pjoin = os.path.join
 
 import zmq
 # from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.traitlets import (HasTraits, Int, Instance, CUnicode, 
                                     Dict, List, Bool, Str, Set)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
-from . import error
-from . import util
-from . import streamsession as ss
+from IPython.parallel import error
+from IPython.parallel import streamsession as ss
+from IPython.parallel import util
+
 from .asyncresult import AsyncResult, AsyncHubResult
-from .clusterdir import ClusterDir, ClusterDirError
+from IPython.parallel.apps.clusterdir import ClusterDir, ClusterDirError
 from .view import DirectView, LoadBalancedView
 
 #--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
 def spin_first(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 @decorator
 def default_block(f, self, *args, **kwargs):
     """Default to self.block; preserve self.block."""
     block = kwargs.get('block',None)
     block = self.block if block is None else block
     saveblock = self.block
     self.block = block
     try:
         ret = f(self, *args, **kwargs)
     finally:
         self.block = saveblock
     return ret
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 class Metadata(dict):
     """Subclass of dict for initializing metadata values.
     
     Attribute access works on keys.
     
     These objects have a strict set of keys - errors will raise if you try
     to add new keys.
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self)
         md = {'msg_id' : None,
               'submitted' : None,
               'started' : None,
               'completed' : None,
               'received' : None,
               'engine_uuid' : None,
               'engine_id' : None,
               'follow' : None,
               'after' : None,
               'status' : None,
 
               'pyin' : None,
               'pyout' : None,
               'pyerr' : None,
               'stdout' : '',
               'stderr' : '',
             }
         self.update(md)
         self.update(dict(*args, **kwargs))
     
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise AttributeError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if key in self.iterkeys():
             self[key] = value
         else:
             raise AttributeError(key)
     
     def __setitem__(self, key, value):
         """strict static key enforcement"""
         if key in self.iterkeys():
             dict.__setitem__(self, key, value)
         else:
             raise KeyError(key)
         
 
 class Client(HasTraits):
     """A semi-synchronous client to the IPython ZMQ cluster
     
     Parameters
     ----------
     
     url_or_file : bytes; zmq url or path to ipcontroller-client.json
         Connection information for the Hub's registration.  If a json connector
         file is given, then likely no further configuration is necessary.
         [Default: use profile]
     profile : bytes
         The name of the Cluster profile to be used to find connector information.
         [Default: 'default']
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
     username : bytes
         set username to be passed to the Session object
     debug : bool
         flag for lots of message printing for debug purposes
  
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However, 
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
     
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to public ssh key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str 
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     paramiko : bool
         flag for whether to use paramiko instead of shell ssh for tunneling.
         [default: True on win32, False else]
     
     ------- exec authentication args -------
     If even localhost is untrusted, you can have some protection against
     unauthorized execution by using a key.  Messages are still sent
     as cleartext, so if someone can snoop your loopback traffic this will
     not help against malicious attacks.
     
     exec_key : str
         an authentication key or file containing a key
         default: None
     
     
     Attributes
     ----------
     
     ids : list of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
     
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
     
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
     
     results : dict
         a dict of all our results, keyed by msg_id
     
     block : bool
         determines default behavior when block not specified
         in execution methods
     
     Methods
     -------
     
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
     
     wait
         wait on one or more msg_ids
     
     execution methods
         apply
         legacy: execute, run
     
     data movement
         push, pull, scatter, gather
     
     query methods
         queue_status, get_result, purge, result_status
     
     control methods
         abort, shutdown
     
     """
     
     
     block = Bool(False)
     outstanding = Set()
     results = Instance('collections.defaultdict', (dict,))
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
     profile=CUnicode('default')
     
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
     _connected=Bool(False)
     _ssh=Bool(False)
     _context = Instance('zmq.Context')
     _config = Dict()
     _engines=Instance(util.ReverseDict, (), {})
     # _hub_socket=Instance('zmq.Socket')
     _query_socket=Instance('zmq.Socket')
     _control_socket=Instance('zmq.Socket')
     _iopub_socket=Instance('zmq.Socket')
     _notification_socket=Instance('zmq.Socket')
     _mux_socket=Instance('zmq.Socket')
     _task_socket=Instance('zmq.Socket')
     _task_scheme=Str()
     _closed = False
     _ignored_control_replies=Int(0)
     _ignored_hub_replies=Int(0)
     
     def __init__(self, url_or_file=None, profile='default', cluster_dir=None, ipython_dir=None,
             context=None, username=None, debug=False, exec_key=None,
             sshserver=None, sshkey=None, password=None, paramiko=None,
             timeout=10
             ):
         super(Client, self).__init__(debug=debug, profile=profile)
         if context is None:
             context = zmq.Context.instance()
         self._context = context
             
         
         self._setup_cluster_dir(profile, cluster_dir, ipython_dir)
         if self._cd is not None:
             if url_or_file is None:
                 url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
         assert url_or_file is not None, "I can't find enough information to connect to a hub!"\
             " Please specify at least one of url_or_file or profile."
         
         try:
             util.validate_url(url_or_file)
         except AssertionError:
             if not os.path.exists(url_or_file):
                 if self._cd:
                     url_or_file = os.path.join(self._cd.security_dir, url_or_file)
                 assert os.path.exists(url_or_file), "Not a valid connection file or url: %r"%url_or_file
             with open(url_or_file) as f:
                 cfg = json.loads(f.read())
         else:
             cfg = {'url':url_or_file}
         
         # sync defaults from args, json:
         if sshserver:
             cfg['ssh'] = sshserver
         if exec_key:
             cfg['exec_key'] = exec_key
         exec_key = cfg['exec_key']
         sshserver=cfg['ssh']
         url = cfg['url']
         location = cfg.setdefault('location', None)
         cfg['url'] = util.disambiguate_url(cfg['url'], location)
         url = cfg['url']
         
         self._config = cfg
         
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to ssh via localhost
             sshserver = url.split('://')[1].split(':')[0]
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
         if exec_key is not None and os.path.isfile(exec_key):
             arg = 'keyfile'
         else:
             arg = 'key'
         key_arg = {arg:exec_key}
         if username is None:
             self.session = ss.StreamSession(**key_arg)
         else:
             self.session = ss.StreamSession(username, **key_arg)
         self._query_socket = self._context.socket(zmq.XREQ)
         self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
         if self._ssh:
             tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs)
         else:
             self._query_socket.connect(url)
         
         self.session.debug = self.debug
         
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
                                     'shutdown_notification' : lambda msg: self.close(),
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
         self._connect(sshserver, ssh_kwargs, timeout)
         
     def __del__(self):
         """cleanup sockets, but _not_ context."""
         self.close()
     
     def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
         if cluster_dir is not None:
             try:
                 self._cd = ClusterDir.find_cluster_dir(cluster_dir)
                 return
             except ClusterDirError:
                 pass
         elif profile is not None:
             try:
                 self._cd = ClusterDir.find_cluster_dir_by_profile(
                     ipython_dir, profile)
                 return
             except ClusterDirError:
                 pass
         self._cd = None
     
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in engines.iteritems():
             eid = int(k)
             self._engines[eid] = bytes(v) # force not unicode
             self._ids.append(eid)
         self._ids = sorted(self._ids)
         if sorted(self._engines.keys()) != range(len(self._engines)) and \
                         self._task_scheme == 'pure' and self._task_socket:
             self._stop_scheduling_tasks()
     
     def _stop_scheduling_tasks(self):
         """Stop scheduling tasks because an engine has been unregistered
         from a pure ZMQ scheduler.
         """
         self._task_socket.close()
         self._task_socket = None
         msg = "An engine has been unregistered, and we are using pure " +\
               "ZMQ task scheduling.  Task farming will be disabled."
         if self.outstanding:
             msg += " If you were running tasks when this happened, " +\
                    "some `outstanding` msg_ids may never resolve."
         warnings.warn(msg, RuntimeWarning)
     
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if targets is None:
             targets = self._ids
         elif isinstance(targets, str):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             if targets < 0:
                 targets = self.ids[targets]
             if targets not in self.ids:
                 raise IndexError("No such engine: %i"%targets)
             targets = [targets]
         
         if isinstance(targets, slice):
             indices = range(len(self._ids))[targets]
             ids = self.ids
             targets = [ ids[i] for i in indices ]
         
         if not isinstance(targets, (tuple, list, xrange)):
             raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
             
         return [self._engines[t] for t in targets], list(targets)
     
     def _connect(self, sshserver, ssh_kwargs, timeout):
         """setup all our socket connections to the cluster. This is called from
         __init__."""
         
         # Maybe allow reconnecting?
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, url):
             url = util.disambiguate_url(url, self._config['location'])
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
                 return s.connect(url)
             
         self.session.send(self._query_socket, 'connection_request')
         r,w,x = zmq.select([self._query_socket],[],[], timeout)
         if not r:
             raise error.TimeoutError("Hub connection request timed out")
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
         msg = ss.Message(msg)
         content = msg.content
         self._config['registration'] = dict(content)
         if content.status == 'ok':
             if content.mux:
                 self._mux_socket = self._context.socket(zmq.XREQ)
                 self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._mux_socket, content.mux)
             if content.task:
                 self._task_scheme, task_addr = content.task
                 self._task_socket = self._context.socket(zmq.XREQ)
                 self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._task_socket, task_addr)
             if content.notification:
                 self._notification_socket = self._context.socket(zmq.SUB)
                 connect_socket(self._notification_socket, content.notification)
                 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
             # if content.query:
             #     self._query_socket = self._context.socket(zmq.XREQ)
             #     self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
             #     connect_socket(self._query_socket, content.query)
             if content.control:
                 self._control_socket = self._context.socket(zmq.XREQ)
                 self._control_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._control_socket, content.control)
             if content.iopub:
                 self._iopub_socket = self._context.socket(zmq.SUB)
                 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
                 self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._iopub_socket, content.iopub)
             self._update_engines(dict(content.engines))
         else:
             self._connected = False
             raise Exception("Failed to connect!")
     
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
     
     def _unwrap_exception(self, content):
         """unwrap exception, and remap engine_id to int."""
         e = error.unwrap_exception(content)
         # print e.traceback
         if e.engine_info:
             e_uuid = e.engine_info['engine_uuid']
             eid = self._engines[e_uuid]
             e.engine_info['engine_id'] = eid
         return e
     
     def _extract_metadata(self, header, parent, content):
         md = {'msg_id' : parent['msg_id'],
               'received' : datetime.now(),
               'engine_uuid' : header.get('engine', None),
               'follow' : parent.get('follow', []),
               'after' : parent.get('after', []),
               'status' : content['status'],
             }
         
         if md['engine_uuid'] is not None:
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
         
         if 'date' in parent:
             md['submitted'] = datetime.strptime(parent['date'], util.ISO8601)
         if 'started' in header:
             md['started'] = datetime.strptime(header['started'], util.ISO8601)
         if 'date' in header:
             md['completed'] = datetime.strptime(header['date'], util.ISO8601)
         return md
     
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['queue']}
         self._update_engines(d)
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             uuid = self._engines.pop(eid)
             
             self._handle_stranded_msgs(eid, uuid)
                 
         if self._task_socket and self._task_scheme == 'pure':
             self._stop_scheduling_tasks()
     
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
         
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         of the unregistration and later receive the successful result.
         """
         
         outstanding = self._outstanding_dict[uuid]
         
         for msg_id in list(outstanding):
             if msg_id in self.results:
                 # we already 
                 continue
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake message:
             parent = {}
             header = {}
             parent['msg_id'] = msg_id
             header['engine'] = uuid
             header['date'] = datetime.now().strftime(util.ISO8601)
             msg = dict(parent_header=parent, header=header, content=content)
             self._handle_apply_reply(msg)
     
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
         
         execute messages are never actually used. apply is used instead.
         """
         
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         self.results[msg_id] = self._unwrap_exception(msg['content'])
     
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
                 print self.results[msg_id]
                 print msg
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
         
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(header, parent, content))
         # is this redundant?
         self.metadata[msg_id] = md
         
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
         
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = util.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
     
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
     
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
         
         Currently: ignore them."""
         if self._ignored_control_replies <= 0:
             return
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_ignored_control(self):
         """flush ignored control replies"""
         while self._ignored_control_replies > 0:
             self.session.recv(self._control_socket)
             self._ignored_control_replies -= 1
     
     def _flush_ignored_hub_replies(self):
         msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
     
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             parent = msg['parent_header']
             msg_id = parent['msg_id']
             content = msg['content']
             header = msg['header']
             msg_type = msg['msg_type']
             
             # init metadata:
             md = self.metadata[msg_id]
             
             if msg_type == 'stream':
                 name = content['name']
                 s = md[name] or ''
                 md[name] = s + content['data']
             elif msg_type == 'pyerr':
                 md.update({'pyerr' : self._unwrap_exception(content)})
             else:
                 md.update({msg_type : content['data']})
             
             # reduntant?
             self.metadata[msg_id] = md
             
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     #--------------------------------------------------------------------------
     # len, getitem
     #--------------------------------------------------------------------------
     
     def __len__(self):
         """len(client) returns # of engines."""
         return len(self.ids)
     
     def __getitem__(self, key):
         """index access returns DirectView multiplexer objects
         
         Must be int, slice, or list/tuple/xrange of ints"""
         if not isinstance(key, (int, slice, tuple, list, xrange)):
             raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
         else:
             return self.direct_view(key)
     
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
     
     @property
     def ids(self):
         """Always up-to-date ids property."""
         self._flush_notifications()
         # always copy:
         return list(self._ids)
         
     def close(self):
         if self._closed:
             return
         snames = filter(lambda n: n.endswith('socket'), dir(self))
         for socket in map(lambda name: getattr(self, name), snames):
             if isinstance(socket, zmq.Socket) and not socket.closed:
                 socket.close()
         self._closed = True
     
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
         if self._mux_socket:
             self._flush_results(self._mux_socket)
         if self._task_socket:
             self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
         if self._iopub_socket:
             self._flush_iopub(self._iopub_socket)
         if self._query_socket:
             self._flush_ignored_hub_replies()
     
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
         
         Parameters
         ----------
         
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
         
         Returns
         -------
         
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if jobs is None:
             theids = self.outstanding
         else:
             if isinstance(jobs, (int, str, AsyncResult)):
                 jobs = [jobs]
             theids = set()
             for job in jobs:
                 if isinstance(job, int):
                     # index access
                     job = self.history[job]
                 elif isinstance(job, AsyncResult):
                     map(theids.add, job.msg_ids)
                     continue
                 theids.add(job)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
     
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
     
     @spin_first
     @default_block
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if self.block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
         
     
     @spin_first
     @default_block
     def abort(self, jobs=None, targets=None, block=None):
         """Abort specific jobs from the execution queues of target(s).
         
         This is a mechanism to prevent jobs that have already been submitted
         from executing.
         
         Parameters
         ----------
         
         jobs : msg_id, list of msg_ids, or AsyncResult
             The jobs to be aborted
         
         
         """
         targets = self._build_targets(targets)[0]
         msg_ids = []
         if isinstance(jobs, (basestring,AsyncResult)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request', 
                     content=content, ident=t)
         error = False
         if self.block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
     
     @spin_first
     @default_block
     def shutdown(self, targets=None, restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub."""
         if hub:
             targets = 'all'
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request', 
                         content={'restart':restart},ident=t)
         error = False
         if block or hub:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket, 0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         
         if hub:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = self._unwrap_exception(msg['content'])
         
         if error:
             raise error
     
     #--------------------------------------------------------------------------
     # Execution methods
     #--------------------------------------------------------------------------
     
     @default_block
     def _execute(self, code, targets='all', block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
         
         ``execute`` is always `bound` (affects engine namespace)
         
         Parameters
         ----------
         
         code : str
                 the code string to be executed
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
         return self[targets].execute(code, block=block)
     
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
         
         return result
     
     def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False,
                             ident=None):
         """construct and send an apply message via a socket.
         
         This is the principal method with which all engine execution is performed by views.
         """
                             
         assert not self._closed, "cannot use me anymore, I'm closed!"
         # defaults:
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
         subheader = subheader if subheader is not None else {}
         
         # validate arguments
         if not callable(f):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
         if not isinstance(subheader, dict):
             raise TypeError("subheader must be dict, not %s"%type(subheader))
         
         if not self._ids:
             # flush notification socket if no engines yet
             any_ids = self.ids
             if not any_ids:
                 raise error.NoEnginesRegistered("Can't execute without any connected engines.")
                 # enforce types of f,args,kwargs
         
         bufs = util.pack_apply_message(f,args,kwargs)
         
         msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
                             subheader=subheader, track=track)
         
         msg_id = msg['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
         
         return msg
 
     #--------------------------------------------------------------------------
     # construct a View object
     #--------------------------------------------------------------------------
     
     def load_balanced_view(self, targets=None):
         """construct a DirectView object.
         
         If no arguments are specified, create a LoadBalancedView
         using all engines.
         
         Parameters
         ----------
         
         targets: list,slice,int,etc. [default: use all engines]
             The subset of engines across which to load-balance
         """
         if targets is not None:
             targets = self._build_targets(targets)[1]
         return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
     
     def direct_view(self, targets='all'):
         """construct a DirectView object.
         
         If no targets are specified, create a DirectView
         using all engines.
         
         Parameters
         ----------
         
         targets: list,slice,int,etc. [default: use all engines]
             The engines to use for the View
         """
         single = isinstance(targets, int)
         targets = self._build_targets(targets)[1]
         if single:
             targets = targets[0]
         return DirectView(client=self, socket=self._mux_socket, targets=targets)
     
     #--------------------------------------------------------------------------
     # Data movement (TO BE REMOVED)
     #--------------------------------------------------------------------------
     
     @default_block
     def _push(self, ns, targets='all', block=None, track=False):
         """Push the contents of `ns` into the namespace on `target`"""
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
         result = self.apply(util._push, kwargs=ns, targets=targets, block=block, bound=True, balanced=False, track=track)
         if not block:
             return result
     
     @default_block
     def _pull(self, keys, targets='all', block=None):
         """Pull objects from `target`'s namespace by `keys`"""
         if isinstance(keys, basestring):
             pass
         elif isinstance(keys, (list,tuple,set)):
             for key in keys:
                 if not isinstance(key, basestring):
                     raise TypeError("keys must be str, not type %r"%type(key))
         else:
             raise TypeError("keys must be strs, not %r"%keys)
         result = self.apply(util._pull, (keys,), targets=targets, block=block, bound=True, balanced=False)
         return result
     
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
     
     @spin_first
     @default_block
     def get_result(self, indices_or_msg_ids=None, block=None):
         """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
         
         If the client already has the results, no request to the Hub will be made.
         
         This is a convenient way to construct AsyncResult objects, which are wrappers
         that include metadata about execution, and allow for awaiting results that
         were not submitted by this Client.
         
         It can also be a convenient way to retrieve the metadata associated with
         blocking execution, since it always retrieves
         
         Examples
         --------
         ::
         
             In [10]: r = client.apply()
         
         Parameters
         ----------
         
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
         
         block : bool
             Whether to wait for the result to be done
         
         Returns
         -------
         
         AsyncResult
             A single AsyncResult object will always be returned.
         
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
         
         """
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
         
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, str):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
         
         local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)
         remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
         
         if remote_ids:
             ar = AsyncHubResult(self, msg_ids=theids)
         else:
             ar = AsyncResult(self, msg_ids=theids)
         
         if block:
             ar.wait()
         
         return ar
     
     @spin_first
     def result_status(self, msg_ids, status_only=True):
         """Check on the status of the result(s) of the apply request with `msg_ids`.
         
         If status_only is False, then the actual results will be retrieved, else
         only the status of the results will be checked.
         
         Parameters
         ----------
         
         msg_ids : list of msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: True)
             if False:
                 Retrieve the actual results of completed tasks.
         
         Returns
         -------
         
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids that are incomplete or complete. If `status_only`
             is False, then completed results will be keyed by their `msg_id`.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
             
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, basestring):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
         
         completed = []
         local_results = {}
         
         # comment this block out to temporarily disable local shortcut:
         for msg_id in theids:
             if msg_id in self.results:
                 completed.append(msg_id)
                 local_results[msg_id] = self.results[msg_id]
                 theids.remove(msg_id)
         
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise self._unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
         
         content['completed'].extend(completed)
         
         if status_only:
             return content
         
         failures = []
         # load cached results into result:
         content.update(local_results)
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = rec['header']
                 header = rec['result_header']
                 rcontent = rec['result_content']
                 iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
                 
                 md = self.metadata[msg_id]
                 md.update(self._extract_metadata(header, parent, rcontent))
                 md.update(iodict)
                 
                 if rcontent['status'] == 'ok':
                     res,buffers = util.unserialize_object(buffers)
                 else:
                     print rcontent
                     res = self._unwrap_exception(rcontent)
                     failures.append(res)
                 
                 self.results[msg_id] = res
                 content[msg_id] = res
         
         if len(theids) == 1 and failures:
                 raise failures[0]
         
         error.collect_exceptions(failures, "result_status")
         return content
 
     @spin_first
     def queue_status(self, targets='all', verbose=False):
         """Fetch the status of engine queues.
         
         Parameters
         ----------
         
         targets : int/str/list of ints/strs
                 the engines whose states are to be queried.
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
         engine_ids = self._build_targets(targets)[1]
         content = dict(targets=engine_ids, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
         content = util.rekey(content)
         if isinstance(targets, int):
             return content[targets]
         else:
             return content
         
     @spin_first
     def purge_results(self, jobs=[], targets=[]):
         """Tell the Hub to forget results.
         
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
         
         Parameters
         ----------
         
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by uuid or int_id, whose entire history is to be purged.
                 Use `targets='all'` to scrub everything from the Hub's memory.
                 
                 default : None
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         if targets:
             targets = self._build_targets(targets)[1]
         
         # construct msg_ids from jobs
         msg_ids = []
         if isinstance(jobs, (basestring,AsyncResult)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         
         content = dict(targets=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
 
 __all__ = [ 'Client' ]

IPython/parallel/client/view.py

@@ -1,1036 +1,1033 b''
 """Views of remote engines."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import imp
 import sys
 import warnings
 from contextlib import contextmanager
 from types import ModuleType
 
 import zmq
 
 from IPython.testing import decorators as testdec
 from IPython.utils.traitlets import HasTraits, Any, Bool, List, Dict, Set, Int, Instance, CFloat
 
 from IPython.external.decorator import decorator
 
+from IPython.parallel import util
+from IPython.parallel.controller.dependency import Dependency, dependent
+
 from . import map as Map
-from . import util
 from .asyncresult import AsyncResult, AsyncMapResult
-from .dependency import Dependency, dependent
 from .remotefunction import ParallelFunction, parallel, remote
 
 #-----------------------------------------------------------------------------
 # Decorators
 #-----------------------------------------------------------------------------
 
 @decorator
 def save_ids(f, self, *args, **kwargs):
     """Keep our history and outstanding attributes up to date after a method call."""
     n_previous = len(self.client.history)
     try:
         ret = f(self, *args, **kwargs)
     finally:
         nmsgs = len(self.client.history) - n_previous
         msg_ids = self.client.history[-nmsgs:]
         self.history.extend(msg_ids)
         map(self.outstanding.add, msg_ids)
     return ret
 
 @decorator
 def sync_results(f, self, *args, **kwargs):
     """sync relevant results from self.client to our results attribute."""
     ret = f(self, *args, **kwargs)
     delta = self.outstanding.difference(self.client.outstanding)
     completed = self.outstanding.intersection(delta)
     self.outstanding = self.outstanding.difference(completed)
     for msg_id in completed:
         self.results[msg_id] = self.client.results[msg_id]
     return ret
 
 @decorator
 def spin_after(f, self, *args, **kwargs):
     """call spin after the method."""
     ret = f(self, *args, **kwargs)
     self.spin()
     return ret
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
[e-mail]_doctest
 class View(HasTraits):
     """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
     
     Don't use this class, use subclasses.
     
     Methods
     -------
     
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
     
     wait
         wait on one or more msg_ids
     
     execution methods
         apply
         legacy: execute, run
     
     data movement
         push, pull, scatter, gather
     
     query methods
         get_result, queue_status, purge_results, result_status
     
     control methods
         abort, shutdown
     
     """
     # flags
     block=Bool(False)
     track=Bool(True)
     targets = Any()
     
     history=List()
     outstanding = Set()
     results = Dict()
-    client = Instance('IPython.parallel.client.Client')
+    client = Instance('IPython.parallel.Client')
     
     _socket = Instance('zmq.Socket')
     _flag_names = List(['targets', 'block', 'track'])
     _targets = Any()
     _idents = Any()
     
     def __init__(self, client=None, socket=None, **flags):
         super(View, self).__init__(client=client, _socket=socket)
         self.block = client.block
         
         self.set_flags(**flags)
         
         assert not self.__class__ is View, "Don't use base View objects, use subclasses"
         
 
     def __repr__(self):
         strtargets = str(self.targets)
         if len(strtargets) > 16:
             strtargets = strtargets[:12]+'...]'
         return "<%s %s>"%(self.__class__.__name__, strtargets)
 
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
         
         Views determine behavior with a few attributes (`block`, `track`, etc.).
         These attributes can be set all at once by name with this method.
         
         Parameters
         ----------
         
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
         """
         for name, value in kwargs.iteritems():
             if name not in self._flag_names:
                 raise KeyError("Invalid name: %r"%name)
             else:
                 setattr(self, name, value)
     
     @contextmanager
     def temp_flags(self, **kwargs):
         """temporarily set flags, for use in `with` statements.
         
         See set_flags for permanent setting of flags
         
         Examples
         --------
         
         >>> view.track=False
         ...
         >>> with view.temp_flags(track=True):
         ...    ar = view.apply(dostuff, my_big_array)
         ...    ar.tracker.wait() # wait for send to finish
         >>> view.track
         False
         
         """
         # preflight: save flags, and set temporaries
         saved_flags = {}
         for f in self._flag_names:
             saved_flags[f] = getattr(self, f)
         self.set_flags(**kwargs)
         # yield to the with-statement block
         try:
             yield
         finally:
             # postflight: restore saved flags
             self.set_flags(**saved_flags)
         
         
     #----------------------------------------------------------------
     # apply
     #----------------------------------------------------------------
     
     @sync_results
     @save_ids
     def _really_apply(self, f, args, kwargs, block=None, **options):
         """wrapper for client.send_apply_message"""
         raise NotImplementedError("Implement in subclasses")
     
     def apply(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines, returning the result.
         
         This method sets all apply flags via this View's attributes.
         
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs)
         """
         return self._really_apply(f, args, kwargs)
 
     def apply_async(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
         
         returns AsyncResult
         """
         return self._really_apply(f, args, kwargs, block=False)
 
     @spin_after
     def apply_sync(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines in a blocking manner,
          returning the result.
         
         returns: actual result of f(*args, **kwargs)
         """
         return self._really_apply(f, args, kwargs, block=True)
 
     #----------------------------------------------------------------
     # wrappers for client and control methods
     #----------------------------------------------------------------
     @sync_results
     def spin(self):
         """spin the client, and sync"""
         self.client.spin()
     
     @sync_results
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
         
         Parameters
         ----------
         
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
         
         Returns
         -------
         
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         if jobs is None:
             jobs = self.history
         return self.client.wait(jobs, timeout)
     
     def abort(self, jobs=None, targets=None, block=None):
         """Abort jobs on my engines.
         
         Parameters
         ----------
         
         jobs : None, str, list of strs, optional
             if None: abort all jobs.
             else: abort specific msg_id(s).
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         return self.client.abort(jobs=jobs, targets=targets, block=block)
 
     def queue_status(self, targets=None, verbose=False):
         """Fetch the Queue status of my engines"""
         targets = targets if targets is not None else self.targets
         return self.client.queue_status(targets=targets, verbose=verbose)
     
     def purge_results(self, jobs=[], targets=[]):
         """Instruct the controller to forget specific results."""
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.purge_results(jobs=jobs, targets=targets)
     
     def shutdown(self, targets=None, restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub.
         """
         block = self.block if block is None else block
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
     
     @spin_after
     def get_result(self, indices_or_msg_ids=None):
         """return one or more results, specified by history index or msg_id.
         
         See client.get_result for details.
         
         """
         
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         if isinstance(indices_or_msg_ids, int):
             indices_or_msg_ids = self.history[indices_or_msg_ids]
         elif isinstance(indices_or_msg_ids, (list,tuple,set)):
             indices_or_msg_ids = list(indices_or_msg_ids)
             for i,index in enumerate(indices_or_msg_ids):
                 if isinstance(index, int):
                     indices_or_msg_ids[i] = self.history[index]
         return self.client.get_result(indices_or_msg_ids)
     
     #-------------------------------------------------------------------
     # Map
     #-------------------------------------------------------------------
     
     def map(self, f, *sequences, **kwargs):
         """override in subclasses"""
         raise NotImplementedError
     
     def map_async(self, f, *sequences, **kwargs):
         """Parallel version of builtin `map`, using this view's engines.
         
         This is equivalent to map(...block=False)
         
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_async doesn't take a `block` keyword argument.")
         kwargs['block'] = False
         return self.map(f,*sequences,**kwargs)
     
     def map_sync(self, f, *sequences, **kwargs):
         """Parallel version of builtin `map`, using this view's engines.
         
         This is equivalent to map(...block=True)
         
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_sync doesn't take a `block` keyword argument.")
         kwargs['block'] = True
         return self.map(f,*sequences,**kwargs)
     
     def imap(self, f, *sequences, **kwargs):
         """Parallel version of `itertools.imap`.
         
         See `self.map` for details.
         
         """
         
         return iter(self.map_async(f,*sequences, **kwargs))
     
     #-------------------------------------------------------------------
     # Decorators
     #-------------------------------------------------------------------
     
     def remote(self, block=True, **flags):
         """Decorator for making a RemoteFunction"""
         block = self.block if block is None else block
         return remote(self, block=block, **flags)
     
     def parallel(self, dist='b', block=None, **flags):
         """Decorator for making a ParallelFunction"""
         block = self.block if block is None else block
         return parallel(self, dist=dist, block=block, **flags)
 
 @testdec.skip_doctest
 class DirectView(View):
     """Direct Multiplexer View of one or more engines.
     
     These are created via indexed access to a client:
     
     >>> dv_1 = client[1]
     >>> dv_all = client[:]
     >>> dv_even = client[::2]
     >>> dv_some = client[1:3]
     
     This object provides dictionary access to engine namespaces:
     
     # push a=5:
     >>> dv['a'] = 5 
     # pull 'foo':
     >>> db['foo']
     
     """
     
     def __init__(self, client=None, socket=None, targets=None):
         super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
     
     @property
     def importer(self):
         """sync_imports(local=True) as a property.
         
         See sync_imports for details.
-
-        In [10]: with v.importer:
-           ....:     import numpy
-           ....:     
-        importing numpy on engine(s)
         
         """
         return self.sync_imports(True)
     
     @contextmanager
     def sync_imports(self, local=True):
         """Context Manager for performing simultaneous local and remote imports.
         
         'import x as y' will *not* work.  The 'as y' part will simply be ignored.
         
         >>> with view.sync_imports():
         ...    from numpy import recarray
         importing recarray from numpy on engine(s)
         
         """
         import __builtin__
         local_import = __builtin__.__import__
         modules = set()
         results = []
         @util.interactive
         def remote_import(name, fromlist, level):
             """the function to be passed to apply, that actually performs the import
             on the engine, and loads up the user namespace.
             """
             import sys
             user_ns = globals()
             mod = __import__(name, fromlist=fromlist, level=level)
             if fromlist:
                 for key in fromlist:
                     user_ns[key] = getattr(mod, key)
             else:
                 user_ns[name] = sys.modules[name]
         
         def view_import(name, globals={}, locals={}, fromlist=[], level=-1):
             """the drop-in replacement for __import__, that optionally imports
             locally as well.
             """
             # don't override nested imports
             save_import = __builtin__.__import__
             __builtin__.__import__ = local_import
             
             if imp.lock_held():
                 # this is a side-effect import, don't do it remotely, or even
                 # ignore the local effects
                 return local_import(name, globals, locals, fromlist, level)
             
             imp.acquire_lock()
             if local:
                 mod = local_import(name, globals, locals, fromlist, level)
             else:
                 raise NotImplementedError("remote-only imports not yet implemented")
             imp.release_lock()
             
             key = name+':'+','.join(fromlist or [])
             if level == -1 and key not in modules:
                 modules.add(key)
                 if fromlist:
                     print "importing %s from %s on engine(s)"%(','.join(fromlist), name)
                 else:
                     print "importing %s on engine(s)"%name
                 results.append(self.apply_async(remote_import, name, fromlist, level))
             # restore override
             __builtin__.__import__ = save_import
                 
             return mod
         
         # override __import__
         __builtin__.__import__ = view_import
         try:
             # enter the block
             yield
         except ImportError:
             if not local:
                 # ignore import errors if not doing local imports
                 pass
         finally:
             # always restore __import__
             __builtin__.__import__ = local_import
         
         for r in results:
             # raise possible remote ImportErrors here
             r.get()
             
     
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
         """calls f(*args, **kwargs) on remote engines, returning the result.
         
         This method sets all of `apply`'s flags via this View's attributes.
         
         Parameters
         ----------
         
         f : callable
         
         args : list [default: empty]
         
         kwargs : dict [default: empty]
         
         targets : target list [default: self.targets]
             where to run
         block : bool [default: self.block]
             whether to block 
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
         
         Returns
         -------
         
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         targets = self.targets if targets is None else targets
         
         _idents = self.client._build_targets(targets)[0]
         msg_ids = []
         trackers = []
         for ident in _idents:
             msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
                                     ident=ident)
             if track:
                 trackers.append(msg['tracker'])
             msg_ids.append(msg['msg_id'])
         tracker = None if track is False else zmq.MessageTracker(*trackers)
         ar = AsyncResult(self.client, msg_ids, fname=f.__name__, targets=targets, tracker=tracker)
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
     
     @spin_after
     def map(self, f, *sequences, **kwargs):
         """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
         
         Parallel version of builtin `map`, using this View's `targets`.
         
         There will be one task per target, so work will be chunked
         if the sequences are longer than `targets`.  
         
         Results can be iterated as they are ready, but will become available in chunks.
         
         Parameters
         ----------
         
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
         
         Returns
         -------
         
         if block=False:
             AsyncMapResult
                 An object like AsyncResult, but which reassembles the sequence of results
                 into a single list. AsyncMapResults can be iterated through before all
                 results are complete.
         else:
             list
                 the result of map(f,*sequences)
         """
         
         block = kwargs.pop('block', self.block)
         for k in kwargs.keys():
             if k not in ['block', 'track']:
                 raise TypeError("invalid keyword arg, %r"%k)
         
         assert len(sequences) > 0, "must have some sequences to map onto!"
         pf = ParallelFunction(self, f, block=block, **kwargs)
         return pf.map(*sequences)
     
     def execute(self, code, targets=None, block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
         
         ``execute`` is always `bound` (affects engine namespace)
         
         Parameters
         ----------
         
         code : str
                 the code string to be executed
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
         return self._really_apply(util._execute, args=(code,), block=block, targets=targets)
     
     def run(self, filename, targets=None, block=None):
         """Execute contents of `filename` on my engine(s). 
         
         This simply reads the contents of the file and calls `execute`.
         
         Parameters
         ----------
         
         filename : str
                 The path to the file
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done
                 default: self.block
         
         """
         with open(filename, 'r') as f:
             # add newline in case of trailing indented whitespace
             # which will cause SyntaxError
             code = f.read()+'\n'
         return self.execute(code, block=block, targets=targets)
     
     def update(self, ns):
         """update remote namespace with dict `ns`
         
         See `push` for details.
         """
         return self.push(ns, block=self.block, track=self.track)
     
     def push(self, ns, targets=None, block=None, track=None):
         """update remote namespace with dict `ns`
         
         Parameters
         ----------
         
         ns : dict
             dict of keys with which to update engine namespace(s)
         block : bool [default : self.block]
             whether to wait to be notified of engine receipt
         
         """
         
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         # applier = self.apply_sync if block else self.apply_async
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
         return self._really_apply(util._push, (ns,), block=block, track=track, targets=targets)
 
     def get(self, key_s):
         """get object(s) by `key_s` from remote namespace
         
         see `pull` for details.
         """
         # block = block if block is not None else self.block
         return self.pull(key_s, block=True)
     
     def pull(self, names, targets=None, block=True):
         """get object(s) by `name` from remote namespace
         
         will return one object if it is a key.
         can also take a list of keys, in which case it will return a list of objects.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         applier = self.apply_sync if block else self.apply_async
         if isinstance(names, basestring):
             pass
         elif isinstance(names, (list,tuple,set)):
             for key in names:
                 if not isinstance(key, basestring):
                     raise TypeError("keys must be str, not type %r"%type(key))
         else:
             raise TypeError("names must be strs, not %r"%names)
         return self._really_apply(util._pull, (names,), block=block, targets=targets)
     
     def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         
         mapObject = Map.dists[dist]()
         nparts = len(targets)
         msg_ids = []
         trackers = []
         for index, engineid in enumerate(targets):
             partition = mapObject.getPartition(seq, index, nparts)
             if flatten and len(partition) == 1:
                 ns = {key: partition[0]}
             else:
                 ns = {key: partition}
             r = self.push(ns, block=False, track=track, targets=engineid)
             msg_ids.extend(r.msg_ids)
             if track:
                 trackers.append(r._tracker)
         
         if track:
             tracker = zmq.MessageTracker(*trackers)
         else:
             tracker = None
         
         r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
         if block:
             r.wait()
         else:
             return r
     
     @sync_results
     @save_ids
     def gather(self, key, dist='b', targets=None, block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         mapObject = Map.dists[dist]()
         msg_ids = []
         
         for index, engineid in enumerate(targets):
             msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
         
         r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
         
         if block:
             try:
                 return r.get()
             except KeyboardInterrupt:
                 pass
         return r
     
     def __getitem__(self, key):
         return self.get(key)
     
     def __setitem__(self,key, value):
         self.update({key:value})
     
     def clear(self, targets=None, block=False):
         """Clear the remote namespaces on my engines."""
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         return self.client.clear(targets=targets, block=block)
     
     def kill(self, targets=None, block=True):
         """Kill my engines."""
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         return self.client.kill(targets=targets, block=block)
     
     #----------------------------------------
     # activate for %px,%autopx magics
     #----------------------------------------
     def activate(self):
         """Make this `View` active for parallel magic commands.
         
         IPython has a magic command syntax to work with `MultiEngineClient` objects.
         In a given IPython session there is a single active one.  While
         there can be many `Views` created and used by the user, 
         there is only one active one.  The active `View` is used whenever 
         the magic commands %px and %autopx are used.
         
         The activate() method is called on a given `View` to make it 
         active.  Once this has been done, the magic commands can be used.
         """
         
         try:
             # This is injected into __builtins__.
             ip = get_ipython()
         except NameError:
             print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
         else:
             pmagic = ip.plugin_manager.get_plugin('parallelmagic')
             if pmagic is not None:
                 pmagic.active_multiengine_client = self
             else:
                 print "You must first load the parallelmagic extension " \
                       "by doing '%load_ext parallelmagic'"
 
 
 @testdec.skip_doctest
 class LoadBalancedView(View):
     """An load-balancing View that only executes via the Task scheduler.
     
     Load-balanced views can be created with the client's `view` method:
     
     >>> v = client.load_balanced_view()
     
     or targets can be specified, to restrict the potential destinations:
     
     >>> v = client.client.load_balanced_view(([1,3])
     
     which would restrict loadbalancing to between engines 1 and 3.
     
     """
     
     follow=Any()
     after=Any()
     timeout=CFloat()
     
     _task_scheme = Any()
     _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout'])
     
     def __init__(self, client=None, socket=None, **flags):
         super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
         self._task_scheme=client._task_scheme
     
     def _validate_dependency(self, dep):
         """validate a dependency.
         
         For use in `set_flags`.
         """
         if dep is None or isinstance(dep, (str, AsyncResult, Dependency)):
             return True
         elif isinstance(dep, (list,set, tuple)):
             for d in dep:
                 if not isinstance(d, (str, AsyncResult)):
                     return False
         elif isinstance(dep, dict):
             if set(dep.keys()) != set(Dependency().as_dict().keys()):
                 return False
             if not isinstance(dep['msg_ids'], list):
                 return False
             for d in dep['msg_ids']:
                 if not isinstance(d, str):
                     return False
         else:
             return False
         
         return True
                 
     def _render_dependency(self, dep):
         """helper for building jsonable dependencies from various input forms."""
         if isinstance(dep, Dependency):
             return dep.as_dict()
         elif isinstance(dep, AsyncResult):
             return dep.msg_ids
         elif dep is None:
             return []
         else:
             # pass to Dependency constructor
             return list(Dependency(dep))
 
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
         
         A View is a wrapper for the Client's apply method, but with attributes
         that specify keyword arguments, those attributes can be set by keyword
         argument with this method.
         
         Parameters
         ----------
         
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
         #
         after : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a time-based dependency.
             This job will only be run *after* the dependencies
             have been met.
 
         follow : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a location-based dependency.
             This job will only be run on an engine where this dependency
             is met.
 
         timeout : float/int or None
             Only for load-balanced execution (targets=None)
             Specify an amount of time (in seconds) for the scheduler to
             wait for dependencies to be met before failing with a
             DependencyTimeout.
         """
         
         super(LoadBalancedView, self).set_flags(**kwargs)
         for name in ('follow', 'after'):
             if name in kwargs:
                 value = kwargs[name]
                 if self._validate_dependency(value):
                     setattr(self, name, value)
                 else:
                     raise ValueError("Invalid dependency: %r"%value)
         if 'timeout' in kwargs:
             t = kwargs['timeout']
             if not isinstance(t, (int, long, float, type(None))):
                 raise TypeError("Invalid type for timeout: %r"%type(t))
             if t is not None:
                 if t < 0:
                     raise ValueError("Invalid timeout: %s"%t)
             self.timeout = t
     
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
                                         after=None, follow=None, timeout=None,
                                         targets=None):
         """calls f(*args, **kwargs) on a remote engine, returning the result.
         
         This method temporarily sets all of `apply`'s flags for a single call.
         
         Parameters
         ----------
         
         f : callable
         
         args : list [default: empty]
         
         kwargs : dict [default: empty]
         
         block : bool [default: self.block]
             whether to block 
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
             
         !!!!!! TODO: THE REST HERE  !!!!
         
         Returns
         -------
         
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
 
         # validate whether we can run
         if self._socket.closed:
             msg = "Task farming is disabled"
             if self._task_scheme == 'pure':
                 msg += " because the pure ZMQ scheduler cannot handle"
                 msg += " disappearing engines."
             raise RuntimeError(msg)
         
         if self._task_scheme == 'pure':
             # pure zmq scheme doesn't support dependencies
             msg = "Pure ZMQ scheduler doesn't support dependencies"
             if (follow or after):
                 # hard fail on DAG dependencies
                 raise RuntimeError(msg)
             if isinstance(f, dependent):
                 # soft warn on functional dependencies
                 warnings.warn(msg, RuntimeWarning)
         
         # build args
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         after = self.after if after is None else after
         follow = self.follow if follow is None else follow
         timeout = self.timeout if timeout is None else timeout
         targets = self.targets if targets is None else targets
         
         if targets is None:
             idents = []
         else:
             idents = self.client._build_targets(targets)[0]
         
         after = self._render_dependency(after)
         follow = self._render_dependency(follow)
         subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents)
         
         msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
                                 subheader=subheader)
         tracker = None if track is False else msg['tracker']
         
         ar = AsyncResult(self.client, msg['msg_id'], fname=f.__name__, targets=None, tracker=tracker)
         
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
         
     @spin_after
     @save_ids
     def map(self, f, *sequences, **kwargs):
         """view.map(f, *sequences, block=self.block, chunksize=1) => list|AsyncMapResult
         
         Parallel version of builtin `map`, load-balanced by this View.
         
         `block`, and `chunksize` can be specified by keyword only.
         
         Each `chunksize` elements will be a separate task, and will be
         load-balanced. This lets individual elements be available for iteration
         as soon as they arrive.
         
         Parameters
         ----------
         
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
         chunksize : int
             how many elements should be in each task [default 1]
         
         Returns
         -------
         
         if block=False:
             AsyncMapResult
                 An object like AsyncResult, but which reassembles the sequence of results
                 into a single list. AsyncMapResults can be iterated through before all
                 results are complete.
             else:
                 the result of map(f,*sequences)
         
         """
         
         # default
         block = kwargs.get('block', self.block)
         chunksize = kwargs.get('chunksize', 1)
         
         keyset = set(kwargs.keys())
         extra_keys = keyset.difference_update(set(['block', 'chunksize']))
         if extra_keys:
             raise TypeError("Invalid kwargs: %s"%list(extra_keys))
         
         assert len(sequences) > 0, "must have some sequences to map onto!"
         
         pf = ParallelFunction(self, f, block=block,  chunksize=chunksize)
         return pf.map(*sequences)
 
 __all__ = ['LoadBalancedView', 'DirectView']
  No newline at end of file

IPython/parallel/controller/controller.py

@@ -1,118 +1,117 b''
 #!/usr/bin/env python
 """The IPython Controller with 0MQ
 This is a collection of one Hub and several Schedulers.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
-import logging
 from multiprocessing import Process
 
 import zmq
 from zmq.devices import ProcessMonitoredQueue
 # internal:
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Int, CStr, Instance, List, Bool
 
-from .entry_point import signal_children
+from IPython.parallel.util import signal_children
 from .hub import Hub, HubFactory
 from .scheduler import launch_scheduler
 
 #-----------------------------------------------------------------------------
 # Configurable
 #-----------------------------------------------------------------------------
 
 
 class ControllerFactory(HubFactory):
     """Configurable for setting up a Hub and Schedulers."""
     
     usethreads = Bool(False, config=True)
     # pure-zmq downstream HWM
     hwm = Int(0, config=True)
     
     # internal
     children = List()
     mq_class = CStr('zmq.devices.ProcessMonitoredQueue')
     
     def _usethreads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
         
     def __init__(self, **kwargs):
         super(ControllerFactory, self).__init__(**kwargs)
         self.subconstructors.append(self.construct_schedulers)
     
     def start(self):
         super(ControllerFactory, self).start()
         child_procs = []
         for child in self.children:
             child.start()
             if isinstance(child, ProcessMonitoredQueue):
                 child_procs.append(child.launcher)
             elif isinstance(child, Process):
                 child_procs.append(child)
         if child_procs:
             signal_children(child_procs)
         
     
     def construct_schedulers(self):
         children = self.children
         mq = import_item(self.mq_class)
         
         maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
         q.bind_in(self.client_info['iopub'])
         q.bind_out(self.engine_info['iopub'])
         q.setsockopt_out(zmq.SUBSCRIBE, '')
         q.connect_mon(maybe_inproc)
         q.daemon=True
         children.append(q)
 
         # Multiplexer Queue (in a Process)
         q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
         q.bind_in(self.client_info['mux'])
         q.setsockopt_in(zmq.IDENTITY, 'mux')
         q.bind_out(self.engine_info['mux'])
         q.connect_mon(maybe_inproc)
         q.daemon=True
         children.append(q)
 
         # Control Queue (in a Process)
         q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
         q.bind_in(self.client_info['control'])
         q.setsockopt_in(zmq.IDENTITY, 'control')
         q.bind_out(self.engine_info['control'])
         q.connect_mon(maybe_inproc)
         q.daemon=True
         children.append(q)
         # Task Queue (in a Process)
         if self.scheme == 'pure':
             self.log.warn("task::using pure XREQ Task scheduler")
             q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
             q.setsockopt_out(zmq.HWM, self.hwm)
             q.bind_in(self.client_info['task'][1])
             q.setsockopt_in(zmq.IDENTITY, 'task')
             q.bind_out(self.engine_info['task'])
             q.connect_mon(maybe_inproc)
             q.daemon=True
             children.append(q)
         elif self.scheme == 'none':
             self.log.warn("task::using no Task scheduler")
             
         else:
             self.log.info("task::using Python %s Task scheduler"%self.scheme)
             sargs = (self.client_info['task'][1], self.engine_info['task'], self.monitor_url, self.client_info['notification'])
             kwargs = dict(scheme=self.scheme,logname=self.log.name, loglevel=self.log.level, config=self.config)
             q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
             q.daemon=True
             children.append(q)
 

IPython/parallel/controller/dependency.py

@@ -1,196 +1,196 b''
 """Dependency utilities"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from types import ModuleType
 
-from .asyncresult import AsyncResult
-from .error import UnmetDependency
-from .util import interactive
+from IPython.parallel.client.asyncresult import AsyncResult
+from IPython.parallel.error import UnmetDependency
+from IPython.parallel.util import interactive
 
 class depend(object):
     """Dependency decorator, for use with tasks.
     
     `@depend` lets you define a function for engine dependencies
     just like you use `apply` for tasks.
     
     
     Examples
     --------
     ::
     
         @depend(df, a,b, c=5)
         def f(m,n,p)
         
         view.apply(f, 1,2,3)
     
     will call df(a,b,c=5) on the engine, and if it returns False or
     raises an UnmetDependency error, then the task will not be run
     and another engine will be tried.
     """
     def __init__(self, f, *args, **kwargs):
         self.f = f
         self.args = args
         self.kwargs = kwargs
     
     def __call__(self, f):
         return dependent(f, self.f, *self.args, **self.kwargs)
 
 class dependent(object):
     """A function that depends on another function.
     This is an object to prevent the closure used
     in traditional decorators, which are not picklable.
     """
 
     def __init__(self, f, df, *dargs, **dkwargs):
         self.f = f
         self.func_name = getattr(f, '__name__', 'f')
         self.df = df
         self.dargs = dargs
         self.dkwargs = dkwargs
 
     def __call__(self, *args, **kwargs):
         # if hasattr(self.f, 'func_globals') and hasattr(self.df, 'func_globals'):
         #     self.df.func_globals = self.f.func_globals
         if self.df(*self.dargs, **self.dkwargs) is False:
             raise UnmetDependency()
         return self.f(*args, **kwargs)
     
     @property
     def __name__(self):
         return self.func_name
 
 @interactive
 def _require(*names):
     """Helper for @require decorator."""
     from IPython.parallel.error import UnmetDependency
     user_ns = globals()
     for name in names:
         if name in user_ns:
             continue
         try:
             exec 'import %s'%name in user_ns
         except ImportError:
             raise UnmetDependency(name)
     return True
 
 def require(*mods):
     """Simple decorator for requiring names to be importable.
     
     Examples
     --------
     
     In [1]: @require('numpy')
        ...: def norm(a):
        ...:     import numpy
        ...:     return numpy.linalg.norm(a,2)
     """
     names = []
     for mod in mods:
         if isinstance(mod, ModuleType):
             mod = mod.__name__
         
         if isinstance(mod, basestring):
             names.append(mod)
         else:
             raise TypeError("names must be modules or module names, not %s"%type(mod))
     
     return depend(_require, *names)
 
 class Dependency(set):
     """An object for representing a set of msg_id dependencies.
     
     Subclassed from set().
     
     Parameters
     ----------
     dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
         The msg_ids to depend on
     all : bool [default True]
         Whether the dependency should be considered met when *all* depending tasks have completed
         or only when *any* have been completed.
     success : bool [default True]
         Whether to consider successes as fulfilling dependencies.
     failure : bool [default False]
         Whether to consider failures as fulfilling dependencies.
     
     If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
         as soon as the first depended-upon task fails.
     """
     
     all=True
     success=True
     failure=True
     
     def __init__(self, dependencies=[], all=True, success=True, failure=False):
         if isinstance(dependencies, dict):
             # load from dict
             all = dependencies.get('all', True)
             success = dependencies.get('success', success)
             failure = dependencies.get('failure', failure)
             dependencies = dependencies.get('dependencies', [])
         ids = []
         
         # extract ids from various sources:
         if isinstance(dependencies, (basestring, AsyncResult)):
             dependencies = [dependencies]
         for d in dependencies:
             if isinstance(d, basestring):
                 ids.append(d)
             elif isinstance(d, AsyncResult):
                 ids.extend(d.msg_ids)
             else:
                 raise TypeError("invalid dependency type: %r"%type(d))
         
         set.__init__(self, ids)
         self.all = all
         if not (success or failure):
             raise ValueError("Must depend on at least one of successes or failures!")
         self.success=success
         self.failure = failure
     
     def check(self, completed, failed=None):
         """check whether our dependencies have been met."""
         if len(self) == 0:
             return True
         against = set()
         if self.success:
             against = completed
         if failed is not None and self.failure:
             against = against.union(failed)
         if self.all:
             return self.issubset(against)
         else:
             return not self.isdisjoint(against)
     
     def unreachable(self, completed, failed=None):
         """return whether this dependency has become impossible."""
         if len(self) == 0:
             return False
         against = set()
         if not self.success:
             against = completed
         if failed is not None and not self.failure:
             against = against.union(failed)
         if self.all:
             return not self.isdisjoint(against)
         else:
             return self.issubset(against)
         
     
     def as_dict(self):
         """Represent this dependency as a dict. For json compatibility."""
         return dict(
             dependencies=list(self),
             all=self.all,
             success=self.success,
             failure=self.failure
         )
 
 
 __all__ = ['depend', 'require', 'dependent', 'Dependency']
 

IPython/parallel/controller/heartmonitor.py

@@ -1,164 +1,163 b''
 #!/usr/bin/env python
 """
 A multi-heart Heartbeat system using PUB and XREP sockets. pings are sent out on the PUB,
 and hearts are tracked based on their XREQ identities.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 import time
-import logging
 import uuid
 
 import zmq
-from zmq.devices import ProcessDevice,ThreadDevice
+from zmq.devices import ProcessDevice, ThreadDevice
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.traitlets import Set, Instance, CFloat, Bool
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 class Heart(object):
     """A basic heart object for responding to a HeartMonitor.
     This is a simple wrapper with defaults for the most common
     Device model for responding to heartbeats.
     
     It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using 
     SUB/XREQ for in/out.
     
     You can specify the XREQ's IDENTITY via the optional heart_id argument."""
     device=None
     id=None
     def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.XREQ, heart_id=None):
         self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
         self.device.daemon=True
         self.device.connect_in(in_addr)
         self.device.connect_out(out_addr)
         if in_type == zmq.SUB:
             self.device.setsockopt_in(zmq.SUBSCRIBE, "")
         if heart_id is None:
             heart_id = str(uuid.uuid4())
         self.device.setsockopt_out(zmq.IDENTITY, heart_id)
         self.id = heart_id
     
     def start(self):
         return self.device.start()
         
 class HeartMonitor(LoggingFactory):
     """A basic HeartMonitor class
     pingstream: a PUB stream
     pongstream: an XREP stream
     period: the period of the heartbeat in milliseconds"""
     
     period=CFloat(1000, config=True) # in milliseconds
     
     pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
     pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         return ioloop.IOLoop.instance()
     debug=Bool(False)
     
     # not settable:
     hearts=Set()
     responses=Set()
     on_probation=Set()
     last_ping=CFloat(0)
     _new_handlers = Set()
     _failure_handlers = Set()
     lifetime = CFloat(0)
     tic = CFloat(0)
     
     def __init__(self, **kwargs):
         super(HeartMonitor, self).__init__(**kwargs)
         
         self.pongstream.on_recv(self.handle_pong)
     
     def start(self):
         self.caller = ioloop.PeriodicCallback(self.beat, self.period, self.loop)
         self.caller.start()
     
     def add_new_heart_handler(self, handler):
         """add a new handler for new hearts"""
         self.log.debug("heartbeat::new_heart_handler: %s"%handler)
         self._new_handlers.add(handler)
         
     def add_heart_failure_handler(self, handler):
         """add a new handler for heart failure"""
         self.log.debug("heartbeat::new heart failure handler: %s"%handler)
         self._failure_handlers.add(handler)
             
     def beat(self):
         self.pongstream.flush() 
         self.last_ping = self.lifetime
         
         toc = time.time()
         self.lifetime += toc-self.tic
         self.tic = toc
         # self.log.debug("heartbeat::%s"%self.lifetime)
         goodhearts = self.hearts.intersection(self.responses)
         missed_beats = self.hearts.difference(goodhearts)
         heartfailures = self.on_probation.intersection(missed_beats)
         newhearts = self.responses.difference(goodhearts)
         map(self.handle_new_heart, newhearts)
         map(self.handle_heart_failure, heartfailures)
         self.on_probation = missed_beats.intersection(self.hearts)
         self.responses = set()
         # print self.on_probation, self.hearts
         # self.log.debug("heartbeat::beat %.3f, %i beating hearts"%(self.lifetime, len(self.hearts)))
         self.pingstream.send(str(self.lifetime))
     
     def handle_new_heart(self, heart):
         if self._new_handlers:
             for handler in self._new_handlers:
                 handler(heart)
         else:
             self.log.info("heartbeat::yay, got new heart %s!"%heart)
         self.hearts.add(heart)
     
     def handle_heart_failure(self, heart):
         if self._failure_handlers:
             for handler in self._failure_handlers:
                 try:
                     handler(heart)
                 except Exception as e:
                     self.log.error("heartbeat::Bad Handler! %s"%handler, exc_info=True)
                     pass
         else:
             self.log.info("heartbeat::Heart %s failed :("%heart)
         self.hearts.remove(heart)
         
     
     def handle_pong(self, msg):
         "a heart just beat"
         if msg[1] == str(self.lifetime):
             delta = time.time()-self.tic
             # self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta))
             self.responses.add(msg[0])
         elif msg[1] == str(self.last_ping):
             delta = time.time()-self.tic + (self.lifetime-self.last_ping)
             self.log.warn("heartbeat::heart %r missed a beat, and took %.2f ms to respond"%(msg[0], 1000*delta))
             self.responses.add(msg[0])
         else:
             self.log.warn("heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)"%
             (msg[1],self.lifetime))
 
 
 if __name__ == '__main__':
     loop = ioloop.IOLoop.instance()
     context = zmq.Context()
     pub = context.socket(zmq.PUB)
     pub.bind('tcp://127.0.0.1:5555')
     xrep = context.socket(zmq.XREP)
     xrep.bind('tcp://127.0.0.1:5556')
     
     outstream = zmqstream.ZMQStream(pub, loop)
     instream = zmqstream.ZMQStream(xrep, loop)
     
     hb = HeartMonitor(loop, outstream, instream)
     
     loop.start()

IPython/parallel/controller/hub.py

@@ -1,1090 +1,1089 b''
 #!/usr/bin/env python
 """The IPython Controller Hub with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import sys
 import time
 from datetime import datetime
 
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 # internal:
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
 
-from .entry_point import select_random_ports
-from .factory import RegistrationFactory, LoggingFactory
+from IPython.parallel import error
+from IPython.parallel.factory import RegistrationFactory, LoggingFactory
+from IPython.parallel.util import select_random_ports, validate_url_container, ISO8601
 
-from . import error
 from .heartmonitor import HeartMonitor
-from .util import validate_url_container, ISO8601
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def _passer(*args, **kwargs):
     return
 
 def _printer(*args, **kwargs):
     print (args)
     print (kwargs)
 
 def empty_record():
     """Return an empty dict with all record keys."""
     return {
         'msg_id' : None,
         'header' : None,
         'content': None,
         'buffers': None,
         'submitted': None,
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'result_header' : None,
         'result_content' : None,
         'result_buffers' : None,
         'queue' : None,
         'pyin' : None,
         'pyout': None,
         'pyerr': None,
         'stdout': '',
         'stderr': '',
     }
     
 def init_record(msg):
     """Initialize a TaskRecord based on a request."""
     header = msg['header']
     return {
         'msg_id' : header['msg_id'],
         'header' : header,
         'content': msg['content'],
         'buffers': msg['buffers'],
         'submitted': datetime.strptime(header['date'], ISO8601),
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'result_header' : None,
         'result_content' : None,
         'result_buffers' : None,
         'queue' : None,
         'pyin' : None,
         'pyout': None,
         'pyerr': None,
         'stdout': '',
         'stderr': '',
     }
 
 
 class EngineConnector(HasTraits):
     """A simple object for accessing the various zmq connections of an object.
     Attributes are:
     id (int): engine ID
     uuid (str): uuid (unused?)
     queue (str): identity of queue's XREQ socket
     registration (str): identity of registration XREQ socket
     heartbeat (str): identity of heartbeat XREQ socket
     """
     id=Int(0)
     queue=Str()
     control=Str()
     registration=Str()
     heartbeat=Str()
     pending=Set()
 
 class HubFactory(RegistrationFactory):
     """The Configurable for setting up a Hub."""
     
     # name of a scheduler scheme
     scheme = Str('leastload', config=True)
     
     # port-pairs for monitoredqueues:
     hb = Instance(list, config=True)
     def _hb_default(self):
         return select_random_ports(2)
     
     mux = Instance(list, config=True)
     def _mux_default(self):
         return select_random_ports(2)
     
     task = Instance(list, config=True)
     def _task_default(self):
         return select_random_ports(2)
     
     control = Instance(list, config=True)
     def _control_default(self):
         return select_random_ports(2)
     
     iopub = Instance(list, config=True)
     def _iopub_default(self):
         return select_random_ports(2)
     
     # single ports:
     mon_port = Instance(int, config=True)
     def _mon_port_default(self):
         return select_random_ports(1)[0]
     
     notifier_port = Instance(int, config=True)
     def _notifier_port_default(self):
         return select_random_ports(1)[0]
     
     ping = Int(1000, config=True) # ping frequency
     
     engine_ip = CStr('127.0.0.1', config=True)
     engine_transport = CStr('tcp', config=True)
     
     client_ip = CStr('127.0.0.1', config=True)
     client_transport = CStr('tcp', config=True)
     
     monitor_ip = CStr('127.0.0.1', config=True)
     monitor_transport = CStr('tcp', config=True)
     
     monitor_url = CStr('')
     
-    db_class = CStr('IPython.parallel.dictdb.DictDB', config=True)
+    db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
     
     # not configurable
-    db = Instance('IPython.parallel.dictdb.BaseDB')
-    heartmonitor = Instance('IPython.parallel.heartmonitor.HeartMonitor')
+    db = Instance('IPython.parallel.controller.dictdb.BaseDB')
+    heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
     subconstructors = List()
     _constructed = Bool(False)
     
     def _ip_changed(self, name, old, new):
         self.engine_ip = new
         self.client_ip = new
         self.monitor_ip = new
         self._update_monitor_url()
     
     def _update_monitor_url(self):
         self.monitor_url = "%s://%s:%i"%(self.monitor_transport, self.monitor_ip, self.mon_port)
     
     def _transport_changed(self, name, old, new):
         self.engine_transport = new
         self.client_transport = new
         self.monitor_transport = new
         self._update_monitor_url()
         
     def __init__(self, **kwargs):
         super(HubFactory, self).__init__(**kwargs)
         self._update_monitor_url()
         # self.on_trait_change(self._sync_ips, 'ip')
         # self.on_trait_change(self._sync_transports, 'transport')
         self.subconstructors.append(self.construct_hub)
     
     
     def construct(self):
         assert not self._constructed, "already constructed!"
         
         for subc in self.subconstructors:
             subc()
         
         self._constructed = True
         
     
     def start(self):
         assert self._constructed, "must be constructed by self.construct() first!"
         self.heartmonitor.start()
         self.log.info("Heartmonitor started")
     
     def construct_hub(self):
         """construct"""
         client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i"
         engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i"
         
         ctx = self.context
         loop = self.loop
         
         # Registrar socket
         q = ZMQStream(ctx.socket(zmq.XREP), loop)
         q.bind(client_iface % self.regport)
         self.log.info("Hub listening on %s for registration."%(client_iface%self.regport))
         if self.client_ip != self.engine_ip:
             q.bind(engine_iface % self.regport)
             self.log.info("Hub listening on %s for registration."%(engine_iface%self.regport))
         
         ### Engine connections ###
 
         # heartbeat
         hpub = ctx.socket(zmq.PUB)
         hpub.bind(engine_iface % self.hb[0])
         hrep = ctx.socket(zmq.XREP)
         hrep.bind(engine_iface % self.hb[1])
         self.heartmonitor = HeartMonitor(loop=loop, pingstream=ZMQStream(hpub,loop), pongstream=ZMQStream(hrep,loop), 
                                 period=self.ping, logname=self.log.name)
 
         ### Client connections ###
         # Notifier socket
         n = ZMQStream(ctx.socket(zmq.PUB), loop)
         n.bind(client_iface%self.notifier_port)
 
         ### build and launch the queues ###
 
         # monitor socket
         sub = ctx.socket(zmq.SUB)
         sub.setsockopt(zmq.SUBSCRIBE, "")
         sub.bind(self.monitor_url)
         sub.bind('inproc://monitor')
         sub = ZMQStream(sub, loop)
         
         # connect the db
         self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1]))
         # cdir = self.config.Global.cluster_dir
         self.db = import_item(self.db_class)(session=self.session.session, config=self.config)
         time.sleep(.25)
 
         # build connection dicts
         self.engine_info = {
             'control' : engine_iface%self.control[1],
             'mux': engine_iface%self.mux[1],
             'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),
             'task' : engine_iface%self.task[1],
             'iopub' : engine_iface%self.iopub[1],
             # 'monitor' : engine_iface%self.mon_port,
             }
 
         self.client_info = {
             'control' : client_iface%self.control[0],
             'mux': client_iface%self.mux[0],
             'task' : (self.scheme, client_iface%self.task[0]),
             'iopub' : client_iface%self.iopub[0],
             'notification': client_iface%self.notifier_port
             }
         self.log.debug("Hub engine addrs: %s"%self.engine_info)
         self.log.debug("Hub client addrs: %s"%self.client_info)
         self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                 query=q, notifier=n, db=self.db,
                 engine_info=self.engine_info, client_info=self.client_info,
                 logname=self.log.name)
     
 
 class Hub(LoggingFactory):
     """The IPython Controller Hub with 0MQ connections
     
     Parameters
     ==========
     loop: zmq IOLoop instance
     session: StreamSession object
     <removed> context: zmq context for creating new connections (?)
     queue: ZMQStream for monitoring the command queue (SUB)
     query: ZMQStream for engine registration and client queries requests (XREP)
     heartbeat: HeartMonitor object checking the pulse of the engines
     notifier: ZMQStream for broadcasting engine registration changes (PUB)
     db: connection to db for out of memory logging of commands
                 NotImplemented
     engine_info: dict of zmq connection information for engines to connect
                 to the queues.
     client_info: dict of zmq connection information for engines to connect
                 to the queues.
     """
     # internal data structures:
     ids=Set() # engine IDs
     keytable=Dict()
     by_ident=Dict()
     engines=Dict()
     clients=Dict()
     hearts=Dict()
     pending=Set()
     queues=Dict()  # pending msg_ids keyed by engine_id
     tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id
     completed=Dict() # completed msg_ids keyed by engine_id
     all_completed=Set() # completed msg_ids keyed by engine_id
     dead_engines=Set() # completed msg_ids keyed by engine_id
     # mia=None
     incoming_registrations=Dict()
     registration_timeout=Int()
     _idcounter=Int(0)
     
     # objects from constructor:
     loop=Instance(ioloop.IOLoop)
     query=Instance(ZMQStream)
     monitor=Instance(ZMQStream)
     heartmonitor=Instance(HeartMonitor)
     notifier=Instance(ZMQStream)
     db=Instance(object)
     client_info=Dict()
     engine_info=Dict()
     
     
     def __init__(self, **kwargs):
         """
         # universal:
         loop: IOLoop for creating future connections
         session: streamsession for sending serialized data
         # engine:
         queue: ZMQStream for monitoring queue messages
         query: ZMQStream for engine+client registration and client requests
         heartbeat: HeartMonitor object for tracking engines
         # extra:
         db: ZMQStream for db connection (NotImplemented)
         engine_info: zmq address/protocol dict for engine connections
         client_info: zmq address/protocol dict for client connections
         """
         
         super(Hub, self).__init__(**kwargs)
         self.registration_timeout = max(5000, 2*self.heartmonitor.period)
         
         # validate connection dicts:
         for k,v in self.client_info.iteritems():
             if k == 'task':
                 validate_url_container(v[1])
             else:
                 validate_url_container(v)
         # validate_url_container(self.client_info)
         validate_url_container(self.engine_info)
         
         # register our callbacks
         self.query.on_recv(self.dispatch_query)
         self.monitor.on_recv(self.dispatch_monitor_traffic)
         
         self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure)
         self.heartmonitor.add_new_heart_handler(self.handle_new_heart)
         
         self.monitor_handlers = { 'in' : self.save_queue_request,
                                 'out': self.save_queue_result,
                                 'intask': self.save_task_request,
                                 'outtask': self.save_task_result,
                                 'tracktask': self.save_task_destination,
                                 'incontrol': _passer,
                                 'outcontrol': _passer,
                                 'iopub': self.save_iopub_message,
         }
         
         self.query_handlers = {'queue_request': self.queue_status,
                                 'result_request': self.get_results,
                                 'purge_request': self.purge_results,
                                 'load_request': self.check_load,
                                 'resubmit_request': self.resubmit_task,
                                 'shutdown_request': self.shutdown_request,
                                 'registration_request' : self.register_engine,
                                 'unregistration_request' : self.unregister_engine,
                                 'connection_request': self.connection_request,
         }
         
         self.log.info("hub::created hub")
     
     @property
     def _next_id(self):
         """gemerate a new ID.
         
         No longer reuse old ids, just count from 0."""
         newid = self._idcounter
         self._idcounter += 1
         return newid
         # newid = 0
         # incoming = [id[0] for id in self.incoming_registrations.itervalues()]
         # # print newid, self.ids, self.incoming_registrations
         # while newid in self.ids or newid in incoming:
         #     newid += 1
         # return newid
     
     #-----------------------------------------------------------------------------
     # message validation
     #-----------------------------------------------------------------------------
     
     def _validate_targets(self, targets):
         """turn any valid targets argument into a list of integer ids"""
         if targets is None:
             # default to all
             targets = self.ids
             
         if isinstance(targets, (int,str,unicode)):
             # only one target specified
             targets = [targets]
         _targets = []
         for t in targets:
             # map raw identities to ids
             if isinstance(t, (str,unicode)):
                 t = self.by_ident.get(t, t)
             _targets.append(t)
         targets = _targets
         bad_targets = [ t for t in targets if t not in self.ids ]
         if bad_targets:
             raise IndexError("No Such Engine: %r"%bad_targets)
         if not targets:
             raise IndexError("No Engines Registered")
         return targets
     
     #-----------------------------------------------------------------------------
     # dispatch methods (1 per stream)
     #-----------------------------------------------------------------------------
     
     # def dispatch_registration_request(self, msg):
     #     """"""
     #     self.log.debug("registration::dispatch_register_request(%s)"%msg)
     #     idents,msg = self.session.feed_identities(msg)
     #     if not idents:
     #         self.log.error("Bad Query Message: %s"%msg, exc_info=True)
     #         return
     #     try:
     #         msg = self.session.unpack_message(msg,content=True)
     #     except:
     #         self.log.error("registration::got bad registration message: %s"%msg, exc_info=True)
     #         return
     #     
     #     msg_type = msg['msg_type']
     #     content = msg['content']
     #     
     #     handler = self.query_handlers.get(msg_type, None)
     #     if handler is None:
     #         self.log.error("registration::got bad registration message: %s"%msg)
     #     else:
     #         handler(idents, msg)
     
     def dispatch_monitor_traffic(self, msg):
         """all ME and Task queue messages come through here, as well as
         IOPub traffic."""
         self.log.debug("monitor traffic: %s"%msg[:2])
         switch = msg[0]
         idents, msg = self.session.feed_identities(msg[1:])
         if not idents:
             self.log.error("Bad Monitor Message: %s"%msg)
             return
         handler = self.monitor_handlers.get(switch, None)
         if handler is not None:
             handler(idents, msg)
         else:
             self.log.error("Invalid monitor topic: %s"%switch)
         
     
     def dispatch_query(self, msg):
         """Route registration requests and queries from clients."""
         idents, msg = self.session.feed_identities(msg)
         if not idents:
             self.log.error("Bad Query Message: %s"%msg)
             return
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             content = error.wrap_exception()
             self.log.error("Bad Query Message: %s"%msg, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id, 
                     content=content)
             return
         
         # print client_id, header, parent, content
         #switch on message type:
         msg_type = msg['msg_type']
         self.log.info("client::client %s requested %s"%(client_id, msg_type))
         handler = self.query_handlers.get(msg_type, None)
         try:
             assert handler is not None, "Bad Message Type: %s"%msg_type
         except:
             content = error.wrap_exception()
             self.log.error("Bad Message Type: %s"%msg_type, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id, 
                     content=content)
             return
         else:
             handler(idents, msg)
             
     def dispatch_db(self, msg):
         """"""
         raise NotImplementedError
     
     #---------------------------------------------------------------------------
     # handler methods (1 per event)
     #---------------------------------------------------------------------------
     
     #----------------------- Heartbeat --------------------------------------
     
     def handle_new_heart(self, heart):
         """handler to attach to heartbeater.
         Called when a new heart starts to beat.
         Triggers completion of registration."""
         self.log.debug("heartbeat::handle_new_heart(%r)"%heart)
         if heart not in self.incoming_registrations:
             self.log.info("heartbeat::ignoring new heart: %r"%heart)
         else:
             self.finish_registration(heart)
         
     
     def handle_heart_failure(self, heart):
         """handler to attach to heartbeater.
         called when a previously registered heart fails to respond to beat request.
         triggers unregistration"""
         self.log.debug("heartbeat::handle_heart_failure(%r)"%heart)
         eid = self.hearts.get(heart, None)
         queue = self.engines[eid].queue
         if eid is None:
             self.log.info("heartbeat::ignoring heart failure %r"%heart)
         else:
             self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))
     
     #----------------------- MUX Queue Traffic ------------------------------
     
     def save_queue_request(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %s"%idents)
             return
         queue_id, client_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("queue::client %r sent invalid message to %r: %s"%(client_id, queue_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::target %r not registered"%queue_id)
             self.log.debug("queue::    valid are: %s"%(self.by_ident.keys()))
             return
             
         header = msg['header']
         msg_id = header['msg_id']
         record = init_record(msg)
         record['engine_uuid'] = queue_id
         record['client_uuid'] = client_id
         record['queue'] = 'mux'
 
         try:
             # it's posible iopub arrived first:
             existing = self.db.get_record(msg_id)
             for key,evalue in existing.iteritems():
                 rvalue = record[key]
                 if evalue and rvalue and evalue != rvalue:
                     self.log.error("conflicting initial state for record: %s:%s <> %s"%(msg_id, rvalue, evalue))
                 elif evalue and not rvalue:
                     record[key] = evalue
             self.db.update_record(msg_id, record)
         except KeyError:
             self.db.add_record(msg_id, record)
         
         self.pending.add(msg_id)
         self.queues[eid].append(msg_id)
     
     def save_queue_result(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %s"%idents)
             return
             
         client_id, queue_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("queue::engine %r sent invalid message to %r: %s"%(
                     queue_id,client_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::unknown engine %r is sending a reply: "%queue_id)
             # self.log.debug("queue::       %s"%msg[2:])
             return
         
         parent = msg['parent_header']
         if not parent:
             return
         msg_id = parent['msg_id']
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             self.queues[eid].remove(msg_id)
             self.completed[eid].append(msg_id)
         elif msg_id not in self.all_completed:
             # it could be a result from a dead engine that died before delivering the
             # result
             self.log.warn("queue:: unknown msg finished %s"%msg_id)
             return
         # update record anyway, because the unregistration could have been premature
         rheader = msg['header']
         completed = datetime.strptime(rheader['date'], ISO8601)
         started = rheader.get('started', None)
         if started is not None:
             started = datetime.strptime(started, ISO8601)
         result = {
             'result_header' : rheader,
             'result_content': msg['content'],
             'started' : started,
             'completed' : completed
         }
 
         result['result_buffers'] = msg['buffers']
         self.db.update_record(msg_id, result)
         
             
     #--------------------- Task Queue Traffic ------------------------------
     
     def save_task_request(self, idents, msg):
         """Save the submission of a task."""
         client_id = idents[0]
         
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("task::client %r sent invalid task message: %s"%(
                     client_id, msg), exc_info=True)
             return
         record = init_record(msg)
 
         record['client_uuid'] = client_id
         record['queue'] = 'task'
         header = msg['header']
         msg_id = header['msg_id']
         self.pending.add(msg_id)
         try:
             # it's posible iopub arrived first:
             existing = self.db.get_record(msg_id)
             for key,evalue in existing.iteritems():
                 rvalue = record[key]
                 if evalue and rvalue and evalue != rvalue:
                     self.log.error("conflicting initial state for record: %s:%s <> %s"%(msg_id, rvalue, evalue))
                 elif evalue and not rvalue:
                     record[key] = evalue
             self.db.update_record(msg_id, record)
         except KeyError:
             self.db.add_record(msg_id, record)
     
     def save_task_result(self, idents, msg):
         """save the result of a completed task."""
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("task::invalid task result message send to %r: %s"%(
                     client_id, msg), exc_info=True)
             raise
             return
         
         parent = msg['parent_header']
         if not parent:
             # print msg
             self.log.warn("Task %r had no parent!"%msg)
             return
         msg_id = parent['msg_id']
         
         header = msg['header']
         engine_uuid = header.get('engine', None)
         eid = self.by_ident.get(engine_uuid, None)
         
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             if eid is not None:
                 self.completed[eid].append(msg_id)
                 if msg_id in self.tasks[eid]:
                     self.tasks[eid].remove(msg_id)
             completed = datetime.strptime(header['date'], ISO8601)
             started = header.get('started', None)
             if started is not None:
                 started = datetime.strptime(started, ISO8601)
             result = {
                 'result_header' : header,
                 'result_content': msg['content'],
                 'started' : started,
                 'completed' : completed,
                 'engine_uuid': engine_uuid
             }
 
             result['result_buffers'] = msg['buffers']
             self.db.update_record(msg_id, result)
             
         else:
             self.log.debug("task::unknown task %s finished"%msg_id)
     
     def save_task_destination(self, idents, msg):
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             self.log.error("task::invalid task tracking message", exc_info=True)
             return
         content = msg['content']
         # print (content)
         msg_id = content['msg_id']
         engine_uuid = content['engine_id']
         eid = self.by_ident[engine_uuid]
         
         self.log.info("task::task %s arrived on %s"%(msg_id, eid))
         # if msg_id in self.mia:
         #     self.mia.remove(msg_id)
         # else:
         #     self.log.debug("task::task %s not listed as MIA?!"%(msg_id))
         
         self.tasks[eid].append(msg_id)
         # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
         self.db.update_record(msg_id, dict(engine_uuid=engine_uuid))
     
     def mia_task_request(self, idents, msg):
         raise NotImplementedError
         client_id = idents[0]
         # content = dict(mia=self.mia,status='ok')
         # self.session.send('mia_reply', content=content, idents=client_id)
         
         
     #--------------------- IOPub Traffic ------------------------------
     
     def save_iopub_message(self, topics, msg):
         """save an iopub message into the db"""
         # print (topics)
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             self.log.error("iopub::invalid IOPub message", exc_info=True)
             return
         
         parent = msg['parent_header']
         if not parent:
             self.log.error("iopub::invalid IOPub message: %s"%msg)
             return
         msg_id = parent['msg_id']
         msg_type = msg['msg_type']
         content = msg['content']
         
         # ensure msg_id is in db
         try:
             rec = self.db.get_record(msg_id)
         except KeyError:
             rec = empty_record()
             rec['msg_id'] = msg_id
             self.db.add_record(msg_id, rec)
         # stream
         d = {}
         if msg_type == 'stream':
             name = content['name']
             s = rec[name] or ''
             d[name] = s + content['data']
             
         elif msg_type == 'pyerr':
             d['pyerr'] = content
         else:
             d[msg_type] = content['data']
         
         self.db.update_record(msg_id, d)
         
     
             
     #-------------------------------------------------------------------------
     # Registration requests
     #-------------------------------------------------------------------------
         
     def connection_request(self, client_id, msg):
         """Reply with connection addresses for clients."""
         self.log.info("client::client %s connected"%client_id)
         content = dict(status='ok')
         content.update(self.client_info)
         jsonable = {}
         for k,v in self.keytable.iteritems():
             if v not in self.dead_engines:
                 jsonable[str(k)] = v
         content['engines'] = jsonable
         self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)
     
     def register_engine(self, reg, msg):
         """Register a new engine."""
         content = msg['content']
         try:
             queue = content['queue']
         except KeyError:
             self.log.error("registration::queue not specified", exc_info=True)
             return
         heart = content.get('heartbeat', None)
         """register a new engine, and create the socket(s) necessary"""
         eid = self._next_id
         # print (eid, queue, reg, heart)
         
         self.log.debug("registration::register_engine(%i, %r, %r, %r)"%(eid, queue, reg, heart))
         
         content = dict(id=eid,status='ok')
         content.update(self.engine_info)
         # check if requesting available IDs:
         if queue in self.by_ident:
             try:
                 raise KeyError("queue_id %r in use"%queue)
             except:
                 content = error.wrap_exception()
                 self.log.error("queue_id %r in use"%queue, exc_info=True)
         elif heart in self.hearts: # need to check unique hearts?
             try:
                 raise KeyError("heart_id %r in use"%heart)
             except:
                 self.log.error("heart_id %r in use"%heart, exc_info=True)
                 content = error.wrap_exception()
         else:
             for h, pack in self.incoming_registrations.iteritems():
                 if heart == h:
                     try:
                         raise KeyError("heart_id %r in use"%heart)
                     except:
                         self.log.error("heart_id %r in use"%heart, exc_info=True)
                         content = error.wrap_exception()
                     break
                 elif queue == pack[1]:
                     try:
                         raise KeyError("queue_id %r in use"%queue)
                     except:
                         self.log.error("queue_id %r in use"%queue, exc_info=True)
                         content = error.wrap_exception()
                     break
         
         msg = self.session.send(self.query, "registration_reply", 
                 content=content, 
                 ident=reg)
         
         if content['status'] == 'ok':
             if heart in self.heartmonitor.hearts:
                 # already beating
                 self.incoming_registrations[heart] = (eid,queue,reg[0],None)
                 self.finish_registration(heart)
             else:
                 purge = lambda : self._purge_stalled_registration(heart)
                 dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                 dc.start()
                 self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
         else:
             self.log.error("registration::registration %i failed: %s"%(eid, content['evalue']))
         return eid
     
     def unregister_engine(self, ident, msg):
         """Unregister an engine that explicitly requested to leave."""
         try:
             eid = msg['content']['id']
         except:
             self.log.error("registration::bad engine id for unregistration: %s"%ident, exc_info=True)
             return
         self.log.info("registration::unregister_engine(%s)"%eid)
         # print (eid)
         uuid = self.keytable[eid]
         content=dict(id=eid, queue=uuid)
         self.dead_engines.add(uuid)
         # self.ids.remove(eid)
         # uuid = self.keytable.pop(eid)
         # 
         # ec = self.engines.pop(eid)
         # self.hearts.pop(ec.heartbeat)
         # self.by_ident.pop(ec.queue)
         # self.completed.pop(eid)
         handleit = lambda : self._handle_stranded_msgs(eid, uuid)
         dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop)
         dc.start()
         ############## TODO: HANDLE IT ################
         
         if self.notifier:
             self.session.send(self.notifier, "unregistration_notification", content=content)
     
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
         
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         that the result failed and later receive the actual result.
         """
         
         outstanding = self.queues[eid]
         
         for msg_id in outstanding:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake header:
             header = {}
             header['engine'] = uuid
             header['date'] = datetime.now().strftime(ISO8601)
             rec = dict(result_content=content, result_header=header, result_buffers=[])
             rec['completed'] = header['date']
             rec['engine_uuid'] = uuid
             self.db.update_record(msg_id, rec)
     
     def finish_registration(self, heart):
         """Second half of engine registration, called after our HeartMonitor
         has received a beat from the Engine's Heart."""
         try: 
             (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
         except KeyError:
             self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
             return
         self.log.info("registration::finished registering engine %i:%r"%(eid,queue))
         if purge is not None:
             purge.stop()
         control = queue
         self.ids.add(eid)
         self.keytable[eid] = queue
         self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg, 
                                     control=control, heartbeat=heart)
         self.by_ident[queue] = eid
         self.queues[eid] = list()
         self.tasks[eid] = list()
         self.completed[eid] = list()
         self.hearts[heart] = eid
         content = dict(id=eid, queue=self.engines[eid].queue)
         if self.notifier:
             self.session.send(self.notifier, "registration_notification", content=content)
         self.log.info("engine::Engine Connected: %i"%eid)
     
     def _purge_stalled_registration(self, heart):
         if heart in self.incoming_registrations:
             eid = self.incoming_registrations.pop(heart)[0]
             self.log.info("registration::purging stalled registration: %i"%eid)
         else:
             pass
             
     #-------------------------------------------------------------------------
     # Client Requests
     #-------------------------------------------------------------------------
     
     def shutdown_request(self, client_id, msg):
         """handle shutdown request."""
         self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
         # also notify other clients of shutdown
         self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
         dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
         dc.start()
     
     def _shutdown(self):
         self.log.info("hub::hub shutting down.")
         time.sleep(0.1)
         sys.exit(0)
         
     
     def check_load(self, client_id, msg):
         content = msg['content']
         try:
             targets = content['targets']
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error", 
                     content=content, ident=client_id)
             return
         
         content = dict(status='ok')
         # loads = {}
         for t in targets:
             content[bytes(t)] = len(self.queues[t])+len(self.tasks[t])
         self.session.send(self.query, "load_reply", content=content, ident=client_id)
             
     
     def queue_status(self, client_id, msg):
         """Return the Queue status of one or more targets.
         if verbose: return the msg_ids
         else: return len of each type.
         keys: queue (pending MUX jobs)
             tasks (pending Task jobs)
             completed (finished jobs from both queues)"""
         content = msg['content']
         targets = content['targets']
         try:
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error", 
                     content=content, ident=client_id)
             return
         verbose = content.get('verbose', False)
         content = dict(status='ok')
         for t in targets:
             queue = self.queues[t]
             completed = self.completed[t]
             tasks = self.tasks[t]
             if not verbose:
                 queue = len(queue)
                 completed = len(completed)
                 tasks = len(tasks)
             content[bytes(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
             # pending
         self.session.send(self.query, "queue_reply", content=content, ident=client_id)
     
     def purge_results(self, client_id, msg):
         """Purge results from memory. This method is more valuable before we move
         to a DB based message storage mechanism."""
         content = msg['content']
         msg_ids = content.get('msg_ids', [])
         reply = dict(status='ok')
         if msg_ids == 'all':
             self.db.drop_matching_records(dict(completed={'$ne':None}))
         else:
             for msg_id in msg_ids:
                 if msg_id in self.all_completed:
                     self.db.drop_record(msg_id)
                 else:
                     if msg_id in self.pending:
                         try:
                             raise IndexError("msg pending: %r"%msg_id)
                         except:
                             reply = error.wrap_exception()
                     else:
                         try:
                             raise IndexError("No such msg: %r"%msg_id)
                         except:
                             reply = error.wrap_exception()
                     break
             eids = content.get('engine_ids', [])
             for eid in eids:
                 if eid not in self.engines:
                     try:
                         raise IndexError("No such engine: %i"%eid)
                     except:
                         reply = error.wrap_exception()
                     break
                 msg_ids = self.completed.pop(eid)
                 uid = self.engines[eid].queue
                 self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
         
         self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)
     
     def resubmit_task(self, client_id, msg, buffers):
         """Resubmit a task."""
         raise NotImplementedError
     
     def get_results(self, client_id, msg):
         """Get the result of 1 or more messages."""
         content = msg['content']
         msg_ids = sorted(set(content['msg_ids']))
         statusonly = content.get('status_only', False)
         pending = []
         completed = []
         content = dict(status='ok')
         content['pending'] = pending
         content['completed'] = completed
         buffers = []
         if not statusonly:
             content['results'] = {}
             records = self.db.find_records(dict(msg_id={'$in':msg_ids}))
         for msg_id in msg_ids:
             if msg_id in self.pending:
                 pending.append(msg_id)
             elif msg_id in self.all_completed:
                 completed.append(msg_id)
                 if not statusonly:
                     rec = records[msg_id]
                     io_dict = {}
                     for key in 'pyin pyout pyerr stdout stderr'.split():
                             io_dict[key] = rec[key]
                     content[msg_id] = { 'result_content': rec['result_content'],
                                         'header': rec['header'],
                                         'result_header' : rec['result_header'],
                                         'io' : io_dict,
                                       }
                     if rec['result_buffers']:
                         buffers.extend(map(str, rec['result_buffers']))
             else:
                 try:
                     raise KeyError('No such message: '+msg_id)
                 except:
                     content = error.wrap_exception()
                 break
         self.session.send(self.query, "result_reply", content=content, 
                                             parent=msg, ident=client_id,
                                             buffers=buffers)
 

IPython/parallel/controller/scheduler.py

@@ -1,592 +1,592 b''
 """The Python scheduler for rich scheduling.
 
 The Pure ZMQ scheduler does not allow routing schemes other than LRU,
 nor does it check msg_id DAG dependencies. For those, a slightly slower
 Python Scheduler exists.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #----------------------------------------------------------------------
 # Imports
 #----------------------------------------------------------------------
 
 from __future__ import print_function
 
 import logging
 import sys
 
 from datetime import datetime, timedelta
 from random import randint, random
 from types import FunctionType
 
 try:
     import numpy
 except ImportError:
     numpy = None
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # local imports
 from IPython.external.decorator import decorator
 from IPython.utils.traitlets import Instance, Dict, List, Set
 
-from . import error
-from .dependency import Dependency
-from .entry_point import connect_logger, local_logger
-from .factory import SessionFactory
+from IPython.parallel import error
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import connect_logger, local_logger
 
+from .dependency import Dependency
 
 @decorator
 def logged(f,self,*args,**kwargs):
     # print ("#--------------------")
     self.log.debug("scheduler::%s(*%s,**%s)"%(f.func_name, args, kwargs))
     # print ("#--")
     return f(self,*args, **kwargs)
 
 #----------------------------------------------------------------------
 # Chooser functions
 #----------------------------------------------------------------------
 
 def plainrandom(loads):
     """Plain random pick."""
     n = len(loads)
     return randint(0,n-1)
 
 def lru(loads):
     """Always pick the front of the line.
     
     The content of `loads` is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     return 0
 
 def twobin(loads):
     """Pick two at random, use the LRU of the two.
     
     The content of loads is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     n = len(loads)
     a = randint(0,n-1)
     b = randint(0,n-1)
     return min(a,b)
 
 def weighted(loads):
     """Pick two at random using inverse load as weight.
     
     Return the less loaded of the two.
     """
     # weight 0 a million times more than 1:
     weights = 1./(1e-6+numpy.array(loads))
     sums = weights.cumsum()
     t = sums[-1]
     x = random()*t
     y = random()*t
     idx = 0
     idy = 0
     while sums[idx] < x:
         idx += 1
     while sums[idy] < y:
         idy += 1
     if weights[idy] > weights[idx]:
         return idy
     else:
         return idx
 
 def leastload(loads):
     """Always choose the lowest load.
     
     If the lowest load occurs more than once, the first
     occurance will be used.  If loads has LRU ordering, this means
     the LRU of those with the lowest load is chosen.
     """
     return loads.index(min(loads))
 
 #---------------------------------------------------------------------
 # Classes
 #---------------------------------------------------------------------
 # store empty default dependency:
 MET = Dependency([])
 
 class TaskScheduler(SessionFactory):
     """Python TaskScheduler object.
     
     This is the simplest object that supports msg_id based
     DAG dependencies. *Only* task msg_ids are checked, not
     msg_ids of jobs submitted via the MUX queue.
     
     """
     
     # input arguments:
     scheme = Instance(FunctionType, default=leastload) # function for determining the destination
     client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
     engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
     notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
     mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream
     
     # internals:
     graph = Dict() # dict by msg_id of [ msg_ids that depend on key ]
     depending = Dict() # dict by msg_id of (msg_id, raw_msg, after, follow)
     pending = Dict() # dict by engine_uuid of submitted tasks
     completed = Dict() # dict by engine_uuid of completed tasks
     failed = Dict() # dict by engine_uuid of failed tasks
     destinations = Dict() # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed)
     clients = Dict() # dict by msg_id for who submitted the task
     targets = List() # list of target IDENTs
     loads = List() # list of engine loads
     all_completed = Set() # set of all completed tasks
     all_failed = Set() # set of all failed tasks
     all_done = Set() # set of all finished tasks=union(completed,failed)
     all_ids = Set() # set of all submitted task IDs
     blacklist = Dict() # dict by msg_id of locations where a job has encountered UnmetDependency
     auditor = Instance('zmq.eventloop.ioloop.PeriodicCallback')
     
     
     def start(self):
         self.engine_stream.on_recv(self.dispatch_result, copy=False)
         self._notification_handlers = dict(
             registration_notification = self._register_engine,
             unregistration_notification = self._unregister_engine
         )
         self.notifier_stream.on_recv(self.dispatch_notification)
         self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz
         self.auditor.start()
         self.log.info("Scheduler started...%r"%self)
     
     def resume_receiving(self):
         """Resume accepting jobs."""
         self.client_stream.on_recv(self.dispatch_submission, copy=False)
     
     def stop_receiving(self):
         """Stop accepting jobs while there are no engines.
         Leave them in the ZMQ queue."""
         self.client_stream.on_recv(None)
     
     #-----------------------------------------------------------------------
     # [Un]Registration Handling
     #-----------------------------------------------------------------------
     
     def dispatch_notification(self, msg):
         """dispatch register/unregister events."""
         idents,msg = self.session.feed_identities(msg)
         msg = self.session.unpack_message(msg)
         msg_type = msg['msg_type']
         handler = self._notification_handlers.get(msg_type, None)
         if handler is None:
             raise Exception("Unhandled message type: %s"%msg_type)
         else:
             try:
                 handler(str(msg['content']['queue']))
             except KeyError:
                 self.log.error("task::Invalid notification msg: %s"%msg)
     
     @logged
     def _register_engine(self, uid):
         """New engine with ident `uid` became available."""
         # head of the line:
         self.targets.insert(0,uid)
         self.loads.insert(0,0)
         # initialize sets
         self.completed[uid] = set()
         self.failed[uid] = set()
         self.pending[uid] = {}
         if len(self.targets) == 1:
             self.resume_receiving()
 
     def _unregister_engine(self, uid):
         """Existing engine with ident `uid` became unavailable."""
         if len(self.targets) == 1:
             # this was our only engine
             self.stop_receiving()
         
         # handle any potentially finished tasks:
         self.engine_stream.flush()
         
         self.completed.pop(uid)
         self.failed.pop(uid)
         # don't pop destinations, because it might be used later
         # map(self.destinations.pop, self.completed.pop(uid))
         # map(self.destinations.pop, self.failed.pop(uid))
         
         idx = self.targets.index(uid)
         self.targets.pop(idx)
         self.loads.pop(idx)
         
         # wait 5 seconds before cleaning up pending jobs, since the results might
         # still be incoming
         if self.pending[uid]:
             dc = ioloop.DelayedCallback(lambda : self.handle_stranded_tasks(uid), 5000, self.loop)
             dc.start()
     
     @logged
     def handle_stranded_tasks(self, engine):
         """Deal with jobs resident in an engine that died."""
         lost = self.pending.pop(engine)
         
         for msg_id, (raw_msg, targets, MET, follow, timeout) in lost.iteritems():
             self.all_failed.add(msg_id)
             self.all_done.add(msg_id)
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, copy=False, content=False)
             parent = msg['header']
             idents = [idents[0],engine]+idents[1:]
             # print (idents)
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(engine, msg_id))
             except:
                 content = error.wrap_exception()
             msg = self.session.send(self.client_stream, 'apply_reply', content, 
                                                     parent=parent, ident=idents)
             self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
             self.update_graph(msg_id)
     
     
     #-----------------------------------------------------------------------
     # Job Submission
     #-----------------------------------------------------------------------
     @logged
     def dispatch_submission(self, raw_msg):
         """Dispatch job submission to appropriate handlers."""
         # ensure targets up to date:
         self.notifier_stream.flush()
         try:
             idents, msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, content=False, copy=False)
         except:
             self.log.error("task::Invaid task: %s"%raw_msg, exc_info=True)
             return
         
         # send to monitor
         self.mon_stream.send_multipart(['intask']+raw_msg, copy=False)
         
         header = msg['header']
         msg_id = header['msg_id']
         self.all_ids.add(msg_id)
         
         # targets
         targets = set(header.get('targets', []))
         
         # time dependencies
         after = Dependency(header.get('after', []))
         if after.all:
             if after.success:
                 after.difference_update(self.all_completed)
             if after.failure:
                 after.difference_update(self.all_failed)
         if after.check(self.all_completed, self.all_failed):
             # recast as empty set, if `after` already met,
             # to prevent unnecessary set comparisons
             after = MET
         
         # location dependencies
         follow = Dependency(header.get('follow', []))
         
         # turn timeouts into datetime objects:
         timeout = header.get('timeout', None)
         if timeout:
             timeout = datetime.now() + timedelta(0,timeout,0)
         
         args = [raw_msg, targets, after, follow, timeout]
         
         # validate and reduce dependencies:
         for dep in after,follow:
             # check valid:
             if msg_id in dep or dep.difference(self.all_ids):
                 self.depending[msg_id] = args
                 return self.fail_unreachable(msg_id, error.InvalidDependency)
             # check if unreachable:
             if dep.unreachable(self.all_completed, self.all_failed):
                 self.depending[msg_id] = args
                 return self.fail_unreachable(msg_id)
         
         if after.check(self.all_completed, self.all_failed):
             # time deps already met, try to run
             if not self.maybe_run(msg_id, *args):
                 # can't run yet
                 self.save_unmet(msg_id, *args)
         else:
             self.save_unmet(msg_id, *args)
     
     # @logged
     def audit_timeouts(self):
         """Audit all waiting tasks for expired timeouts."""
         now = datetime.now()
         for msg_id in self.depending.keys():
             # must recheck, in case one failure cascaded to another:
             if msg_id in self.depending:
                 raw,after,targets,follow,timeout = self.depending[msg_id]
                 if timeout and timeout < now:
                     self.fail_unreachable(msg_id, timeout=True)
                 
     @logged
     def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
         """a task has become unreachable, send a reply with an ImpossibleDependency
         error."""
         if msg_id not in self.depending:
             self.log.error("msg %r already failed!"%msg_id)
             return
         raw_msg,targets,after,follow,timeout = self.depending.pop(msg_id)
         for mid in follow.union(after):
             if mid in self.graph:
                 self.graph[mid].remove(msg_id)
         
         # FIXME: unpacking a message I've already unpacked, but didn't save:
         idents,msg = self.session.feed_identities(raw_msg, copy=False)
         msg = self.session.unpack_message(msg, copy=False, content=False)
         header = msg['header']
         
         try:
             raise why()
         except:
             content = error.wrap_exception()
         
         self.all_done.add(msg_id)
         self.all_failed.add(msg_id)
         
         msg = self.session.send(self.client_stream, 'apply_reply', content, 
                                                 parent=header, ident=idents)
         self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
         
         self.update_graph(msg_id, success=False)
     
     @logged
     def maybe_run(self, msg_id, raw_msg, targets, after, follow, timeout):
         """check location dependencies, and run if they are met."""
         blacklist = self.blacklist.setdefault(msg_id, set())
         if follow or targets or blacklist:
             # we need a can_run filter
             def can_run(idx):
                 target = self.targets[idx]
                 # check targets
                 if targets and target not in targets:
                     return False
                 # check blacklist
                 if target in blacklist:
                     return False
                 # check follow
                 return follow.check(self.completed[target], self.failed[target])
             
             indices = filter(can_run, range(len(self.targets)))
             if not indices:
                 # couldn't run
                 if follow.all:
                     # check follow for impossibility
                     dests = set()
                     relevant = set()
                     if follow.success:
                         relevant = self.all_completed
                     if follow.failure:
                         relevant = relevant.union(self.all_failed)
                     for m in follow.intersection(relevant):
                         dests.add(self.destinations[m])
                     if len(dests) > 1:
                         self.fail_unreachable(msg_id)
                         return False
                 if targets:
                     # check blacklist+targets for impossibility
                     targets.difference_update(blacklist)
                     if not targets or not targets.intersection(self.targets):
                         self.fail_unreachable(msg_id)
                         return False
                 return False
         else:
             indices = None
             
         self.submit_task(msg_id, raw_msg, targets, follow, timeout, indices)
         return True
             
     @logged
     def save_unmet(self, msg_id, raw_msg, targets, after, follow, timeout):
         """Save a message for later submission when its dependencies are met."""
         self.depending[msg_id] = [raw_msg,targets,after,follow,timeout]
         # track the ids in follow or after, but not those already finished
         for dep_id in after.union(follow).difference(self.all_done):
             if dep_id not in self.graph:
                 self.graph[dep_id] = set()
             self.graph[dep_id].add(msg_id)
     
     @logged
     def submit_task(self, msg_id, raw_msg, targets, follow, timeout, indices=None):
         """Submit a task to any of a subset of our targets."""
         if indices:
             loads = [self.loads[i] for i in indices]
         else:
             loads = self.loads
         idx = self.scheme(loads)
         if indices:
             idx = indices[idx]
         target = self.targets[idx]
         # print (target, map(str, msg[:3]))
         self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
         self.engine_stream.send_multipart(raw_msg, copy=False)
         self.add_job(idx)
         self.pending[target][msg_id] = (raw_msg, targets, MET, follow, timeout)
         content = dict(msg_id=msg_id, engine_id=target)
         self.session.send(self.mon_stream, 'task_destination', content=content, 
                         ident=['tracktask',self.session.session])
     
     #-----------------------------------------------------------------------
     # Result Handling
     #-----------------------------------------------------------------------
     @logged
     def dispatch_result(self, raw_msg):
         """dispatch method for result replies"""
         try:
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, content=False, copy=False)
         except:
             self.log.error("task::Invaid result: %s"%raw_msg, exc_info=True)
             return
         
         header = msg['header']
         if header.get('dependencies_met', True):
             success = (header['status'] == 'ok')
             self.handle_result(idents, msg['parent_header'], raw_msg, success)
             # send to Hub monitor
             self.mon_stream.send_multipart(['outtask']+raw_msg, copy=False)
         else:
             self.handle_unmet_dependency(idents, msg['parent_header'])
         
     @logged
     def handle_result(self, idents, parent, raw_msg, success=True):
         """handle a real task result, either success or failure"""
         # first, relay result to client
         engine = idents[0]
         client = idents[1]
         # swap_ids for XREP-XREP mirror
         raw_msg[:2] = [client,engine]
         # print (map(str, raw_msg[:4]))
         self.client_stream.send_multipart(raw_msg, copy=False)
         # now, update our data structures
         msg_id = parent['msg_id']
         self.blacklist.pop(msg_id, None)
         self.pending[engine].pop(msg_id)
         if success:
             self.completed[engine].add(msg_id)
             self.all_completed.add(msg_id)
         else:
             self.failed[engine].add(msg_id)
             self.all_failed.add(msg_id)
         self.all_done.add(msg_id)
         self.destinations[msg_id] = engine
         
         self.update_graph(msg_id, success)
         
     @logged
     def handle_unmet_dependency(self, idents, parent):
         """handle an unmet dependency"""
         engine = idents[0]
         msg_id = parent['msg_id']
         
         if msg_id not in self.blacklist:
             self.blacklist[msg_id] = set()
         self.blacklist[msg_id].add(engine)
         
         args = self.pending[engine].pop(msg_id)
         raw,targets,after,follow,timeout = args
         
         if self.blacklist[msg_id] == targets:
             self.depending[msg_id] = args
             return self.fail_unreachable(msg_id)
         
         elif not self.maybe_run(msg_id, *args):
             # resubmit failed, put it back in our dependency tree
             self.save_unmet(msg_id, *args)
         
     
     @logged
     def update_graph(self, dep_id, success=True):
         """dep_id just finished. Update our dependency
         graph and submit any jobs that just became runable."""
         # print ("\n\n***********")
         # pprint (dep_id)
         # pprint (self.graph)
         # pprint (self.depending)
         # pprint (self.all_completed)
         # pprint (self.all_failed)
         # print ("\n\n***********\n\n")
         if dep_id not in self.graph:
             return
         jobs = self.graph.pop(dep_id)
         
         for msg_id in jobs:
             raw_msg, targets, after, follow, timeout = self.depending[msg_id]
             
             if after.unreachable(self.all_completed, self.all_failed) or follow.unreachable(self.all_completed, self.all_failed):
                 self.fail_unreachable(msg_id)
             
             elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run
                 if self.maybe_run(msg_id, raw_msg, targets, MET, follow, timeout):
                     
                     self.depending.pop(msg_id)
                     for mid in follow.union(after):
                         if mid in self.graph:
                             self.graph[mid].remove(msg_id)
     
     #----------------------------------------------------------------------
     # methods to be overridden by subclasses
     #----------------------------------------------------------------------
     
     def add_job(self, idx):
         """Called after self.targets[idx] just got the job with header.
         Override with subclasses.  The default ordering is simple LRU.
         The default loads are the number of outstanding jobs."""
         self.loads[idx] += 1
         for lis in (self.targets, self.loads):
             lis.append(lis.pop(idx))
             
     
     def finish_job(self, idx):
         """Called after self.targets[idx] just finished a job.
         Override with subclasses."""
         self.loads[idx] -= 1
     
 
 
 def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
                             log_addr=None, loglevel=logging.DEBUG, scheme='lru',
                             identity=b'task'):
     from zmq.eventloop import ioloop
     from zmq.eventloop.zmqstream import ZMQStream
     
     ctx = zmq.Context()
     loop = ioloop.IOLoop()
     print (in_addr, out_addr, mon_addr, not_addr)
     ins = ZMQStream(ctx.socket(zmq.XREP),loop)
     ins.setsockopt(zmq.IDENTITY, identity)
     ins.bind(in_addr)
     
     outs = ZMQStream(ctx.socket(zmq.XREP),loop)
     outs.setsockopt(zmq.IDENTITY, identity)
     outs.bind(out_addr)
     mons = ZMQStream(ctx.socket(zmq.PUB),loop)
     mons.connect(mon_addr)
     nots = ZMQStream(ctx.socket(zmq.SUB),loop)
     nots.setsockopt(zmq.SUBSCRIBE, '')
     nots.connect(not_addr)
     
     scheme = globals().get(scheme, None)
     # setup logging
     if log_addr:
         connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
     else:
         local_logger(logname, loglevel)
     
     scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                             mon_stream=mons, notifier_stream=nots,
                             scheme=scheme, loop=loop, logname=logname,
                             config=config)
     scheduler.start()
     try:
         loop.start()
     except KeyboardInterrupt:
         print ("interrupted, exiting...", file=sys.__stderr__)
 

IPython/parallel/controller/sqlitedb.py

@@ -1,284 +1,284 b''
 """A TaskRecord backend using sqlite3"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 import json
 import os
 import cPickle as pickle
 from datetime import datetime
 
 import sqlite3
 
 from zmq.eventloop import ioloop
 
 from IPython.utils.traitlets import CUnicode, CStr, Instance, List
 from .dictdb import BaseDB
-from .util import ISO8601
+from IPython.parallel.util import ISO8601
 
 #-----------------------------------------------------------------------------
 # SQLite operators, adapters, and converters
 #-----------------------------------------------------------------------------
 
 operators = {
  '$lt' : lambda a,b: "%s < ?",
  '$gt' : ">",
  # null is handled weird with ==,!=
  '$eq' : "IS",
  '$ne' : "IS NOT",
  '$lte': "<=",
  '$gte': ">=",
  '$in' : ('IS', ' OR '),
  '$nin': ('IS NOT', ' AND '),
  # '$all': None,
  # '$mod': None,
  # '$exists' : None
 }
 
 def _adapt_datetime(dt):
     return dt.strftime(ISO8601)
 
 def _convert_datetime(ds):
     if ds is None:
         return ds
     else:
         return datetime.strptime(ds, ISO8601)
 
 def _adapt_dict(d):
     return json.dumps(d)
 
 def _convert_dict(ds):
     if ds is None:
         return ds
     else:
         return json.loads(ds)
 
 def _adapt_bufs(bufs):
     # this is *horrible*
     # copy buffers into single list and pickle it:
     if bufs and isinstance(bufs[0], (bytes, buffer)):
         return sqlite3.Binary(pickle.dumps(map(bytes, bufs),-1))
     elif bufs:
         return bufs
     else:
         return None
 
 def _convert_bufs(bs):
     if bs is None:
         return []
     else:
         return pickle.loads(bytes(bs))
 
 #-----------------------------------------------------------------------------
 # SQLiteDB class
 #-----------------------------------------------------------------------------
 
 class SQLiteDB(BaseDB):
     """SQLite3 TaskRecord backend."""
     
     filename = CUnicode('tasks.db', config=True)
     location = CUnicode('', config=True)
     table = CUnicode("", config=True)
     
     _db = Instance('sqlite3.Connection')
     _keys = List(['msg_id' ,
             'header' ,
             'content',
             'buffers',
             'submitted',
             'client_uuid' ,
             'engine_uuid' ,
             'started',
             'completed',
             'resubmitted',
             'result_header' ,
             'result_content' ,
             'result_buffers' ,
             'queue' ,
             'pyin' ,
             'pyout',
             'pyerr',
             'stdout',
             'stderr',
         ])
     
     def __init__(self, **kwargs):
         super(SQLiteDB, self).__init__(**kwargs)
         if not self.table:
             # use session, and prefix _, since starting with # is illegal
             self.table = '_'+self.session.replace('-','_')
         if not self.location:
             if hasattr(self.config.Global, 'cluster_dir'):
                 self.location = self.config.Global.cluster_dir
             else:
                 self.location = '.'
         self._init_db()
         
         # register db commit as 2s periodic callback
         # to prevent clogging pipes
         # assumes we are being run in a zmq ioloop app
         loop = ioloop.IOLoop.instance()
         pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
         pc.start()
     
     def _defaults(self):
         """create an empty record"""
         d = {}
         for key in self._keys:
             d[key] = None
         return d
     
     def _init_db(self):
         """Connect to the database and get new session number."""
         # register adapters
         sqlite3.register_adapter(datetime, _adapt_datetime)
         sqlite3.register_converter('datetime', _convert_datetime)
         sqlite3.register_adapter(dict, _adapt_dict)
         sqlite3.register_converter('dict', _convert_dict)
         sqlite3.register_adapter(list, _adapt_bufs)
         sqlite3.register_converter('bufs', _convert_bufs)
         # connect to the db
         dbfile = os.path.join(self.location, self.filename)
         self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES, 
             # isolation_level = None)#,
              cached_statements=64)
         # print dir(self._db)
         
         self._db.execute("""CREATE TABLE IF NOT EXISTS %s 
                 (msg_id text PRIMARY KEY,
                 header dict text,
                 content dict text,
                 buffers bufs blob,
                 submitted datetime text,
                 client_uuid text,
                 engine_uuid text,
                 started datetime text,
                 completed datetime text,
                 resubmitted datetime text,
                 result_header dict text,
                 result_content dict text,
                 result_buffers bufs blob,
                 queue text,
                 pyin text,
                 pyout text,
                 pyerr text,
                 stdout text,
                 stderr text)
                 """%self.table)
         # self._db.execute("""CREATE TABLE IF NOT EXISTS %s_buffers 
         #         (msg_id text, result integer, buffer blob)
         #         """%self.table)
         self._db.commit()
     
     def _dict_to_list(self, d):
         """turn a mongodb-style record dict into a list."""
         
         return [ d[key] for key in self._keys ]
     
     def _list_to_dict(self, line):
         """Inverse of dict_to_list"""
         d = self._defaults()
         for key,value in zip(self._keys, line):
             d[key] = value
         
         return d
     
     def _render_expression(self, check):
         """Turn a mongodb-style search dict into an SQL query."""
         expressions = []
         args = []
         
         skeys = set(check.keys())
         skeys.difference_update(set(self._keys))
         skeys.difference_update(set(['buffers', 'result_buffers']))
         if skeys:
             raise KeyError("Illegal testing key(s): %s"%skeys)
         
         for name,sub_check in check.iteritems():
             if isinstance(sub_check, dict):
                 for test,value in sub_check.iteritems():
                     try:
                         op = operators[test]
                     except KeyError:
                         raise KeyError("Unsupported operator: %r"%test)
                     if isinstance(op, tuple):
                         op, join = op
                     expr = "%s %s ?"%(name, op)
                     if isinstance(value, (tuple,list)):
                         expr = '( %s )'%( join.join([expr]*len(value)) )
                         args.extend(value)
                     else:
                         args.append(value)
                     expressions.append(expr)
             else:
                 # it's an equality check
                 expressions.append("%s IS ?"%name)
                 args.append(sub_check)
         
         expr = " AND ".join(expressions)
         return expr, args
     
     def add_record(self, msg_id, rec):
         """Add a new Task Record, by msg_id."""
         d = self._defaults()
         d.update(rec)
         d['msg_id'] = msg_id
         line = self._dict_to_list(d)
         tups = '(%s)'%(','.join(['?']*len(line)))
         self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
         # self._db.commit()
     
     def get_record(self, msg_id):
         """Get a specific Task Record, by msg_id."""
         cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
         line = cursor.fetchone()
         if line is None:
             raise KeyError("No such msg: %r"%msg_id)
         return self._list_to_dict(line)
     
     def update_record(self, msg_id, rec):
         """Update the data in an existing record."""
         query = "UPDATE %s SET "%self.table
         sets = []
         keys = sorted(rec.keys())
         values = []
         for key in keys:
             sets.append('%s = ?'%key)
             values.append(rec[key])
         query += ', '.join(sets)
         query += ' WHERE msg_id == %r'%msg_id
         self._db.execute(query, values)
         # self._db.commit()
     
     def drop_record(self, msg_id):
         """Remove a record from the DB."""
         self._db.execute("""DELETE FROM %s WHERE mgs_id==?"""%self.table, (msg_id,))
         # self._db.commit()
     
     def drop_matching_records(self, check):
         """Remove a record from the DB."""
         expr,args = self._render_expression(check)
         query = "DELETE FROM %s WHERE %s"%(self.table, expr)
         self._db.execute(query,args)
         # self._db.commit()
         
     def find_records(self, check, id_only=False):
         """Find records matching a query dict."""
         req = 'msg_id' if id_only else '*'
         expr,args = self._render_expression(check)
         query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
         cursor = self._db.execute(query, args)
         matches = cursor.fetchall()
         if id_only:
             return [ m[0] for m in matches ]
         else:
             records = {}
             for line in matches:
                 rec = self._list_to_dict(line)
                 records[rec['msg_id']] = rec
             return records
 
 __all__ = ['SQLiteDB']
  No newline at end of file

IPython/parallel/engine/engine.py

@@ -1,155 +1,156 b''
 #!/usr/bin/env python
 """A simple engine that talks to a controller over 0MQ.
 it handles registration, etc. and launches a kernel
 connected to the Controller's Schedulers.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import sys
 import time
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # internal
 from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat
 # from IPython.utils.localinterfaces import LOCALHOST 
 
-from . import heartmonitor
-from .factory import RegistrationFactory
+from IPython.parallel.controller.heartmonitor import Heart
+from IPython.parallel.factory import RegistrationFactory
+from IPython.parallel.streamsession import Message
+from IPython.parallel.util import disambiguate_url
+
 from .streamkernel import Kernel
-from .streamsession import Message
-from .util import disambiguate_url
 
 class EngineFactory(RegistrationFactory):
     """IPython engine"""
     
     # configurables:
     user_ns=Dict(config=True)
     out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True)
     display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True)
     location=Str(config=True)
     timeout=CFloat(2,config=True)
     
     # not configurable:
     id=Int(allow_none=True)
     registrar=Instance('zmq.eventloop.zmqstream.ZMQStream')
     kernel=Instance(Kernel)
     
     
     def __init__(self, **kwargs):
         super(EngineFactory, self).__init__(**kwargs)
         ctx = self.context
         
         reg = ctx.socket(zmq.XREQ)
         reg.setsockopt(zmq.IDENTITY, self.ident)
         reg.connect(self.url)
         self.registrar = zmqstream.ZMQStream(reg, self.loop)
         
     def register(self):
         """send the registration_request"""
         
         self.log.info("registering")
         content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident)
         self.registrar.on_recv(self.complete_registration)
         # print (self.session.key)
         self.session.send(self.registrar, "registration_request",content=content)
     
     def complete_registration(self, msg):
         # print msg
         self._abort_dc.stop()
         ctx = self.context
         loop = self.loop
         identity = self.ident
         
         idents,msg = self.session.feed_identities(msg)
         msg = Message(self.session.unpack_message(msg))
         
         if msg.content.status == 'ok':
             self.id = int(msg.content.id)
             
             # create Shell Streams (MUX, Task, etc.):
             queue_addr = msg.content.mux
             shell_addrs = [ str(queue_addr) ]
             task_addr = msg.content.task
             if task_addr:
                 shell_addrs.append(str(task_addr))
             
             # Uncomment this to go back to two-socket model
             # shell_streams = []
             # for addr in shell_addrs:
             #     stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
             #     stream.setsockopt(zmq.IDENTITY, identity)
             #     stream.connect(disambiguate_url(addr, self.location))
             #     shell_streams.append(stream)
             
             # Now use only one shell stream for mux and tasks
             stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
             stream.setsockopt(zmq.IDENTITY, identity)
             shell_streams = [stream]
             for addr in shell_addrs:
                 stream.connect(disambiguate_url(addr, self.location))
             # end single stream-socket
             
             # control stream:
             control_addr = str(msg.content.control)
             control_stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
             control_stream.setsockopt(zmq.IDENTITY, identity)
             control_stream.connect(disambiguate_url(control_addr, self.location))
             
             # create iopub stream:
             iopub_addr = msg.content.iopub
             iopub_stream = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
             iopub_stream.setsockopt(zmq.IDENTITY, identity)
             iopub_stream.connect(disambiguate_url(iopub_addr, self.location))
             
             # launch heartbeat
             hb_addrs = msg.content.heartbeat
             # print (hb_addrs)
             
             # # Redirect input streams and set a display hook.
             if self.out_stream_factory:
                 sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout')
                 sys.stdout.topic = 'engine.%i.stdout'%self.id
                 sys.stderr = self.out_stream_factory(self.session, iopub_stream, u'stderr')
                 sys.stderr.topic = 'engine.%i.stderr'%self.id
             if self.display_hook_factory:
                 sys.displayhook = self.display_hook_factory(self.session, iopub_stream)
                 sys.displayhook.topic = 'engine.%i.pyout'%self.id
             
             self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, 
                     control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, 
                     loop=loop, user_ns = self.user_ns, logname=self.log.name)
             self.kernel.start()
             hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
-            heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
+            heart = Heart(*map(str, hb_addrs), heart_id=identity)
             # ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
             heart.start()
             
             
         else:
             self.log.fatal("Registration Failed: %s"%msg)
             raise Exception("Registration Failed: %s"%msg)
         
         self.log.info("Completed registration with id %i"%self.id)
     
     
     def abort(self):
         self.log.fatal("Registration timed out")
         self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id))
         time.sleep(1)
         sys.exit(255)
     
     def start(self):
         dc = ioloop.DelayedCallback(self.register, 0, self.loop)
         dc.start()
         self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop)
         self._abort_dc.start()
 

IPython/parallel/engine/kernelstarter.py

@@ -1,225 +1,225 b''
 """KernelStarter class that intercepts Control Queue messages, and handles process management."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from zmq.eventloop import ioloop
 
-from .streamsession import StreamSession
+from IPython.parallel.streamsession import StreamSession
 
 class KernelStarter(object):
     """Object for resetting/killing the Kernel."""
     
     
     def __init__(self, session, upstream, downstream, *kernel_args, **kernel_kwargs):
         self.session = session
         self.upstream = upstream
         self.downstream = downstream
         self.kernel_args = kernel_args
         self.kernel_kwargs = kernel_kwargs
         self.handlers = {}
         for method in 'shutdown_request shutdown_reply'.split():
             self.handlers[method] = getattr(self, method)
     
     def start(self):
         self.upstream.on_recv(self.dispatch_request)
         self.downstream.on_recv(self.dispatch_reply)
         
     #--------------------------------------------------------------------------
     # Dispatch methods
     #--------------------------------------------------------------------------
     
     def dispatch_request(self, raw_msg):
         idents, msg = self.session.feed_identities()
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             print ("bad msg: %s"%msg)
         
         msgtype = msg['msg_type']
         handler = self.handlers.get(msgtype, None)
         if handler is None:
             self.downstream.send_multipart(raw_msg, copy=False)
         else:
             handler(msg)
         
     def dispatch_reply(self, raw_msg):
         idents, msg = self.session.feed_identities()
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             print ("bad msg: %s"%msg)
         
         msgtype = msg['msg_type']
         handler = self.handlers.get(msgtype, None)
         if handler is None:
             self.upstream.send_multipart(raw_msg, copy=False)
         else:
             handler(msg)
     
     #--------------------------------------------------------------------------
     # Handlers
     #--------------------------------------------------------------------------
     
     def shutdown_request(self, msg):
         """"""
         self.downstream.send_multipart(msg)
     
     #--------------------------------------------------------------------------
     # Kernel process management methods, from KernelManager:
     #--------------------------------------------------------------------------
     
     def _check_local(addr):
         if isinstance(addr, tuple):
             addr = addr[0]
         return addr in LOCAL_IPS
     
     def start_kernel(self, **kw):
         """Starts a kernel process and configures the manager to use it.
 
         If random ports (port=0) are being used, this method must be called
         before the channels are created.
 
         Parameters:
         -----------
         ipython : bool, optional (default True)
              Whether to use an IPython kernel instead of a plain Python kernel.
         """
         self.kernel = Process(target=make_kernel, args=self.kernel_args,
                                             kwargs=self.kernel_kwargs)
 
     def shutdown_kernel(self, restart=False):
         """ Attempts to the stop the kernel process cleanly. If the kernel
         cannot be stopped, it is killed, if possible.
         """
         # FIXME: Shutdown does not work on Windows due to ZMQ errors!
         if sys.platform == 'win32':
             self.kill_kernel()
             return
 
         # Don't send any additional kernel kill messages immediately, to give
         # the kernel a chance to properly execute shutdown actions. Wait for at
         # most 1s, checking every 0.1s.
         self.xreq_channel.shutdown(restart=restart)
         for i in range(10):
             if self.is_alive:
                 time.sleep(0.1)
             else:
                 break
         else:
             # OK, we've waited long enough.
             if self.has_kernel:
                 self.kill_kernel()
     
     def restart_kernel(self, now=False):
         """Restarts a kernel with the same arguments that were used to launch
         it. If the old kernel was launched with random ports, the same ports
         will be used for the new kernel.
 
         Parameters
         ----------
         now : bool, optional
           If True, the kernel is forcefully restarted *immediately*, without
           having a chance to do any cleanup action.  Otherwise the kernel is
           given 1s to clean up before a forceful restart is issued.
 
           In all cases the kernel is restarted, the only difference is whether
           it is given a chance to perform a clean shutdown or not.
         """
         if self._launch_args is None:
             raise RuntimeError("Cannot restart the kernel. "
                                "No previous call to 'start_kernel'.")
         else:
             if self.has_kernel:
                 if now:
                     self.kill_kernel()
                 else:
                     self.shutdown_kernel(restart=True)
             self.start_kernel(**self._launch_args)
 
             # FIXME: Messages get dropped in Windows due to probable ZMQ bug
             # unless there is some delay here.
             if sys.platform == 'win32':
                 time.sleep(0.2)
 
     @property
     def has_kernel(self):
         """Returns whether a kernel process has been specified for the kernel
         manager.
         """
         return self.kernel is not None
 
     def kill_kernel(self):
         """ Kill the running kernel. """
         if self.has_kernel:
             # Pause the heart beat channel if it exists.
             if self._hb_channel is not None:
                 self._hb_channel.pause()
 
             # Attempt to kill the kernel.
             try:
                 self.kernel.kill()
             except OSError, e:
                 # In Windows, we will get an Access Denied error if the process
                 # has already terminated. Ignore it.
                 if not (sys.platform == 'win32' and e.winerror == 5):
                     raise
             self.kernel = None
         else:
             raise RuntimeError("Cannot kill kernel. No kernel is running!")
 
     def interrupt_kernel(self):
         """ Interrupts the kernel. Unlike ``signal_kernel``, this operation is
         well supported on all platforms.
         """
         if self.has_kernel:
             if sys.platform == 'win32':
                 from parentpoller import ParentPollerWindows as Poller
                 Poller.send_interrupt(self.kernel.win32_interrupt_event)
             else:
                 self.kernel.send_signal(signal.SIGINT)
         else:
             raise RuntimeError("Cannot interrupt kernel. No kernel is running!")
 
     def signal_kernel(self, signum):
         """ Sends a signal to the kernel. Note that since only SIGTERM is
         supported on Windows, this function is only useful on Unix systems.
         """
         if self.has_kernel:
             self.kernel.send_signal(signum)
         else:
             raise RuntimeError("Cannot signal kernel. No kernel is running!")
 
     @property
     def is_alive(self):
         """Is the kernel process still running?"""
         # FIXME: not using a heartbeat means this method is broken for any
         # remote kernel, it's only capable of handling local kernels.
         if self.has_kernel:
             if self.kernel.poll() is None:
                 return True
             else:
                 return False
         else:
             # We didn't start the kernel with this KernelManager so we don't
             # know if it is running. We should use a heartbeat for this case.
             return True
 
 
 def make_starter(up_addr, down_addr, *args, **kwargs):
     """entry point function for launching a kernelstarter in a subprocess"""
     loop = ioloop.IOLoop.instance()
     ctx = zmq.Context()
     session = StreamSession()
     upstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)
     upstream.connect(up_addr)
     downstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)
     downstream.connect(down_addr)
     
     starter = KernelStarter(session, upstream, downstream, *args, **kwargs)
     starter.start()
     loop.start()
     
  No newline at end of file

IPython/parallel/engine/streamkernel.py

@@ -1,489 +1,423 b''
 #!/usr/bin/env python
 """
 Kernel adapted from kernel.py to use ZMQ Streams
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports.
 from __future__ import print_function
 
 import sys
 import time
 
 from code import CommandCompiler
 from datetime import datetime
 from pprint import pprint
-from signal import SIGTERM, SIGKILL
 
 # System library imports.
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports.
-from IPython.core import ultratb
 from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Str
 from IPython.zmq.completer import KernelCompleter
-from IPython.zmq.iostream import OutStream
-from IPython.zmq.displayhook import DisplayHook
 
-from . import heartmonitor
-from .client import Client
-from .error import wrap_exception
-from .factory import SessionFactory
-from .streamsession import StreamSession
-from .util import serialize_object, unpack_apply_message, ISO8601, Namespace
+from IPython.parallel.error import wrap_exception
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import serialize_object, unpack_apply_message, ISO8601
 
 def printer(*args):
     pprint(args, stream=sys.__stdout__)
 
 
 class _Passer:
     """Empty class that implements `send()` that does nothing."""
     def send(self, *args, **kwargs):
         pass
     send_multipart = send
     
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(SessionFactory):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
     
     # kwargs:
     int_id = Int(-1, config=True)
     user_ns = Dict(config=True)
     exec_lines = List(config=True)
     
     control_stream = Instance(zmqstream.ZMQStream)
     task_stream = Instance(zmqstream.ZMQStream)
     iopub_stream = Instance(zmqstream.ZMQStream)
-    client = Instance('IPython.parallel.client.Client')
+    client = Instance('IPython.parallel.Client')
     
     # internals
     shell_streams = List()
     compiler = Instance(CommandCompiler, (), {})
     completer = Instance(KernelCompleter)
     
     aborted = Set()
     shell_handlers = Dict()
     control_handlers = Dict()
     
     def _set_prefix(self):
         self.prefix = "engine.%s"%self.int_id
     
     def _connect_completer(self):
         self.completer = KernelCompleter(self.user_ns)
     
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
         self._set_prefix()
         self._connect_completer()
         
         self.on_trait_change(self._set_prefix, 'id')
         self.on_trait_change(self._connect_completer, 'user_ns')
         
         # Build dict of handlers for message types
         for msg_type in ['execute_request', 'complete_request', 'apply_request', 
                 'clear_request']:
             self.shell_handlers[msg_type] = getattr(self, msg_type)
         
         for msg_type in ['shutdown_request', 'abort_request']+self.shell_handlers.keys():
             self.control_handlers[msg_type] = getattr(self, msg_type)
         
         self._initial_exec_lines()
     
     def _wrap_exception(self, method=None):
         e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
         content=wrap_exception(e_info)
         return content
     
     def _initial_exec_lines(self):
         s = _Passer()
         content = dict(silent=True, user_variable=[],user_expressions=[])
         for line in self.exec_lines:
             self.log.debug("executing initialization: %s"%line)
             content.update({'code':line})
             msg = self.session.msg('execute_request', content)
             self.execute_request(s, [], msg)
         
         
     #-------------------- control handlers -----------------------------
     def abort_queues(self):
         for stream in self.shell_streams:
             if stream:
                 self.abort_queue(stream)
     
     def abort_queue(self, stream):
         while True:
             try:
                 msg = self.session.recv(stream, zmq.NOBLOCK,content=True)
             except zmq.ZMQError as e:
                 if e.errno == zmq.EAGAIN:
                     break
                 else:
                     return
             else:
                 if msg is None:
                     return
                 else:
                     idents,msg = msg
                 
                 # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part."
                 # msg = self.reply_socket.recv_json()
             self.log.info("Aborting:")
             self.log.info(str(msg))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
             # self.reply_socket.send(ident,zmq.SNDMORE)
             # self.reply_socket.send_json(reply_msg)
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)[0]
             self.log.debug(str(reply_msg))
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.05)
     
     def abort_request(self, stream, ident, parent):
         """abort a specifig msg by id"""
         msg_ids = parent['content'].get('msg_ids', None)
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         if not msg_ids:
             self.abort_queues()
         for mid in msg_ids:
             self.aborted.add(str(mid))
         
         content = dict(status='ok')
         reply_msg = self.session.send(stream, 'abort_reply', content=content, 
                 parent=parent, ident=ident)
         self.log.debug(str(reply_msg))
     
     def shutdown_request(self, stream, ident, parent):
         """kill ourself.  This should really be handled in an external process"""
         try:
             self.abort_queues()
         except:
             content = self._wrap_exception('shutdown')
         else:
             content = dict(parent['content'])
             content['status'] = 'ok'
         msg = self.session.send(stream, 'shutdown_reply',
                                 content=content, parent=parent, ident=ident)
         self.log.debug(str(msg))
         dc = ioloop.DelayedCallback(lambda : sys.exit(0), 1000, self.loop)
         dc.start()
     
     def dispatch_control(self, msg):
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
         
         header = msg['header']
         msg_id = header['msg_id']
         
         handler = self.control_handlers.get(msg['msg_type'], None)
         if handler is None:
             self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r"%msg['msg_type'])
         else:
             handler(self.control_stream, idents, msg)
     
 
     #-------------------- queue helpers ------------------------------
     
     def check_dependencies(self, dependencies):
         if not dependencies:
             return True
         if len(dependencies) == 2 and dependencies[0] in 'any all'.split():
             anyorall = dependencies[0]
             dependencies = dependencies[1]
         else:
             anyorall = 'all'
         results = self.client.get_results(dependencies,status_only=True)
         if results['status'] != 'ok':
             return False
         
         if anyorall == 'any':
             if not results['completed']:
                 return False
         else:
             if results['pending']:
                 return False
         
         return True
     
     def check_aborted(self, msg_id):
         return msg_id in self.aborted
     
     #-------------------- queue handlers -----------------------------
     
     def clear_request(self, stream, idents, parent):
         """Clear our namespace."""
         self.user_ns = {}
         msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, 
                 content = dict(status='ok'))
         self._initial_exec_lines()
     
     def execute_request(self, stream, ident, parent):
         self.log.debug('execute request %s'%parent)
         try:
             code = parent[u'content'][u'code']
         except:
             self.log.error("Got bad msg: %s"%parent, exc_info=True)
             return
         self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent,
                             ident='%s.pyin'%self.prefix)
         started = datetime.now().strftime(ISO8601)
         try:
             comp_code = self.compiler(code, '<zmq-kernel>')
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
                 sys.stdout.set_parent(parent)
                 sys.stderr.set_parent(parent)
             exec comp_code in self.user_ns, self.user_ns
         except:
             exc_content = self._wrap_exception('execute')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
             self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
                             ident='%s.pyerr'%self.prefix)
             reply_content = exc_content
         else:
             reply_content = {'status' : 'ok'}
         
         reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, 
                     ident=ident, subheader = dict(started=started))
         self.log.debug(str(reply_msg))
         if reply_msg['content']['status'] == u'error':
             self.abort_queues()
 
     def complete_request(self, stream, ident, parent):
         matches = {'matches' : self.complete(parent),
                    'status' : 'ok'}
         completion_msg = self.session.send(stream, 'complete_reply',
                                            matches, parent, ident)
         # print >> sys.__stdout__, completion_msg
 
     def complete(self, msg):
         return self.completer.complete(msg.content.line, msg.content.text)
     
     def apply_request(self, stream, ident, parent):
         # flush previous reply, so this request won't block it
         stream.flush(zmq.POLLOUT)
         
         try:
             content = parent[u'content']
             bufs = parent[u'buffers']
             msg_id = parent['header']['msg_id']
             # bound = parent['header'].get('bound', False)
         except:
             self.log.error("Got bad msg: %s"%parent, exc_info=True)
             return
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_stream.send(pyin_msg)
         # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
         sub = {'dependencies_met' : True, 'engine' : self.ident,
                 'started': datetime.now().strftime(ISO8601)}
         try:
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
                 sys.stdout.set_parent(parent)
                 sys.stderr.set_parent(parent)
             # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns
             working = self.user_ns
             # suffix = 
             prefix = "_"+str(msg_id).replace("-","")+"_"
             
             f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
             # if bound:
             #     bound_ns = Namespace(working)
             #     args = [bound_ns]+list(args)
 
             fname = getattr(f, '__name__', 'f')
             
             fname = prefix+"f"
             argname = prefix+"args"
             kwargname = prefix+"kwargs"
             resultname = prefix+"result"
             
             ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
             # print ns
             working.update(ns)
             code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname)
             try:
                 exec code in working,working
                 result = working.get(resultname)
             finally:
                 for key in ns.iterkeys():
                     working.pop(key)
             # if bound:
             #     working.update(bound_ns)
             
             packed_result,buf = serialize_object(result)
             result_buf = [packed_result]+buf
         except:
             exc_content = self._wrap_exception('apply')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
             self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
                                 ident='%s.pyerr'%self.prefix)
             reply_content = exc_content
             result_buf = []
             
             if exc_content['ename'] == 'UnmetDependency':
                 sub['dependencies_met'] = False
         else:
             reply_content = {'status' : 'ok'}
         
         # put 'ok'/'error' status in header, for scheduler introspection:
         sub['status'] = reply_content['status']
         
         reply_msg = self.session.send(stream, u'apply_reply', reply_content, 
                     parent=parent, ident=ident,buffers=result_buf, subheader=sub)
         
         # flush i/o
         # should this be before reply_msg is sent, like in the single-kernel code, 
         # or should nothing get in the way of real results?
         sys.stdout.flush()
         sys.stderr.flush()
     
     def dispatch_queue(self, stream, msg):
         self.control_stream.flush()
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
             
         
         header = msg['header']
         msg_id = header['msg_id']
         if self.check_aborted(msg_id):
             self.aborted.remove(msg_id)
             # is it safe to assume a msg_id will not be resubmitted?
             reply_type = msg['msg_type'].split('_')[0] + '_reply'
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)
             return
         handler = self.shell_handlers.get(msg['msg_type'], None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE: %r"%msg['msg_type'])
         else:
             handler(stream, idents, msg)
     
     def start(self):
         #### stream mode:
         if self.control_stream:
             self.control_stream.on_recv(self.dispatch_control, copy=False)
             self.control_stream.on_err(printer)
         
         def make_dispatcher(stream):
             def dispatcher(msg):
                 return self.dispatch_queue(stream, msg)
             return dispatcher
         
         for s in self.shell_streams:
             s.on_recv(make_dispatcher(s), copy=False)
             s.on_err(printer)
         
         if self.iopub_stream:
             self.iopub_stream.on_err(printer)
         
         #### while True mode:
         # while True:
         #     idle = True
         #     try:
         #         msg = self.shell_stream.socket.recv_multipart(
         #                     zmq.NOBLOCK, copy=False)
         #     except zmq.ZMQError, e:
         #         if e.errno != zmq.EAGAIN:
         #             raise e
         #     else:
         #         idle=False
         #         self.dispatch_queue(self.shell_stream, msg)
         #             
         #     if not self.task_stream.empty():
         #         idle=False
         #         msg = self.task_stream.recv_multipart()
         #         self.dispatch_queue(self.task_stream, msg)
         #     if idle:
         #         # don't busywait
         #         time.sleep(1e-3)
 
-def make_kernel(int_id, identity, control_addr, shell_addrs, iopub_addr, hb_addrs, 
-                client_addr=None, loop=None, context=None, key=None,
-                out_stream_factory=OutStream, display_hook_factory=DisplayHook):
-    """NO LONGER IN USE"""
-    # create loop, context, and session:
-    if loop is None:
-        loop = ioloop.IOLoop.instance()
-    if context is None:
-        context = zmq.Context()
-    c = context
-    session = StreamSession(key=key)
-    # print (session.key)
-    # print (control_addr, shell_addrs, iopub_addr, hb_addrs)
-    
-    # create Control Stream
-    control_stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
-    control_stream.setsockopt(zmq.IDENTITY, identity)
-    control_stream.connect(control_addr)
-    
-    # create Shell Streams (MUX, Task, etc.):
-    shell_streams = []
-    for addr in shell_addrs:
-        stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
-        stream.setsockopt(zmq.IDENTITY, identity)
-        stream.connect(addr)
-        shell_streams.append(stream)
-    
-    # create iopub stream:
-    iopub_stream = zmqstream.ZMQStream(c.socket(zmq.PUB), loop)
-    iopub_stream.setsockopt(zmq.IDENTITY, identity)
-    iopub_stream.connect(iopub_addr)
-    
-    # Redirect input streams and set a display hook.
-    if out_stream_factory:
-        sys.stdout = out_stream_factory(session, iopub_stream, u'stdout')
-        sys.stdout.topic = 'engine.%i.stdout'%int_id
-        sys.stderr = out_stream_factory(session, iopub_stream, u'stderr')
-        sys.stderr.topic = 'engine.%i.stderr'%int_id
-    if display_hook_factory:
-        sys.displayhook = display_hook_factory(session, iopub_stream)
-        sys.displayhook.topic = 'engine.%i.pyout'%int_id
-    
-    
-    # launch heartbeat
-    heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
-    heart.start()
-    
-    # create (optional) Client
-    if client_addr:
-        client = Client(client_addr, username=identity)
-    else:
-        client = None
-    
-    kernel = Kernel(id=int_id, session=session, control_stream=control_stream,
-            shell_streams=shell_streams, iopub_stream=iopub_stream, 
-            client=client, loop=loop)
-    kernel.start()
-    return loop, c, kernel
-

IPython/parallel/factory.py

@@ -1,152 +1,152 b''
 """Base config factories."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
 import logging
 import os
 import uuid
 
 from zmq.eventloop.ioloop import IOLoop
 
 from IPython.config.configurable import Configurable
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Str,Int,Instance, CUnicode, CStr
 
 import IPython.parallel.streamsession as ss
-from IPython.parallel.entry_point import select_random_ports
+from IPython.parallel.util import select_random_ports
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 class LoggingFactory(Configurable):
     """A most basic class, that has a `log` (type:`Logger`) attribute, set via a `logname` Trait."""
     log = Instance('logging.Logger', ('ZMQ', logging.WARN))
     logname = CUnicode('ZMQ')
     def _logname_changed(self, name, old, new):
         self.log = logging.getLogger(new)
     
 
 class SessionFactory(LoggingFactory):
     """The Base factory from which every factory in IPython.parallel inherits"""
     
     packer = Str('',config=True)
     unpacker = Str('',config=True)
     ident = CStr('',config=True)
     def _ident_default(self):
         return str(uuid.uuid4())
     username = CUnicode(os.environ.get('USER','username'),config=True)
     exec_key = CUnicode('',config=True)
     # not configurable:
     context = Instance('zmq.Context', (), {})
     session = Instance('IPython.parallel.streamsession.StreamSession')
     loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
     def _loop_default(self):
         return IOLoop.instance()
     
     
     def __init__(self, **kwargs):
         super(SessionFactory, self).__init__(**kwargs)
         exec_key = self.exec_key or None
         # set the packers:
         if not self.packer:
             packer_f = unpacker_f = None
         elif self.packer.lower() == 'json':
             packer_f = ss.json_packer
             unpacker_f = ss.json_unpacker
         elif self.packer.lower() == 'pickle':
             packer_f = ss.pickle_packer
             unpacker_f = ss.pickle_unpacker
         else:
             packer_f = import_item(self.packer)
             unpacker_f = import_item(self.unpacker)
         
         # construct the session
         self.session = ss.StreamSession(self.username, self.ident, packer=packer_f, unpacker=unpacker_f, key=exec_key)
     
 
 class RegistrationFactory(SessionFactory):
     """The Base Configurable for objects that involve registration."""
     
     url = Str('', config=True) # url takes precedence over ip,regport,transport
     transport = Str('tcp', config=True)
     ip = Str('127.0.0.1', config=True)
     regport = Instance(int, config=True)
     def _regport_default(self):
         # return 10101
         return select_random_ports(1)[0]
     
     def __init__(self, **kwargs):
         super(RegistrationFactory, self).__init__(**kwargs)
         self._propagate_url()
         self._rebuild_url()
         self.on_trait_change(self._propagate_url, 'url')
         self.on_trait_change(self._rebuild_url, 'ip')
         self.on_trait_change(self._rebuild_url, 'transport')
         self.on_trait_change(self._rebuild_url, 'regport')
     
     def _rebuild_url(self):
         self.url = "%s://%s:%i"%(self.transport, self.ip, self.regport)
         
     def _propagate_url(self):
         """Ensure self.url contains full transport://interface:port"""
         if self.url:
             iface = self.url.split('://',1)
             if len(iface) == 2:
                 self.transport,iface = iface
             iface = iface.split(':')
             self.ip = iface[0]
             if iface[1]:
                 self.regport = int(iface[1])
 
 #-----------------------------------------------------------------------------
 # argparse argument extenders
 #-----------------------------------------------------------------------------
 
 
 def add_session_arguments(parser):
     paa = parser.add_argument
     paa('--ident',
         type=str, dest='SessionFactory.ident', 
         help='set the ZMQ and session identity [default: random uuid]',
         metavar='identity')
     # paa('--execkey',
     #     type=str, dest='SessionFactory.exec_key', 
     #     help='path to a file containing an execution key.',
     #     metavar='execkey')
     paa('--packer',
         type=str, dest='SessionFactory.packer', 
         help='method to serialize messages: {json,pickle} [default: json]',
         metavar='packer')
     paa('--unpacker',
         type=str, dest='SessionFactory.unpacker', 
         help='inverse function of `packer`.  Only necessary when using something other than json|pickle',
         metavar='packer')
 
 def add_registration_arguments(parser):
     paa = parser.add_argument
     paa('--ip',
         type=str, dest='RegistrationFactory.ip',
         help="The IP used for registration [default: localhost]",
         metavar='ip')
     paa('--transport',
         type=str, dest='RegistrationFactory.transport',
         help="The ZeroMQ transport used for registration [default: tcp]",
         metavar='transport')
     paa('--url',
         type=str, dest='RegistrationFactory.url', 
         help='set transport,ip,regport in one go, e.g. tcp://127.0.0.1:10101',
         metavar='url')
     paa('--regport',
         type=int, dest='RegistrationFactory.regport',
         help="The port used for registration [default: 10101]",
         metavar='ip')

IPython/parallel/scripts/ipcluster

@@ -1,18 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
-from IPython.parallel.ipclusterapp import launch_new_instance
+from IPython.parallel.apps.ipclusterapp import launch_new_instance
 
 launch_new_instance()

IPython/parallel/scripts/ipcontroller

@@ -1,18 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
-from IPython.parallel.ipcontrollerapp import launch_new_instance
+from IPython.parallel.apps.ipcontrollerapp import launch_new_instance
 
 launch_new_instance()

IPython/parallel/scripts/ipengine

@@ -1,20 +1,20 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
-from IPython.parallel.ipengineapp import launch_new_instance
+from IPython.parallel.apps.ipengineapp import launch_new_instance
 
 launch_new_instance()
 
 

IPython/parallel/scripts/iplogger

@@ -1,20 +1,20 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
-from IPython.parallel.iploggerapp import launch_new_instance
+from IPython.parallel.apps.iploggerapp import launch_new_instance
 
 launch_new_instance()
 
 

IPython/parallel/tests/__init__.py

@@ -1,69 +1,69 b''
 """toplevel setup/teardown for parallel tests."""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import tempfile
 import time
 from subprocess import Popen, PIPE, STDOUT
 
-from IPython.parallel import client
+from IPython.parallel import Client
 
 processes = []
 blackhole = tempfile.TemporaryFile()
 
 # nose setup/teardown
 
 def setup():
     cp = Popen('ipcontroller --profile iptest -r --log-level 10 --log-to-file'.split(), stdout=blackhole, stderr=STDOUT)
     processes.append(cp)
     time.sleep(.5)
     add_engines(1)
-    c = client.Client(profile='iptest')
+    c = Client(profile='iptest')
     while not c.ids:
         time.sleep(.1)
         c.spin()
     c.close()
 
 def add_engines(n=1, profile='iptest'):
-    rc = client.Client(profile=profile)
+    rc = Client(profile=profile)
     base = len(rc)
     eps = []
     for i in range(n):
         ep = Popen(['ipengine']+ ['--profile', profile, '--log-level', '10', '--log-to-file'], stdout=blackhole, stderr=STDOUT)
         # ep.start()
         processes.append(ep)
         eps.append(ep)
     while len(rc) < base+n:
         time.sleep(.1)
         rc.spin()
     rc.close()
     return eps
 
 def teardown():
     time.sleep(1)
     while processes:
         p = processes.pop()
         if p.poll() is None:
             try:
                 p.terminate()
             except Exception, e:
                 print e
                 pass
         if p.poll() is None:
             time.sleep(.25)
         if p.poll() is None:
             try:
                 print 'killing'
                 p.kill()
             except:
                 print "couldn't shutdown process: ", p
     

IPython/parallel/tests/clienttest.py

@@ -1,119 +1,115 b''
 """base class for parallel client tests"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 import sys
 import tempfile
 import time
-from signal import SIGINT
-from multiprocessing import Process
 
 from nose import SkipTest
 
 import zmq
 from zmq.tests import BaseZMQTestCase
 
 from IPython.external.decorator import decorator
 
 from IPython.parallel import error
-from IPython.parallel.client import Client
-from IPython.parallel.ipcluster import launch_process
-from IPython.parallel.entry_point import select_random_ports
+from IPython.parallel import Client
 from IPython.parallel.tests import processes,add_engines
 
 # simple tasks for use in apply tests
 
 def segfault():
     """this will segfault"""
     import ctypes
     ctypes.memset(-1,0,1)
 
 def wait(n):
     """sleep for a time"""
     import time
     time.sleep(n)
     return n
 
 def raiser(eclass):
     """raise an exception"""
     raise eclass()
 
 # test decorator for skipping tests when libraries are unavailable
 def skip_without(*names):
     """skip a test if some names are not importable"""
     @decorator
     def skip_without_names(f, *args, **kwargs):
         """decorator to skip tests in the absence of numpy."""
         for name in names:
             try:
                 __import__(name)
             except ImportError:
                 raise SkipTest
         return f(*args, **kwargs)
     return skip_without_names
 
 class ClusterTestCase(BaseZMQTestCase):
     
     def add_engines(self, n=1, block=True):
         """add multiple engines to our cluster"""
         self.engines.extend(add_engines(n))
         if block:
             self.wait_on_engines()
     
     def wait_on_engines(self, timeout=5):
         """wait for our engines to connect."""
         n = len(self.engines)+self.base_engine_count
         tic = time.time()
         while time.time()-tic < timeout and len(self.client.ids) < n:
             time.sleep(0.1)
         
         assert not len(self.client.ids) < n, "waiting for engines timed out"
     
     def connect_client(self):
         """connect a client with my Context, and track its sockets for cleanup"""
         c = Client(profile='iptest', context=self.context)
         for name in filter(lambda n:n.endswith('socket'), dir(c)):
             s = getattr(c, name)
             s.setsockopt(zmq.LINGER, 0)
             self.sockets.append(s)
         return c
     
     def assertRaisesRemote(self, etype, f, *args, **kwargs):
         try:
             try:
                 f(*args, **kwargs)
             except error.CompositeError as e:
                 e.raise_exception()
         except error.RemoteError as e:
             self.assertEquals(etype.__name__, e.ename, "Should have raised %r, but raised %r"%(e.ename, etype.__name__))
         else:
             self.fail("should have raised a RemoteError")
             
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.client = self.connect_client()
         self.base_engine_count=len(self.client.ids)
         self.engines=[]
     
     def tearDown(self):
         # self.client.clear(block=True)
         # close fds:
         for e in filter(lambda e: e.poll() is not None, processes):
             processes.remove(e)
         
         # allow flushing of incoming messages to prevent crash on socket close
         self.client.wait(timeout=2)
         # time.sleep(2)
         self.client.spin()
         self.client.close()
         BaseZMQTestCase.tearDown(self)
         # this will be redundant when pyzmq merges PR #88
         # self.context.term()
         # print tempfile.TemporaryFile().fileno(),
         # sys.stdout.flush()
         
  No newline at end of file

IPython/parallel/tests/test_client.py

@@ -1,147 +1,147 b''
 """Tests for parallel client.py"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import time
 from tempfile import mktemp
 
 import zmq
 
-from IPython.parallel import client as clientmod
+from IPython.parallel.client import client as clientmod
 from IPython.parallel import error
-from IPython.parallel.asyncresult import AsyncResult, AsyncHubResult
-from IPython.parallel.view import LoadBalancedView, DirectView
+from IPython.parallel import AsyncResult, AsyncHubResult
+from IPython.parallel import LoadBalancedView, DirectView
 
 from clienttest import ClusterTestCase, segfault, wait, add_engines
 
 def setup():
     add_engines(4)
 
 class TestClient(ClusterTestCase):
     
     def test_ids(self):
         n = len(self.client.ids)
         self.add_engines(3)
         self.assertEquals(len(self.client.ids), n+3)
     
     def test_view_indexing(self):
         """test index access for views"""
         self.add_engines(2)
         targets = self.client._build_targets('all')[-1]
         v = self.client[:]
         self.assertEquals(v.targets, targets)
         t = self.client.ids[2]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         t = self.client.ids[2:4]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         v = self.client[::2]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[::2])
         v = self.client[1::3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[1::3])
         v = self.client[:-3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[:-3])
         v = self.client[-1]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[-1])
         self.assertRaises(TypeError, lambda : self.client[None])
     
     def test_lbview_targets(self):
         """test load_balanced_view targets"""
         v = self.client.load_balanced_view()
         self.assertEquals(v.targets, None)
         v = self.client.load_balanced_view(-1)
         self.assertEquals(v.targets, [self.client.ids[-1]])
         v = self.client.load_balanced_view('all')
         self.assertEquals(v.targets, self.client.ids)
     
     def test_targets(self):
         """test various valid targets arguments"""
         build = self.client._build_targets
         ids = self.client.ids
         idents,targets = build(None)
         self.assertEquals(ids, targets)
     
     def test_clear(self):
         """test clear behavior"""
         # self.add_engines(2)
         v = self.client[:]
         v.block=True
         v.push(dict(a=5))
         v.pull('a')
         id0 = self.client.ids[-1]
         self.client.clear(targets=id0)
         self.client[:-1].pull('a')
         self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
         self.client.clear(block=True)
         for i in self.client.ids:
             # print i
             self.assertRaisesRemote(NameError, self.client[i].get, 'a')
     
     def test_get_result(self):
         """test getting results from the Hub."""
         c = clientmod.Client(profile='iptest')
         # self.add_engines(1)
         t = c.ids[-1]
         ar = c[t].apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = self.client.get_result(ar.msg_ids)
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEquals(ahr.get(), ar.get())
         ar2 = self.client.get_result(ar.msg_ids)
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.close()
     
     def test_ids_list(self):
         """test client.ids"""
         # self.add_engines(2)
         ids = self.client.ids
         self.assertEquals(ids, self.client._ids)
         self.assertFalse(ids is self.client._ids)
         ids.remove(ids[-1])
         self.assertNotEquals(ids, self.client._ids)
     
     def test_queue_status(self):
         # self.addEngine(4)
         ids = self.client.ids
         id0 = ids[0]
         qs = self.client.queue_status(targets=id0)
         self.assertTrue(isinstance(qs, dict))
         self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
         allqs = self.client.queue_status()
         self.assertTrue(isinstance(allqs, dict))
         self.assertEquals(sorted(allqs.keys()), self.client.ids)
         for eid,qs in allqs.items():
             self.assertTrue(isinstance(qs, dict))
             self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
 
     def test_shutdown(self):
         # self.addEngine(4)
         ids = self.client.ids
         id0 = ids[0]
         self.client.shutdown(id0, block=True)
         while id0 in self.client.ids:
             time.sleep(0.1)
             self.client.spin()
         
         self.assertRaises(IndexError, lambda : self.client[id0])
         
     def test_result_status(self):
         pass
         # to be written

IPython/parallel/tests/test_dependency.py

@@ -1,101 +1,101 b''
 """Tests for dependency.py"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # import
 import os
 
 from IPython.utils.pickleutil import can, uncan
 
-from IPython.parallel import dependency as dmod
+import IPython.parallel as pmod
 from IPython.parallel.util import interactive
 
 from IPython.parallel.tests import add_engines
 from .clienttest import ClusterTestCase
 
 def setup():
     add_engines(1)
 
[e-mail]('time')
[e-mail]('time')
 def wait(n):
     time.sleep(n)
     return n
 
 mixed = map(str, range(10))
 completed = map(str, range(0,10,2))
 failed = map(str, range(1,10,2))
 
 class DependencyTest(ClusterTestCase):
     
     def setUp(self):
         ClusterTestCase.setUp(self)
         self.user_ns = {'__builtins__' : __builtins__}
         self.view = self.client.load_balanced_view()
         self.dview = self.client[-1]
         self.succeeded = set(map(str, range(0,25,2)))
         self.failed = set(map(str, range(1,25,2)))
     
     def assertMet(self, dep):
         self.assertTrue(dep.check(self.succeeded, self.failed), "Dependency should be met")
         
     def assertUnmet(self, dep):
         self.assertFalse(dep.check(self.succeeded, self.failed), "Dependency should not be met")
         
     def assertUnreachable(self, dep):
         self.assertTrue(dep.unreachable(self.succeeded, self.failed), "Dependency should be unreachable")
     
     def assertReachable(self, dep):
         self.assertFalse(dep.unreachable(self.succeeded, self.failed), "Dependency should be reachable")
     
     def cancan(self, f):
         """decorator to pass through canning into self.user_ns"""
         return uncan(can(f), self.user_ns)
     
     def test_require_imports(self):
         """test that @require imports names"""
         @self.cancan
-        @dmod.require('urllib')
+        @pmod.require('urllib')
         @interactive
         def encode(dikt):
             return urllib.urlencode(dikt)
         # must pass through canning to properly connect namespaces
         self.assertEquals(encode(dict(a=5)), 'a=5')
     
     def test_success_only(self):
-        dep = dmod.Dependency(mixed, success=True, failure=False)
+        dep = pmod.Dependency(mixed, success=True, failure=False)
         self.assertUnmet(dep)
         self.assertUnreachable(dep)
         dep.all=False
         self.assertMet(dep)
         self.assertReachable(dep)
-        dep = dmod.Dependency(completed, success=True, failure=False)
+        dep = pmod.Dependency(completed, success=True, failure=False)
         self.assertMet(dep)
         self.assertReachable(dep)
         dep.all=False
         self.assertMet(dep)
         self.assertReachable(dep)
 
     def test_failure_only(self):
-        dep = dmod.Dependency(mixed, success=False, failure=True)
+        dep = pmod.Dependency(mixed, success=False, failure=True)
         self.assertUnmet(dep)
         self.assertUnreachable(dep)
         dep.all=False
         self.assertMet(dep)
         self.assertReachable(dep)
-        dep = dmod.Dependency(completed, success=False, failure=True)
+        dep = pmod.Dependency(completed, success=False, failure=True)
         self.assertUnmet(dep)
         self.assertUnreachable(dep)
         dep.all=False
         self.assertUnmet(dep)
         self.assertUnreachable(dep)

IPython/parallel/tests/test_view.py

@@ -1,301 +1,301 b''
 """test View objects"""
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import time
 from tempfile import mktemp
 
 import zmq
 
 from IPython import parallel  as pmod
 from IPython.parallel import error
-from IPython.parallel.asyncresult import AsyncResult, AsyncHubResult, AsyncMapResult
-from IPython.parallel.view import LoadBalancedView, DirectView
+from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
+from IPython.parallel import LoadBalancedView, DirectView
 from IPython.parallel.util import interactive
 
 from IPython.parallel.tests import add_engines
 
 from .clienttest import ClusterTestCase, segfault, wait, skip_without
 
 def setup():
     add_engines(3)
 
 class TestView(ClusterTestCase):
     
     def test_segfault_task(self):
         """test graceful handling of engine death (balanced)"""
         # self.add_engines(1)
         ar = self.client[-1].apply_async(segfault)
         self.assertRaisesRemote(error.EngineError, ar.get)
         eid = ar.engine_id
         while eid in self.client.ids:
             time.sleep(.01)
             self.client.spin()
     
     def test_segfault_mux(self):
         """test graceful handling of engine death (direct)"""
         # self.add_engines(1)
         eid = self.client.ids[-1]
         ar = self.client[eid].apply_async(segfault)
         self.assertRaisesRemote(error.EngineError, ar.get)
         eid = ar.engine_id
         while eid in self.client.ids:
             time.sleep(.01)
             self.client.spin()
     
     def test_push_pull(self):
         """test pushing and pulling"""
         data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
         t = self.client.ids[-1]
         v = self.client[t]
         push = v.push
         pull = v.pull
         v.block=True
         nengines = len(self.client)
         push({'data':data})
         d = pull('data')
         self.assertEquals(d, data)
         self.client[:].push({'data':data})
         d = self.client[:].pull('data', block=True)
         self.assertEquals(d, nengines*[data])
         ar = push({'data':data}, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         ar = self.client[:].pull('data', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         self.assertEquals(r, nengines*[data])
         self.client[:].push(dict(a=10,b=20))
         r = self.client[:].pull(('a','b'))
         self.assertEquals(r, nengines*[[10,20]])
     
     def test_push_pull_function(self):
         "test pushing and pulling functions"
         def testf(x):
             return 2.0*x
         
         t = self.client.ids[-1]
         self.client[t].block=True
         push = self.client[t].push
         pull = self.client[t].pull
         execute = self.client[t].execute
         push({'testf':testf})
         r = pull('testf')
         self.assertEqual(r(1.0), testf(1.0))
         execute('r = testf(10)')
         r = pull('r')
         self.assertEquals(r, testf(10))
         ar = self.client[:].push({'testf':testf}, block=False)
         ar.get()
         ar = self.client[:].pull('testf', block=False)
         rlist = ar.get()
         for r in rlist:
             self.assertEqual(r(1.0), testf(1.0))
         execute("def g(x): return x*x")
         r = pull(('testf','g'))
         self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
     
     def test_push_function_globals(self):
         """test that pushed functions have access to globals"""
         @interactive
         def geta():
             return a
         # self.add_engines(1)
         v = self.client[-1]
         v.block=True
         v['f'] = geta
         self.assertRaisesRemote(NameError, v.execute, 'b=f()')
         v.execute('a=5')
         v.execute('b=f()')
         self.assertEquals(v['b'], 5)
     
     def test_push_function_defaults(self):
         """test that pushed functions preserve default args"""
         def echo(a=10):
             return a
         v = self.client[-1]
         v.block=True
         v['f'] = echo
         v.execute('b=f()')
         self.assertEquals(v['b'], 10)
 
     def test_get_result(self):
         """test getting results from the Hub."""
         c = pmod.Client(profile='iptest')
         # self.add_engines(1)
         t = c.ids[-1]
         v = c[t]
         v2 = self.client[t]
         ar = v.apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = v2.get_result(ar.msg_ids)
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEquals(ahr.get(), ar.get())
         ar2 = v2.get_result(ar.msg_ids)
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.spin()
         c.close()
     
     def test_run_newline(self):
         """test that run appends newline to files"""
         tmpfile = mktemp()
         with open(tmpfile, 'w') as f:
             f.write("""def g():
                 return 5
                 """)
         v = self.client[-1]
         v.run(tmpfile, block=True)
         self.assertEquals(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
 
     def test_apply_tracked(self):
         """test tracking for apply"""
         # self.add_engines(1)
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=False
         def echo(n=1024*1024, **kwargs):
             with v.temp_flags(**kwargs):
                 return v.apply(lambda x: x, 'x'*n)
         ar = echo(1, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         ar = echo(track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEquals(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         
     def test_push_tracked(self):
         t = self.client.ids[-1]
         ns = dict(x='x'*1024*1024)
         v = self.client[t]
         ar = v.push(ns, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = v.push(ns, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEquals(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         ar.get()
         
     def test_scatter_tracked(self):
         t = self.client.ids
         x='x'*1024*1024
         ar = self.client[t].scatter('x', x, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = self.client[t].scatter('x', x, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEquals(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         ar.get()
     
     def test_remote_reference(self):
         v = self.client[-1]
         v['a'] = 123
         ra = pmod.Reference('a')
         b = v.apply_sync(lambda x: x, ra)
         self.assertEquals(b, 123)
 
 
     def test_scatter_gather(self):
         view = self.client[:]
         seq1 = range(16)
         view.scatter('a', seq1)
         seq2 = view.gather('a', block=True)
         self.assertEquals(seq2, seq1)
         self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         view = self.client[:]
         a = numpy.arange(64)
         view.scatter('a', a)
         b = view.gather('a', block=True)
         assert_array_equal(b, a)
     
     def test_map(self):
         view = self.client[:]
         def f(x):
             return x**2
         data = range(16)
         r = view.map_sync(f, data)
         self.assertEquals(r, map(f, data))
     
     def test_scatterGatherNonblocking(self):
         data = range(16)
         view = self.client[:]
         view.scatter('a', data, block=False)
         ar = view.gather('a', block=False)
         self.assertEquals(ar.get(), data)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy_nonblocking(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         a = numpy.arange(64)
         view = self.client[:]
         ar = view.scatter('a', a, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         amr = view.gather('a', block=False)
         self.assertTrue(isinstance(amr, AsyncMapResult))
         assert_array_equal(amr.get(), a)
 
     def test_execute(self):
         view = self.client[:]
         # self.client.debug=True
         execute = view.execute
         ar = execute('c=30', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         ar = execute('d=[0,1,2]', block=False)
         self.client.wait(ar, 1)
         self.assertEquals(len(ar.get()), len(self.client))
         for c in view['c']:
             self.assertEquals(c, 30)
     
     def test_abort(self):
         view = self.client[-1]
         ar = view.execute('import time; time.sleep(0.25)', block=False)
         ar2 = view.apply_async(lambda : 2)
         ar3 = view.apply_async(lambda : 3)
         view.abort(ar2)
         view.abort(ar3.msg_ids)
         self.assertRaises(error.TaskAborted, ar2.get)
         self.assertRaises(error.TaskAborted, ar3.get)
         
     def test_temp_flags(self):
         view = self.client[-1]
         view.block=True
         with view.temp_flags(block=False):
             self.assertFalse(view.block)
         self.assertTrue(view.block)
     
     def test_importer(self):
         view = self.client[-1]
         view.clear(block=True)
         with view.importer:
             import re
         
         @interactive
         def findall(pat, s):
             # this globals() step isn't necessary in real code
             # only to prevent a closure in the test
             return globals()['re'].findall(pat, s)
         
         self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
     

IPython/parallel/util.py

@@ -1,354 +1,462 b''
 """some generic utilities for dealing with classes, urls, and serialization"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+# Standard library imports.
+import logging
+import os
 import re
+import stat
 import socket
+import sys
+from signal import signal, SIGINT, SIGABRT, SIGTERM
+try:
+    from signal import SIGKILL
+except ImportError:
+    SIGKILL=None
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
+# System library imports
+import zmq
+from zmq.log import handlers
 
+# IPython imports
 from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
 from IPython.utils.newserialized import serialize, unserialize
+from IPython.zmq.log import EnginePUBHandler
 
+# globals
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class Namespace(dict):
     """Subclass of dict for attribute access to keys."""
     
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise NameError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if hasattr(dict, key):
             raise KeyError("Cannot override dict keys %r"%key)
         self[key] = value
     
 
 class ReverseDict(dict):
     """simple double-keyed subset of dict methods."""
     
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
         self._reverse = dict()
         for key, value in self.iteritems():
             self._reverse[value] = key
     
     def __getitem__(self, key):
         try:
             return dict.__getitem__(self, key)
         except KeyError:
             return self._reverse[key]
     
     def __setitem__(self, key, value):
         if key in self._reverse:
             raise KeyError("Can't have key %r on both sides!"%key)
         dict.__setitem__(self, key, value)
         self._reverse[value] = key
     
     def pop(self, key):
         value = dict.pop(self, key)
         self._reverse.pop(value)
         return value
     
     def get(self, key, default=None):
         try:
             return self[key]
         except KeyError:
             return default
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def validate_url(url):
     """validate a url for zeromq"""
     if not isinstance(url, basestring):
         raise TypeError("url must be a string, not %r"%type(url))
     url = url.lower()
     
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     assert proto in ['tcp','pgm','epgm','ipc','inproc'], "Invalid protocol: %r"%proto
     
     # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391
     # author: Remi Sabourin
     pat = re.compile(r'^([\w\d]([\w\d\-]{0,61}[\w\d])?\.)*[\w\d]([\w\d\-]{0,61}[\w\d])?$')
     
     if proto == 'tcp':
         lis = addr.split(':')
         assert len(lis) == 2, 'Invalid url: %r'%url
         addr,s_port = lis
         try:
             port = int(s_port)
         except ValueError:
             raise AssertionError("Invalid port %r in url: %r"%(port, url))
         
         assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url
         
     else:
         # only validate tcp urls currently
         pass
     
     return True
 
 
 def validate_url_container(container):
     """validate a potentially nested collection of urls."""
     if isinstance(container, basestring):
         url = container
         return validate_url(url)
     elif isinstance(container, dict):
         container = container.itervalues()
     
     for element in container:
         validate_url_container(element)
 
 
 def split_url(url):
     """split a zmq url (tcp://ip:port) into ('tcp','ip','port')."""
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     lis = addr.split(':')
     assert len(lis) == 2, 'Invalid url: %r'%url
     addr,s_port = lis
     return proto,addr,s_port
     
 def disambiguate_ip_address(ip, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation of location is localhost)."""
     if ip in ('0.0.0.0', '*'):
         external_ips = socket.gethostbyname_ex(socket.gethostname())[2]
         if location is None or location in external_ips:
             ip='127.0.0.1'
         elif location:
             return location
     return ip
 
 def disambiguate_url(url, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation is localhost).
     
     This is for zeromq urls, such as tcp://*:10101."""
     try:
         proto,ip,port = split_url(url)
     except AssertionError:
         # probably not tcp url; could be ipc, etc.
         return url
     
     ip = disambiguate_ip_address(ip,location)
     
     return "%s://%s:%s"%(proto,ip,port)
 
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json.  This belongs in the jsonutil added by fperez."""
     for k in dikt.iterkeys():
         if isinstance(k, str):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 def serialize_object(obj, threshold=64e-6):
     """Serialize an object into a list of sendable buffers.
     
     Parameters
     ----------
     
     obj : object
         The object to be serialized
     threshold : float
         The threshold for not double-pickling the content.
         
     
     Returns
     -------
     ('pmd', [bufs]) :
         where pmd is the pickled metadata wrapper,
         bufs is a list of data buffers
     """
     databuffers = []
     if isinstance(obj, (list, tuple)):
         clist = canSequence(obj)
         slist = map(serialize, clist)
         for s in slist:
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
         return pickle.dumps(slist,-1), databuffers
     elif isinstance(obj, dict):
         sobj = {}
         for k in sorted(obj.iterkeys()):
             s = serialize(can(obj[k]))
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
             sobj[k] = s
         return pickle.dumps(sobj,-1),databuffers
     else:
         s = serialize(can(obj))
         if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
             databuffers.append(s.getData())
             s.data = None
         return pickle.dumps(s,-1),databuffers
             
         
 def unserialize_object(bufs):
     """reconstruct an object serialized by serialize_object from data buffers."""
     bufs = list(bufs)
     sobj = pickle.loads(bufs.pop(0))
     if isinstance(sobj, (list, tuple)):
         for s in sobj:
             if s.data is None:
                 s.data = bufs.pop(0)
         return uncanSequence(map(unserialize, sobj)), bufs
     elif isinstance(sobj, dict):
         newobj = {}
         for k in sorted(sobj.iterkeys()):
             s = sobj[k]
             if s.data is None:
                 s.data = bufs.pop(0)
             newobj[k] = uncan(unserialize(s))
         return newobj, bufs
     else:
         if sobj.data is None:
             sobj.data = bufs.pop(0)
         return uncan(unserialize(sobj)), bufs
 
 def pack_apply_message(f, args, kwargs, threshold=64e-6):
     """pack up a function, args, and kwargs to be sent over the wire
     as a series of buffers. Any object whose data is larger than `threshold`
     will not have their data copied (currently only numpy arrays support zero-copy)"""
     msg = [pickle.dumps(can(f),-1)]
     databuffers = [] # for large objects
     sargs, bufs = serialize_object(args,threshold)
     msg.append(sargs)
     databuffers.extend(bufs)
     skwargs, bufs = serialize_object(kwargs,threshold)
     msg.append(skwargs)
     databuffers.extend(bufs)
     msg.extend(databuffers)
     return msg
 
 def unpack_apply_message(bufs, g=None, copy=True):
     """unpack f,args,kwargs from buffers packed by pack_apply_message()
     Returns: original f,args,kwargs"""
     bufs = list(bufs) # allow us to pop
     assert len(bufs) >= 3, "not enough buffers!"
     if not copy:
         for i in range(3):
             bufs[i] = bufs[i].bytes
     cf = pickle.loads(bufs.pop(0))
     sargs = list(pickle.loads(bufs.pop(0)))
     skwargs = dict(pickle.loads(bufs.pop(0)))
     # print sargs, skwargs
     f = uncan(cf, g)
     for sa in sargs:
         if sa.data is None:
             m = bufs.pop(0)
             if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                 if copy:
                     sa.data = buffer(m)
                 else:
                     sa.data = m.buffer
             else:
                 if copy:
                     sa.data = m
                 else:
                     sa.data = m.bytes
     
     args = uncanSequence(map(unserialize, sargs), g)
     kwargs = {}
     for k in sorted(skwargs.iterkeys()):
         sa = skwargs[k]
         if sa.data is None:
             m = bufs.pop(0)
             if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                 if copy:
                     sa.data = buffer(m)
                 else:
                     sa.data = m.buffer
             else:
                 if copy:
                     sa.data = m
                 else:
                     sa.data = m.bytes
 
         kwargs[k] = uncan(unserialize(sa), g)
     
     return f,args,kwargs
 
 #--------------------------------------------------------------------------
 # helpers for implementing old MEC API via view.apply
 #--------------------------------------------------------------------------
 
 def interactive(f):
     """decorator for making functions appear as interactively defined.
     This results in the function being linked to the user_ns as globals()
     instead of the module globals().
     """
     f.__module__ = '__main__'
     return f
 
 @interactive
 def _push(ns):
     """helper method for implementing `client.push` via `client.apply`"""
     globals().update(ns)
 
 @interactive
 def _pull(keys):
     """helper method for implementing `client.pull` via `client.apply`"""
     user_ns = globals()
     if isinstance(keys, (list,tuple, set)):
         for key in keys:
             if not user_ns.has_key(key):
                 raise NameError("name '%s' is not defined"%key)
         return map(user_ns.get, keys)
     else:
         if not user_ns.has_key(keys):
             raise NameError("name '%s' is not defined"%keys)
         return user_ns.get(keys)
 
 @interactive
 def _execute(code):
     """helper method for implementing `client.execute` via `client.apply`"""
     exec code in globals()
 
+#--------------------------------------------------------------------------
+# extra process management utilities
+#--------------------------------------------------------------------------
+
+_random_ports = set()
+
+def select_random_ports(n):
+    """Selects and return n random ports that are available."""
+    ports = []
+    for i in xrange(n):
+        sock = socket.socket()
+        sock.bind(('', 0))
+        while sock.getsockname()[1] in _random_ports:
+            sock.close()
+            sock = socket.socket()
+            sock.bind(('', 0))
+        ports.append(sock)
+    for i, sock in enumerate(ports):
+        port = sock.getsockname()[1]
+        sock.close()
+        ports[i] = port
+        _random_ports.add(port)
+    return ports
+
+def signal_children(children):
+    """Relay interupt/term signals to children, for more solid process cleanup."""
+    def terminate_children(sig, frame):
+        logging.critical("Got signal %i, terminating children..."%sig)
+        for child in children:
+            child.terminate()
+        
+        sys.exit(sig != SIGINT)
+        # sys.exit(sig)
+    for sig in (SIGINT, SIGABRT, SIGTERM):
+        signal(sig, terminate_children)
+
+def generate_exec_key(keyfile):
+    import uuid
+    newkey = str(uuid.uuid4())
+    with open(keyfile, 'w') as f:
+        # f.write('ipython-key ')
+        f.write(newkey+'\n')
+    # set user-only RW permissions (0600)
+    # this will have no effect on Windows
+    os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
+
+
+def integer_loglevel(loglevel):
+    try:
+        loglevel = int(loglevel)
+    except ValueError:
+        if isinstance(loglevel, str):
+            loglevel = getattr(logging, loglevel)
+    return loglevel
+
+def connect_logger(logname, context, iface, root="ip", loglevel=logging.DEBUG):
+    logger = logging.getLogger(logname)
+    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
+        # don't add a second PUBHandler
+        return
+    loglevel = integer_loglevel(loglevel)
+    lsock = context.socket(zmq.PUB)
+    lsock.connect(iface)
+    handler = handlers.PUBHandler(lsock)
+    handler.setLevel(loglevel)
+    handler.root_topic = root
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)
+
+def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):
+    logger = logging.getLogger()
+    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
+        # don't add a second PUBHandler
+        return
+    loglevel = integer_loglevel(loglevel)
+    lsock = context.socket(zmq.PUB)
+    lsock.connect(iface)
+    handler = EnginePUBHandler(engine, lsock)
+    handler.setLevel(loglevel)
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)
+
+def local_logger(logname, loglevel=logging.DEBUG):
+    loglevel = integer_loglevel(loglevel)
+    logger = logging.getLogger(logname)
+    if any([isinstance(h, logging.StreamHandler) for h in logger.handlers]):
+        # don't add a second StreamHandler
+        return
+    handler = logging.StreamHandler()
+    handler.setLevel(loglevel)
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)

IPython/testing/iptest.py

@@ -1,420 +1,423 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
 import signal
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
 # Note: monkeypatch!
 # We need to monkeypatch a small problem in nose itself first, before importing
 # it for actual use.  This should get into nose upstream, but its release cycle
 # is slow and we need it for our parametric tests to work correctly.
 from IPython.testing import nosepatch
 # Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.core import TestProgram
 
 # Our own imports
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv
 from IPython.utils.sysinfo import sys_info
 
 from IPython.testing import globalipapp
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 from IPython.external.decorators import KnownFailure
 
 pjoin = path.join
 
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 # Wx on Fedora11 spits these out
 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
                         UserWarning)
 
 #-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
 def test_for(mod, min_version=None):
     """Test to see if mod is importable."""
     try:
         __import__(mod)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if  it's
         # importable.
         return False
     else:
         if min_version:
             return sys.modules[mod].__version__ >= min_version
         else:
             return True
 
 # Global dict where we can store information on what we have and what we don't
 # have available at test run time
 have = {}
 
 have['curses'] = test_for('_curses')
 have['wx'] = test_for('wx')
 have['wx.aui'] = test_for('wx.aui')
 have['pexpect'] = test_for('pexpect')
 have['zmq'] = test_for('zmq', '2.0.10')
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def report():
     """Return a string with a summary report of test-related variables."""
 
     out = [ sys_info(), '\n']
 
     avail = []
     not_avail = []
     
     for k, is_avail in have.items():
         if is_avail:
             avail.append(k)
         else:
             not_avail.append(k)
 
     if avail:
         out.append('\nTools and libraries available at test time:\n')
         avail.sort()
         out.append('   ' + ' '.join(avail)+'\n')
 
     if not_avail:
         out.append('\nTools and libraries NOT available at test time:\n')
         not_avail.sort()
         out.append('   ' + ' '.join(not_avail)+'\n')
         
     return ''.join(out)
 
 
 def make_exclude():
     """Make patterns of modules and packages to exclude from testing.
 
     For the IPythonDoctest plugin, we need to exclude certain patterns that
     cause testing problems.  We should strive to minimize the number of
     skipped modules, since this means untested code.
 
     These modules and packages will NOT get scanned by nose at all for tests.
     """
     # Simple utility to make IPython paths more readably, we need a lot of
     # these below
     ipjoin = lambda *paths: pjoin('IPython', *paths)
     
     exclusions = [ipjoin('external'),
                   pjoin('IPython_doctest_plugin'),
                   ipjoin('quarantine'),
                   ipjoin('deathrow'),
                   ipjoin('testing', 'attic'),
                   # This guy is probably attic material
                   ipjoin('testing', 'mkdoctests'),
                   # Testing inputhook will need a lot of thought, to figure out
                   # how to have tests that don't lock up with the gui event
                   # loops in the picture
                   ipjoin('lib', 'inputhook'),
                   # Config files aren't really importable stand-alone
                   ipjoin('config', 'default'),
                   ipjoin('config', 'profile'),
                   ]
 
     if not have['wx']:
         exclusions.append(ipjoin('lib', 'inputhookwx'))
         
     # We do this unconditionally, so that the test suite doesn't import
     # gtk, changing the default encoding and masking some unicode bugs.
     exclusions.append(ipjoin('lib', 'inputhookgtk'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
     if not have['pexpect']:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
 
     if not have['zmq']:
         exclusions.append(ipjoin('zmq'))
         exclusions.append(ipjoin('parallel'))
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
         exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
     return exclusions
 
 
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
     #: string, name of test runner that will be called
     runner = None
     #: list, parameters for test runner
     params = None
     #: list, arguments of system call to be made to call test runner
     call_args = None
     #: list, process ids of subprocesses we start (for cleanup)
     pids = None
     
     def __init__(self, runner='iptest', params=None):
         """Create new test runner."""
         p = os.path
         if runner == 'iptest':
             iptest_app = get_ipython_module_path('IPython.testing.iptest')
             self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
         else:
             raise Exception('Not a valid test runner: %s' % repr(runner))
         if params is None:
             params = []
         if isinstance(params, str):
             params = [params]
         self.params = params
 
         # Assemble call
         self.call_args = self.runner+self.params
 
         # Store pids of anything we start to clean up on deletion, if possible
         # (on posix only, since win32 has no os.kill)
         self.pids = []
 
     if sys.platform == 'win32':
         def _run_cmd(self):
             # On Windows, use os.system instead of subprocess.call, because I
             # was having problems with subprocess and I just don't know enough
             # about win32 to debug this reliably.  Os.system may be the 'old
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
             # What types of problems are you having. They may be related to
             # running Python in unboffered mode. BG.
             return os.system(' '.join(self.call_args))
     else:
         def _run_cmd(self):
             # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
             subp = subprocess.Popen(self.call_args)
             self.pids.append(subp.pid)
             # If this fails, the pid will be left in self.pids and cleaned up
             # later, but if the wait call succeeds, then we can clear the
             # stored pid.
             retcode = subp.wait()
             self.pids.pop()
             return retcode
         
     def run(self):
         """Run the stored commands"""
         try:
             return self._run_cmd()
         except:
             import traceback
             traceback.print_exc()
             return 1  # signal failure
 
     def __del__(self):
         """Cleanup on exit by killing any leftover processes."""
 
         if not hasattr(os, 'kill'):
             return
         
         for pid in self.pids:
             try:
                 print 'Cleaning stale PID:', pid
                 os.kill(pid, signal.SIGKILL)
             except OSError:
                 # This is just a best effort, if we fail or the process was
                 # really gone, ignore it.
                 pass     
 
 
 def make_runners():
     """Define the top-level packages that need to be tested.
     """
 
     # Packages to be tested via nose, that only depend on the stdlib
     nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils' ]
-
+    
+    if have['zmq']:
+        nose_pkg_names.append('parallel')
+    
     # For debugging this code, only load quick stuff
     #nose_pkg_names = ['core', 'extensions']  # dbg
 
     # Make fully qualified package names prepending 'IPython.' to our name lists
     nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
 
     # Make runners
     runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
     
     return runners
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
     
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
                                                 
                         # Loading ipdoctest causes problems with Twisted, but
                         # our test suite runner now separates things and runs
                         # all Twisted tests with trial.
                         '--with-ipdoctest',
                         '--ipdoctest-tests','--ipdoctest-extension=txt',
                         
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
 
     if nose.__version__ >= '0.11':
         # I don't fully understand why we need this one, but depending on what
         # directory the test suite is run from, if we don't give it, 0 tests
         # get run.  Specifically, if the test suite is run from the source dir
         # with an argument (like 'iptest.py IPython.core', 0 tests are run,
         # even if the same call done in this directory works fine).  It appears
         # that if the requested package is in the current dir, nose bails early
         # by default.  Since it's otherwise harmless, leave it in by default
         # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
         argv.append('--traverse-namespace')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
     plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
     # We need a global ipython running in this process
     globalipapp.start_ipython()
     # Now nose can run
     TestProgram(argv=argv, plugins=plugins)
 
 
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
     This function constructs :class:`IPTester` instances for all IPython
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
 
     runners = make_runners()
 
     # Run the test runners in a temporary dir so we can nuke it when finished
     # to clean up any junk files left over by accident.  This also makes it
     # robust against being run in non-writeable directories by mistake, as the
     # temp dir will always be user-writeable.
     curdir = os.getcwd()
     testdir = tempfile.gettempdir()
     os.chdir(testdir)
 
     # Run all test runners, tracking execution time
     failed = []
     t_start = time.time()
     try:
         for (name, runner) in runners:
             print '*'*70
             print 'IPython test group:',name
             res = runner.run()
             if res:
                 failed.append( (name, runner) )
     finally:
         os.chdir(curdir)
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)
     nfail = len(failed)
     # summarize results
     print
     print '*'*70
     print 'Test suite completed for system with the following information:'
     print report()
     print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     print 'Status:'
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
         print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name, failed_runner in failed:
             print '-'*40
             print 'Runner failed:',name
             print 'You may wish to rerun this one individually, with:'
             print ' '.join(failed_runner.call_args)
             print
 
 
 def main():
     for arg in sys.argv[1:]:
         if arg.startswith('IPython'):
             # This is in-process
             run_iptest()
     else:
         # This starts subprocesses
         run_iptestall()
 
 
 if __name__ == '__main__':
     main()

IPython/utils/pickleutil.py

@@ -1,153 +1,153 b''
 # encoding: utf-8
 
 """Pickle related utilities. Perhaps this should be called 'can'."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import copy
 import sys
 from types import FunctionType
 
 import codeutil
 
 #-------------------------------------------------------------------------------
 # Classes
 #-------------------------------------------------------------------------------
 
 
 class CannedObject(object):
     def __init__(self, obj, keys=[]):
         self.keys = keys
         self.obj = copy.copy(obj)
         for key in keys:
             setattr(self.obj, key, can(getattr(obj, key)))
             
     
     def getObject(self, g=None):
         if g is None:
             g = globals()
         for key in self.keys:
             setattr(self.obj, key, uncan(getattr(self.obj, key), g))
         return self.obj
 
 class Reference(CannedObject):
     """object for wrapping a remote reference by name."""
     def __init__(self, name):
         if not isinstance(name, basestring):
             raise TypeError("illegal name: %r"%name)
         self.name = name
     
     def __repr__(self):
         return "<Reference: %r>"%self.name
     
     def getObject(self, g=None):
         if g is None:
             g = globals()
         try:
             return g[self.name]
         except KeyError:
             raise NameError("name %r is not defined"%self.name)
     
 
 class CannedFunction(CannedObject):
     
     def __init__(self, f):
         self._checkType(f)    
         self.code = f.func_code
         self.defaults = f.func_defaults
         self.module = f.__module__ or '__main__'
         self.__name__ = f.__name__
     
     def _checkType(self, obj):
         assert isinstance(obj, FunctionType), "Not a function type"
     
     def getObject(self, g=None):
         # try to load function back into its module:
         if not self.module.startswith('__'):
             try:
                 __import__(self.module)
             except ImportError:
                 pass
             else:
                 g = sys.modules[self.module].__dict__
             
         if g is None:
             g = globals()
         newFunc = FunctionType(self.code, g, self.__name__, self.defaults)
         return newFunc
 
 #-------------------------------------------------------------------------------
 # Functions
 #-------------------------------------------------------------------------------
 
 def can(obj):
     # import here to prevent module-level circular imports
-    from IPython.parallel.dependency import dependent
+    from IPython.parallel import dependent
     if isinstance(obj, dependent):
         keys = ('f','df')
         return CannedObject(obj, keys=keys)
     elif isinstance(obj, FunctionType):
         return CannedFunction(obj)
     elif isinstance(obj,dict):
         return canDict(obj)
     elif isinstance(obj, (list,tuple)):
         return canSequence(obj)
     else:
         return obj
 
 def canDict(obj):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = can(v)
         return newobj
     else:
         return obj
 
 def canSequence(obj):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([can(i) for i in obj])
     else:
         return obj
 
 def uncan(obj, g=None):
     if isinstance(obj, CannedObject):
         return obj.getObject(g)
     elif isinstance(obj,dict):
         return uncanDict(obj, g)
     elif isinstance(obj, (list,tuple)):
         return uncanSequence(obj, g)
     else:
         return obj
 
 def uncanDict(obj, g=None):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = uncan(v,g)
         return newobj
     else:
         return obj
 
 def uncanSequence(obj, g=None):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([uncan(i,g) for i in obj])
     else:
         return obj
 
 
 def rebindFunctionGlobals(f, glbls):
     return FunctionType(f.func_code, glbls)

docs/source/parallel/parallel_details.txt

@@ -1,621 +1,621 b''
 .. _parallel_details:
 
 ==========================================
 Details of Parallel Computing with IPython
 ==========================================
 
 .. note::
 
     There are still many sections to fill out
 
 
 Caveats
 =======
 
 First, some caveats about the detailed workings of parallel computing with 0MQ and IPython.
 
 Non-copying sends and numpy arrays
 ----------------------------------
 
 When numpy arrays are passed as arguments to apply or via data-movement methods, they are not
 copied. This means that you must be careful if you are sending an array that you intend to work
 on. PyZMQ does allow you to track when a message has been sent so you can know when it is safe
 to edit the buffer, but IPython only allows for this.
 
 It is also important to note that the non-copying receive of a message is *read-only*. That
 means that if you intend to work in-place on an array that you have sent or received, you must
 copy it. This is true for both numpy arrays sent to engines and numpy arrays retrieved as
 results.
 
 The following will fail:
 
 .. sourcecode:: ipython
 
     In [3]: A = numpy.zeros(2)
     
     In [4]: def setter(a):
        ...: a[0]=1
        ...: return a
 
     In [5]: rc[0].apply_sync(setter, A)
     ---------------------------------------------------------------------------
     RemoteError                               Traceback (most recent call last)
     ...
     RemoteError: RuntimeError(array is not writeable)
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/streamkernel.py", line 329, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "<ipython-input-14-736187483856>", line 2, in setter
     RuntimeError: array is not writeable
 
 If you do need to edit the array in-place, just remember to copy the array if it's read-only.
 The :attr:`ndarray.flags.writeable` flag will tell you if you can write to an array.
 
 .. sourcecode:: ipython
 
     In [3]: A = numpy.zeros(2)
     
     In [4]: def setter(a):
        ...:     """only copy read-only arrays"""
        ...:     if not a.flags.writeable:
        ...:         a=a.copy()
        ...:     a[0]=1
        ...:     return a
 
     In [5]: rc[0].apply_sync(setter, A)
     Out[5]: array([ 1.,  0.])
     
     # note that results will also be read-only:
     In [6]: _.flags.writeable
     Out[6]: False
 
 If you want to safely edit an array in-place after *sending* it, you must use the `track=True` flag.  IPython always performs non-copying sends of arrays, which return immediately.  You
 must instruct IPython track those messages *at send time* in order to know for sure that the send has completed.  AsyncResults have a :attr:`sent` property, and :meth:`wait_on_send` method
 for checking and waiting for 0MQ to finish with a buffer.
 
 .. sourcecode:: ipython
 
     In [5]: A = numpy.random.random((1024,1024))
     
     In [6]: view.track=True
     
     In [7]: ar = view.apply_async(lambda x: 2*x, A)
     
     In [8]: ar.sent
     Out[8]: False
     
     In [9]: ar.wait_on_send() # blocks until sent is True
 
 
 What is sendable?
 -----------------
 
 If IPython doesn't know what to do with an object, it will pickle it. There is a short list of
 objects that are not pickled: ``buffers``, ``str/bytes`` objects, and ``numpy``
 arrays. These are handled specially by IPython in order to prevent the copying of data. Sending
 bytes or numpy arrays will result in exactly zero in-memory copies of your data (unless the data
 is very small).
 
 If you have an object that provides a Python buffer interface, then you can always send that
 buffer without copying - and reconstruct the object on the other side in your own code. It is
 possible that the object reconstruction will become extensible, so you can add your own
 non-copying types, but this does not yet exist.
 
 Closures
 ********
 
 Just about anything in Python is pickleable. The one notable exception is objects (generally
 functions) with *closures*. Closures can be a complicated topic, but the basic principal is that
 functions that refer to variables in their parent scope have closures.
 
 An example of a function that uses a closure:
 
 .. sourcecode:: python
 
     def f(a):
         def inner():
             # inner will have a closure
             return a
         return echo
     
     f1 = f(1)
     f2 = f(2)
     f1() # returns 1
     f2() # returns 2
 
 f1 and f2 will have closures referring to the scope in which `inner` was defined, because they
 use the variable 'a'. As a result, you would not be able to send ``f1`` or ``f2`` with IPython.
 Note that you *would* be able to send `f`. This is only true for interactively defined
 functions (as are often used in decorators), and only when there are variables used inside the
 inner function, that are defined in the outer function. If the names are *not* in the outer
 function, then there will not be a closure, and the generated function will look in
 ``globals()`` for the name:
 
 .. sourcecode:: python
 
     def g(b):
         # note that `b` is not referenced in inner's scope
         def inner():
             # this inner will *not* have a closure
             return a
         return echo
     g1 = g(1)
     g2 = g(2)
     g1() # raises NameError on 'a'
     a=5
     g2() # returns 5
 
 `g1` and `g2` *will* be sendable with IPython, and will treat the engine's namespace as
 globals().  The :meth:`pull` method is implemented based on this principal.  If we did not
 provide pull, you could implement it yourself with `apply`, by simply returning objects out
 of the global namespace:
 
 .. sourcecode:: ipython
 
     In [10]: view.apply(lambda : a)
     
     # is equivalent to
     In [11]: view.pull('a')
 
 Running Code
 ============
 
 There are two principal units of execution in Python: strings of Python code (e.g. 'a=5'),
 and Python functions.  IPython is designed around the use of functions via the core
 Client method, called `apply`.
 
 Apply
 -----
 
 The principal method of remote execution is :meth:`apply`, of View objects. The Client provides
 the full execution and communication API for engines via its low-level
 :meth:`send_apply_message` method.
 
 f : function
     The fuction to be called remotely
 args : tuple/list
     The positional arguments passed to `f`
 kwargs : dict
     The keyword arguments passed to `f`
 
 flags for all views:
 
 block : bool (default: view.block)
     Whether to wait for the result, or return immediately.
     False:
         returns AsyncResult
     True:
         returns actual result(s) of f(*args, **kwargs)
         if multiple targets:
             list of results, matching `targets`
 track : bool [default view.track]
     whether to track non-copying sends.
 
 targets : int,list of ints, 'all', None [default view.targets]
     Specify the destination of the job.
     if 'all' or None:
         Run on all active engines
     if list:
         Run on each specified engine
     if int:
         Run on single engine
 
 Note that LoadBalancedView uses targets to restrict possible destinations.  LoadBalanced calls
 will always execute in just one location.
 
 flags only in LoadBalancedViews:
 
 after : Dependency or collection of msg_ids
     Only for load-balanced execution (targets=None)
     Specify a list of msg_ids as a time-based dependency.
     This job will only be run *after* the dependencies
     have been met.
     
 follow : Dependency or collection of msg_ids
     Only for load-balanced execution (targets=None)
     Specify a list of msg_ids as a location-based dependency.
     This job will only be run on an engine where this dependency
     is met.
 
 timeout : float/int or None
     Only for load-balanced execution (targets=None)
     Specify an amount of time (in seconds) for the scheduler to
     wait for dependencies to be met before failing with a
     DependencyTimeout.
 
 execute and run
 ---------------
 
 For executing strings of Python code, :class:`DirectView`s also provide an :meth:`execute` and a
 :meth:`run` method, which rather than take functions and arguments, take simple strings.
 `execute` simply takes a string of Python code to execute, and sends it to the Engine(s). `run`
 is the same as `execute`, but for a *file*, rather than a string. It is simply a wrapper that
 does something very similar to ``execute(open(f).read())``.
 
 .. note::
 
     TODO: Example
 
 Views
 =====
 
 The principal extension of the :class:`~parallel.Client` is the
-:class:`~parallel.view.View` class. The client
+:class:`~parallel.View` class. The client
 
 
 DirectView
 ----------
 
 The :class:`.DirectView` is the class for the IPython :ref:`Multiplexing Interface
 <parallel_multiengine>`.
 
 Creating a DirectView
 *********************
 
 DirectViews can be created in two ways, by index access to a client, or by a client's
 :meth:`view` method.  Index access to a Client works in a few ways.  First, you can create
 DirectViews to single engines simply by accessing the client by engine id:
 
 .. sourcecode:: ipython
 
     In [2]: rc[0]
     Out[2]: <DirectView 0>
 
 You can also create a DirectView with a list of engines:
 
 .. sourcecode:: ipython
 
     In [2]: rc[0,1,2]
     Out[2]: <DirectView [0,1,2]>
 
 Other methods for accessing elements, such as slicing and negative indexing, work by passing
 the index directly to the client's :attr:`ids` list, so:
 
 .. sourcecode:: ipython
 
     # negative index
     In [2]: rc[-1]
     Out[2]: <DirectView 3>
     
     # or slicing:
     In [3]: rc[::2]
     Out[3]: <DirectView [0,2]>
 
 are always the same as:
 
 .. sourcecode:: ipython
 
     In [2]: rc[rc.ids[-1]]
     Out[2]: <DirectView 3>
     
     In [3]: rc[rc.ids[::2]]
     Out[3]: <DirectView [0,2]>
 
 Also note that the slice is evaluated at the time of construction of the DirectView, so the 
 targets will not change over time if engines are added/removed from the cluster.
 
 Execution via DirectView
 ************************
 
 The DirectView is the simplest way to work with one or more engines directly (hence the name).
 
 
 Data movement via DirectView
 ****************************
 
 Since a Python namespace is just a :class:`dict`, :class:`DirectView` objects provide
 dictionary-style access by key and methods such as :meth:`get` and
 :meth:`update` for convenience. This make the remote namespaces of the engines
 appear as a local dictionary. Underneath, these methods call :meth:`apply`:
 
 .. sourcecode:: ipython
 
     In [51]: dview['a']=['foo','bar']
 
     In [52]: dview['a']
     Out[52]: [ ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'] ]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`Client` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI should be used:
 
 .. sourcecode:: ipython
 
     In [58]: dview.scatter('a',range(16))
     Out[58]: [None,None,None,None]
 
     In [59]: dview['a']
     Out[59]: [ [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15] ]
 
     In [60]: dview.gather('a')
     Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 Push and pull
 -------------
 
 push
 
 pull
 
 
 
 
     
 LoadBalancedView
 ----------------
 
 The :class:`.LoadBalancedView` 
 
 
 Data Movement
 =============
 
 Reference
 
 Results
 =======
 
 AsyncResults
 ------------
 
 Our primary representation is the AsyncResult object, based on the object of the same name in
 the built-in :mod:`multiprocessing.pool` module. Our version provides a superset of that
 interface.
 
 The basic principle of the AsyncResult is the encapsulation of one or more results not yet completed.  Execution methods (including data movement, such as push/pull) will all return
 AsyncResults when `block=False`.
 
 The mp.pool.AsyncResult interface
 ---------------------------------
 
 The basic interface of the AsyncResult is exactly that of the AsyncResult in :mod:`multiprocessing.pool`, and consists of four methods:
 
 .. AsyncResult spec directly from docs.python.org
 
 .. class:: AsyncResult
 
    The stdlib AsyncResult spec
    
    .. method:: wait([timeout])
 
       Wait until the result is available or until *timeout* seconds pass. This
       method always returns ``None``.
 
    .. method:: ready()
 
       Return whether the call has completed.
 
    .. method:: successful()
 
       Return whether the call completed without raising an exception.  Will
       raise :exc:`AssertionError` if the result is not ready.
 
    .. method:: get([timeout])
 
       Return the result when it arrives.  If *timeout* is not ``None`` and the
       result does not arrive within *timeout* seconds then
       :exc:`TimeoutError` is raised.  If the remote call raised
       an exception then that exception will be reraised as a :exc:`RemoteError`
       by :meth:`get`.
 
 
 While an AsyncResult is not done, you can check on it with its :meth:`ready` method, which will
 return whether the AR is done. You can also wait on an AsyncResult with its :meth:`wait` method.
 This method blocks until the result arrives. If you don't want to wait forever, you can pass a
 timeout (in seconds) as an argument to :meth:`wait`. :meth:`wait` will *always return None*, and
 should never raise an error.
 
 :meth:`ready` and :meth:`wait` are insensitive to the success or failure of the call. After a
 result is done, :meth:`successful` will tell you whether the call completed without raising an
 exception.
 
 If you actually want the result of the call, you can use :meth:`get`. Initially, :meth:`get`
 behaves just like :meth:`wait`, in that it will block until the result is ready, or until a
 timeout is met. However, unlike :meth:`wait`, :meth:`get` will raise a :exc:`TimeoutError` if
 the timeout is reached and the result is still not ready. If the result arrives before the
 timeout is reached, then :meth:`get` will return the result itself if no exception was raised,
 and will raise an exception if there was.
 
 Here is where we start to expand on the multiprocessing interface. Rather than raising the
 original exception, a RemoteError will be raised, encapsulating the remote exception with some
 metadata. If the AsyncResult represents multiple calls (e.g. any time `targets` is plural), then
 a CompositeError, a subclass of RemoteError, will be raised.
 
 .. seealso::
 
     For more information on remote exceptions, see :ref:`the section in the Direct Interface
     <Parallel_exceptions>`.
 
 Extended interface
 ******************
 
 
 Other extensions of the AsyncResult interface include convenience wrappers for :meth:`get`.
 AsyncResults have a property, :attr:`result`, with the short alias :attr:`r`, which simply call
 :meth:`get`. Since our object is designed for representing *parallel* results, it is expected
 that many calls (any of those submitted via DirectView) will map results to engine IDs. We
 provide a :meth:`get_dict`, which is also a wrapper on :meth:`get`, which returns a dictionary
 of the individual results, keyed by engine ID.
 
 You can also prevent a submitted job from actually executing, via the AsyncResult's :meth:`abort` method.  This will instruct engines to not execute the job when it arrives.
 
 The larger extension of the AsyncResult API is the :attr:`metadata` attribute.  The metadata
 is a dictionary (with attribute access) that contains, logically enough, metadata about the
 execution.
 
 Metadata keys:
 
 timestamps
 
 submitted
     When the task left the Client
 started
     When the task started execution on the engine
 completed
     When execution finished on the engine
 received
     When the result arrived on the Client
     
     note that it is not known when the result arrived in 0MQ on the client, only when it
     arrived in Python via :meth:`Client.spin`, so in interactive use, this may not be
     strictly informative.
 
 Information about the engine
 
 engine_id
     The integer id
 engine_uuid
     The UUID of the engine
 
 output of the call
 
 pyerr
     Python exception, if there was one
 pyout
     Python output
 stderr
     stderr stream
 stdout
     stdout (e.g. print) stream
 
 And some extended information
 
 status
     either 'ok' or 'error'
 msg_id
     The UUID of the message
 after
     For tasks: the time-based msg_id dependencies
 follow
     For tasks: the location-based msg_id dependencies
 
 While in most cases, the Clients that submitted a request will be the ones using the results,
 other Clients can also request results directly from the Hub. This is done via the Client's
 :meth:`get_result` method. This method will *always* return an AsyncResult object. If the call
 was not submitted by the client, then it will be a subclass, called :class:`AsyncHubResult`.
 These behave in the same way as an AsyncResult, but if the result is not ready, waiting on an
 AsyncHubResult polls the Hub, which is much more expensive than the passive polling used
 in regular AsyncResults.
 
 
 The Client keeps track of all results
 history, results, metadata
 
 Querying the Hub
 ================
 
 The Hub sees all traffic that may pass through the schedulers between engines and clients.
 It does this so that it can track state, allowing multiple clients to retrieve results of
 computations submitted by their peers, as well as persisting the state to a database.
 
 queue_status
 
     You can check the status of the queues of the engines with this command.
 
 result_status
 
     check on results
 
 purge_results
 
     forget results (conserve resources)
 
 Controlling the Engines
 =======================
 
 There are a few actions you can do with Engines that do not involve execution.  These
 messages are sent via the Control socket, and bypass any long queues of waiting execution
 jobs
 
 abort
 
     Sometimes you may want to prevent a job you have submitted from actually running. The method
     for this is :meth:`abort`. It takes a container of msg_ids, and instructs the Engines to not
     run the jobs if they arrive. The jobs will then fail with an AbortedTask error.
 
 clear
 
     You may want to purge the Engine(s) namespace of any data you have left in it.  After
     running `clear`, there will be no names in the Engine's namespace
 
 shutdown
 
     You can also instruct engines (and the Controller) to terminate from a Client.  This 
     can be useful when a job is finished, since you can shutdown all the processes with a
     single command.
 
 Synchronization
 ===============
 
 Since the Client is a synchronous object, events do not automatically trigger in your
 interactive session - you must poll the 0MQ sockets for incoming messages.  Note that 
 this polling *does not* actually make any network requests.  It simply performs a `select`
 operation, to check if messages are already in local memory, waiting to be handled.
 
 The method that handles incoming messages is :meth:`spin`. This method flushes any waiting
 messages on the various incoming sockets, and updates the state of the Client.
 
 If you need to wait for particular results to finish, you can use the :meth:`wait` method,
 which will call :meth:`spin` until the messages are no longer outstanding. Anything that
 represents a collection of messages, such as a list of msg_ids or one or more AsyncResult
 objects, can be passed as argument to wait. A timeout can be specified, which will prevent
 the call from blocking for more than a specified time, but the default behavior is to wait
 forever.
 
 
 
 The client also has an `outstanding` attribute - a ``set`` of msg_ids that are awaiting replies.
 This is the default if wait is called with no arguments - i.e. wait on *all* outstanding
 messages.
 
 
 .. note::
 
     TODO wait example
 
 Map
 ===
 
 Many parallel computing problems can be expressed as a `map`, or running a single program with a
 variety of different inputs. Python has a built-in :py-func:`map`, which does exactly this, and
 many parallel execution tools in Python, such as the built-in :py-class:`multiprocessing.Pool`
 object provide implementations of `map`. All View objects provide a :meth:`map` method as well,
 but the load-balanced and direct implementations differ.
 
 Views' map methods can be called on any number of sequences, but they can also take the `block`
 and `bound` keyword arguments, just like :meth:`~client.apply`, but *only as keywords*.
 
 .. sourcecode:: python
 
     dview.map(*sequences, block=None)
 
 
 * iter, map_async, reduce
 
 Decorators and RemoteFunctions
 ==============================
 
 @parallel
 
 @remote
 
 RemoteFunction
 
 ParallelFunction
 
 Dependencies
 ============
 
 @depend
 
 @require
 
 Dependency

docs/source/parallel/parallel_intro.txt

@@ -1,253 +1,253 b''
 .. _ip1par:
 
 ============================
 Overview and getting started
 ============================
 
 Introduction
 ============
 
 This section gives an overview of IPython's sophisticated and powerful
 architecture for parallel and distributed computing. This architecture
 abstracts out parallelism in a very general way, which enables IPython to
 support many different styles of parallelism including:
 
 * Single program, multiple data (SPMD) parallelism.
 * Multiple program, multiple data (MPMD) parallelism.
 * Message passing using MPI.
 * Task farming.
 * Data parallel.
 * Combinations of these approaches.
 * Custom user defined approaches.
 
 Most importantly, IPython enables all types of parallel applications to
 be developed, executed, debugged and monitored *interactively*. Hence,
 the ``I`` in IPython.  The following are some example usage cases for IPython:
 
 * Quickly parallelize algorithms that are embarrassingly parallel
   using a number of simple approaches.  Many simple things can be
   parallelized interactively in one or two lines of code.
 
 * Steer traditional MPI applications on a supercomputer from an
   IPython session on your laptop.
 
 * Analyze and visualize large datasets (that could be remote and/or
   distributed) interactively using IPython and tools like
   matplotlib/TVTK.
 
 * Develop, test and debug new parallel algorithms 
   (that may use MPI) interactively.
 
 * Tie together multiple MPI jobs running on different systems into
   one giant distributed and parallel system.
 
 * Start a parallel job on your cluster and then have a remote 	
   collaborator connect to it and pull back data into their 
   local IPython session for plotting and analysis.
 
 * Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
 =====================
 
 The IPython architecture consists of four components:
 
 * The IPython engine.
 * The IPython hub.
 * The IPython schedulers.
 * The controller client.
 
 These components live in the :mod:`IPython.parallel` package and are
 installed with IPython.  They do, however, have additional dependencies
 that must be installed.  For more information, see our
 :ref:`installation documentation <install_index>`.
 
 .. TODO: include zmq in install_index
 
 IPython engine
 ---------------
 
 The IPython engine is a Python instance that takes Python commands over a
 network connection. Eventually, the IPython engine will be a full IPython
 interpreter, but for now, it is a regular Python interpreter. The engine
 can also handle incoming and outgoing Python objects sent over a network
 connection.  When multiple engines are started, parallel and distributed
 computing becomes possible. An important feature of an IPython engine is
 that it blocks while user code is being executed. Read on for how the
 IPython controller solves this problem to expose a clean asynchronous API
 to the user.
 
 IPython controller
 ------------------
 
 The IPython controller processes provide an interface for working with a set of engines.
 At a general level, the controller is a collection of processes to which IPython engines
 and clients can connect. The controller is composed of a :class:`Hub` and a collection of
 :class:`Schedulers`. These Schedulers are typically run in separate processes but on the
 same machine as the Hub, but can be run anywhere from local threads or on remote machines.
 
 The controller also provides a single point of contact for users who wish to
 utilize the engines connected to the controller. There are different ways of
 working with a controller. In IPython, all of these models are implemented via
 the client's :meth:`.View.apply` method, with various arguments, or
 constructing :class:`.View` objects to represent subsets of engines. The two
 primary models for interacting with engines are:
 
 * A **Direct** interface, where engines are addressed explicitly.
 * A **LoadBalanced** interface, where the Scheduler is trusted with assigning work to
   appropriate engines.
 
 Advanced users can readily extend the View models to enable other
 styles of parallelism. 
 
 .. note:: 
 
     A single controller and set of engines can be used with multiple models
     simultaneously. This opens the door for lots of interesting things.
 
 
 The Hub
 *******
 
 The center of an IPython cluster is the Hub. This is the process that keeps
 track of engine connections, schedulers, clients, as well as all task requests and
 results. The primary role of the Hub is to facilitate queries of the cluster state, and
 minimize the necessary information required to establish the many connections involved in
 connecting new clients and engines.
 
 
 Schedulers
 **********
 
 All actions that can be performed on the engine go through a Scheduler. While the engines
 themselves block when user code is run, the schedulers hide that from the user to provide
 a fully asynchronous interface to a set of engines.
 
 
 IPython client and views
 ------------------------
 
 There is one primary object, the :class:`~.parallel.Client`, for connecting to a cluster.
-For each execution model, there is a corresponding :class:`~.parallel.view.View`. These views
+For each execution model, there is a corresponding :class:`~.parallel.View`. These views
 allow users to interact with a set of engines through the interface. Here are the two default
 views:
 
 * The :class:`DirectView` class for explicit addressing.
 * The :class:`LoadBalancedView` class for destination-agnostic scheduling.
 
 Security
 --------
 
 IPython uses ZeroMQ for networking, which has provided many advantages, but
 one of the setbacks is its utter lack of security [ZeroMQ]_. By default, no IPython
 connections are encrypted, but open ports only listen on localhost. The only
 source of security for IPython is via ssh-tunnel. IPython supports both shell
 (`openssh`) and `paramiko` based tunnels for connections.  There is a key necessary
 to submit requests, but due to the lack of encryption, it does not provide
 significant security if loopback traffic is compromised.
 
 In our architecture, the controller is the only process that listens on
 network ports, and is thus the main point of vulnerability. The standard model
 for secure connections is to designate that the controller listen on
 localhost, and use ssh-tunnels to connect clients and/or
 engines.
 
 To connect and authenticate to the controller an engine or client needs
 some information that the controller has stored in a JSON file.
 Thus, the JSON files need to be copied to a location where
 the clients and engines can find them. Typically, this is the
 :file:`~/.ipython/cluster_default/security` directory on the host where the 
 client/engine is running (which could be a different host than the controller). 
 Once the JSON files are copied over, everything should work fine.
 
 Currently, there are two JSON files that the controller creates:
 
 ipcontroller-engine.json
     This JSON file has the information necessary for an engine to connect
     to a controller.
 
 ipcontroller-client.json
     The client's connection information.  This may not differ from the engine's,
     but since the controller may listen on different ports for clients and
     engines, it is stored separately.
 
 More details of how these JSON files are used are given below.
 
 A detailed description of the security model and its implementation in IPython
 can be found :ref:`here <parallelsecurity>`.
 
 .. warning::
 
     Even at its most secure, the Controller listens on ports on localhost, and
     every time you make a tunnel, you open a localhost port on the connecting
     machine that points to the Controller. If localhost on the Controller's
     machine, or the machine of any client or engine, is untrusted, then your
     Controller is insecure. There is no way around this with ZeroMQ.
 
 
 
 Getting Started
 ===============
 
 To use IPython for parallel computing, you need to start one instance of the
 controller and one or more instances of the engine. Initially, it is best to
 simply start a controller and engines on a single host using the
 :command:`ipcluster` command. To start a controller and 4 engines on your
 localhost, just do::
 
     $ ipcluster start -n 4
 
 More details about starting the IPython controller and engines can be found
 :ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
 are ready to use the engines to do something useful. To make sure
 everything is working correctly, try the following commands:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.parallel import Client
 	
 	In [2]: c = Client()
 	
 	In [4]: c.ids
 	Out[4]: set([0, 1, 2, 3])
 	
 	In [5]: c[:].apply_sync(lambda : "Hello, World")
 	Out[5]: [ 'Hello, World', 'Hello, World', 'Hello, World', 'Hello, World' ]
 
 
 When a client is created with no arguments, the client tries to find the corresponding JSON file
 in the local `~/.ipython/cluster_default/security` directory. Or if you specified a profile,
 you can use that with the Client.  This should cover most cases:
 
 .. sourcecode:: ipython
 
     In [2]: c = Client(profile='myprofile')
 
 If you have put the JSON file in a different location or it has a different name, create the
 client like this:
 
 .. sourcecode:: ipython
 
     In [2]: c = Client('/path/to/my/ipcontroller-client.json')
 
 Remember, a client needs to be able to see the Hub's ports to connect. So if they are on a
 different machine, you may need to use an ssh server to tunnel access to that machine,
 then you would connect to it with:
 
 .. sourcecode:: ipython
 
     In [2]: c = Client(sshserver='myhub.example.com')
 
 Where 'myhub.example.com' is the url or IP address of the machine on
 which the Hub process is running (or another machine that has direct access to the Hub's ports).
 
 The SSH server may already be specified in ipcontroller-client.json, if the controller was
 instructed at its launch time.
 
 You are now ready to learn more about the :ref:`Direct
 <parallel_multiengine>` and :ref:`LoadBalanced <parallel_task>` interfaces to the
 controller.
 
 .. [ZeroMQ] ZeroMQ.  http://www.zeromq.org