From c03e562be19f19e74b18da82dcac0567537a0489 2014-06-24 23:14:37 From: Thomas Kluyver Date: 2014-06-24 23:14:37 Subject: [PATCH] Merge pull request #6042 from minrk/import-less-than-everything reduce a few implicit imports --- diff --git a/IPython/html/services/clusters/clustermanager.py b/IPython/html/services/clusters/clustermanager.py index 66fb529..fecde70 100644 --- a/IPython/html/services/clusters/clustermanager.py +++ b/IPython/html/services/clusters/clustermanager.py @@ -21,7 +21,6 @@ from zmq.eventloop import ioloop from IPython.config.configurable import LoggingConfigurable from IPython.utils.traitlets import Dict, Instance, CFloat -from IPython.parallel.apps.ipclusterapp import IPClusterStart from IPython.core.profileapp import list_profiles_in from IPython.core.profiledir import ProfileDir from IPython.utils import py3compat @@ -33,17 +32,6 @@ from IPython.utils.path import get_ipython_dir #----------------------------------------------------------------------------- -class DummyIPClusterStart(IPClusterStart): - """Dummy subclass to skip init steps that conflict with global app. - - Instantiating and initializing this class should result in fully configured - launchers, but no other side effects or state. - """ - - def init_signal(self): - pass - def reinit_logging(self): - pass class ClusterManager(LoggingConfigurable): @@ -59,6 +47,20 @@ class ClusterManager(LoggingConfigurable): return IOLoop.instance() def build_launchers(self, profile_dir): + from IPython.parallel.apps.ipclusterapp import IPClusterStart + + class DummyIPClusterStart(IPClusterStart): + """Dummy subclass to skip init steps that conflict with global app. + + Instantiating and initializing this class should result in fully configured + launchers, but no other side effects or state. + """ + + def init_signal(self): + pass + def reinit_logging(self): + pass + starter = DummyIPClusterStart(log=self.log) starter.initialize(['--profile-dir', profile_dir]) cl = starter.controller_launcher diff --git a/IPython/kernel/connect.py b/IPython/kernel/connect.py index c5b82a2..a2799bc 100644 --- a/IPython/kernel/connect.py +++ b/IPython/kernel/connect.py @@ -22,7 +22,6 @@ from subprocess import Popen, PIPE import tempfile import zmq -from zmq.ssh import tunnel # IPython imports from IPython.config import LoggingConfigurable @@ -342,6 +341,7 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ + from zmq.ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 6ad1689..f313329 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -18,7 +18,6 @@ from pprint import pprint pjoin = os.path.join import zmq -from zmq.ssh import tunnel from IPython.config.configurable import MultipleInstanceError from IPython.core.application import BaseIPythonApplication @@ -443,6 +442,7 @@ class Client(HasTraits): # default to ssh via localhost sshserver = addr if self._ssh and password is None: + from zmq.ssh import tunnel if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko): password=False else: @@ -467,6 +467,7 @@ class Client(HasTraits): self._query_socket = self._context.socket(zmq.DEALER) if self._ssh: + from zmq.ssh import tunnel tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs) else: self._query_socket.connect(cfg['registration']) @@ -589,6 +590,7 @@ class Client(HasTraits): def connect_socket(s, url): if self._ssh: + from zmq.ssh import tunnel return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) else: return s.connect(url) diff --git a/IPython/parallel/client/map.py b/IPython/parallel/client/map.py index 1fc6d65..67b5b29 100644 --- a/IPython/parallel/client/map.py +++ b/IPython/parallel/client/map.py @@ -4,56 +4,33 @@ Scattering consists of partitioning a sequence and sending the various pieces to individual nodes in a cluster. - - -Authors: - -* Brian Granger -* MinRK - """ -#------------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#------------------------------------------------------------------------------- - -#------------------------------------------------------------------------------- -# Imports -#------------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import division +import sys from itertools import islice from IPython.utils.data import flatten as utils_flatten -#------------------------------------------------------------------------------- -# Figure out which array packages are present and their array types -#------------------------------------------------------------------------------- - -arrayModules = [] -try: - import Numeric -except ImportError: - pass -else: - arrayModules.append({'module':Numeric, 'type':Numeric.arraytype}) -try: - import numpy -except ImportError: - pass -else: - arrayModules.append({'module':numpy, 'type':numpy.ndarray}) -try: - import numarray -except ImportError: - pass -else: - arrayModules.append({'module':numarray, - 'type':numarray.numarraycore.NumArray}) + +numpy = None + +def is_array(obj): + """Is an object a numpy array? + + Avoids importing numpy until it is requested + """ + global numpy + if 'numpy' not in sys.modules: + return False + + if numpy is None: + import numpy + return isinstance(obj, numpy.ndarray) class Map(object): """A class for partitioning a sequence using a map.""" @@ -90,14 +67,12 @@ class Map(object): def joinPartitions(self, listOfPartitions): return self.concatenate(listOfPartitions) - + def concatenate(self, listOfPartitions): testObject = listOfPartitions[0] # First see if we have a known array type - for m in arrayModules: - #print m - if isinstance(testObject, m['type']): - return m['module'].concatenate(listOfPartitions) + if is_array(testObject): + return numpy.concatenate(listOfPartitions) # Next try for Python sequence types if isinstance(testObject, (list, tuple)): return utils_flatten(listOfPartitions) @@ -117,19 +92,17 @@ class RoundRobinMap(Map): def joinPartitions(self, listOfPartitions): testObject = listOfPartitions[0] # First see if we have a known array type - for m in arrayModules: - #print m - if isinstance(testObject, m['type']): - return self.flatten_array(m['type'], listOfPartitions) + if is_array(testObject): + return self.flatten_array(listOfPartitions) if isinstance(testObject, (list, tuple)): return self.flatten_list(listOfPartitions) return listOfPartitions - def flatten_array(self, klass, listOfPartitions): + def flatten_array(self, listOfPartitions): test = listOfPartitions[0] shape = list(test.shape) shape[0] = sum([ p.shape[0] for p in listOfPartitions]) - A = klass(shape) + A = numpy.ndarray(shape) N = shape[0] q = len(listOfPartitions) for p,part in enumerate(listOfPartitions): @@ -141,23 +114,13 @@ class RoundRobinMap(Map): for i in range(len(listOfPartitions[0])): flat.extend([ part[i] for part in listOfPartitions if len(part) > i ]) return flat - #lengths = [len(x) for x in listOfPartitions] - #maxPartitionLength = len(listOfPartitions[0]) - #numberOfPartitions = len(listOfPartitions) - #concat = self.concatenate(listOfPartitions) - #totalLength = len(concat) - #result = [] - #for i in range(maxPartitionLength): - # result.append(concat[i:totalLength:maxPartitionLength]) - # return self.concatenate(listOfPartitions) def mappable(obj): """return whether an object is mappable or not.""" if isinstance(obj, (tuple,list)): return True - for m in arrayModules: - if isinstance(obj,m['type']): - return True + if is_array(obj): + return True return False dists = {'b':Map,'r':RoundRobinMap} diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 4ae801c..bc0486b 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -14,11 +14,10 @@ from getpass import getpass import zmq from zmq.eventloop import ioloop, zmqstream -from zmq.ssh import tunnel from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import ( - Instance, Dict, Integer, Type, Float, Integer, Unicode, CBytes, Bool + Instance, Dict, Integer, Type, Float, Unicode, CBytes, Bool ) from IPython.utils.py3compat import cast_bytes @@ -58,6 +57,11 @@ class EngineFactory(RegistrationFactory): help="""The SSH private key file to use when tunneling connections to the Controller.""") paramiko=Bool(sys.platform == 'win32', config=True, help="""Whether to use paramiko instead of openssh for tunnels.""") + + @property + def tunnel_mod(self): + from zmq.ssh import tunnel + return tunnel # not configurable: @@ -97,7 +101,7 @@ class EngineFactory(RegistrationFactory): self.sshserver = self.url.split('://')[1].split(':')[0] if self.using_ssh: - if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): + if self.tunnel_mod.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): password=False else: password = getpass("SSH Password for %s: "%self.sshserver) @@ -108,7 +112,7 @@ class EngineFactory(RegistrationFactory): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) - return tunnel.tunnel_connection(s, url, self.sshserver, + return self.tunnel_mod.tunnel_connection(s, url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) @@ -120,7 +124,7 @@ class EngineFactory(RegistrationFactory): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) - url,tunnelobj = tunnel.open_tunnel(url, self.sshserver, + url, tunnelobj = self.tunnel_mod.open_tunnel(url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, )