Show More
@@ -21,7 +21,6 b' from zmq.eventloop import ioloop' | |||
|
21 | 21 | |
|
22 | 22 | from IPython.config.configurable import LoggingConfigurable |
|
23 | 23 | from IPython.utils.traitlets import Dict, Instance, CFloat |
|
24 | from IPython.parallel.apps.ipclusterapp import IPClusterStart | |
|
25 | 24 | from IPython.core.profileapp import list_profiles_in |
|
26 | 25 | from IPython.core.profiledir import ProfileDir |
|
27 | 26 | from IPython.utils import py3compat |
@@ -33,17 +32,6 b' from IPython.utils.path import get_ipython_dir' | |||
|
33 | 32 | #----------------------------------------------------------------------------- |
|
34 | 33 | |
|
35 | 34 | |
|
36 | class DummyIPClusterStart(IPClusterStart): | |
|
37 | """Dummy subclass to skip init steps that conflict with global app. | |
|
38 | ||
|
39 | Instantiating and initializing this class should result in fully configured | |
|
40 | launchers, but no other side effects or state. | |
|
41 | """ | |
|
42 | ||
|
43 | def init_signal(self): | |
|
44 | pass | |
|
45 | def reinit_logging(self): | |
|
46 | pass | |
|
47 | 35 | |
|
48 | 36 | |
|
49 | 37 | class ClusterManager(LoggingConfigurable): |
@@ -59,6 +47,20 b' class ClusterManager(LoggingConfigurable):' | |||
|
59 | 47 | return IOLoop.instance() |
|
60 | 48 | |
|
61 | 49 | def build_launchers(self, profile_dir): |
|
50 | from IPython.parallel.apps.ipclusterapp import IPClusterStart | |
|
51 | ||
|
52 | class DummyIPClusterStart(IPClusterStart): | |
|
53 | """Dummy subclass to skip init steps that conflict with global app. | |
|
54 | ||
|
55 | Instantiating and initializing this class should result in fully configured | |
|
56 | launchers, but no other side effects or state. | |
|
57 | """ | |
|
58 | ||
|
59 | def init_signal(self): | |
|
60 | pass | |
|
61 | def reinit_logging(self): | |
|
62 | pass | |
|
63 | ||
|
62 | 64 | starter = DummyIPClusterStart(log=self.log) |
|
63 | 65 | starter.initialize(['--profile-dir', profile_dir]) |
|
64 | 66 | cl = starter.controller_launcher |
@@ -22,7 +22,6 b' from subprocess import Popen, PIPE' | |||
|
22 | 22 | import tempfile |
|
23 | 23 | |
|
24 | 24 | import zmq |
|
25 | from zmq.ssh import tunnel | |
|
26 | 25 | |
|
27 | 26 | # IPython imports |
|
28 | 27 | from IPython.config import LoggingConfigurable |
@@ -342,6 +341,7 b' def tunnel_to_kernel(connection_info, sshserver, sshkey=None):' | |||
|
342 | 341 | (shell, iopub, stdin, hb) : ints |
|
343 | 342 | The four ports on localhost that have been forwarded to the kernel. |
|
344 | 343 | """ |
|
344 | from zmq.ssh import tunnel | |
|
345 | 345 | if isinstance(connection_info, string_types): |
|
346 | 346 | # it's a path, unpack it |
|
347 | 347 | with open(connection_info) as f: |
@@ -18,7 +18,6 b' from pprint import pprint' | |||
|
18 | 18 | pjoin = os.path.join |
|
19 | 19 | |
|
20 | 20 | import zmq |
|
21 | from zmq.ssh import tunnel | |
|
22 | 21 | |
|
23 | 22 | from IPython.config.configurable import MultipleInstanceError |
|
24 | 23 | from IPython.core.application import BaseIPythonApplication |
@@ -443,6 +442,7 b' class Client(HasTraits):' | |||
|
443 | 442 | # default to ssh via localhost |
|
444 | 443 | sshserver = addr |
|
445 | 444 | if self._ssh and password is None: |
|
445 | from zmq.ssh import tunnel | |
|
446 | 446 | if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko): |
|
447 | 447 | password=False |
|
448 | 448 | else: |
@@ -467,6 +467,7 b' class Client(HasTraits):' | |||
|
467 | 467 | self._query_socket = self._context.socket(zmq.DEALER) |
|
468 | 468 | |
|
469 | 469 | if self._ssh: |
|
470 | from zmq.ssh import tunnel | |
|
470 | 471 | tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs) |
|
471 | 472 | else: |
|
472 | 473 | self._query_socket.connect(cfg['registration']) |
@@ -589,6 +590,7 b' class Client(HasTraits):' | |||
|
589 | 590 | |
|
590 | 591 | def connect_socket(s, url): |
|
591 | 592 | if self._ssh: |
|
593 | from zmq.ssh import tunnel | |
|
592 | 594 | return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) |
|
593 | 595 | else: |
|
594 | 596 | return s.connect(url) |
@@ -4,56 +4,33 b'' | |||
|
4 | 4 | |
|
5 | 5 | Scattering consists of partitioning a sequence and sending the various |
|
6 | 6 | pieces to individual nodes in a cluster. |
|
7 | ||
|
8 | ||
|
9 | Authors: | |
|
10 | ||
|
11 | * Brian Granger | |
|
12 | * MinRK | |
|
13 | ||
|
14 | 7 | """ |
|
15 | 8 | |
|
16 | #------------------------------------------------------------------------------- | |
|
17 | # Copyright (C) 2008-2011 The IPython Development Team | |
|
18 | # | |
|
19 | # Distributed under the terms of the BSD License. The full license is in | |
|
20 | # the file COPYING, distributed as part of this software. | |
|
21 | #------------------------------------------------------------------------------- | |
|
22 | ||
|
23 | #------------------------------------------------------------------------------- | |
|
24 | # Imports | |
|
25 | #------------------------------------------------------------------------------- | |
|
9 | # Copyright (c) IPython Development Team. | |
|
10 | # Distributed under the terms of the Modified BSD License. | |
|
26 | 11 | |
|
27 | 12 | from __future__ import division |
|
28 | 13 | |
|
14 | import sys | |
|
29 | 15 | from itertools import islice |
|
30 | 16 | |
|
31 | 17 | from IPython.utils.data import flatten as utils_flatten |
|
32 | 18 | |
|
33 | #------------------------------------------------------------------------------- | |
|
34 | # Figure out which array packages are present and their array types | |
|
35 | #------------------------------------------------------------------------------- | |
|
36 | 19 | |
|
37 | arrayModules = [] | |
|
38 | try: | |
|
39 | import Numeric | |
|
40 | except ImportError: | |
|
41 |
|
|
|
42 | else: | |
|
43 | arrayModules.append({'module':Numeric, 'type':Numeric.arraytype}) | |
|
44 | try: | |
|
20 | numpy = None | |
|
21 | ||
|
22 | def is_array(obj): | |
|
23 | """Is an object a numpy array? | |
|
24 | ||
|
25 | Avoids importing numpy until it is requested | |
|
26 | """ | |
|
27 | global numpy | |
|
28 | if 'numpy' not in sys.modules: | |
|
29 | return False | |
|
30 | ||
|
31 | if numpy is None: | |
|
45 | 32 | import numpy |
|
46 | except ImportError: | |
|
47 | pass | |
|
48 | else: | |
|
49 | arrayModules.append({'module':numpy, 'type':numpy.ndarray}) | |
|
50 | try: | |
|
51 | import numarray | |
|
52 | except ImportError: | |
|
53 | pass | |
|
54 | else: | |
|
55 | arrayModules.append({'module':numarray, | |
|
56 | 'type':numarray.numarraycore.NumArray}) | |
|
33 | return isinstance(obj, numpy.ndarray) | |
|
57 | 34 | |
|
58 | 35 | class Map(object): |
|
59 | 36 | """A class for partitioning a sequence using a map.""" |
@@ -94,10 +71,8 b' class Map(object):' | |||
|
94 | 71 | def concatenate(self, listOfPartitions): |
|
95 | 72 | testObject = listOfPartitions[0] |
|
96 | 73 | # First see if we have a known array type |
|
97 |
f |
|
|
98 | #print m | |
|
99 | if isinstance(testObject, m['type']): | |
|
100 | return m['module'].concatenate(listOfPartitions) | |
|
74 | if is_array(testObject): | |
|
75 | return numpy.concatenate(listOfPartitions) | |
|
101 | 76 | # Next try for Python sequence types |
|
102 | 77 | if isinstance(testObject, (list, tuple)): |
|
103 | 78 | return utils_flatten(listOfPartitions) |
@@ -117,19 +92,17 b' class RoundRobinMap(Map):' | |||
|
117 | 92 | def joinPartitions(self, listOfPartitions): |
|
118 | 93 | testObject = listOfPartitions[0] |
|
119 | 94 | # First see if we have a known array type |
|
120 |
f |
|
|
121 | #print m | |
|
122 | if isinstance(testObject, m['type']): | |
|
123 | return self.flatten_array(m['type'], listOfPartitions) | |
|
95 | if is_array(testObject): | |
|
96 | return self.flatten_array(listOfPartitions) | |
|
124 | 97 | if isinstance(testObject, (list, tuple)): |
|
125 | 98 | return self.flatten_list(listOfPartitions) |
|
126 | 99 | return listOfPartitions |
|
127 | 100 | |
|
128 |
def flatten_array(self, |
|
|
101 | def flatten_array(self, listOfPartitions): | |
|
129 | 102 | test = listOfPartitions[0] |
|
130 | 103 | shape = list(test.shape) |
|
131 | 104 | shape[0] = sum([ p.shape[0] for p in listOfPartitions]) |
|
132 |
A = |
|
|
105 | A = numpy.ndarray(shape) | |
|
133 | 106 | N = shape[0] |
|
134 | 107 | q = len(listOfPartitions) |
|
135 | 108 | for p,part in enumerate(listOfPartitions): |
@@ -141,22 +114,12 b' class RoundRobinMap(Map):' | |||
|
141 | 114 | for i in range(len(listOfPartitions[0])): |
|
142 | 115 | flat.extend([ part[i] for part in listOfPartitions if len(part) > i ]) |
|
143 | 116 | return flat |
|
144 | #lengths = [len(x) for x in listOfPartitions] | |
|
145 | #maxPartitionLength = len(listOfPartitions[0]) | |
|
146 | #numberOfPartitions = len(listOfPartitions) | |
|
147 | #concat = self.concatenate(listOfPartitions) | |
|
148 | #totalLength = len(concat) | |
|
149 | #result = [] | |
|
150 | #for i in range(maxPartitionLength): | |
|
151 | # result.append(concat[i:totalLength:maxPartitionLength]) | |
|
152 | # return self.concatenate(listOfPartitions) | |
|
153 | 117 | |
|
154 | 118 | def mappable(obj): |
|
155 | 119 | """return whether an object is mappable or not.""" |
|
156 | 120 | if isinstance(obj, (tuple,list)): |
|
157 | 121 | return True |
|
158 | for m in arrayModules: | |
|
159 | if isinstance(obj,m['type']): | |
|
122 | if is_array(obj): | |
|
160 | 123 |
|
|
161 | 124 | return False |
|
162 | 125 |
@@ -14,11 +14,10 b' from getpass import getpass' | |||
|
14 | 14 | |
|
15 | 15 | import zmq |
|
16 | 16 | from zmq.eventloop import ioloop, zmqstream |
|
17 | from zmq.ssh import tunnel | |
|
18 | 17 | |
|
19 | 18 | from IPython.utils.localinterfaces import localhost |
|
20 | 19 | from IPython.utils.traitlets import ( |
|
21 |
Instance, Dict, Integer, Type, Float, |
|
|
20 | Instance, Dict, Integer, Type, Float, Unicode, CBytes, Bool | |
|
22 | 21 | ) |
|
23 | 22 | from IPython.utils.py3compat import cast_bytes |
|
24 | 23 | |
@@ -59,6 +58,11 b' class EngineFactory(RegistrationFactory):' | |||
|
59 | 58 | paramiko=Bool(sys.platform == 'win32', config=True, |
|
60 | 59 | help="""Whether to use paramiko instead of openssh for tunnels.""") |
|
61 | 60 | |
|
61 | @property | |
|
62 | def tunnel_mod(self): | |
|
63 | from zmq.ssh import tunnel | |
|
64 | return tunnel | |
|
65 | ||
|
62 | 66 | |
|
63 | 67 | # not configurable: |
|
64 | 68 | connection_info = Dict() |
@@ -97,7 +101,7 b' class EngineFactory(RegistrationFactory):' | |||
|
97 | 101 | self.sshserver = self.url.split('://')[1].split(':')[0] |
|
98 | 102 | |
|
99 | 103 | if self.using_ssh: |
|
100 | if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): | |
|
104 | if self.tunnel_mod.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): | |
|
101 | 105 | password=False |
|
102 | 106 | else: |
|
103 | 107 | password = getpass("SSH Password for %s: "%self.sshserver) |
@@ -108,7 +112,7 b' class EngineFactory(RegistrationFactory):' | |||
|
108 | 112 | url = disambiguate_url(url, self.location) |
|
109 | 113 | if self.using_ssh: |
|
110 | 114 | self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) |
|
111 | return tunnel.tunnel_connection(s, url, self.sshserver, | |
|
115 | return self.tunnel_mod.tunnel_connection(s, url, self.sshserver, | |
|
112 | 116 | keyfile=self.sshkey, paramiko=self.paramiko, |
|
113 | 117 | password=password, |
|
114 | 118 | ) |
@@ -120,7 +124,7 b' class EngineFactory(RegistrationFactory):' | |||
|
120 | 124 | url = disambiguate_url(url, self.location) |
|
121 | 125 | if self.using_ssh: |
|
122 | 126 | self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) |
|
123 | url,tunnelobj = tunnel.open_tunnel(url, self.sshserver, | |
|
127 | url, tunnelobj = self.tunnel_mod.open_tunnel(url, self.sshserver, | |
|
124 | 128 | keyfile=self.sshkey, paramiko=self.paramiko, |
|
125 | 129 | password=password, |
|
126 | 130 | ) |
General Comments 0
You need to be logged in to leave comments.
Login now