Show More
@@ -100,7 +100,7 class SerializeIt(object): | |||||
100 | self.typeDescriptor = 'ndarray' |
|
100 | self.typeDescriptor = 'ndarray' | |
101 | self.metadata = {'shape':self.obj.shape, |
|
101 | self.metadata = {'shape':self.obj.shape, | |
102 | 'dtype':self.obj.dtype.str} |
|
102 | 'dtype':self.obj.dtype.str} | |
103 |
elif isinstance(self.obj, |
|
103 | elif isinstance(self.obj, bytes): | |
104 | self.typeDescriptor = 'bytes' |
|
104 | self.typeDescriptor = 'bytes' | |
105 | self.metadata = {} |
|
105 | self.metadata = {} | |
106 | elif isinstance(self.obj, buffer): |
|
106 | elif isinstance(self.obj, buffer): | |
@@ -109,7 +109,7 class SerializeIt(object): | |||||
109 | else: |
|
109 | else: | |
110 | self.typeDescriptor = 'pickle' |
|
110 | self.typeDescriptor = 'pickle' | |
111 | self.metadata = {} |
|
111 | self.metadata = {} | |
112 |
self._generateData() |
|
112 | self._generateData() | |
113 |
|
113 | |||
114 | def _generateData(self): |
|
114 | def _generateData(self): | |
115 | if self.typeDescriptor == 'ndarray': |
|
115 | if self.typeDescriptor == 'ndarray': | |
@@ -145,11 +145,13 class UnSerializeIt(UnSerialized): | |||||
145 | def getObject(self): |
|
145 | def getObject(self): | |
146 | typeDescriptor = self.serialized.getTypeDescriptor() |
|
146 | typeDescriptor = self.serialized.getTypeDescriptor() | |
147 | if globals().has_key('numpy') and typeDescriptor == 'ndarray': |
|
147 | if globals().has_key('numpy') and typeDescriptor == 'ndarray': | |
148 | result = numpy.frombuffer(self.serialized.getData(), dtype = self.serialized.metadata['dtype']) |
|
148 | buf = self.serialized.getData() | |
|
149 | if isinstance(buf, buffer): | |||
|
150 | result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype']) | |||
|
151 | else: | |||
|
152 | # memoryview | |||
|
153 | result = numpy.array(buf, dtype = self.serialized.metadata['dtype']) | |||
149 | result.shape = self.serialized.metadata['shape'] |
|
154 | result.shape = self.serialized.metadata['shape'] | |
150 | # numpy arrays with frombuffer are read-only. We are working with |
|
|||
151 | # the numpy folks to address this issue. |
|
|||
152 | # result = result.copy() |
|
|||
153 | elif typeDescriptor == 'pickle': |
|
155 | elif typeDescriptor == 'pickle': | |
154 | result = pickle.loads(self.serialized.getData()) |
|
156 | result = pickle.loads(self.serialized.getData()) | |
155 | elif typeDescriptor in ('bytes', 'buffer'): |
|
157 | elif typeDescriptor in ('bytes', 'buffer'): |
@@ -13,7 +13,7 | |||||
13 | import time |
|
13 | import time | |
14 |
|
14 | |||
15 | from IPython.external.decorator import decorator |
|
15 | from IPython.external.decorator import decorator | |
16 | import error |
|
16 | from . import error | |
17 |
|
17 | |||
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | # Classes |
|
19 | # Classes |
@@ -29,15 +29,15 from IPython.utils.traitlets import (HasTraits, Int, Instance, CUnicode, | |||||
29 | from IPython.external.decorator import decorator |
|
29 | from IPython.external.decorator import decorator | |
30 | from IPython.external.ssh import tunnel |
|
30 | from IPython.external.ssh import tunnel | |
31 |
|
31 | |||
32 | import error |
|
32 | from . import error | |
33 | import map as Map |
|
33 | from . import map as Map | |
34 | import streamsession as ss |
|
34 | from . import streamsession as ss | |
35 | from asyncresult import AsyncResult, AsyncMapResult, AsyncHubResult |
|
35 | from .asyncresult import AsyncResult, AsyncMapResult, AsyncHubResult | |
36 | from clusterdir import ClusterDir, ClusterDirError |
|
36 | from .clusterdir import ClusterDir, ClusterDirError | |
37 | from dependency import Dependency, depend, require, dependent |
|
37 | from .dependency import Dependency, depend, require, dependent | |
38 | from remotefunction import remote,parallel,ParallelFunction,RemoteFunction |
|
38 | from .remotefunction import remote,parallel,ParallelFunction,RemoteFunction | |
39 | from util import ReverseDict, disambiguate_url, validate_url |
|
39 | from .util import ReverseDict, disambiguate_url, validate_url | |
40 | from view import DirectView, LoadBalancedView |
|
40 | from .view import DirectView, LoadBalancedView | |
41 |
|
41 | |||
42 | #-------------------------------------------------------------------------- |
|
42 | #-------------------------------------------------------------------------- | |
43 | # helpers for implementing old MEC API via client.apply |
|
43 | # helpers for implementing old MEC API via client.apply |
@@ -23,9 +23,9 from zmq.devices import ProcessMonitoredQueue | |||||
23 | from IPython.utils.importstring import import_item |
|
23 | from IPython.utils.importstring import import_item | |
24 | from IPython.utils.traitlets import Int, Str, Instance, List, Bool |
|
24 | from IPython.utils.traitlets import Int, Str, Instance, List, Bool | |
25 |
|
25 | |||
26 | from entry_point import signal_children |
|
26 | from .entry_point import signal_children | |
27 | from hub import Hub, HubFactory |
|
27 | from .hub import Hub, HubFactory | |
28 | from scheduler import launch_scheduler |
|
28 | from .scheduler import launch_scheduler | |
29 |
|
29 | |||
30 | #----------------------------------------------------------------------------- |
|
30 | #----------------------------------------------------------------------------- | |
31 | # Configurable |
|
31 | # Configurable |
@@ -2,8 +2,8 | |||||
2 |
|
2 | |||
3 | from IPython.external.decorator import decorator |
|
3 | from IPython.external.decorator import decorator | |
4 |
|
4 | |||
5 | from asyncresult import AsyncResult |
|
5 | from .asyncresult import AsyncResult | |
6 | from error import UnmetDependency |
|
6 | from .error import UnmetDependency | |
7 |
|
7 | |||
8 |
|
8 | |||
9 | class depend(object): |
|
9 | class depend(object): |
@@ -19,11 +19,11 from IPython.config.configurable import Configurable | |||||
19 | from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat |
|
19 | from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat | |
20 | # from IPython.utils.localinterfaces import LOCALHOST |
|
20 | # from IPython.utils.localinterfaces import LOCALHOST | |
21 |
|
21 | |||
22 | import heartmonitor |
|
22 | from . import heartmonitor | |
23 | from factory import RegistrationFactory |
|
23 | from .factory import RegistrationFactory | |
24 | from streamkernel import Kernel |
|
24 | from .streamkernel import Kernel | |
25 | from streamsession import Message |
|
25 | from .streamsession import Message | |
26 | from util import disambiguate_url |
|
26 | from .util import disambiguate_url | |
27 |
|
27 | |||
28 | def printer(*msg): |
|
28 | def printer(*msg): | |
29 | # print (self.log.handlers, file=sys.__stdout__) |
|
29 | # print (self.log.handlers, file=sys.__stdout__) |
@@ -23,6 +23,7 except ImportError: | |||||
23 | # System library imports. |
|
23 | # System library imports. | |
24 | import zmq |
|
24 | import zmq | |
25 | from zmq.log import handlers |
|
25 | from zmq.log import handlers | |
|
26 | ||||
26 | # Local imports. |
|
27 | # Local imports. | |
27 | from IPython.core.ultratb import FormattedTB |
|
28 | from IPython.core.ultratb import FormattedTB | |
28 | from IPython.external.argparse import ArgumentParser |
|
29 | from IPython.external.argparse import ArgumentParser |
@@ -215,7 +215,7 class CompositeError(RemoteError): | |||||
215 | if not ei: |
|
215 | if not ei: | |
216 | return '[Engine Exception]' |
|
216 | return '[Engine Exception]' | |
217 | else: |
|
217 | else: | |
218 | return '[%s:%s]: ' % (ei['engineid'], ei['method']) |
|
218 | return '[%s:%s]: ' % (ei['engine_id'], ei['method']) | |
219 |
|
219 | |||
220 | def _get_traceback(self, ev): |
|
220 | def _get_traceback(self, ev): | |
221 | try: |
|
221 | try: |
@@ -14,7 +14,7 from zmq.devices import ProcessDevice,ThreadDevice | |||||
14 | from zmq.eventloop import ioloop, zmqstream |
|
14 | from zmq.eventloop import ioloop, zmqstream | |
15 |
|
15 | |||
16 | from IPython.utils.traitlets import Set, Instance, CFloat, Bool |
|
16 | from IPython.utils.traitlets import Set, Instance, CFloat, Bool | |
17 | from factory import LoggingFactory |
|
17 | from .factory import LoggingFactory | |
18 |
|
18 | |||
19 | class Heart(object): |
|
19 | class Heart(object): | |
20 | """A basic heart object for responding to a HeartMonitor. |
|
20 | """A basic heart object for responding to a HeartMonitor. |
@@ -29,12 +29,12 from IPython.config.configurable import Configurable | |||||
29 | from IPython.utils.importstring import import_item |
|
29 | from IPython.utils.importstring import import_item | |
30 | from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool |
|
30 | from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool | |
31 |
|
31 | |||
32 | from entry_point import select_random_ports |
|
32 | from .entry_point import select_random_ports | |
33 | from factory import RegistrationFactory, LoggingFactory |
|
33 | from .factory import RegistrationFactory, LoggingFactory | |
34 |
|
34 | |||
35 | from heartmonitor import HeartMonitor |
|
35 | from .heartmonitor import HeartMonitor | |
36 | from streamsession import Message, wrap_exception, ISO8601 |
|
36 | from .streamsession import Message, wrap_exception, ISO8601 | |
37 | from util import validate_url_container |
|
37 | from .util import validate_url_container | |
38 |
|
38 | |||
39 | try: |
|
39 | try: | |
40 | from pymongo.binary import Binary |
|
40 | from pymongo.binary import Binary |
@@ -36,10 +36,10 from IPython.zmq.parallel.clusterdir import ( | |||||
36 | ApplicationWithClusterDir, |
|
36 | ApplicationWithClusterDir, | |
37 | ClusterDirConfigLoader |
|
37 | ClusterDirConfigLoader | |
38 | ) |
|
38 | ) | |
|
39 | from IPython.zmq.parallel.util import disambiguate_ip_address, split_url | |||
39 | # from IPython.kernel.fcutil import FCServiceFactory, FURLError |
|
40 | # from IPython.kernel.fcutil import FCServiceFactory, FURLError | |
40 | from IPython.utils.traitlets import Instance, Unicode |
|
41 | from IPython.utils.traitlets import Instance, Unicode | |
41 |
|
42 | |||
42 | from util import disambiguate_ip_address, split_url |
|
|||
43 |
|
43 | |||
44 |
|
44 | |||
45 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- |
@@ -31,9 +31,9 from IPython.zmq.log import EnginePUBHandler | |||||
31 | from IPython.zmq.parallel import factory |
|
31 | from IPython.zmq.parallel import factory | |
32 | from IPython.zmq.parallel.engine import EngineFactory |
|
32 | from IPython.zmq.parallel.engine import EngineFactory | |
33 | from IPython.zmq.parallel.streamkernel import Kernel |
|
33 | from IPython.zmq.parallel.streamkernel import Kernel | |
|
34 | from IPython.zmq.parallel.util import disambiguate_url | |||
34 | from IPython.utils.importstring import import_item |
|
35 | from IPython.utils.importstring import import_item | |
35 |
|
36 | |||
36 | from util import disambiguate_url |
|
|||
37 |
|
37 | |||
38 | #----------------------------------------------------------------------------- |
|
38 | #----------------------------------------------------------------------------- | |
39 | # Module level variables |
|
39 | # Module level variables |
@@ -24,7 +24,7 from IPython.zmq.parallel.clusterdir import ( | |||||
24 | ApplicationWithClusterDir, |
|
24 | ApplicationWithClusterDir, | |
25 | ClusterDirConfigLoader |
|
25 | ClusterDirConfigLoader | |
26 | ) |
|
26 | ) | |
27 |
from |
|
27 | from .logwatcher import LogWatcher | |
28 |
|
28 | |||
29 | #----------------------------------------------------------------------------- |
|
29 | #----------------------------------------------------------------------------- | |
30 | # Module level variables |
|
30 | # Module level variables |
@@ -1,7 +1,8 | |||||
1 | """KernelStarter class that intercepts Control Queue messages, and handles process management.""" |
|
1 | """KernelStarter class that intercepts Control Queue messages, and handles process management.""" | |
2 |
|
2 | |||
3 | from zmq.eventloop import ioloop |
|
3 | from zmq.eventloop import ioloop | |
4 | from streamsession import StreamSession |
|
4 | ||
|
5 | from .streamsession import StreamSession | |||
5 |
|
6 | |||
6 | class KernelStarter(object): |
|
7 | class KernelStarter(object): | |
7 | """Object for resetting/killing the Kernel.""" |
|
8 | """Object for resetting/killing the Kernel.""" |
@@ -48,7 +48,7 from IPython.utils.traitlets import Str, Int, List, Unicode, Instance | |||||
48 | from IPython.utils.path import get_ipython_module_path |
|
48 | from IPython.utils.path import get_ipython_module_path | |
49 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError |
|
49 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError | |
50 |
|
50 | |||
51 | from factory import LoggingFactory |
|
51 | from .factory import LoggingFactory | |
52 |
|
52 | |||
53 | # load winhpcjob from IPython.kernel |
|
53 | # load winhpcjob from IPython.kernel | |
54 | try: |
|
54 | try: |
@@ -21,7 +21,7 from zmq.eventloop import ioloop, zmqstream | |||||
21 |
|
21 | |||
22 | from IPython.utils.traitlets import Int, Str, Instance, List |
|
22 | from IPython.utils.traitlets import Int, Str, Instance, List | |
23 |
|
23 | |||
24 | from factory import LoggingFactory |
|
24 | from .factory import LoggingFactory | |
25 |
|
25 | |||
26 | #----------------------------------------------------------------------------- |
|
26 | #----------------------------------------------------------------------------- | |
27 | # Classes |
|
27 | # Classes |
@@ -10,7 +10,7 from datetime import datetime | |||||
10 |
|
10 | |||
11 | from pymongo import Connection |
|
11 | from pymongo import Connection | |
12 |
|
12 | |||
13 | from dictdb import BaseDB |
|
13 | from .dictdb import BaseDB | |
14 |
|
14 | |||
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 | # MongoDB class |
|
16 | # MongoDB class |
@@ -14,8 +14,8 import warnings | |||||
14 |
|
14 | |||
15 | from IPython.testing import decorators as testdec |
|
15 | from IPython.testing import decorators as testdec | |
16 |
|
16 | |||
17 | import map as Map |
|
17 | from . import map as Map | |
18 | from asyncresult import AsyncMapResult |
|
18 | from .asyncresult import AsyncMapResult | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Decorators |
|
21 | # Decorators |
@@ -30,11 +30,11 from zmq.eventloop import ioloop, zmqstream | |||||
30 | from IPython.external.decorator import decorator |
|
30 | from IPython.external.decorator import decorator | |
31 | from IPython.utils.traitlets import Instance, Dict, List, Set |
|
31 | from IPython.utils.traitlets import Instance, Dict, List, Set | |
32 |
|
32 | |||
33 | import error |
|
33 | from . import error | |
34 | import streamsession as ss |
|
34 | from . import streamsession as ss | |
35 | from dependency import Dependency |
|
35 | from .dependency import Dependency | |
36 | from entry_point import connect_logger, local_logger |
|
36 | from .entry_point import connect_logger, local_logger | |
37 | from factory import SessionFactory |
|
37 | from .factory import SessionFactory | |
38 |
|
38 | |||
39 |
|
39 | |||
40 | @decorator |
|
40 | @decorator |
@@ -33,10 +33,10 from IPython.zmq.completer import KernelCompleter | |||||
33 | from IPython.zmq.iostream import OutStream |
|
33 | from IPython.zmq.iostream import OutStream | |
34 | from IPython.zmq.displayhook import DisplayHook |
|
34 | from IPython.zmq.displayhook import DisplayHook | |
35 |
|
35 | |||
36 | import heartmonitor |
|
36 | from . import heartmonitor | |
37 | from client import Client |
|
37 | from .client import Client | |
38 | from factory import SessionFactory |
|
38 | from .factory import SessionFactory | |
39 | from streamsession import StreamSession, Message, extract_header, serialize_object,\ |
|
39 | from .streamsession import StreamSession, Message, extract_header, serialize_object,\ | |
40 | unpack_apply_message, ISO8601, wrap_exception |
|
40 | unpack_apply_message, ISO8601, wrap_exception | |
41 |
|
41 | |||
42 | def printer(*args): |
|
42 | def printer(*args): |
@@ -24,7 +24,7 from zmq.eventloop.zmqstream import ZMQStream | |||||
24 | from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence |
|
24 | from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence | |
25 | from IPython.utils.newserialized import serialize, unserialize |
|
25 | from IPython.utils.newserialized import serialize, unserialize | |
26 |
|
26 | |||
27 |
from |
|
27 | from .error import RemoteError | |
28 |
|
28 | |||
29 | # packer priority: jsonlib[2], cPickle, simplejson/json, pickle |
|
29 | # packer priority: jsonlib[2], cPickle, simplejson/json, pickle | |
30 | json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__ |
|
30 | json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__ |
@@ -14,9 +14,10 from IPython.testing import decorators as testdec | |||||
14 | from IPython.utils.traitlets import HasTraits, Bool, List, Dict, Set, Int, Instance |
|
14 | from IPython.utils.traitlets import HasTraits, Bool, List, Dict, Set, Int, Instance | |
15 |
|
15 | |||
16 | from IPython.external.decorator import decorator |
|
16 | from IPython.external.decorator import decorator | |
17 | from IPython.zmq.parallel.asyncresult import AsyncResult |
|
17 | ||
18 | from IPython.zmq.parallel.dependency import Dependency |
|
18 | from .asyncresult import AsyncResult | |
19 | from IPython.zmq.parallel.remotefunction import ParallelFunction, parallel, remote |
|
19 | from .dependency import Dependency | |
|
20 | from .remotefunction import ParallelFunction, parallel, remote | |||
20 |
|
21 | |||
21 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
22 | # Decorators |
|
23 | # Decorators | |
@@ -405,15 +406,32 class DirectView(View): | |||||
405 |
|
406 | |||
406 | @sync_results |
|
407 | @sync_results | |
407 | @save_ids |
|
408 | @save_ids | |
408 |
def execute(self, code, block= |
|
409 | def execute(self, code, block=None): | |
409 | """execute some code on my targets.""" |
|
410 | """execute some code on my targets.""" | |
|
411 | ||||
|
412 | block = block if block is not None else self.block | |||
|
413 | ||||
410 | return self.client.execute(code, block=block, targets=self._targets) |
|
414 | return self.client.execute(code, block=block, targets=self._targets) | |
411 |
|
415 | |||
|
416 | @sync_results | |||
|
417 | @save_ids | |||
|
418 | def run(self, fname, block=None): | |||
|
419 | """execute the code in a file on my targets.""" | |||
|
420 | ||||
|
421 | block = block if block is not None else self.block | |||
|
422 | ||||
|
423 | return self.client.run(fname, block=block, targets=self._targets) | |||
|
424 | ||||
412 | def update(self, ns): |
|
425 | def update(self, ns): | |
413 | """update remote namespace with dict `ns`""" |
|
426 | """update remote namespace with dict `ns`""" | |
414 | return self.client.push(ns, targets=self._targets, block=self.block) |
|
427 | return self.client.push(ns, targets=self._targets, block=self.block) | |
415 |
|
428 | |||
416 | push = update |
|
429 | def push(self, ns, block=None): | |
|
430 | """update remote namespace with dict `ns`""" | |||
|
431 | ||||
|
432 | block = block if block is not None else self.block | |||
|
433 | ||||
|
434 | return self.client.push(ns, targets=self._targets, block=block) | |||
417 |
|
435 | |||
418 | def get(self, key_s): |
|
436 | def get(self, key_s): | |
419 | """get object(s) by `key_s` from remote namespace |
|
437 | """get object(s) by `key_s` from remote namespace |
@@ -170,14 +170,26 communicate with the engines are built on top of it), is :meth:`Client.apply`. | |||||
170 | Ideally, :meth:`apply` would have the signature ``apply(f,*args,**kwargs)``, |
|
170 | Ideally, :meth:`apply` would have the signature ``apply(f,*args,**kwargs)``, | |
171 | which would call ``f(*args,**kwargs)`` remotely. However, since :class:`Clients` |
|
171 | which would call ``f(*args,**kwargs)`` remotely. However, since :class:`Clients` | |
172 | require some more options, they cannot easily provide this interface. |
|
172 | require some more options, they cannot easily provide this interface. | |
173 |
Instead, they provide the signature: |
|
173 | Instead, they provide the signature: | |
|
174 | ||||
|
175 | .. sourcecode:: python | |||
174 |
|
176 | |||
175 | c.apply(f, args=None, kwargs=None, bound=True, block=None, targets=None, |
|
177 | c.apply(f, args=None, kwargs=None, bound=True, block=None, targets=None, | |
176 | after=None, follow=None, timeout=None) |
|
178 | after=None, follow=None, timeout=None) | |
177 |
|
179 | |||
|
180 | Where various behavior is controlled via keyword arguments. This means that in the client, | |||
|
181 | you must pass `args` as a tuple, and `kwargs` as a dict. | |||
|
182 | ||||
178 | In order to provide the nicer interface, we have :class:`View` classes, which wrap |
|
183 | In order to provide the nicer interface, we have :class:`View` classes, which wrap | |
179 | :meth:`Client.apply` by using attributes and extra :meth:`apply_x` methods to determine |
|
184 | :meth:`Client.apply` by using attributes and extra :meth:`apply_x` methods to determine | |
180 | the extra arguments. For instance, performing index-access on a client creates a |
|
185 | the extra keyword arguments. This means that the views can have the desired pattern: | |
|
186 | ||||
|
187 | .. sourcecode:: python | |||
|
188 | ||||
|
189 | v.apply(f, *args, **kwargs) | |||
|
190 | ||||
|
191 | ||||
|
192 | For instance, performing index-access on a client creates a | |||
181 | :class:`.DirectView`. |
|
193 | :class:`.DirectView`. | |
182 |
|
194 | |||
183 | .. sourcecode:: ipython |
|
195 | .. sourcecode:: ipython | |
@@ -221,7 +233,7 blocks until the engines are done executing the command: | |||||
221 | In [5]: dview['b'] = 10 |
|
233 | In [5]: dview['b'] = 10 | |
222 |
|
234 | |||
223 | In [6]: dview.apply_bound(lambda x: a+b+x, 27) |
|
235 | In [6]: dview.apply_bound(lambda x: a+b+x, 27) | |
224 | Out[6]: [42, 42, 42, 42] |
|
236 | Out[6]: [42, 42, 42, 42]%exit | |
225 |
|
237 | |||
226 | Python commands can be executed on specific engines by calling execute using the ``targets`` |
|
238 | Python commands can be executed on specific engines by calling execute using the ``targets`` | |
227 | keyword argument in :meth:`client.execute`, or creating a :class:`DirectView` instance by |
|
239 | keyword argument in :meth:`client.execute`, or creating a :class:`DirectView` instance by | |
@@ -553,15 +565,14 In non-blocking mode :meth:`push` and :meth:`pull` also return | |||||
553 | Dictionary interface |
|
565 | Dictionary interface | |
554 | -------------------- |
|
566 | -------------------- | |
555 |
|
567 | |||
556 | Since a namespace is just a :class:`dict`, :class:`DirectView` objects provide |
|
568 | Since a Python namespace is just a :class:`dict`, :class:`DirectView` objects provide | |
557 | dictionary-style access by key and methods such as :meth:`get` and |
|
569 | dictionary-style access by key and methods such as :meth:`get` and | |
558 | :meth:`update` for convenience. This make the remote namespaces of the engines |
|
570 | :meth:`update` for convenience. This make the remote namespaces of the engines | |
559 |
appear as a local dictionary. Underneath, th |
|
571 | appear as a local dictionary. Underneath, these methods call :meth:`apply`: | |
560 | :meth:`pull`: |
|
|||
561 |
|
572 | |||
562 | .. sourcecode:: ipython |
|
573 | .. sourcecode:: ipython | |
563 |
|
574 | |||
564 |
In [50]: |
|
575 | In [50]: dview.block=True | |
565 |
|
576 | |||
566 | In [51]: dview['a']=['foo','bar'] |
|
577 | In [51]: dview['a']=['foo','bar'] | |
567 |
|
578 | |||
@@ -603,7 +614,6 basic effect using :meth:`scatter` and :meth:`gather`: | |||||
603 | .. sourcecode:: ipython |
|
614 | .. sourcecode:: ipython | |
604 |
|
615 | |||
605 | In [66]: dview.scatter('x',range(64)) |
|
616 | In [66]: dview.scatter('x',range(64)) | |
606 | Out[66]: [None,None,None,None] |
|
|||
607 |
|
617 | |||
608 | In [67]: px y = [i**10 for i in x] |
|
618 | In [67]: px y = [i**10 for i in x] | |
609 | Parallel execution on engines: [0, 1, 2, 3] |
|
619 | Parallel execution on engines: [0, 1, 2, 3] | |
@@ -620,40 +630,49 Parallel exceptions | |||||
620 | In the multiengine interface, parallel commands can raise Python exceptions, |
|
630 | In the multiengine interface, parallel commands can raise Python exceptions, | |
621 | just like serial commands. But, it is a little subtle, because a single |
|
631 | just like serial commands. But, it is a little subtle, because a single | |
622 | parallel command can actually raise multiple exceptions (one for each engine |
|
632 | parallel command can actually raise multiple exceptions (one for each engine | |
623 |
the command was run on). To express this idea, |
|
633 | the command was run on). To express this idea, we have a | |
624 | :exc:`CompositeError` exception class that will be raised in most cases. The |
|
634 | :exc:`CompositeError` exception class that will be raised in most cases. The | |
625 | :exc:`CompositeError` class is a special type of exception that wraps one or |
|
635 | :exc:`CompositeError` class is a special type of exception that wraps one or | |
626 | more other types of exceptions. Here is how it works: |
|
636 | more other types of exceptions. Here is how it works: | |
627 |
|
637 | |||
628 | .. sourcecode:: ipython |
|
638 | .. sourcecode:: ipython | |
629 |
|
639 | |||
630 |
In [76]: |
|
640 | In [76]: dview.block=True | |
631 |
|
641 | |||
632 |
In [77]: |
|
642 | In [77]: dview.execute('1/0') | |
633 | --------------------------------------------------------------------------- |
|
643 | --------------------------------------------------------------------------- | |
634 | CompositeError Traceback (most recent call last) |
|
644 | CompositeError Traceback (most recent call last) | |
|
645 | /Users/minrk/<ipython-input-10-5d56b303a66c> in <module>() | |||
|
646 | ----> 1 dview.execute('1/0') | |||
635 |
|
647 | |||
636 | /ipython1-client-r3021/docs/examples/<ipython console> in <module>() |
|
648 | ... | |
637 |
|
649 | |||
638 | /ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block) |
|
650 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout) | |
639 | 432 targets, block = self._findTargetsAndBlock(targets, block) |
|
651 | 1012 raise ValueError(msg) | |
640 | 433 result = blockingCallFromThread(self.smultiengine.execute, lines, |
|
652 | 1013 else: | |
641 | --> 434 targets=targets, block=block) |
|
653 | -> 1014 return self._apply_direct(f, args, kwargs, **options) | |
642 | 435 if block: |
|
654 | 1015 | |
643 | 436 result = ResultList(result) |
|
655 | 1016 def _apply_balanced(self, f, args, kwargs, bound=None, block=None, targets=None, | |
644 |
|
656 | |||
645 | /ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw) |
|
657 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in _apply_direct(self, f, args, kwargs, bound, block, targets) | |
646 | 72 result.raiseException() |
|
658 | 1100 if block: | |
647 | 73 except Exception, e: |
|
659 | 1101 try: | |
648 |
- |
|
660 | -> 1102 return ar.get() | |
649 | 75 return result |
|
661 | 1103 except KeyboardInterrupt: | |
650 | 76 |
|
662 | 1104 return ar | |
651 |
|
663 | |||
652 | CompositeError: one or more exceptions from call to method: execute |
|
664 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout) | |
653 | [0:execute]: ZeroDivisionError: integer division or modulo by zero |
|
665 | 78 return self._result | |
654 | [1:execute]: ZeroDivisionError: integer division or modulo by zero |
|
666 | 79 else: | |
655 | [2:execute]: ZeroDivisionError: integer division or modulo by zero |
|
667 | ---> 80 raise self._exception | |
656 | [3:execute]: ZeroDivisionError: integer division or modulo by zero |
|
668 | 81 else: | |
|
669 | 82 raise error.TimeoutError("Result not ready.") | |||
|
670 | ||||
|
671 | CompositeError: one or more exceptions from call to method: _execute | |||
|
672 | [0:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
673 | [1:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
674 | [2:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
675 | [3:apply]: ZeroDivisionError: integer division or modulo by zero | |||
657 |
|
676 | |||
658 | Notice how the error message printed when :exc:`CompositeError` is raised has |
|
677 | Notice how the error message printed when :exc:`CompositeError` is raised has | |
659 | information about the individual exceptions that were raised on each engine. |
|
678 | information about the individual exceptions that were raised on each engine. | |
@@ -690,37 +709,68 instance: | |||||
690 | In [81]: rc.execute('1/0') |
|
709 | In [81]: rc.execute('1/0') | |
691 | --------------------------------------------------------------------------- |
|
710 | --------------------------------------------------------------------------- | |
692 | CompositeError Traceback (most recent call last) |
|
711 | CompositeError Traceback (most recent call last) | |
693 |
|
712 | /Users/minrk/<ipython-input-5-b0c7a2b62c52> in <module>() | ||
694 | /ipython1-client-r3021/docs/examples/<ipython console> in <module>() |
|
713 | ----> 1 rc.execute('1/0') | |
695 |
|
714 | |||
696 | /ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block) |
|
715 | /Users/minrk/<string> in execute(self, code, targets, block) | |
697 | 432 targets, block = self._findTargetsAndBlock(targets, block) |
|
716 | ||
698 | 433 result = blockingCallFromThread(self.smultiengine.execute, lines, |
|
717 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in defaultblock(f, self, *args, **kwargs) | |
699 | --> 434 targets=targets, block=block) |
|
718 | 88 self.block = block | |
700 | 435 if block: |
|
719 | 89 try: | |
701 | 436 result = ResultList(result) |
|
720 | ---> 90 ret = f(self, *args, **kwargs) | |
702 |
|
721 | 91 finally: | ||
703 | /ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw) |
|
722 | 92 self.block = saveblock | |
704 | 72 result.raiseException() |
|
723 | ||
705 | 73 except Exception, e: |
|
724 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in execute(self, code, targets, block) | |
706 | ---> 74 raise e |
|
725 | 855 default: self.block | |
707 | 75 return result |
|
726 | 856 """ | |
708 | 76 |
|
727 | --> 857 result = self.apply(_execute, (code,), targets=targets, block=block, bound=True, balanced=False) | |
709 |
|
728 | 858 if not block: | ||
710 | CompositeError: one or more exceptions from call to method: execute |
|
729 | 859 return result | |
711 | [0:execute]: ZeroDivisionError: integer division or modulo by zero |
|
730 | ||
712 | [1:execute]: ZeroDivisionError: integer division or modulo by zero |
|
731 | /Users/minrk/<string> in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout) | |
713 | [2:execute]: ZeroDivisionError: integer division or modulo by zero |
|
732 | ||
714 | [3:execute]: ZeroDivisionError: integer division or modulo by zero |
|
733 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in defaultblock(f, self, *args, **kwargs) | |
715 |
|
734 | 88 self.block = block | ||
|
735 | 89 try: | |||
|
736 | ---> 90 ret = f(self, *args, **kwargs) | |||
|
737 | 91 finally: | |||
|
738 | 92 self.block = saveblock | |||
|
739 | ||||
|
740 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout) | |||
|
741 | 1012 raise ValueError(msg) | |||
|
742 | 1013 else: | |||
|
743 | -> 1014 return self._apply_direct(f, args, kwargs, **options) | |||
|
744 | 1015 | |||
|
745 | 1016 def _apply_balanced(self, f, args, kwargs, bound=None, block=None, targets=None, | |||
|
746 | ||||
|
747 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in _apply_direct(self, f, args, kwargs, bound, block, targets) | |||
|
748 | 1100 if block: | |||
|
749 | 1101 try: | |||
|
750 | -> 1102 return ar.get() | |||
|
751 | 1103 except KeyboardInterrupt: | |||
|
752 | 1104 return ar | |||
|
753 | ||||
|
754 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout) | |||
|
755 | 78 return self._result | |||
|
756 | 79 else: | |||
|
757 | ---> 80 raise self._exception | |||
|
758 | 81 else: | |||
|
759 | 82 raise error.TimeoutError("Result not ready.") | |||
|
760 | ||||
|
761 | CompositeError: one or more exceptions from call to method: _execute | |||
|
762 | [0:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
763 | [1:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
764 | [2:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
765 | [3:apply]: ZeroDivisionError: integer division or modulo by zero | |||
|
766 | ||||
716 | In [82]: %debug |
|
767 | In [82]: %debug | |
717 | > |
|
768 | > /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.py(80)get() | |
|
769 | 79 else: | |||
|
770 | ---> 80 raise self._exception | |||
|
771 | 81 else: | |||
|
772 | ||||
718 |
|
773 | |||
719 | /ipython1-client-r3021/ipython1/kernel/twistedutil.py(74)blockingCallFromThread() |
|
|||
720 | 73 except Exception, e: |
|
|||
721 | ---> 74 raise e |
|
|||
722 | 75 return result |
|
|||
723 |
|
||||
724 | # With the debugger running, e is the exceptions instance. We can tab complete |
|
774 | # With the debugger running, e is the exceptions instance. We can tab complete | |
725 | # on it and see the extra methods that are available. |
|
775 | # on it and see the extra methods that are available. | |
726 | ipdb> e. |
|
776 | ipdb> e. | |
@@ -730,37 +780,49 instance: | |||||
730 | e.__doc__ e.__init__ e.__repr__ e._get_engine_str e.print_tracebacks |
|
780 | e.__doc__ e.__init__ e.__repr__ e._get_engine_str e.print_tracebacks | |
731 | e.__getattribute__ e.__module__ e.__setattr__ e._get_traceback e.raise_exception |
|
781 | e.__getattribute__ e.__module__ e.__setattr__ e._get_traceback e.raise_exception | |
732 | ipdb> e.print_tracebacks() |
|
782 | ipdb> e.print_tracebacks() | |
733 |
[0: |
|
783 | [0:apply]: | |
734 | --------------------------------------------------------------------------- |
|
784 | Traceback (most recent call last): | |
735 | ZeroDivisionError Traceback (most recent call last) |
|
785 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request | |
736 |
|
786 | exec code in working, working | ||
737 | /ipython1-client-r3021/docs/examples/<string> in <module>() |
|
787 | File "<string>", line 1, in <module> | |
738 |
|
788 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute | ||
|
789 | exec code in globals() | |||
|
790 | File "<string>", line 1, in <module> | |||
739 | ZeroDivisionError: integer division or modulo by zero |
|
791 | ZeroDivisionError: integer division or modulo by zero | |
740 |
|
792 | |||
741 | [1:execute]: |
|
|||
742 | --------------------------------------------------------------------------- |
|
|||
743 | ZeroDivisionError Traceback (most recent call last) |
|
|||
744 |
|
||||
745 | /ipython1-client-r3021/docs/examples/<string> in <module>() |
|
|||
746 |
|
793 | |||
|
794 | [1:apply]: | |||
|
795 | Traceback (most recent call last): | |||
|
796 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request | |||
|
797 | exec code in working, working | |||
|
798 | File "<string>", line 1, in <module> | |||
|
799 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute | |||
|
800 | exec code in globals() | |||
|
801 | File "<string>", line 1, in <module> | |||
747 | ZeroDivisionError: integer division or modulo by zero |
|
802 | ZeroDivisionError: integer division or modulo by zero | |
748 |
|
803 | |||
749 | [2:execute]: |
|
|||
750 | --------------------------------------------------------------------------- |
|
|||
751 | ZeroDivisionError Traceback (most recent call last) |
|
|||
752 |
|
||||
753 | /ipython1-client-r3021/docs/examples/<string> in <module>() |
|
|||
754 |
|
804 | |||
|
805 | [2:apply]: | |||
|
806 | Traceback (most recent call last): | |||
|
807 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request | |||
|
808 | exec code in working, working | |||
|
809 | File "<string>", line 1, in <module> | |||
|
810 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute | |||
|
811 | exec code in globals() | |||
|
812 | File "<string>", line 1, in <module> | |||
755 | ZeroDivisionError: integer division or modulo by zero |
|
813 | ZeroDivisionError: integer division or modulo by zero | |
756 |
|
814 | |||
757 | [3:execute]: |
|
|||
758 | --------------------------------------------------------------------------- |
|
|||
759 | ZeroDivisionError Traceback (most recent call last) |
|
|||
760 |
|
||||
761 | /ipython1-client-r3021/docs/examples/<string> in <module>() |
|
|||
762 |
|
815 | |||
|
816 | [3:apply]: | |||
|
817 | Traceback (most recent call last): | |||
|
818 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request | |||
|
819 | exec code in working, working | |||
|
820 | File "<string>", line 1, in <module> | |||
|
821 | File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute | |||
|
822 | exec code in globals() | |||
|
823 | File "<string>", line 1, in <module> | |||
763 | ZeroDivisionError: integer division or modulo by zero |
|
824 | ZeroDivisionError: integer division or modulo by zero | |
|
825 | ||||
764 |
|
826 | |||
765 |
|
827 | |||
766 | All of this same error handling magic even works in non-blocking mode: |
|
828 | All of this same error handling magic even works in non-blocking mode: | |
@@ -774,41 +836,19 All of this same error handling magic even works in non-blocking mode: | |||||
774 | In [85]: ar.get() |
|
836 | In [85]: ar.get() | |
775 | --------------------------------------------------------------------------- |
|
837 | --------------------------------------------------------------------------- | |
776 | CompositeError Traceback (most recent call last) |
|
838 | CompositeError Traceback (most recent call last) | |
777 |
|
839 | /Users/minrk/<ipython-input-3-8531eb3d26fb> in <module>() | ||
778 | /ipython1-client-r3021/docs/examples/<ipython console> in <module>() |
|
840 | ----> 1 ar.get() | |
779 |
|
841 | |||
780 | /ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in _get_r(self) |
|
842 | /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout) | |
781 | 170 |
|
843 | 78 return self._result | |
782 | 171 def _get_r(self): |
|
844 | 79 else: | |
783 | --> 172 return self.get_result(block=True) |
|
845 | ---> 80 raise self._exception | |
784 | 173 |
|
846 | 81 else: | |
785 | 174 r = property(_get_r) |
|
847 | 82 raise error.TimeoutError("Result not ready.") | |
786 |
|
848 | |||
787 | /ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_result(self, default, block) |
|
849 | CompositeError: one or more exceptions from call to method: _execute | |
788 | 131 return self.result |
|
850 | [0:apply]: ZeroDivisionError: integer division or modulo by zero | |
789 | 132 try: |
|
851 | [1:apply]: ZeroDivisionError: integer division or modulo by zero | |
790 | --> 133 result = self.client.get_pending_deferred(self.result_id, block) |
|
852 | [2:apply]: ZeroDivisionError: integer division or modulo by zero | |
791 | 134 except error.ResultNotCompleted: |
|
853 | [3:apply]: ZeroDivisionError: integer division or modulo by zero | |
792 | 135 return default |
|
|||
793 |
|
||||
794 | /ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_pending_deferred(self, deferredID, block) |
|
|||
795 | 385 |
|
|||
796 | 386 def get_pending_deferred(self, deferredID, block): |
|
|||
797 | --> 387 return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block) |
|
|||
798 | 388 |
|
|||
799 | 389 def barrier(self, pendingResults): |
|
|||
800 |
|
||||
801 | /ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw) |
|
|||
802 | 72 result.raiseException() |
|
|||
803 | 73 except Exception, e: |
|
|||
804 | ---> 74 raise e |
|
|||
805 | 75 return result |
|
|||
806 | 76 |
|
|||
807 |
|
||||
808 | CompositeError: one or more exceptions from call to method: execute |
|
|||
809 | [0:execute]: ZeroDivisionError: integer division or modulo by zero |
|
|||
810 | [1:execute]: ZeroDivisionError: integer division or modulo by zero |
|
|||
811 | [2:execute]: ZeroDivisionError: integer division or modulo by zero |
|
|||
812 | [3:execute]: ZeroDivisionError: integer division or modulo by zero |
|
|||
813 |
|
||||
814 |
|
854 |
General Comments 0
You need to be logged in to leave comments.
Login now