Show More
@@ -1,896 +1,951 b'' | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 | # -*- test-case-name: IPython.kernel.test.test_multiengineclient -*- |
|
2 | # -*- test-case-name: IPython.kernel.test.test_multiengineclient -*- | |
3 |
|
3 | |||
4 | """General Classes for IMultiEngine clients.""" |
|
4 | """General Classes for IMultiEngine clients.""" | |
5 |
|
5 | |||
6 | __docformat__ = "restructuredtext en" |
|
6 | __docformat__ = "restructuredtext en" | |
7 |
|
7 | |||
8 | #------------------------------------------------------------------------------- |
|
8 | #------------------------------------------------------------------------------- | |
9 | # Copyright (C) 2008 The IPython Development Team |
|
9 | # Copyright (C) 2008 The IPython Development Team | |
10 | # |
|
10 | # | |
11 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | # Distributed under the terms of the BSD License. The full license is in | |
12 | # the file COPYING, distributed as part of this software. |
|
12 | # the file COPYING, distributed as part of this software. | |
13 | #------------------------------------------------------------------------------- |
|
13 | #------------------------------------------------------------------------------- | |
14 |
|
14 | |||
15 | #------------------------------------------------------------------------------- |
|
15 | #------------------------------------------------------------------------------- | |
16 | # Imports |
|
16 | # Imports | |
17 | #------------------------------------------------------------------------------- |
|
17 | #------------------------------------------------------------------------------- | |
18 |
|
18 | |||
19 | import sys |
|
19 | import sys | |
20 | import cPickle as pickle |
|
20 | import cPickle as pickle | |
21 | from types import FunctionType |
|
21 | from types import FunctionType | |
22 | import linecache |
|
22 | import linecache | |
23 |
|
23 | |||
24 | from twisted.internet import reactor |
|
24 | from twisted.internet import reactor | |
25 | from twisted.python import components, log |
|
25 | from twisted.python import components, log | |
26 | from twisted.python.failure import Failure |
|
26 | from twisted.python.failure import Failure | |
27 | from zope.interface import Interface, implements, Attribute |
|
27 | from zope.interface import Interface, implements, Attribute | |
28 |
|
28 | |||
29 | from IPython.ColorANSI import TermColors |
|
29 | from IPython.ColorANSI import TermColors | |
30 |
|
30 | |||
31 | from IPython.kernel.twistedutil import blockingCallFromThread |
|
31 | from IPython.kernel.twistedutil import blockingCallFromThread | |
32 | from IPython.kernel import error |
|
32 | from IPython.kernel import error | |
33 | from IPython.kernel.parallelfunction import ParallelFunction |
|
33 | from IPython.kernel.parallelfunction import ParallelFunction | |
34 | from IPython.kernel.mapper import ( |
|
34 | from IPython.kernel.mapper import ( | |
35 | MultiEngineMapper, |
|
35 | MultiEngineMapper, | |
36 | IMultiEngineMapperFactory, |
|
36 | IMultiEngineMapperFactory, | |
37 | IMapper |
|
37 | IMapper | |
38 | ) |
|
38 | ) | |
39 | from IPython.kernel import map as Map |
|
39 | from IPython.kernel import map as Map | |
40 | from IPython.kernel import multiengine as me |
|
40 | from IPython.kernel import multiengine as me | |
41 | from IPython.kernel.multiengine import (IFullMultiEngine, |
|
41 | from IPython.kernel.multiengine import (IFullMultiEngine, | |
42 | IFullSynchronousMultiEngine) |
|
42 | IFullSynchronousMultiEngine) | |
43 |
|
43 | |||
44 |
|
44 | |||
45 | #------------------------------------------------------------------------------- |
|
45 | #------------------------------------------------------------------------------- | |
46 | # Pending Result things |
|
46 | # Pending Result things | |
47 | #------------------------------------------------------------------------------- |
|
47 | #------------------------------------------------------------------------------- | |
48 |
|
48 | |||
49 | class IPendingResult(Interface): |
|
49 | class IPendingResult(Interface): | |
50 | """A representation of a result that is pending. |
|
50 | """A representation of a result that is pending. | |
51 |
|
51 | |||
52 | This class is similar to Twisted's `Deferred` object, but is designed to be |
|
52 | This class is similar to Twisted's `Deferred` object, but is designed to be | |
53 | used in a synchronous context. |
|
53 | used in a synchronous context. | |
54 | """ |
|
54 | """ | |
55 |
|
55 | |||
56 | result_id=Attribute("ID of the deferred on the other side") |
|
56 | result_id=Attribute("ID of the deferred on the other side") | |
57 | client=Attribute("A client that I came from") |
|
57 | client=Attribute("A client that I came from") | |
58 | r=Attribute("An attribute that is a property that calls and returns get_result") |
|
58 | r=Attribute("An attribute that is a property that calls and returns get_result") | |
59 |
|
59 | |||
60 | def get_result(default=None, block=True): |
|
60 | def get_result(default=None, block=True): | |
61 | """ |
|
61 | """ | |
62 | Get a result that is pending. |
|
62 | Get a result that is pending. | |
63 |
|
63 | |||
64 | :Parameters: |
|
64 | :Parameters: | |
65 | default |
|
65 | default | |
66 | The value to return if the result is not ready. |
|
66 | The value to return if the result is not ready. | |
67 | block : boolean |
|
67 | block : boolean | |
68 | Should I block for the result. |
|
68 | Should I block for the result. | |
69 |
|
69 | |||
70 | :Returns: The actual result or the default value. |
|
70 | :Returns: The actual result or the default value. | |
71 | """ |
|
71 | """ | |
72 |
|
72 | |||
73 | def add_callback(f, *args, **kwargs): |
|
73 | def add_callback(f, *args, **kwargs): | |
74 | """ |
|
74 | """ | |
75 | Add a callback that is called with the result. |
|
75 | Add a callback that is called with the result. | |
76 |
|
76 | |||
77 | If the original result is foo, adding a callback will cause |
|
77 | If the original result is foo, adding a callback will cause | |
78 | f(foo, *args, **kwargs) to be returned instead. If multiple |
|
78 | f(foo, *args, **kwargs) to be returned instead. If multiple | |
79 | callbacks are registered, they are chained together: the result of |
|
79 | callbacks are registered, they are chained together: the result of | |
80 | one is passed to the next and so on. |
|
80 | one is passed to the next and so on. | |
81 |
|
81 | |||
82 | Unlike Twisted's Deferred object, there is no errback chain. Thus |
|
82 | Unlike Twisted's Deferred object, there is no errback chain. Thus | |
83 | any exception raised will not be caught and handled. User must |
|
83 | any exception raised will not be caught and handled. User must | |
84 | catch these by hand when calling `get_result`. |
|
84 | catch these by hand when calling `get_result`. | |
85 | """ |
|
85 | """ | |
86 |
|
86 | |||
87 |
|
87 | |||
88 | class PendingResult(object): |
|
88 | class PendingResult(object): | |
89 | """A representation of a result that is not yet ready. |
|
89 | """A representation of a result that is not yet ready. | |
90 |
|
90 | |||
91 | A user should not create a `PendingResult` instance by hand. |
|
91 | A user should not create a `PendingResult` instance by hand. | |
92 |
|
92 | |||
93 | Methods |
|
93 | Methods | |
94 | ======= |
|
94 | ======= | |
95 |
|
95 | |||
96 | * `get_result` |
|
96 | * `get_result` | |
97 | * `add_callback` |
|
97 | * `add_callback` | |
98 |
|
98 | |||
99 | Properties |
|
99 | Properties | |
100 | ========== |
|
100 | ========== | |
101 | * `r` |
|
101 | * `r` | |
102 | """ |
|
102 | """ | |
103 |
|
103 | |||
104 | def __init__(self, client, result_id): |
|
104 | def __init__(self, client, result_id): | |
105 | """Create a PendingResult with a result_id and a client instance. |
|
105 | """Create a PendingResult with a result_id and a client instance. | |
106 |
|
106 | |||
107 | The client should implement `_getPendingResult(result_id, block)`. |
|
107 | The client should implement `_getPendingResult(result_id, block)`. | |
108 | """ |
|
108 | """ | |
109 | self.client = client |
|
109 | self.client = client | |
110 | self.result_id = result_id |
|
110 | self.result_id = result_id | |
111 | self.called = False |
|
111 | self.called = False | |
112 | self.raised = False |
|
112 | self.raised = False | |
113 | self.callbacks = [] |
|
113 | self.callbacks = [] | |
114 |
|
114 | |||
115 | def get_result(self, default=None, block=True): |
|
115 | def get_result(self, default=None, block=True): | |
116 | """Get a result that is pending. |
|
116 | """Get a result that is pending. | |
117 |
|
117 | |||
118 | This method will connect to an IMultiEngine adapted controller |
|
118 | This method will connect to an IMultiEngine adapted controller | |
119 | and see if the result is ready. If the action triggers an exception |
|
119 | and see if the result is ready. If the action triggers an exception | |
120 | raise it and record it. This method records the result/exception once it is |
|
120 | raise it and record it. This method records the result/exception once it is | |
121 | retrieved. Calling `get_result` again will get this cached result or will |
|
121 | retrieved. Calling `get_result` again will get this cached result or will | |
122 | re-raise the exception. The .r attribute is a property that calls |
|
122 | re-raise the exception. The .r attribute is a property that calls | |
123 | `get_result` with block=True. |
|
123 | `get_result` with block=True. | |
124 |
|
124 | |||
125 | :Parameters: |
|
125 | :Parameters: | |
126 | default |
|
126 | default | |
127 | The value to return if the result is not ready. |
|
127 | The value to return if the result is not ready. | |
128 | block : boolean |
|
128 | block : boolean | |
129 | Should I block for the result. |
|
129 | Should I block for the result. | |
130 |
|
130 | |||
131 | :Returns: The actual result or the default value. |
|
131 | :Returns: The actual result or the default value. | |
132 | """ |
|
132 | """ | |
133 |
|
133 | |||
134 | if self.called: |
|
134 | if self.called: | |
135 | if self.raised: |
|
135 | if self.raised: | |
136 | raise self.result[0], self.result[1], self.result[2] |
|
136 | raise self.result[0], self.result[1], self.result[2] | |
137 | else: |
|
137 | else: | |
138 | return self.result |
|
138 | return self.result | |
139 | try: |
|
139 | try: | |
140 | result = self.client.get_pending_deferred(self.result_id, block) |
|
140 | result = self.client.get_pending_deferred(self.result_id, block) | |
141 | except error.ResultNotCompleted: |
|
141 | except error.ResultNotCompleted: | |
142 | return default |
|
142 | return default | |
143 | except: |
|
143 | except: | |
144 | # Reraise other error, but first record them so they can be reraised |
|
144 | # Reraise other error, but first record them so they can be reraised | |
145 | # later if .r or get_result is called again. |
|
145 | # later if .r or get_result is called again. | |
146 | self.result = sys.exc_info() |
|
146 | self.result = sys.exc_info() | |
147 | self.called = True |
|
147 | self.called = True | |
148 | self.raised = True |
|
148 | self.raised = True | |
149 | raise |
|
149 | raise | |
150 | else: |
|
150 | else: | |
151 | for cb in self.callbacks: |
|
151 | for cb in self.callbacks: | |
152 | result = cb[0](result, *cb[1], **cb[2]) |
|
152 | result = cb[0](result, *cb[1], **cb[2]) | |
153 | self.result = result |
|
153 | self.result = result | |
154 | self.called = True |
|
154 | self.called = True | |
155 | return result |
|
155 | return result | |
156 |
|
156 | |||
157 | def add_callback(self, f, *args, **kwargs): |
|
157 | def add_callback(self, f, *args, **kwargs): | |
158 | """Add a callback that is called with the result. |
|
158 | """Add a callback that is called with the result. | |
159 |
|
159 | |||
160 | If the original result is result, adding a callback will cause |
|
160 | If the original result is result, adding a callback will cause | |
161 | f(result, *args, **kwargs) to be returned instead. If multiple |
|
161 | f(result, *args, **kwargs) to be returned instead. If multiple | |
162 | callbacks are registered, they are chained together: the result of |
|
162 | callbacks are registered, they are chained together: the result of | |
163 | one is passed to the next and so on. |
|
163 | one is passed to the next and so on. | |
164 |
|
164 | |||
165 | Unlike Twisted's Deferred object, there is no errback chain. Thus |
|
165 | Unlike Twisted's Deferred object, there is no errback chain. Thus | |
166 | any exception raised will not be caught and handled. User must |
|
166 | any exception raised will not be caught and handled. User must | |
167 | catch these by hand when calling `get_result`. |
|
167 | catch these by hand when calling `get_result`. | |
168 | """ |
|
168 | """ | |
169 | assert callable(f) |
|
169 | assert callable(f) | |
170 | self.callbacks.append((f, args, kwargs)) |
|
170 | self.callbacks.append((f, args, kwargs)) | |
171 |
|
171 | |||
172 | def __cmp__(self, other): |
|
172 | def __cmp__(self, other): | |
173 | if self.result_id < other.result_id: |
|
173 | if self.result_id < other.result_id: | |
174 | return -1 |
|
174 | return -1 | |
175 | else: |
|
175 | else: | |
176 | return 1 |
|
176 | return 1 | |
177 |
|
177 | |||
178 | def _get_r(self): |
|
178 | def _get_r(self): | |
179 | return self.get_result(block=True) |
|
179 | return self.get_result(block=True) | |
180 |
|
180 | |||
181 | r = property(_get_r) |
|
181 | r = property(_get_r) | |
182 | """This property is a shortcut to a `get_result(block=True)`.""" |
|
182 | """This property is a shortcut to a `get_result(block=True)`.""" | |
183 |
|
183 | |||
184 |
|
184 | |||
185 | #------------------------------------------------------------------------------- |
|
185 | #------------------------------------------------------------------------------- | |
186 | # Pretty printing wrappers for certain lists |
|
186 | # Pretty printing wrappers for certain lists | |
187 | #------------------------------------------------------------------------------- |
|
187 | #------------------------------------------------------------------------------- | |
188 |
|
188 | |||
189 | class ResultList(list): |
|
189 | class ResultList(list): | |
190 | """A subclass of list that pretty prints the output of `execute`/`get_result`.""" |
|
190 | """A subclass of list that pretty prints the output of `execute`/`get_result`.""" | |
191 |
|
191 | |||
192 | def __repr__(self): |
|
192 | def __repr__(self): | |
193 | output = [] |
|
193 | output = [] | |
194 | # These colored prompts were not working on Windows |
|
194 | # These colored prompts were not working on Windows | |
195 | if sys.platform == 'win32': |
|
195 | if sys.platform == 'win32': | |
196 | blue = normal = red = green = '' |
|
196 | blue = normal = red = green = '' | |
197 | else: |
|
197 | else: | |
198 | blue = TermColors.Blue |
|
198 | blue = TermColors.Blue | |
199 | normal = TermColors.Normal |
|
199 | normal = TermColors.Normal | |
200 | red = TermColors.Red |
|
200 | red = TermColors.Red | |
201 | green = TermColors.Green |
|
201 | green = TermColors.Green | |
202 | output.append("<Results List>\n") |
|
202 | output.append("<Results List>\n") | |
203 | for cmd in self: |
|
203 | for cmd in self: | |
204 | if isinstance(cmd, Failure): |
|
204 | if isinstance(cmd, Failure): | |
205 | output.append(cmd) |
|
205 | output.append(cmd) | |
206 | else: |
|
206 | else: | |
207 | target = cmd.get('id',None) |
|
207 | target = cmd.get('id',None) | |
208 | cmd_num = cmd.get('number',None) |
|
208 | cmd_num = cmd.get('number',None) | |
209 | cmd_stdin = cmd.get('input',{}).get('translated','No Input') |
|
209 | cmd_stdin = cmd.get('input',{}).get('translated','No Input') | |
210 | cmd_stdout = cmd.get('stdout', None) |
|
210 | cmd_stdout = cmd.get('stdout', None) | |
211 | cmd_stderr = cmd.get('stderr', None) |
|
211 | cmd_stderr = cmd.get('stderr', None) | |
212 | output.append("%s[%i]%s In [%i]:%s %s\n" % \ |
|
212 | output.append("%s[%i]%s In [%i]:%s %s\n" % \ | |
213 | (green, target, |
|
213 | (green, target, | |
214 | blue, cmd_num, normal, cmd_stdin)) |
|
214 | blue, cmd_num, normal, cmd_stdin)) | |
215 | if cmd_stdout: |
|
215 | if cmd_stdout: | |
216 | output.append("%s[%i]%s Out[%i]:%s %s\n" % \ |
|
216 | output.append("%s[%i]%s Out[%i]:%s %s\n" % \ | |
217 | (green, target, |
|
217 | (green, target, | |
218 | red, cmd_num, normal, cmd_stdout)) |
|
218 | red, cmd_num, normal, cmd_stdout)) | |
219 | if cmd_stderr: |
|
219 | if cmd_stderr: | |
220 | output.append("%s[%i]%s Err[%i]:\n%s %s" % \ |
|
220 | output.append("%s[%i]%s Err[%i]:\n%s %s" % \ | |
221 | (green, target, |
|
221 | (green, target, | |
222 | red, cmd_num, normal, cmd_stderr)) |
|
222 | red, cmd_num, normal, cmd_stderr)) | |
223 | return ''.join(output) |
|
223 | return ''.join(output) | |
224 |
|
224 | |||
225 |
|
225 | |||
226 | def wrapResultList(result): |
|
226 | def wrapResultList(result): | |
227 | """A function that wraps the output of `execute`/`get_result` -> `ResultList`.""" |
|
227 | """A function that wraps the output of `execute`/`get_result` -> `ResultList`.""" | |
228 | if len(result) == 0: |
|
228 | if len(result) == 0: | |
229 | result = [result] |
|
229 | result = [result] | |
230 | return ResultList(result) |
|
230 | return ResultList(result) | |
231 |
|
231 | |||
232 |
|
232 | |||
233 | class QueueStatusList(list): |
|
233 | class QueueStatusList(list): | |
234 | """A subclass of list that pretty prints the output of `queue_status`.""" |
|
234 | """A subclass of list that pretty prints the output of `queue_status`.""" | |
235 |
|
235 | |||
236 | def __repr__(self): |
|
236 | def __repr__(self): | |
237 | output = [] |
|
237 | output = [] | |
238 | output.append("<Queue Status List>\n") |
|
238 | output.append("<Queue Status List>\n") | |
239 | for e in self: |
|
239 | for e in self: | |
240 | output.append("Engine: %s\n" % repr(e[0])) |
|
240 | output.append("Engine: %s\n" % repr(e[0])) | |
241 | output.append(" Pending: %s\n" % repr(e[1]['pending'])) |
|
241 | output.append(" Pending: %s\n" % repr(e[1]['pending'])) | |
242 | for q in e[1]['queue']: |
|
242 | for q in e[1]['queue']: | |
243 | output.append(" Command: %s\n" % repr(q)) |
|
243 | output.append(" Command: %s\n" % repr(q)) | |
244 | return ''.join(output) |
|
244 | return ''.join(output) | |
245 |
|
245 | |||
246 |
|
246 | |||
247 | #------------------------------------------------------------------------------- |
|
247 | #------------------------------------------------------------------------------- | |
248 | # InteractiveMultiEngineClient |
|
248 | # InteractiveMultiEngineClient | |
249 | #------------------------------------------------------------------------------- |
|
249 | #------------------------------------------------------------------------------- | |
250 |
|
250 | |||
251 | class InteractiveMultiEngineClient(object): |
|
251 | class InteractiveMultiEngineClient(object): | |
252 | """A mixin class that add a few methods to a multiengine client. |
|
252 | """A mixin class that add a few methods to a multiengine client. | |
253 |
|
253 | |||
254 | The methods in this mixin class are designed for interactive usage. |
|
254 | The methods in this mixin class are designed for interactive usage. | |
255 | """ |
|
255 | """ | |
256 |
|
256 | |||
257 | def activate(self): |
|
257 | def activate(self): | |
258 | """Make this `MultiEngineClient` active for parallel magic commands. |
|
258 | """Make this `MultiEngineClient` active for parallel magic commands. | |
259 |
|
259 | |||
260 | IPython has a magic command syntax to work with `MultiEngineClient` objects. |
|
260 | IPython has a magic command syntax to work with `MultiEngineClient` objects. | |
261 | In a given IPython session there is a single active one. While |
|
261 | In a given IPython session there is a single active one. While | |
262 | there can be many `MultiEngineClient` created and used by the user, |
|
262 | there can be many `MultiEngineClient` created and used by the user, | |
263 | there is only one active one. The active `MultiEngineClient` is used whenever |
|
263 | there is only one active one. The active `MultiEngineClient` is used whenever | |
264 | the magic commands %px and %autopx are used. |
|
264 | the magic commands %px and %autopx are used. | |
265 |
|
265 | |||
266 | The activate() method is called on a given `MultiEngineClient` to make it |
|
266 | The activate() method is called on a given `MultiEngineClient` to make it | |
267 | active. Once this has been done, the magic commands can be used. |
|
267 | active. Once this has been done, the magic commands can be used. | |
268 | """ |
|
268 | """ | |
269 |
|
269 | |||
270 | try: |
|
270 | try: | |
271 | __IPYTHON__.activeController = self |
|
271 | __IPYTHON__.activeController = self | |
272 | except NameError: |
|
272 | except NameError: | |
273 | print "The IPython Controller magics only work within IPython." |
|
273 | print "The IPython Controller magics only work within IPython." | |
274 |
|
274 | |||
275 | def __setitem__(self, key, value): |
|
275 | def __setitem__(self, key, value): | |
276 | """Add a dictionary interface for pushing/pulling. |
|
276 | """Add a dictionary interface for pushing/pulling. | |
277 |
|
277 | |||
278 | This functions as a shorthand for `push`. |
|
278 | This functions as a shorthand for `push`. | |
279 |
|
279 | |||
280 | :Parameters: |
|
280 | :Parameters: | |
281 | key : str |
|
281 | key : str | |
282 | What to call the remote object. |
|
282 | What to call the remote object. | |
283 | value : object |
|
283 | value : object | |
284 | The local Python object to push. |
|
284 | The local Python object to push. | |
285 | """ |
|
285 | """ | |
286 | targets, block = self._findTargetsAndBlock() |
|
286 | targets, block = self._findTargetsAndBlock() | |
287 | return self.push({key:value}, targets=targets, block=block) |
|
287 | return self.push({key:value}, targets=targets, block=block) | |
288 |
|
288 | |||
289 | def __getitem__(self, key): |
|
289 | def __getitem__(self, key): | |
290 | """Add a dictionary interface for pushing/pulling. |
|
290 | """Add a dictionary interface for pushing/pulling. | |
291 |
|
291 | |||
292 | This functions as a shorthand to `pull`. |
|
292 | This functions as a shorthand to `pull`. | |
293 |
|
293 | |||
294 | :Parameters: |
|
294 | :Parameters: | |
295 | - `key`: A string representing the key. |
|
295 | - `key`: A string representing the key. | |
296 | """ |
|
296 | """ | |
297 | if isinstance(key, str): |
|
297 | if isinstance(key, str): | |
298 | targets, block = self._findTargetsAndBlock() |
|
298 | targets, block = self._findTargetsAndBlock() | |
299 | return self.pull(key, targets=targets, block=block) |
|
299 | return self.pull(key, targets=targets, block=block) | |
300 | else: |
|
300 | else: | |
301 | raise TypeError("__getitem__ only takes strs") |
|
301 | raise TypeError("__getitem__ only takes strs") | |
302 |
|
302 | |||
303 | def __len__(self): |
|
303 | def __len__(self): | |
304 | """Return the number of available engines.""" |
|
304 | """Return the number of available engines.""" | |
305 | return len(self.get_ids()) |
|
305 | return len(self.get_ids()) | |
306 |
|
306 | |||
307 | #--------------------------------------------------------------------------- |
|
307 | #--------------------------------------------------------------------------- | |
308 | # Make this a context manager for with |
|
308 | # Make this a context manager for with | |
309 | #--------------------------------------------------------------------------- |
|
309 | #--------------------------------------------------------------------------- | |
310 |
|
310 | |||
311 | def findsource_file(self,f): |
|
311 | def findsource_file(self,f): | |
312 | linecache.checkcache() |
|
312 | linecache.checkcache() | |
313 | s = findsource(f.f_code) |
|
313 | s = findsource(f.f_code) | |
314 | lnum = f.f_lineno |
|
314 | lnum = f.f_lineno | |
315 | wsource = s[0][f.f_lineno:] |
|
315 | wsource = s[0][f.f_lineno:] | |
316 | return strip_whitespace(wsource) |
|
316 | return strip_whitespace(wsource) | |
317 |
|
317 | |||
318 | def findsource_ipython(self,f): |
|
318 | def findsource_ipython(self,f): | |
319 | from IPython import ipapi |
|
319 | from IPython import ipapi | |
320 | self.ip = ipapi.get() |
|
320 | self.ip = ipapi.get() | |
321 | wsource = [l+'\n' for l in |
|
321 | wsource = [l+'\n' for l in | |
322 | self.ip.IP.input_hist_raw[-1].splitlines()[1:]] |
|
322 | self.ip.IP.input_hist_raw[-1].splitlines()[1:]] | |
323 | return strip_whitespace(wsource) |
|
323 | return strip_whitespace(wsource) | |
324 |
|
324 | |||
325 | def __enter__(self): |
|
325 | def __enter__(self): | |
326 | f = sys._getframe(1) |
|
326 | f = sys._getframe(1) | |
327 | local_ns = f.f_locals |
|
327 | local_ns = f.f_locals | |
328 | global_ns = f.f_globals |
|
328 | global_ns = f.f_globals | |
329 | if f.f_code.co_filename == '<ipython console>': |
|
329 | if f.f_code.co_filename == '<ipython console>': | |
330 | s = self.findsource_ipython(f) |
|
330 | s = self.findsource_ipython(f) | |
331 | else: |
|
331 | else: | |
332 | s = self.findsource_file(f) |
|
332 | s = self.findsource_file(f) | |
333 |
|
333 | |||
334 | self._with_context_result = self.execute(s) |
|
334 | self._with_context_result = self.execute(s) | |
335 |
|
335 | |||
336 | def __exit__ (self, etype, value, tb): |
|
336 | def __exit__ (self, etype, value, tb): | |
337 | if issubclass(etype,error.StopLocalExecution): |
|
337 | if issubclass(etype,error.StopLocalExecution): | |
338 | return True |
|
338 | return True | |
339 |
|
339 | |||
340 |
|
340 | |||
341 | def remote(): |
|
341 | def remote(): | |
342 | m = 'Special exception to stop local execution of parallel code.' |
|
342 | m = 'Special exception to stop local execution of parallel code.' | |
343 | raise error.StopLocalExecution(m) |
|
343 | raise error.StopLocalExecution(m) | |
344 |
|
344 | |||
345 | def strip_whitespace(source): |
|
345 | def strip_whitespace(source): | |
346 | # Expand tabs to avoid any confusion. |
|
346 | # Expand tabs to avoid any confusion. | |
347 | wsource = [l.expandtabs(4) for l in source] |
|
347 | wsource = [l.expandtabs(4) for l in source] | |
348 | # Detect the indentation level |
|
348 | # Detect the indentation level | |
349 | done = False |
|
349 | done = False | |
350 | for line in wsource: |
|
350 | for line in wsource: | |
351 | if line.isspace(): |
|
351 | if line.isspace(): | |
352 | continue |
|
352 | continue | |
353 | for col,char in enumerate(line): |
|
353 | for col,char in enumerate(line): | |
354 | if char != ' ': |
|
354 | if char != ' ': | |
355 | done = True |
|
355 | done = True | |
356 | break |
|
356 | break | |
357 | if done: |
|
357 | if done: | |
358 | break |
|
358 | break | |
359 | # Now we know how much leading space there is in the code. Next, we |
|
359 | # Now we know how much leading space there is in the code. Next, we | |
360 | # extract up to the first line that has less indentation. |
|
360 | # extract up to the first line that has less indentation. | |
361 | # WARNINGS: we skip comments that may be misindented, but we do NOT yet |
|
361 | # WARNINGS: we skip comments that may be misindented, but we do NOT yet | |
362 | # detect triple quoted strings that may have flush left text. |
|
362 | # detect triple quoted strings that may have flush left text. | |
363 | for lno,line in enumerate(wsource): |
|
363 | for lno,line in enumerate(wsource): | |
364 | lead = line[:col] |
|
364 | lead = line[:col] | |
365 | if lead.isspace(): |
|
365 | if lead.isspace(): | |
366 | continue |
|
366 | continue | |
367 | else: |
|
367 | else: | |
368 | if not lead.lstrip().startswith('#'): |
|
368 | if not lead.lstrip().startswith('#'): | |
369 | break |
|
369 | break | |
370 | # The real 'with' source is up to lno |
|
370 | # The real 'with' source is up to lno | |
371 | src_lines = [l[col:] for l in wsource[:lno+1]] |
|
371 | src_lines = [l[col:] for l in wsource[:lno+1]] | |
372 |
|
372 | |||
373 | # Finally, check that the source's first non-comment line begins with the |
|
373 | # Finally, check that the source's first non-comment line begins with the | |
374 | # special call 'remote()' |
|
374 | # special call 'remote()' | |
375 | for nline,line in enumerate(src_lines): |
|
375 | for nline,line in enumerate(src_lines): | |
376 | if line.isspace() or line.startswith('#'): |
|
376 | if line.isspace() or line.startswith('#'): | |
377 | continue |
|
377 | continue | |
378 | if 'remote()' in line: |
|
378 | if 'remote()' in line: | |
379 | break |
|
379 | break | |
380 | else: |
|
380 | else: | |
381 | raise ValueError('remote() call missing at the start of code') |
|
381 | raise ValueError('remote() call missing at the start of code') | |
382 | src = ''.join(src_lines[nline+1:]) |
|
382 | src = ''.join(src_lines[nline+1:]) | |
383 | #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src # dbg |
|
383 | #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src # dbg | |
384 | return src |
|
384 | return src | |
385 |
|
385 | |||
386 |
|
386 | |||
387 | #------------------------------------------------------------------------------- |
|
387 | #------------------------------------------------------------------------------- | |
388 | # The top-level MultiEngine client adaptor |
|
388 | # The top-level MultiEngine client adaptor | |
389 | #------------------------------------------------------------------------------- |
|
389 | #------------------------------------------------------------------------------- | |
390 |
|
390 | |||
391 |
|
391 | |||
392 | class IFullBlockingMultiEngineClient(Interface): |
|
392 | class IFullBlockingMultiEngineClient(Interface): | |
393 | pass |
|
393 | pass | |
394 |
|
394 | |||
395 |
|
395 | |||
396 | class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): |
|
396 | class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): | |
397 | """ |
|
397 | """ | |
398 | A blocking client to the `IMultiEngine` controller interface. |
|
398 | A blocking client to the `IMultiEngine` controller interface. | |
399 |
|
399 | |||
400 | This class allows users to use a set of engines for a parallel |
|
400 | This class allows users to use a set of engines for a parallel | |
401 | computation through the `IMultiEngine` interface. In this interface, |
|
401 | computation through the `IMultiEngine` interface. In this interface, | |
402 | each engine has a specific id (an int) that is used to refer to the |
|
402 | each engine has a specific id (an int) that is used to refer to the | |
403 | engine, run code on it, etc. |
|
403 | engine, run code on it, etc. | |
404 | """ |
|
404 | """ | |
405 |
|
405 | |||
406 | implements( |
|
406 | implements( | |
407 | IFullBlockingMultiEngineClient, |
|
407 | IFullBlockingMultiEngineClient, | |
408 | IMultiEngineMapperFactory, |
|
408 | IMultiEngineMapperFactory, | |
409 | IMapper |
|
409 | IMapper | |
410 | ) |
|
410 | ) | |
411 |
|
411 | |||
412 | def __init__(self, smultiengine): |
|
412 | def __init__(self, smultiengine): | |
413 | self.smultiengine = smultiengine |
|
413 | self.smultiengine = smultiengine | |
414 | self.block = True |
|
414 | self.block = True | |
415 | self.targets = 'all' |
|
415 | self.targets = 'all' | |
416 |
|
416 | |||
417 | def _findBlock(self, block=None): |
|
417 | def _findBlock(self, block=None): | |
418 | if block is None: |
|
418 | if block is None: | |
419 | return self.block |
|
419 | return self.block | |
420 | else: |
|
420 | else: | |
421 | if block in (True, False): |
|
421 | if block in (True, False): | |
422 | return block |
|
422 | return block | |
423 | else: |
|
423 | else: | |
424 | raise ValueError("block must be True or False") |
|
424 | raise ValueError("block must be True or False") | |
425 |
|
425 | |||
426 | def _findTargets(self, targets=None): |
|
426 | def _findTargets(self, targets=None): | |
427 | if targets is None: |
|
427 | if targets is None: | |
428 | return self.targets |
|
428 | return self.targets | |
429 | else: |
|
429 | else: | |
430 | if not isinstance(targets, (str,list,tuple,int)): |
|
430 | if not isinstance(targets, (str,list,tuple,int)): | |
431 | raise ValueError("targets must be a str, list, tuple or int") |
|
431 | raise ValueError("targets must be a str, list, tuple or int") | |
432 | return targets |
|
432 | return targets | |
433 |
|
433 | |||
434 | def _findTargetsAndBlock(self, targets=None, block=None): |
|
434 | def _findTargetsAndBlock(self, targets=None, block=None): | |
435 | return self._findTargets(targets), self._findBlock(block) |
|
435 | return self._findTargets(targets), self._findBlock(block) | |
436 |
|
436 | |||
437 | def _blockFromThread(self, function, *args, **kwargs): |
|
437 | def _blockFromThread(self, function, *args, **kwargs): | |
438 | block = kwargs.get('block', None) |
|
438 | block = kwargs.get('block', None) | |
439 | if block is None: |
|
439 | if block is None: | |
440 | raise error.MissingBlockArgument("'block' keyword argument is missing") |
|
440 | raise error.MissingBlockArgument("'block' keyword argument is missing") | |
441 | result = blockingCallFromThread(function, *args, **kwargs) |
|
441 | result = blockingCallFromThread(function, *args, **kwargs) | |
442 | if not block: |
|
442 | if not block: | |
443 | result = PendingResult(self, result) |
|
443 | result = PendingResult(self, result) | |
444 | return result |
|
444 | return result | |
445 |
|
445 | |||
446 | def get_pending_deferred(self, deferredID, block): |
|
446 | def get_pending_deferred(self, deferredID, block): | |
447 | return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block) |
|
447 | return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block) | |
448 |
|
448 | |||
449 | def barrier(self, pendingResults): |
|
449 | def barrier(self, pendingResults): | |
450 | """Synchronize a set of `PendingResults`. |
|
450 | """Synchronize a set of `PendingResults`. | |
451 |
|
451 | |||
452 | This method is a synchronization primitive that waits for a set of |
|
452 | This method is a synchronization primitive that waits for a set of | |
453 | `PendingResult` objects to complete. More specifically, barier does |
|
453 | `PendingResult` objects to complete. More specifically, barier does | |
454 | the following. |
|
454 | the following. | |
455 |
|
455 | |||
456 | * The `PendingResult`s are sorted by result_id. |
|
456 | * The `PendingResult`s are sorted by result_id. | |
457 | * The `get_result` method is called for each `PendingResult` sequentially |
|
457 | * The `get_result` method is called for each `PendingResult` sequentially | |
458 | with block=True. |
|
458 | with block=True. | |
459 | * If a `PendingResult` gets a result that is an exception, it is |
|
459 | * If a `PendingResult` gets a result that is an exception, it is | |
460 | trapped and can be re-raised later by calling `get_result` again. |
|
460 | trapped and can be re-raised later by calling `get_result` again. | |
461 | * The `PendingResult`s are flushed from the controller. |
|
461 | * The `PendingResult`s are flushed from the controller. | |
462 |
|
462 | |||
463 | After barrier has been called on a `PendingResult`, its results can |
|
463 | After barrier has been called on a `PendingResult`, its results can | |
464 | be retrieved by calling `get_result` again or accesing the `r` attribute |
|
464 | be retrieved by calling `get_result` again or accesing the `r` attribute | |
465 | of the instance. |
|
465 | of the instance. | |
466 | """ |
|
466 | """ | |
467 |
|
467 | |||
468 | # Convert to list for sorting and check class type |
|
468 | # Convert to list for sorting and check class type | |
469 | prList = list(pendingResults) |
|
469 | prList = list(pendingResults) | |
470 | for pr in prList: |
|
470 | for pr in prList: | |
471 | if not isinstance(pr, PendingResult): |
|
471 | if not isinstance(pr, PendingResult): | |
472 | raise error.NotAPendingResult("Objects passed to barrier must be PendingResult instances") |
|
472 | raise error.NotAPendingResult("Objects passed to barrier must be PendingResult instances") | |
473 |
|
473 | |||
474 | # Sort the PendingResults so they are in order |
|
474 | # Sort the PendingResults so they are in order | |
475 | prList.sort() |
|
475 | prList.sort() | |
476 | # Block on each PendingResult object |
|
476 | # Block on each PendingResult object | |
477 | for pr in prList: |
|
477 | for pr in prList: | |
478 | try: |
|
478 | try: | |
479 | result = pr.get_result(block=True) |
|
479 | result = pr.get_result(block=True) | |
480 | except Exception: |
|
480 | except Exception: | |
481 | pass |
|
481 | pass | |
482 |
|
482 | |||
483 | def flush(self): |
|
483 | def flush(self): | |
484 | """ |
|
484 | """ | |
485 | Clear all pending deferreds/results from the controller. |
|
485 | Clear all pending deferreds/results from the controller. | |
486 |
|
486 | |||
487 | For each `PendingResult` that is created by this client, the controller |
|
487 | For each `PendingResult` that is created by this client, the controller | |
488 | holds on to the result for that `PendingResult`. This can be a problem |
|
488 | holds on to the result for that `PendingResult`. This can be a problem | |
489 | if there are a large number of `PendingResult` objects that are created. |
|
489 | if there are a large number of `PendingResult` objects that are created. | |
490 |
|
490 | |||
491 | Once the result of the `PendingResult` has been retrieved, the result |
|
491 | Once the result of the `PendingResult` has been retrieved, the result | |
492 | is removed from the controller, but if a user doesn't get a result ( |
|
492 | is removed from the controller, but if a user doesn't get a result ( | |
493 | they just ignore the `PendingResult`) the result is kept forever on the |
|
493 | they just ignore the `PendingResult`) the result is kept forever on the | |
494 | controller. This method allows the user to clear out all un-retrieved |
|
494 | controller. This method allows the user to clear out all un-retrieved | |
495 | results on the controller. |
|
495 | results on the controller. | |
496 | """ |
|
496 | """ | |
497 | r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds) |
|
497 | r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds) | |
498 | return r |
|
498 | return r | |
499 |
|
499 | |||
500 | clear_pending_results = flush |
|
500 | clear_pending_results = flush | |
501 |
|
501 | |||
502 | #--------------------------------------------------------------------------- |
|
502 | #--------------------------------------------------------------------------- | |
503 | # IEngineMultiplexer related methods |
|
503 | # IEngineMultiplexer related methods | |
504 | #--------------------------------------------------------------------------- |
|
504 | #--------------------------------------------------------------------------- | |
505 |
|
505 | |||
506 | def execute(self, lines, targets=None, block=None): |
|
506 | def execute(self, lines, targets=None, block=None): | |
507 | """ |
|
507 | """ | |
508 | Execute code on a set of engines. |
|
508 | Execute code on a set of engines. | |
509 |
|
509 | |||
510 | :Parameters: |
|
510 | :Parameters: | |
511 | lines : str |
|
511 | lines : str | |
512 | The Python code to execute as a string |
|
512 | The Python code to execute as a string | |
513 | targets : id or list of ids |
|
513 | targets : id or list of ids | |
514 | The engine to use for the execution |
|
514 | The engine to use for the execution | |
515 | block : boolean |
|
515 | block : boolean | |
516 | If False, this method will return the actual result. If False, |
|
516 | If False, this method will return the actual result. If False, | |
517 | a `PendingResult` is returned which can be used to get the result |
|
517 | a `PendingResult` is returned which can be used to get the result | |
518 | at a later time. |
|
518 | at a later time. | |
519 | """ |
|
519 | """ | |
520 | targets, block = self._findTargetsAndBlock(targets, block) |
|
520 | targets, block = self._findTargetsAndBlock(targets, block) | |
521 | result = blockingCallFromThread(self.smultiengine.execute, lines, |
|
521 | result = blockingCallFromThread(self.smultiengine.execute, lines, | |
522 | targets=targets, block=block) |
|
522 | targets=targets, block=block) | |
523 | if block: |
|
523 | if block: | |
524 | result = ResultList(result) |
|
524 | result = ResultList(result) | |
525 | else: |
|
525 | else: | |
526 | result = PendingResult(self, result) |
|
526 | result = PendingResult(self, result) | |
527 | result.add_callback(wrapResultList) |
|
527 | result.add_callback(wrapResultList) | |
528 | return result |
|
528 | return result | |
529 |
|
529 | |||
530 | def push(self, namespace, targets=None, block=None): |
|
530 | def push(self, namespace, targets=None, block=None): | |
531 | """ |
|
531 | """ | |
532 | Push a dictionary of keys and values to engines namespace. |
|
532 | Push a dictionary of keys and values to engines namespace. | |
533 |
|
533 | |||
534 | Each engine has a persistent namespace. This method is used to push |
|
534 | Each engine has a persistent namespace. This method is used to push | |
535 | Python objects into that namespace. |
|
535 | Python objects into that namespace. | |
536 |
|
536 | |||
537 | The objects in the namespace must be pickleable. |
|
537 | The objects in the namespace must be pickleable. | |
538 |
|
538 | |||
539 | :Parameters: |
|
539 | :Parameters: | |
540 | namespace : dict |
|
540 | namespace : dict | |
541 | A dict that contains Python objects to be injected into |
|
541 | A dict that contains Python objects to be injected into | |
542 | the engine persistent namespace. |
|
542 | the engine persistent namespace. | |
543 | targets : id or list of ids |
|
543 | targets : id or list of ids | |
544 | The engine to use for the execution |
|
544 | The engine to use for the execution | |
545 | block : boolean |
|
545 | block : boolean | |
546 | If False, this method will return the actual result. If False, |
|
546 | If False, this method will return the actual result. If False, | |
547 | a `PendingResult` is returned which can be used to get the result |
|
547 | a `PendingResult` is returned which can be used to get the result | |
548 | at a later time. |
|
548 | at a later time. | |
549 | """ |
|
549 | """ | |
550 | targets, block = self._findTargetsAndBlock(targets, block) |
|
550 | targets, block = self._findTargetsAndBlock(targets, block) | |
551 | return self._blockFromThread(self.smultiengine.push, namespace, |
|
551 | return self._blockFromThread(self.smultiengine.push, namespace, | |
552 | targets=targets, block=block) |
|
552 | targets=targets, block=block) | |
553 |
|
553 | |||
554 | def pull(self, keys, targets=None, block=None): |
|
554 | def pull(self, keys, targets=None, block=None): | |
555 | """ |
|
555 | """ | |
556 | Pull Python objects by key out of engines namespaces. |
|
556 | Pull Python objects by key out of engines namespaces. | |
557 |
|
557 | |||
558 | :Parameters: |
|
558 | :Parameters: | |
559 | keys : str or list of str |
|
559 | keys : str or list of str | |
560 | The names of the variables to be pulled |
|
560 | The names of the variables to be pulled | |
561 | targets : id or list of ids |
|
561 | targets : id or list of ids | |
562 | The engine to use for the execution |
|
562 | The engine to use for the execution | |
563 | block : boolean |
|
563 | block : boolean | |
564 | If False, this method will return the actual result. If False, |
|
564 | If False, this method will return the actual result. If False, | |
565 | a `PendingResult` is returned which can be used to get the result |
|
565 | a `PendingResult` is returned which can be used to get the result | |
566 | at a later time. |
|
566 | at a later time. | |
567 | """ |
|
567 | """ | |
568 | targets, block = self._findTargetsAndBlock(targets, block) |
|
568 | targets, block = self._findTargetsAndBlock(targets, block) | |
569 | return self._blockFromThread(self.smultiengine.pull, keys, targets=targets, block=block) |
|
569 | return self._blockFromThread(self.smultiengine.pull, keys, targets=targets, block=block) | |
570 |
|
570 | |||
571 | def push_function(self, namespace, targets=None, block=None): |
|
571 | def push_function(self, namespace, targets=None, block=None): | |
572 | """ |
|
572 | """ | |
573 | Push a Python function to an engine. |
|
573 | Push a Python function to an engine. | |
574 |
|
574 | |||
575 | This method is used to push a Python function to an engine. This |
|
575 | This method is used to push a Python function to an engine. This | |
576 | method can then be used in code on the engines. Closures are not supported. |
|
576 | method can then be used in code on the engines. Closures are not supported. | |
577 |
|
577 | |||
578 | :Parameters: |
|
578 | :Parameters: | |
579 | namespace : dict |
|
579 | namespace : dict | |
580 | A dict whose values are the functions to be pushed. The keys give |
|
580 | A dict whose values are the functions to be pushed. The keys give | |
581 | that names that the function will appear as in the engines |
|
581 | that names that the function will appear as in the engines | |
582 | namespace. |
|
582 | namespace. | |
583 | targets : id or list of ids |
|
583 | targets : id or list of ids | |
584 | The engine to use for the execution |
|
584 | The engine to use for the execution | |
585 | block : boolean |
|
585 | block : boolean | |
586 | If False, this method will return the actual result. If False, |
|
586 | If False, this method will return the actual result. If False, | |
587 | a `PendingResult` is returned which can be used to get the result |
|
587 | a `PendingResult` is returned which can be used to get the result | |
588 | at a later time. |
|
588 | at a later time. | |
589 | """ |
|
589 | """ | |
590 | targets, block = self._findTargetsAndBlock(targets, block) |
|
590 | targets, block = self._findTargetsAndBlock(targets, block) | |
591 | return self._blockFromThread(self.smultiengine.push_function, namespace, targets=targets, block=block) |
|
591 | return self._blockFromThread(self.smultiengine.push_function, namespace, targets=targets, block=block) | |
592 |
|
592 | |||
593 | def pull_function(self, keys, targets=None, block=None): |
|
593 | def pull_function(self, keys, targets=None, block=None): | |
594 | """ |
|
594 | """ | |
595 | Pull a Python function from an engine. |
|
595 | Pull a Python function from an engine. | |
596 |
|
596 | |||
597 | This method is used to pull a Python function from an engine. |
|
597 | This method is used to pull a Python function from an engine. | |
598 | Closures are not supported. |
|
598 | Closures are not supported. | |
599 |
|
599 | |||
600 | :Parameters: |
|
600 | :Parameters: | |
601 | keys : str or list of str |
|
601 | keys : str or list of str | |
602 | The names of the functions to be pulled |
|
602 | The names of the functions to be pulled | |
603 | targets : id or list of ids |
|
603 | targets : id or list of ids | |
604 | The engine to use for the execution |
|
604 | The engine to use for the execution | |
605 | block : boolean |
|
605 | block : boolean | |
606 | If False, this method will return the actual result. If False, |
|
606 | If False, this method will return the actual result. If False, | |
607 | a `PendingResult` is returned which can be used to get the result |
|
607 | a `PendingResult` is returned which can be used to get the result | |
608 | at a later time. |
|
608 | at a later time. | |
609 | """ |
|
609 | """ | |
610 | targets, block = self._findTargetsAndBlock(targets, block) |
|
610 | targets, block = self._findTargetsAndBlock(targets, block) | |
611 | return self._blockFromThread(self.smultiengine.pull_function, keys, targets=targets, block=block) |
|
611 | return self._blockFromThread(self.smultiengine.pull_function, keys, targets=targets, block=block) | |
612 |
|
612 | |||
613 | def push_serialized(self, namespace, targets=None, block=None): |
|
613 | def push_serialized(self, namespace, targets=None, block=None): | |
614 | targets, block = self._findTargetsAndBlock(targets, block) |
|
614 | targets, block = self._findTargetsAndBlock(targets, block) | |
615 | return self._blockFromThread(self.smultiengine.push_serialized, namespace, targets=targets, block=block) |
|
615 | return self._blockFromThread(self.smultiengine.push_serialized, namespace, targets=targets, block=block) | |
616 |
|
616 | |||
617 | def pull_serialized(self, keys, targets=None, block=None): |
|
617 | def pull_serialized(self, keys, targets=None, block=None): | |
618 | targets, block = self._findTargetsAndBlock(targets, block) |
|
618 | targets, block = self._findTargetsAndBlock(targets, block) | |
619 | return self._blockFromThread(self.smultiengine.pull_serialized, keys, targets=targets, block=block) |
|
619 | return self._blockFromThread(self.smultiengine.pull_serialized, keys, targets=targets, block=block) | |
620 |
|
620 | |||
621 | def get_result(self, i=None, targets=None, block=None): |
|
621 | def get_result(self, i=None, targets=None, block=None): | |
622 | """ |
|
622 | """ | |
623 | Get a previous result. |
|
623 | Get a previous result. | |
624 |
|
624 | |||
625 | When code is executed in an engine, a dict is created and returned. This |
|
625 | When code is executed in an engine, a dict is created and returned. This | |
626 | method retrieves that dict for previous commands. |
|
626 | method retrieves that dict for previous commands. | |
627 |
|
627 | |||
628 | :Parameters: |
|
628 | :Parameters: | |
629 | i : int |
|
629 | i : int | |
630 | The number of the result to get |
|
630 | The number of the result to get | |
631 | targets : id or list of ids |
|
631 | targets : id or list of ids | |
632 | The engine to use for the execution |
|
632 | The engine to use for the execution | |
633 | block : boolean |
|
633 | block : boolean | |
634 | If False, this method will return the actual result. If False, |
|
634 | If False, this method will return the actual result. If False, | |
635 | a `PendingResult` is returned which can be used to get the result |
|
635 | a `PendingResult` is returned which can be used to get the result | |
636 | at a later time. |
|
636 | at a later time. | |
637 | """ |
|
637 | """ | |
638 | targets, block = self._findTargetsAndBlock(targets, block) |
|
638 | targets, block = self._findTargetsAndBlock(targets, block) | |
639 | result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block) |
|
639 | result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block) | |
640 | if block: |
|
640 | if block: | |
641 | result = ResultList(result) |
|
641 | result = ResultList(result) | |
642 | else: |
|
642 | else: | |
643 | result = PendingResult(self, result) |
|
643 | result = PendingResult(self, result) | |
644 | result.add_callback(wrapResultList) |
|
644 | result.add_callback(wrapResultList) | |
645 | return result |
|
645 | return result | |
646 |
|
646 | |||
647 | def reset(self, targets=None, block=None): |
|
647 | def reset(self, targets=None, block=None): | |
648 | """ |
|
648 | """ | |
649 | Reset an engine. |
|
649 | Reset an engine. | |
650 |
|
650 | |||
651 | This method clears out the namespace of an engine. |
|
651 | This method clears out the namespace of an engine. | |
652 |
|
652 | |||
653 | :Parameters: |
|
653 | :Parameters: | |
654 | targets : id or list of ids |
|
654 | targets : id or list of ids | |
655 | The engine to use for the execution |
|
655 | The engine to use for the execution | |
656 | block : boolean |
|
656 | block : boolean | |
657 | If False, this method will return the actual result. If False, |
|
657 | If False, this method will return the actual result. If False, | |
658 | a `PendingResult` is returned which can be used to get the result |
|
658 | a `PendingResult` is returned which can be used to get the result | |
659 | at a later time. |
|
659 | at a later time. | |
660 | """ |
|
660 | """ | |
661 | targets, block = self._findTargetsAndBlock(targets, block) |
|
661 | targets, block = self._findTargetsAndBlock(targets, block) | |
662 | return self._blockFromThread(self.smultiengine.reset, targets=targets, block=block) |
|
662 | return self._blockFromThread(self.smultiengine.reset, targets=targets, block=block) | |
663 |
|
663 | |||
664 | def keys(self, targets=None, block=None): |
|
664 | def keys(self, targets=None, block=None): | |
665 | """ |
|
665 | """ | |
666 | Get a list of all the variables in an engine's namespace. |
|
666 | Get a list of all the variables in an engine's namespace. | |
667 |
|
667 | |||
668 | :Parameters: |
|
668 | :Parameters: | |
669 | targets : id or list of ids |
|
669 | targets : id or list of ids | |
670 | The engine to use for the execution |
|
670 | The engine to use for the execution | |
671 | block : boolean |
|
671 | block : boolean | |
672 | If False, this method will return the actual result. If False, |
|
672 | If False, this method will return the actual result. If False, | |
673 | a `PendingResult` is returned which can be used to get the result |
|
673 | a `PendingResult` is returned which can be used to get the result | |
674 | at a later time. |
|
674 | at a later time. | |
675 | """ |
|
675 | """ | |
676 | targets, block = self._findTargetsAndBlock(targets, block) |
|
676 | targets, block = self._findTargetsAndBlock(targets, block) | |
677 | return self._blockFromThread(self.smultiengine.keys, targets=targets, block=block) |
|
677 | return self._blockFromThread(self.smultiengine.keys, targets=targets, block=block) | |
678 |
|
678 | |||
679 | def kill(self, controller=False, targets=None, block=None): |
|
679 | def kill(self, controller=False, targets=None, block=None): | |
680 | """ |
|
680 | """ | |
681 | Kill the engines and controller. |
|
681 | Kill the engines and controller. | |
682 |
|
682 | |||
683 | This method is used to stop the engine and controller by calling |
|
683 | This method is used to stop the engine and controller by calling | |
684 | `reactor.stop`. |
|
684 | `reactor.stop`. | |
685 |
|
685 | |||
686 | :Parameters: |
|
686 | :Parameters: | |
687 | controller : boolean |
|
687 | controller : boolean | |
688 | If True, kill the engines and controller. If False, just the |
|
688 | If True, kill the engines and controller. If False, just the | |
689 | engines |
|
689 | engines | |
690 | targets : id or list of ids |
|
690 | targets : id or list of ids | |
691 | The engine to use for the execution |
|
691 | The engine to use for the execution | |
692 | block : boolean |
|
692 | block : boolean | |
693 | If False, this method will return the actual result. If False, |
|
693 | If False, this method will return the actual result. If False, | |
694 | a `PendingResult` is returned which can be used to get the result |
|
694 | a `PendingResult` is returned which can be used to get the result | |
695 | at a later time. |
|
695 | at a later time. | |
696 | """ |
|
696 | """ | |
697 | targets, block = self._findTargetsAndBlock(targets, block) |
|
697 | targets, block = self._findTargetsAndBlock(targets, block) | |
698 | return self._blockFromThread(self.smultiengine.kill, controller, targets=targets, block=block) |
|
698 | return self._blockFromThread(self.smultiengine.kill, controller, targets=targets, block=block) | |
699 |
|
699 | |||
700 | def clear_queue(self, targets=None, block=None): |
|
700 | def clear_queue(self, targets=None, block=None): | |
701 | """ |
|
701 | """ | |
702 | Clear out the controller's queue for an engine. |
|
702 | Clear out the controller's queue for an engine. | |
703 |
|
703 | |||
704 | The controller maintains a queue for each engine. This clear it out. |
|
704 | The controller maintains a queue for each engine. This clear it out. | |
705 |
|
705 | |||
706 | :Parameters: |
|
706 | :Parameters: | |
707 | targets : id or list of ids |
|
707 | targets : id or list of ids | |
708 | The engine to use for the execution |
|
708 | The engine to use for the execution | |
709 | block : boolean |
|
709 | block : boolean | |
710 | If False, this method will return the actual result. If False, |
|
710 | If False, this method will return the actual result. If False, | |
711 | a `PendingResult` is returned which can be used to get the result |
|
711 | a `PendingResult` is returned which can be used to get the result | |
712 | at a later time. |
|
712 | at a later time. | |
713 | """ |
|
713 | """ | |
714 | targets, block = self._findTargetsAndBlock(targets, block) |
|
714 | targets, block = self._findTargetsAndBlock(targets, block) | |
715 | return self._blockFromThread(self.smultiengine.clear_queue, targets=targets, block=block) |
|
715 | return self._blockFromThread(self.smultiengine.clear_queue, targets=targets, block=block) | |
716 |
|
716 | |||
717 | def queue_status(self, targets=None, block=None): |
|
717 | def queue_status(self, targets=None, block=None): | |
718 | """ |
|
718 | """ | |
719 | Get the status of an engines queue. |
|
719 | Get the status of an engines queue. | |
720 |
|
720 | |||
721 | :Parameters: |
|
721 | :Parameters: | |
722 | targets : id or list of ids |
|
722 | targets : id or list of ids | |
723 | The engine to use for the execution |
|
723 | The engine to use for the execution | |
724 | block : boolean |
|
724 | block : boolean | |
725 | If False, this method will return the actual result. If False, |
|
725 | If False, this method will return the actual result. If False, | |
726 | a `PendingResult` is returned which can be used to get the result |
|
726 | a `PendingResult` is returned which can be used to get the result | |
727 | at a later time. |
|
727 | at a later time. | |
728 | """ |
|
728 | """ | |
729 | targets, block = self._findTargetsAndBlock(targets, block) |
|
729 | targets, block = self._findTargetsAndBlock(targets, block) | |
730 | return self._blockFromThread(self.smultiengine.queue_status, targets=targets, block=block) |
|
730 | return self._blockFromThread(self.smultiengine.queue_status, targets=targets, block=block) | |
731 |
|
731 | |||
732 | def set_properties(self, properties, targets=None, block=None): |
|
732 | def set_properties(self, properties, targets=None, block=None): | |
733 | targets, block = self._findTargetsAndBlock(targets, block) |
|
733 | targets, block = self._findTargetsAndBlock(targets, block) | |
734 | return self._blockFromThread(self.smultiengine.set_properties, properties, targets=targets, block=block) |
|
734 | return self._blockFromThread(self.smultiengine.set_properties, properties, targets=targets, block=block) | |
735 |
|
735 | |||
736 | def get_properties(self, keys=None, targets=None, block=None): |
|
736 | def get_properties(self, keys=None, targets=None, block=None): | |
737 | targets, block = self._findTargetsAndBlock(targets, block) |
|
737 | targets, block = self._findTargetsAndBlock(targets, block) | |
738 | return self._blockFromThread(self.smultiengine.get_properties, keys, targets=targets, block=block) |
|
738 | return self._blockFromThread(self.smultiengine.get_properties, keys, targets=targets, block=block) | |
739 |
|
739 | |||
740 | def has_properties(self, keys, targets=None, block=None): |
|
740 | def has_properties(self, keys, targets=None, block=None): | |
741 | targets, block = self._findTargetsAndBlock(targets, block) |
|
741 | targets, block = self._findTargetsAndBlock(targets, block) | |
742 | return self._blockFromThread(self.smultiengine.has_properties, keys, targets=targets, block=block) |
|
742 | return self._blockFromThread(self.smultiengine.has_properties, keys, targets=targets, block=block) | |
743 |
|
743 | |||
744 | def del_properties(self, keys, targets=None, block=None): |
|
744 | def del_properties(self, keys, targets=None, block=None): | |
745 | targets, block = self._findTargetsAndBlock(targets, block) |
|
745 | targets, block = self._findTargetsAndBlock(targets, block) | |
746 | return self._blockFromThread(self.smultiengine.del_properties, keys, targets=targets, block=block) |
|
746 | return self._blockFromThread(self.smultiengine.del_properties, keys, targets=targets, block=block) | |
747 |
|
747 | |||
748 | def clear_properties(self, targets=None, block=None): |
|
748 | def clear_properties(self, targets=None, block=None): | |
749 | targets, block = self._findTargetsAndBlock(targets, block) |
|
749 | targets, block = self._findTargetsAndBlock(targets, block) | |
750 | return self._blockFromThread(self.smultiengine.clear_properties, targets=targets, block=block) |
|
750 | return self._blockFromThread(self.smultiengine.clear_properties, targets=targets, block=block) | |
751 |
|
751 | |||
752 | #--------------------------------------------------------------------------- |
|
752 | #--------------------------------------------------------------------------- | |
753 | # IMultiEngine related methods |
|
753 | # IMultiEngine related methods | |
754 | #--------------------------------------------------------------------------- |
|
754 | #--------------------------------------------------------------------------- | |
755 |
|
755 | |||
756 | def get_ids(self): |
|
756 | def get_ids(self): | |
757 | """ |
|
757 | """ | |
758 | Returns the ids of currently registered engines. |
|
758 | Returns the ids of currently registered engines. | |
759 | """ |
|
759 | """ | |
760 | result = blockingCallFromThread(self.smultiengine.get_ids) |
|
760 | result = blockingCallFromThread(self.smultiengine.get_ids) | |
761 | return result |
|
761 | return result | |
762 |
|
762 | |||
763 | #--------------------------------------------------------------------------- |
|
763 | #--------------------------------------------------------------------------- | |
764 | # IMultiEngineCoordinator |
|
764 | # IMultiEngineCoordinator | |
765 | #--------------------------------------------------------------------------- |
|
765 | #--------------------------------------------------------------------------- | |
766 |
|
766 | |||
767 | def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None): |
|
767 | def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None): | |
768 | """ |
|
768 | """ | |
769 | Partition a Python sequence and send the partitions to a set of engines. |
|
769 | Partition a Python sequence and send the partitions to a set of engines. | |
770 | """ |
|
770 | """ | |
771 | targets, block = self._findTargetsAndBlock(targets, block) |
|
771 | targets, block = self._findTargetsAndBlock(targets, block) | |
772 | return self._blockFromThread(self.smultiengine.scatter, key, seq, |
|
772 | return self._blockFromThread(self.smultiengine.scatter, key, seq, | |
773 | dist, flatten, targets=targets, block=block) |
|
773 | dist, flatten, targets=targets, block=block) | |
774 |
|
774 | |||
775 | def gather(self, key, dist='b', targets=None, block=None): |
|
775 | def gather(self, key, dist='b', targets=None, block=None): | |
776 | """ |
|
776 | """ | |
777 | Gather a partitioned sequence on a set of engines as a single local seq. |
|
777 | Gather a partitioned sequence on a set of engines as a single local seq. | |
778 | """ |
|
778 | """ | |
779 | targets, block = self._findTargetsAndBlock(targets, block) |
|
779 | targets, block = self._findTargetsAndBlock(targets, block) | |
780 | return self._blockFromThread(self.smultiengine.gather, key, dist, |
|
780 | return self._blockFromThread(self.smultiengine.gather, key, dist, | |
781 | targets=targets, block=block) |
|
781 | targets=targets, block=block) | |
782 |
|
782 | |||
783 | def raw_map(self, func, seq, dist='b', targets=None, block=None): |
|
783 | def raw_map(self, func, seq, dist='b', targets=None, block=None): | |
784 | """ |
|
784 | """ | |
785 | A parallelized version of Python's builtin map. |
|
785 | A parallelized version of Python's builtin map. | |
786 |
|
786 | |||
787 | This has a slightly different syntax than the builtin `map`. |
|
787 | This has a slightly different syntax than the builtin `map`. | |
788 | This is needed because we need to have keyword arguments and thus |
|
788 | This is needed because we need to have keyword arguments and thus | |
789 | can't use *args to capture all the sequences. Instead, they must |
|
789 | can't use *args to capture all the sequences. Instead, they must | |
790 | be passed in a list or tuple. |
|
790 | be passed in a list or tuple. | |
791 |
|
791 | |||
792 | raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...) |
|
792 | raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...) | |
793 |
|
793 | |||
794 | Most users will want to use parallel functions or the `mapper` |
|
794 | Most users will want to use parallel functions or the `mapper` | |
795 | and `map` methods for an API that follows that of the builtin |
|
795 | and `map` methods for an API that follows that of the builtin | |
796 | `map`. |
|
796 | `map`. | |
797 | """ |
|
797 | """ | |
798 | targets, block = self._findTargetsAndBlock(targets, block) |
|
798 | targets, block = self._findTargetsAndBlock(targets, block) | |
799 | return self._blockFromThread(self.smultiengine.raw_map, func, seq, |
|
799 | return self._blockFromThread(self.smultiengine.raw_map, func, seq, | |
800 | dist, targets=targets, block=block) |
|
800 | dist, targets=targets, block=block) | |
801 |
|
801 | |||
802 | def map(self, func, *sequences): |
|
802 | def map(self, func, *sequences): | |
803 | """ |
|
803 | """ | |
804 | A parallel version of Python's builtin `map` function. |
|
804 | A parallel version of Python's builtin `map` function. | |
805 |
|
805 | |||
806 | This method applies a function to sequences of arguments. It |
|
806 | This method applies a function to sequences of arguments. It | |
807 | follows the same syntax as the builtin `map`. |
|
807 | follows the same syntax as the builtin `map`. | |
808 |
|
808 | |||
809 | This method creates a mapper objects by calling `self.mapper` with |
|
809 | This method creates a mapper objects by calling `self.mapper` with | |
810 | no arguments and then uses that mapper to do the mapping. See |
|
810 | no arguments and then uses that mapper to do the mapping. See | |
811 | the documentation of `mapper` for more details. |
|
811 | the documentation of `mapper` for more details. | |
812 | """ |
|
812 | """ | |
813 | return self.mapper().map(func, *sequences) |
|
813 | return self.mapper().map(func, *sequences) | |
814 |
|
814 | |||
815 | def mapper(self, dist='b', targets='all', block=None): |
|
815 | def mapper(self, dist='b', targets='all', block=None): | |
816 | """ |
|
816 | """ | |
817 | Create a mapper object that has a `map` method. |
|
817 | Create a mapper object that has a `map` method. | |
818 |
|
818 | |||
819 | This method returns an object that implements the `IMapper` |
|
819 | This method returns an object that implements the `IMapper` | |
820 | interface. This method is a factory that is used to control how |
|
820 | interface. This method is a factory that is used to control how | |
821 | the map happens. |
|
821 | the map happens. | |
822 |
|
822 | |||
823 | :Parameters: |
|
823 | :Parameters: | |
824 | dist : str |
|
824 | dist : str | |
825 | What decomposition to use, 'b' is the only one supported |
|
825 | What decomposition to use, 'b' is the only one supported | |
826 | currently |
|
826 | currently | |
827 | targets : str, int, sequence of ints |
|
827 | targets : str, int, sequence of ints | |
828 | Which engines to use for the map |
|
828 | Which engines to use for the map | |
829 | block : boolean |
|
829 | block : boolean | |
830 | Should calls to `map` block or not |
|
830 | Should calls to `map` block or not | |
831 | """ |
|
831 | """ | |
832 | return MultiEngineMapper(self, dist, targets, block) |
|
832 | return MultiEngineMapper(self, dist, targets, block) | |
833 |
|
833 | |||
834 | def parallel(self, dist='b', targets=None, block=None): |
|
834 | def parallel(self, dist='b', targets=None, block=None): | |
835 | """ |
|
835 | """ | |
836 | A decorator that turns a function into a parallel function. |
|
836 | A decorator that turns a function into a parallel function. | |
837 |
|
837 | |||
838 | This can be used as: |
|
838 | This can be used as: | |
839 |
|
839 | |||
840 | @parallel() |
|
840 | @parallel() | |
841 | def f(x, y) |
|
841 | def f(x, y) | |
842 | ... |
|
842 | ... | |
843 |
|
843 | |||
844 | f(range(10), range(10)) |
|
844 | f(range(10), range(10)) | |
845 |
|
845 | |||
846 | This causes f(0,0), f(1,1), ... to be called in parallel. |
|
846 | This causes f(0,0), f(1,1), ... to be called in parallel. | |
847 |
|
847 | |||
848 | :Parameters: |
|
848 | :Parameters: | |
849 | dist : str |
|
849 | dist : str | |
850 | What decomposition to use, 'b' is the only one supported |
|
850 | What decomposition to use, 'b' is the only one supported | |
851 | currently |
|
851 | currently | |
852 | targets : str, int, sequence of ints |
|
852 | targets : str, int, sequence of ints | |
853 | Which engines to use for the map |
|
853 | Which engines to use for the map | |
854 | block : boolean |
|
854 | block : boolean | |
855 | Should calls to `map` block or not |
|
855 | Should calls to `map` block or not | |
856 | """ |
|
856 | """ | |
857 | targets, block = self._findTargetsAndBlock(targets, block) |
|
857 | targets, block = self._findTargetsAndBlock(targets, block) | |
858 | mapper = self.mapper(dist, targets, block) |
|
858 | mapper = self.mapper(dist, targets, block) | |
859 | pf = ParallelFunction(mapper) |
|
859 | pf = ParallelFunction(mapper) | |
860 | return pf |
|
860 | return pf | |
861 |
|
861 | |||
862 | #--------------------------------------------------------------------------- |
|
862 | #--------------------------------------------------------------------------- | |
863 | # IMultiEngineExtras |
|
863 | # IMultiEngineExtras | |
864 | #--------------------------------------------------------------------------- |
|
864 | #--------------------------------------------------------------------------- | |
865 |
|
865 | |||
866 | def zip_pull(self, keys, targets=None, block=None): |
|
866 | def zip_pull(self, keys, targets=None, block=None): | |
867 | targets, block = self._findTargetsAndBlock(targets, block) |
|
867 | targets, block = self._findTargetsAndBlock(targets, block) | |
868 | return self._blockFromThread(self.smultiengine.zip_pull, keys, |
|
868 | return self._blockFromThread(self.smultiengine.zip_pull, keys, | |
869 | targets=targets, block=block) |
|
869 | targets=targets, block=block) | |
870 |
|
870 | |||
871 | def run(self, filename, targets=None, block=None): |
|
871 | def run(self, filename, targets=None, block=None): | |
872 | """ |
|
872 | """ | |
873 | Run a Python code in a file on the engines. |
|
873 | Run a Python code in a file on the engines. | |
874 |
|
874 | |||
875 | :Parameters: |
|
875 | :Parameters: | |
876 | filename : str |
|
876 | filename : str | |
877 | The name of the local file to run |
|
877 | The name of the local file to run | |
878 | targets : id or list of ids |
|
878 | targets : id or list of ids | |
879 | The engine to use for the execution |
|
879 | The engine to use for the execution | |
880 | block : boolean |
|
880 | block : boolean | |
881 | If False, this method will return the actual result. If False, |
|
881 | If False, this method will return the actual result. If False, | |
882 | a `PendingResult` is returned which can be used to get the result |
|
882 | a `PendingResult` is returned which can be used to get the result | |
883 | at a later time. |
|
883 | at a later time. | |
884 | """ |
|
884 | """ | |
885 | targets, block = self._findTargetsAndBlock(targets, block) |
|
885 | targets, block = self._findTargetsAndBlock(targets, block) | |
886 | return self._blockFromThread(self.smultiengine.run, filename, |
|
886 | return self._blockFromThread(self.smultiengine.run, filename, | |
887 | targets=targets, block=block) |
|
887 | targets=targets, block=block) | |
888 |
|
888 | |||
|
889 | def benchmark(self, push_size=10000): | |||
|
890 | """ | |||
|
891 | Run performance benchmarks for the current IPython cluster. | |||
|
892 | ||||
|
893 | This method tests both the latency of sending command and data to the | |||
|
894 | engines as well as the throughput of sending large objects to the | |||
|
895 | engines using push. The latency is measured by having one or more | |||
|
896 | engines execute the command 'pass'. The throughput is measure by | |||
|
897 | sending an NumPy array of size `push_size` to one or more engines. | |||
|
898 | ||||
|
899 | These benchmarks will vary widely on different hardware and networks | |||
|
900 | and thus can be used to get an idea of the performance characteristics | |||
|
901 | of a particular configuration of an IPython controller and engines. | |||
|
902 | ||||
|
903 | This function is not testable within our current testing framework. | |||
|
904 | """ | |||
|
905 | import timeit, __builtin__ | |||
|
906 | __builtin__._mec_self = self | |||
|
907 | benchmarks = {} | |||
|
908 | repeat = 3 | |||
|
909 | count = 10 | |||
|
910 | ||||
|
911 | timer = timeit.Timer('_mec_self.execute("pass",0)') | |||
|
912 | result = 1000*min(timer.repeat(repeat,count))/count | |||
|
913 | benchmarks['single_engine_latency'] = (result,'msec') | |||
|
914 | ||||
|
915 | timer = timeit.Timer('_mec_self.execute("pass")') | |||
|
916 | result = 1000*min(timer.repeat(repeat,count))/count | |||
|
917 | benchmarks['all_engine_latency'] = (result,'msec') | |||
|
918 | ||||
|
919 | try: | |||
|
920 | import numpy as np | |||
|
921 | except: | |||
|
922 | pass | |||
|
923 | else: | |||
|
924 | timer = timeit.Timer( | |||
|
925 | "_mec_self.push(d)", | |||
|
926 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |||
|
927 | ) | |||
|
928 | result = min(timer.repeat(repeat,count))/count | |||
|
929 | benchmarks['all_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |||
|
930 | ||||
|
931 | try: | |||
|
932 | import numpy as np | |||
|
933 | except: | |||
|
934 | pass | |||
|
935 | else: | |||
|
936 | timer = timeit.Timer( | |||
|
937 | "_mec_self.push(d,0)", | |||
|
938 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |||
|
939 | ) | |||
|
940 | result = min(timer.repeat(repeat,count))/count | |||
|
941 | benchmarks['single_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |||
|
942 | ||||
|
943 | return benchmarks | |||
889 |
|
944 | |||
890 |
|
945 | |||
891 | components.registerAdapter(FullBlockingMultiEngineClient, |
|
946 | components.registerAdapter(FullBlockingMultiEngineClient, | |
892 | IFullSynchronousMultiEngine, IFullBlockingMultiEngineClient) |
|
947 | IFullSynchronousMultiEngine, IFullBlockingMultiEngineClient) | |
893 |
|
948 | |||
894 |
|
949 | |||
895 |
|
950 | |||
896 |
|
951 |
@@ -1,723 +1,783 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 |
|
3 | |||
4 | """Start an IPython cluster = (controller + engines).""" |
|
4 | """Start an IPython cluster = (controller + engines).""" | |
5 |
|
5 | |||
6 | #----------------------------------------------------------------------------- |
|
6 | #----------------------------------------------------------------------------- | |
7 | # Copyright (C) 2008 The IPython Development Team |
|
7 | # Copyright (C) 2008 The IPython Development Team | |
8 | # |
|
8 | # | |
9 | # Distributed under the terms of the BSD License. The full license is in |
|
9 | # Distributed under the terms of the BSD License. The full license is in | |
10 | # the file COPYING, distributed as part of this software. |
|
10 | # the file COPYING, distributed as part of this software. | |
11 | #----------------------------------------------------------------------------- |
|
11 | #----------------------------------------------------------------------------- | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Imports |
|
14 | # Imports | |
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 |
|
16 | |||
17 | import os |
|
17 | import os | |
18 | import re |
|
18 | import re | |
19 | import sys |
|
19 | import sys | |
20 | import signal |
|
20 | import signal | |
21 | import tempfile |
|
21 | import tempfile | |
22 | pjoin = os.path.join |
|
22 | pjoin = os.path.join | |
23 |
|
23 | |||
24 | from twisted.internet import reactor, defer |
|
24 | from twisted.internet import reactor, defer | |
25 | from twisted.internet.protocol import ProcessProtocol |
|
25 | from twisted.internet.protocol import ProcessProtocol | |
26 | from twisted.internet.error import ProcessDone, ProcessTerminated |
|
26 | from twisted.internet.error import ProcessDone, ProcessTerminated | |
27 | from twisted.internet.utils import getProcessOutput |
|
27 | from twisted.internet.utils import getProcessOutput | |
28 | from twisted.python import failure, log |
|
28 | from twisted.python import failure, log | |
29 |
|
29 | |||
30 | from IPython.external import argparse |
|
30 | from IPython.external import argparse | |
31 | from IPython.external import Itpl |
|
31 | from IPython.external import Itpl | |
32 | from IPython.genutils import get_ipython_dir, num_cpus |
|
32 | from IPython.genutils import get_ipython_dir, num_cpus | |
33 | from IPython.kernel.fcutil import have_crypto |
|
33 | from IPython.kernel.fcutil import have_crypto | |
34 | from IPython.kernel.error import SecurityError |
|
34 | from IPython.kernel.error import SecurityError | |
35 | from IPython.kernel.fcutil import have_crypto |
|
35 | from IPython.kernel.fcutil import have_crypto | |
36 | from IPython.kernel.twistedutil import gatherBoth |
|
36 | from IPython.kernel.twistedutil import gatherBoth | |
37 | from IPython.kernel.util import printer |
|
37 | from IPython.kernel.util import printer | |
38 |
|
38 | |||
39 |
|
39 | |||
40 | #----------------------------------------------------------------------------- |
|
40 | #----------------------------------------------------------------------------- | |
41 | # General process handling code |
|
41 | # General process handling code | |
42 | #----------------------------------------------------------------------------- |
|
42 | #----------------------------------------------------------------------------- | |
43 |
|
43 | |||
44 | def find_exe(cmd): |
|
44 | def find_exe(cmd): | |
45 | try: |
|
45 | try: | |
46 | import win32api |
|
46 | import win32api | |
47 | except ImportError: |
|
47 | except ImportError: | |
48 | raise ImportError('you need to have pywin32 installed for this to work') |
|
48 | raise ImportError('you need to have pywin32 installed for this to work') | |
49 | else: |
|
49 | else: | |
50 | try: |
|
50 | try: | |
51 | (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe') |
|
51 | (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe') | |
52 | except: |
|
52 | except: | |
53 | (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat') |
|
53 | (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat') | |
54 | return path |
|
54 | return path | |
55 |
|
55 | |||
56 | class ProcessStateError(Exception): |
|
56 | class ProcessStateError(Exception): | |
57 | pass |
|
57 | pass | |
58 |
|
58 | |||
59 | class UnknownStatus(Exception): |
|
59 | class UnknownStatus(Exception): | |
60 | pass |
|
60 | pass | |
61 |
|
61 | |||
62 | class LauncherProcessProtocol(ProcessProtocol): |
|
62 | class LauncherProcessProtocol(ProcessProtocol): | |
63 | """ |
|
63 | """ | |
64 | A ProcessProtocol to go with the ProcessLauncher. |
|
64 | A ProcessProtocol to go with the ProcessLauncher. | |
65 | """ |
|
65 | """ | |
66 | def __init__(self, process_launcher): |
|
66 | def __init__(self, process_launcher): | |
67 | self.process_launcher = process_launcher |
|
67 | self.process_launcher = process_launcher | |
68 |
|
68 | |||
69 | def connectionMade(self): |
|
69 | def connectionMade(self): | |
70 | self.process_launcher.fire_start_deferred(self.transport.pid) |
|
70 | self.process_launcher.fire_start_deferred(self.transport.pid) | |
71 |
|
71 | |||
72 | def processEnded(self, status): |
|
72 | def processEnded(self, status): | |
73 | value = status.value |
|
73 | value = status.value | |
74 | if isinstance(value, ProcessDone): |
|
74 | if isinstance(value, ProcessDone): | |
75 | self.process_launcher.fire_stop_deferred(0) |
|
75 | self.process_launcher.fire_stop_deferred(0) | |
76 | elif isinstance(value, ProcessTerminated): |
|
76 | elif isinstance(value, ProcessTerminated): | |
77 | self.process_launcher.fire_stop_deferred( |
|
77 | self.process_launcher.fire_stop_deferred( | |
78 | {'exit_code':value.exitCode, |
|
78 | {'exit_code':value.exitCode, | |
79 | 'signal':value.signal, |
|
79 | 'signal':value.signal, | |
80 | 'status':value.status |
|
80 | 'status':value.status | |
81 | } |
|
81 | } | |
82 | ) |
|
82 | ) | |
83 | else: |
|
83 | else: | |
84 | raise UnknownStatus("unknown exit status, this is probably a bug in Twisted") |
|
84 | raise UnknownStatus("unknown exit status, this is probably a bug in Twisted") | |
85 |
|
85 | |||
86 | def outReceived(self, data): |
|
86 | def outReceived(self, data): | |
87 | log.msg(data) |
|
87 | log.msg(data) | |
88 |
|
88 | |||
89 | def errReceived(self, data): |
|
89 | def errReceived(self, data): | |
90 | log.err(data) |
|
90 | log.err(data) | |
91 |
|
91 | |||
92 | class ProcessLauncher(object): |
|
92 | class ProcessLauncher(object): | |
93 | """ |
|
93 | """ | |
94 | Start and stop an external process in an asynchronous manner. |
|
94 | Start and stop an external process in an asynchronous manner. | |
95 |
|
95 | |||
96 | Currently this uses deferreds to notify other parties of process state |
|
96 | Currently this uses deferreds to notify other parties of process state | |
97 | changes. This is an awkward design and should be moved to using |
|
97 | changes. This is an awkward design and should be moved to using | |
98 | a formal NotificationCenter. |
|
98 | a formal NotificationCenter. | |
99 | """ |
|
99 | """ | |
100 | def __init__(self, cmd_and_args): |
|
100 | def __init__(self, cmd_and_args): | |
101 | self.cmd = cmd_and_args[0] |
|
101 | self.cmd = cmd_and_args[0] | |
102 | self.args = cmd_and_args |
|
102 | self.args = cmd_and_args | |
103 | self._reset() |
|
103 | self._reset() | |
104 |
|
104 | |||
105 | def _reset(self): |
|
105 | def _reset(self): | |
106 | self.process_protocol = None |
|
106 | self.process_protocol = None | |
107 | self.pid = None |
|
107 | self.pid = None | |
108 | self.start_deferred = None |
|
108 | self.start_deferred = None | |
109 | self.stop_deferreds = [] |
|
109 | self.stop_deferreds = [] | |
110 | self.state = 'before' # before, running, or after |
|
110 | self.state = 'before' # before, running, or after | |
111 |
|
111 | |||
112 | @property |
|
112 | @property | |
113 | def running(self): |
|
113 | def running(self): | |
114 | if self.state == 'running': |
|
114 | if self.state == 'running': | |
115 | return True |
|
115 | return True | |
116 | else: |
|
116 | else: | |
117 | return False |
|
117 | return False | |
118 |
|
118 | |||
119 | def fire_start_deferred(self, pid): |
|
119 | def fire_start_deferred(self, pid): | |
120 | self.pid = pid |
|
120 | self.pid = pid | |
121 | self.state = 'running' |
|
121 | self.state = 'running' | |
122 | log.msg('Process %r has started with pid=%i' % (self.args, pid)) |
|
122 | log.msg('Process %r has started with pid=%i' % (self.args, pid)) | |
123 | self.start_deferred.callback(pid) |
|
123 | self.start_deferred.callback(pid) | |
124 |
|
124 | |||
125 | def start(self): |
|
125 | def start(self): | |
126 | if self.state == 'before': |
|
126 | if self.state == 'before': | |
127 | self.process_protocol = LauncherProcessProtocol(self) |
|
127 | self.process_protocol = LauncherProcessProtocol(self) | |
128 | self.start_deferred = defer.Deferred() |
|
128 | self.start_deferred = defer.Deferred() | |
129 | self.process_transport = reactor.spawnProcess( |
|
129 | self.process_transport = reactor.spawnProcess( | |
130 | self.process_protocol, |
|
130 | self.process_protocol, | |
131 | self.cmd, |
|
131 | self.cmd, | |
132 | self.args, |
|
132 | self.args, | |
133 | env=os.environ |
|
133 | env=os.environ | |
134 | ) |
|
134 | ) | |
135 | return self.start_deferred |
|
135 | return self.start_deferred | |
136 | else: |
|
136 | else: | |
137 | s = 'the process has already been started and has state: %r' % \ |
|
137 | s = 'the process has already been started and has state: %r' % \ | |
138 | self.state |
|
138 | self.state | |
139 | return defer.fail(ProcessStateError(s)) |
|
139 | return defer.fail(ProcessStateError(s)) | |
140 |
|
140 | |||
141 | def get_stop_deferred(self): |
|
141 | def get_stop_deferred(self): | |
142 | if self.state == 'running' or self.state == 'before': |
|
142 | if self.state == 'running' or self.state == 'before': | |
143 | d = defer.Deferred() |
|
143 | d = defer.Deferred() | |
144 | self.stop_deferreds.append(d) |
|
144 | self.stop_deferreds.append(d) | |
145 | return d |
|
145 | return d | |
146 | else: |
|
146 | else: | |
147 | s = 'this process is already complete' |
|
147 | s = 'this process is already complete' | |
148 | return defer.fail(ProcessStateError(s)) |
|
148 | return defer.fail(ProcessStateError(s)) | |
149 |
|
149 | |||
150 | def fire_stop_deferred(self, exit_code): |
|
150 | def fire_stop_deferred(self, exit_code): | |
151 | log.msg('Process %r has stopped with %r' % (self.args, exit_code)) |
|
151 | log.msg('Process %r has stopped with %r' % (self.args, exit_code)) | |
152 | self.state = 'after' |
|
152 | self.state = 'after' | |
153 | for d in self.stop_deferreds: |
|
153 | for d in self.stop_deferreds: | |
154 | d.callback(exit_code) |
|
154 | d.callback(exit_code) | |
155 |
|
155 | |||
156 | def signal(self, sig): |
|
156 | def signal(self, sig): | |
157 | """ |
|
157 | """ | |
158 | Send a signal to the process. |
|
158 | Send a signal to the process. | |
159 |
|
159 | |||
160 | The argument sig can be ('KILL','INT', etc.) or any signal number. |
|
160 | The argument sig can be ('KILL','INT', etc.) or any signal number. | |
161 | """ |
|
161 | """ | |
162 | if self.state == 'running': |
|
162 | if self.state == 'running': | |
163 | self.process_transport.signalProcess(sig) |
|
163 | self.process_transport.signalProcess(sig) | |
164 |
|
164 | |||
165 | # def __del__(self): |
|
165 | # def __del__(self): | |
166 | # self.signal('KILL') |
|
166 | # self.signal('KILL') | |
167 |
|
167 | |||
168 | def interrupt_then_kill(self, delay=1.0): |
|
168 | def interrupt_then_kill(self, delay=1.0): | |
169 | self.signal('INT') |
|
169 | self.signal('INT') | |
170 | reactor.callLater(delay, self.signal, 'KILL') |
|
170 | reactor.callLater(delay, self.signal, 'KILL') | |
171 |
|
171 | |||
172 |
|
172 | |||
173 | #----------------------------------------------------------------------------- |
|
173 | #----------------------------------------------------------------------------- | |
174 | # Code for launching controller and engines |
|
174 | # Code for launching controller and engines | |
175 | #----------------------------------------------------------------------------- |
|
175 | #----------------------------------------------------------------------------- | |
176 |
|
176 | |||
177 |
|
177 | |||
178 | class ControllerLauncher(ProcessLauncher): |
|
178 | class ControllerLauncher(ProcessLauncher): | |
179 |
|
179 | |||
180 | def __init__(self, extra_args=None): |
|
180 | def __init__(self, extra_args=None): | |
181 | if sys.platform == 'win32': |
|
181 | if sys.platform == 'win32': | |
182 | # This logic is needed because the ipcontroller script doesn't |
|
182 | # This logic is needed because the ipcontroller script doesn't | |
183 | # always get installed in the same way or in the same location. |
|
183 | # always get installed in the same way or in the same location. | |
184 | from IPython.kernel.scripts import ipcontroller |
|
184 | from IPython.kernel.scripts import ipcontroller | |
185 | script_location = ipcontroller.__file__.replace('.pyc', '.py') |
|
185 | script_location = ipcontroller.__file__.replace('.pyc', '.py') | |
186 | # The -u option here turns on unbuffered output, which is required |
|
186 | # The -u option here turns on unbuffered output, which is required | |
187 | # on Win32 to prevent wierd conflict and problems with Twisted |
|
187 | # on Win32 to prevent wierd conflict and problems with Twisted | |
188 | args = [find_exe('python'), '-u', script_location] |
|
188 | args = [find_exe('python'), '-u', script_location] | |
189 | else: |
|
189 | else: | |
190 | args = ['ipcontroller'] |
|
190 | args = ['ipcontroller'] | |
191 | self.extra_args = extra_args |
|
191 | self.extra_args = extra_args | |
192 | if extra_args is not None: |
|
192 | if extra_args is not None: | |
193 | args.extend(extra_args) |
|
193 | args.extend(extra_args) | |
194 |
|
194 | |||
195 | ProcessLauncher.__init__(self, args) |
|
195 | ProcessLauncher.__init__(self, args) | |
196 |
|
196 | |||
197 |
|
197 | |||
198 | class EngineLauncher(ProcessLauncher): |
|
198 | class EngineLauncher(ProcessLauncher): | |
199 |
|
199 | |||
200 | def __init__(self, extra_args=None): |
|
200 | def __init__(self, extra_args=None): | |
201 | if sys.platform == 'win32': |
|
201 | if sys.platform == 'win32': | |
202 | # This logic is needed because the ipcontroller script doesn't |
|
202 | # This logic is needed because the ipcontroller script doesn't | |
203 | # always get installed in the same way or in the same location. |
|
203 | # always get installed in the same way or in the same location. | |
204 | from IPython.kernel.scripts import ipengine |
|
204 | from IPython.kernel.scripts import ipengine | |
205 | script_location = ipengine.__file__.replace('.pyc', '.py') |
|
205 | script_location = ipengine.__file__.replace('.pyc', '.py') | |
206 | # The -u option here turns on unbuffered output, which is required |
|
206 | # The -u option here turns on unbuffered output, which is required | |
207 | # on Win32 to prevent wierd conflict and problems with Twisted |
|
207 | # on Win32 to prevent wierd conflict and problems with Twisted | |
208 | args = [find_exe('python'), '-u', script_location] |
|
208 | args = [find_exe('python'), '-u', script_location] | |
209 | else: |
|
209 | else: | |
210 | args = ['ipengine'] |
|
210 | args = ['ipengine'] | |
211 | self.extra_args = extra_args |
|
211 | self.extra_args = extra_args | |
212 | if extra_args is not None: |
|
212 | if extra_args is not None: | |
213 | args.extend(extra_args) |
|
213 | args.extend(extra_args) | |
214 |
|
214 | |||
215 | ProcessLauncher.__init__(self, args) |
|
215 | ProcessLauncher.__init__(self, args) | |
216 |
|
216 | |||
217 |
|
217 | |||
218 | class LocalEngineSet(object): |
|
218 | class LocalEngineSet(object): | |
219 |
|
219 | |||
220 | def __init__(self, extra_args=None): |
|
220 | def __init__(self, extra_args=None): | |
221 | self.extra_args = extra_args |
|
221 | self.extra_args = extra_args | |
222 | self.launchers = [] |
|
222 | self.launchers = [] | |
223 |
|
223 | |||
224 | def start(self, n): |
|
224 | def start(self, n): | |
225 | dlist = [] |
|
225 | dlist = [] | |
226 | for i in range(n): |
|
226 | for i in range(n): | |
227 | el = EngineLauncher(extra_args=self.extra_args) |
|
227 | el = EngineLauncher(extra_args=self.extra_args) | |
228 | d = el.start() |
|
228 | d = el.start() | |
229 | self.launchers.append(el) |
|
229 | self.launchers.append(el) | |
230 | dlist.append(d) |
|
230 | dlist.append(d) | |
231 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
231 | dfinal = gatherBoth(dlist, consumeErrors=True) | |
232 | dfinal.addCallback(self._handle_start) |
|
232 | dfinal.addCallback(self._handle_start) | |
233 | return dfinal |
|
233 | return dfinal | |
234 |
|
234 | |||
235 | def _handle_start(self, r): |
|
235 | def _handle_start(self, r): | |
236 | log.msg('Engines started with pids: %r' % r) |
|
236 | log.msg('Engines started with pids: %r' % r) | |
237 | return r |
|
237 | return r | |
238 |
|
238 | |||
239 | def _handle_stop(self, r): |
|
239 | def _handle_stop(self, r): | |
240 | log.msg('Engines received signal: %r' % r) |
|
240 | log.msg('Engines received signal: %r' % r) | |
241 | return r |
|
241 | return r | |
242 |
|
242 | |||
243 | def signal(self, sig): |
|
243 | def signal(self, sig): | |
244 | dlist = [] |
|
244 | dlist = [] | |
245 | for el in self.launchers: |
|
245 | for el in self.launchers: | |
246 | d = el.get_stop_deferred() |
|
246 | d = el.get_stop_deferred() | |
247 | dlist.append(d) |
|
247 | dlist.append(d) | |
248 | el.signal(sig) |
|
248 | el.signal(sig) | |
249 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
249 | dfinal = gatherBoth(dlist, consumeErrors=True) | |
250 | dfinal.addCallback(self._handle_stop) |
|
250 | dfinal.addCallback(self._handle_stop) | |
251 | return dfinal |
|
251 | return dfinal | |
252 |
|
252 | |||
253 | def interrupt_then_kill(self, delay=1.0): |
|
253 | def interrupt_then_kill(self, delay=1.0): | |
254 | dlist = [] |
|
254 | dlist = [] | |
255 | for el in self.launchers: |
|
255 | for el in self.launchers: | |
256 | d = el.get_stop_deferred() |
|
256 | d = el.get_stop_deferred() | |
257 | dlist.append(d) |
|
257 | dlist.append(d) | |
258 | el.interrupt_then_kill(delay) |
|
258 | el.interrupt_then_kill(delay) | |
259 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
259 | dfinal = gatherBoth(dlist, consumeErrors=True) | |
260 | dfinal.addCallback(self._handle_stop) |
|
260 | dfinal.addCallback(self._handle_stop) | |
261 | return dfinal |
|
261 | return dfinal | |
262 |
|
262 | |||
263 |
|
263 | |||
264 | class BatchEngineSet(object): |
|
264 | class BatchEngineSet(object): | |
265 |
|
265 | |||
266 | # Subclasses must fill these in. See PBSEngineSet |
|
266 | # Subclasses must fill these in. See PBSEngineSet | |
267 | submit_command = '' |
|
267 | submit_command = '' | |
268 | delete_command = '' |
|
268 | delete_command = '' | |
269 | job_id_regexp = '' |
|
269 | job_id_regexp = '' | |
270 |
|
270 | |||
271 | def __init__(self, template_file, **kwargs): |
|
271 | def __init__(self, template_file, **kwargs): | |
272 | self.template_file = template_file |
|
272 | self.template_file = template_file | |
273 | self.context = {} |
|
273 | self.context = {} | |
274 | self.context.update(kwargs) |
|
274 | self.context.update(kwargs) | |
275 | self.batch_file = self.template_file+'-run' |
|
275 | self.batch_file = self.template_file+'-run' | |
276 |
|
276 | |||
277 | def parse_job_id(self, output): |
|
277 | def parse_job_id(self, output): | |
278 | m = re.match(self.job_id_regexp, output) |
|
278 | m = re.match(self.job_id_regexp, output) | |
279 | if m is not None: |
|
279 | if m is not None: | |
280 | job_id = m.group() |
|
280 | job_id = m.group() | |
281 | else: |
|
281 | else: | |
282 | raise Exception("job id couldn't be determined: %s" % output) |
|
282 | raise Exception("job id couldn't be determined: %s" % output) | |
283 | self.job_id = job_id |
|
283 | self.job_id = job_id | |
284 | log.msg('Job started with job id: %r' % job_id) |
|
284 | log.msg('Job started with job id: %r' % job_id) | |
285 | return job_id |
|
285 | return job_id | |
286 |
|
286 | |||
287 | def write_batch_script(self, n): |
|
287 | def write_batch_script(self, n): | |
288 | self.context['n'] = n |
|
288 | self.context['n'] = n | |
289 | template = open(self.template_file, 'r').read() |
|
289 | template = open(self.template_file, 'r').read() | |
290 | log.msg('Using template for batch script: %s' % self.template_file) |
|
290 | log.msg('Using template for batch script: %s' % self.template_file) | |
291 | script_as_string = Itpl.itplns(template, self.context) |
|
291 | script_as_string = Itpl.itplns(template, self.context) | |
292 | log.msg('Writing instantiated batch script: %s' % self.batch_file) |
|
292 | log.msg('Writing instantiated batch script: %s' % self.batch_file) | |
293 | f = open(self.batch_file,'w') |
|
293 | f = open(self.batch_file,'w') | |
294 | f.write(script_as_string) |
|
294 | f.write(script_as_string) | |
295 | f.close() |
|
295 | f.close() | |
296 |
|
296 | |||
297 | def handle_error(self, f): |
|
297 | def handle_error(self, f): | |
298 | f.printTraceback() |
|
298 | f.printTraceback() | |
299 | f.raiseException() |
|
299 | f.raiseException() | |
300 |
|
300 | |||
301 | def start(self, n): |
|
301 | def start(self, n): | |
302 | self.write_batch_script(n) |
|
302 | self.write_batch_script(n) | |
303 | d = getProcessOutput(self.submit_command, |
|
303 | d = getProcessOutput(self.submit_command, | |
304 | [self.batch_file],env=os.environ) |
|
304 | [self.batch_file],env=os.environ) | |
305 | d.addCallback(self.parse_job_id) |
|
305 | d.addCallback(self.parse_job_id) | |
306 | d.addErrback(self.handle_error) |
|
306 | d.addErrback(self.handle_error) | |
307 | return d |
|
307 | return d | |
308 |
|
308 | |||
309 | def kill(self): |
|
309 | def kill(self): | |
310 | d = getProcessOutput(self.delete_command, |
|
310 | d = getProcessOutput(self.delete_command, | |
311 | [self.job_id],env=os.environ) |
|
311 | [self.job_id],env=os.environ) | |
312 | return d |
|
312 | return d | |
313 |
|
313 | |||
314 | class PBSEngineSet(BatchEngineSet): |
|
314 | class PBSEngineSet(BatchEngineSet): | |
315 |
|
315 | |||
316 | submit_command = 'qsub' |
|
316 | submit_command = 'qsub' | |
317 | delete_command = 'qdel' |
|
317 | delete_command = 'qdel' | |
318 | job_id_regexp = '\d+' |
|
318 | job_id_regexp = '\d+' | |
319 |
|
319 | |||
320 | def __init__(self, template_file, **kwargs): |
|
320 | def __init__(self, template_file, **kwargs): | |
321 | BatchEngineSet.__init__(self, template_file, **kwargs) |
|
321 | BatchEngineSet.__init__(self, template_file, **kwargs) | |
322 |
|
322 | |||
323 |
|
323 | |||
324 | sshx_template="""#!/bin/sh |
|
324 | sshx_template="""#!/bin/sh | |
325 | "$@" &> /dev/null & |
|
325 | "$@" &> /dev/null & | |
326 | echo $! |
|
326 | echo $! | |
327 | """ |
|
327 | """ | |
328 |
|
328 | |||
329 | engine_killer_template="""#!/bin/sh |
|
329 | engine_killer_template="""#!/bin/sh | |
330 | ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM |
|
330 | ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM | |
331 | """ |
|
331 | """ | |
332 |
|
332 | |||
333 | class SSHEngineSet(object): |
|
333 | class SSHEngineSet(object): | |
334 | sshx_template=sshx_template |
|
334 | sshx_template=sshx_template | |
335 | engine_killer_template=engine_killer_template |
|
335 | engine_killer_template=engine_killer_template | |
336 |
|
336 | |||
337 | def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"): |
|
337 | def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"): | |
338 | """Start a controller on localhost and engines using ssh. |
|
338 | """Start a controller on localhost and engines using ssh. | |
339 |
|
339 | |||
340 | The engine_hosts argument is a dict with hostnames as keys and |
|
340 | The engine_hosts argument is a dict with hostnames as keys and | |
341 | the number of engine (int) as values. sshx is the name of a local |
|
341 | the number of engine (int) as values. sshx is the name of a local | |
342 | file that will be used to run remote commands. This file is used |
|
342 | file that will be used to run remote commands. This file is used | |
343 | to setup the environment properly. |
|
343 | to setup the environment properly. | |
344 | """ |
|
344 | """ | |
345 |
|
345 | |||
346 | self.temp_dir = tempfile.gettempdir() |
|
346 | self.temp_dir = tempfile.gettempdir() | |
347 | if sshx is not None: |
|
347 | if sshx is not None: | |
348 | self.sshx = sshx |
|
348 | self.sshx = sshx | |
349 | else: |
|
349 | else: | |
350 | # Write the sshx.sh file locally from our template. |
|
350 | # Write the sshx.sh file locally from our template. | |
351 | self.sshx = os.path.join( |
|
351 | self.sshx = os.path.join( | |
352 | self.temp_dir, |
|
352 | self.temp_dir, | |
353 | '%s-main-sshx.sh' % os.environ['USER'] |
|
353 | '%s-main-sshx.sh' % os.environ['USER'] | |
354 | ) |
|
354 | ) | |
355 | f = open(self.sshx, 'w') |
|
355 | f = open(self.sshx, 'w') | |
356 | f.writelines(self.sshx_template) |
|
356 | f.writelines(self.sshx_template) | |
357 | f.close() |
|
357 | f.close() | |
358 | self.engine_command = ipengine |
|
358 | self.engine_command = ipengine | |
359 | self.engine_hosts = engine_hosts |
|
359 | self.engine_hosts = engine_hosts | |
360 | # Write the engine killer script file locally from our template. |
|
360 | # Write the engine killer script file locally from our template. | |
361 | self.engine_killer = os.path.join( |
|
361 | self.engine_killer = os.path.join( | |
362 | self.temp_dir, |
|
362 | self.temp_dir, | |
363 | '%s-local-engine_killer.sh' % os.environ['USER'] |
|
363 | '%s-local-engine_killer.sh' % os.environ['USER'] | |
364 | ) |
|
364 | ) | |
365 | f = open(self.engine_killer, 'w') |
|
365 | f = open(self.engine_killer, 'w') | |
366 | f.writelines(self.engine_killer_template) |
|
366 | f.writelines(self.engine_killer_template) | |
367 | f.close() |
|
367 | f.close() | |
368 |
|
368 | |||
369 | def start(self, send_furl=False): |
|
369 | def start(self, send_furl=False): | |
370 | dlist = [] |
|
370 | dlist = [] | |
371 | for host in self.engine_hosts.keys(): |
|
371 | for host in self.engine_hosts.keys(): | |
372 | count = self.engine_hosts[host] |
|
372 | count = self.engine_hosts[host] | |
373 | d = self._start(host, count, send_furl) |
|
373 | d = self._start(host, count, send_furl) | |
374 | dlist.append(d) |
|
374 | dlist.append(d) | |
375 | return gatherBoth(dlist, consumeErrors=True) |
|
375 | return gatherBoth(dlist, consumeErrors=True) | |
376 |
|
376 | |||
377 | def _start(self, hostname, count=1, send_furl=False): |
|
377 | def _start(self, hostname, count=1, send_furl=False): | |
378 | if send_furl: |
|
378 | if send_furl: | |
379 | d = self._scp_furl(hostname) |
|
379 | d = self._scp_furl(hostname) | |
380 | else: |
|
380 | else: | |
381 | d = defer.succeed(None) |
|
381 | d = defer.succeed(None) | |
382 | d.addCallback(lambda r: self._scp_sshx(hostname)) |
|
382 | d.addCallback(lambda r: self._scp_sshx(hostname)) | |
383 | d.addCallback(lambda r: self._ssh_engine(hostname, count)) |
|
383 | d.addCallback(lambda r: self._ssh_engine(hostname, count)) | |
384 | return d |
|
384 | return d | |
385 |
|
385 | |||
386 | def _scp_furl(self, hostname): |
|
386 | def _scp_furl(self, hostname): | |
387 | scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname) |
|
387 | scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname) | |
388 | cmd_list = scp_cmd.split() |
|
388 | cmd_list = scp_cmd.split() | |
389 | cmd_list[1] = os.path.expanduser(cmd_list[1]) |
|
389 | cmd_list[1] = os.path.expanduser(cmd_list[1]) | |
390 | log.msg('Copying furl file: %s' % scp_cmd) |
|
390 | log.msg('Copying furl file: %s' % scp_cmd) | |
391 | d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) |
|
391 | d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) | |
392 | return d |
|
392 | return d | |
393 |
|
393 | |||
394 | def _scp_sshx(self, hostname): |
|
394 | def _scp_sshx(self, hostname): | |
395 | scp_cmd = "scp %s %s:%s/%s-sshx.sh" % ( |
|
395 | scp_cmd = "scp %s %s:%s/%s-sshx.sh" % ( | |
396 | self.sshx, hostname, |
|
396 | self.sshx, hostname, | |
397 | self.temp_dir, os.environ['USER'] |
|
397 | self.temp_dir, os.environ['USER'] | |
398 | ) |
|
398 | ) | |
399 |
|
399 | |||
400 | log.msg("Copying sshx: %s" % scp_cmd) |
|
400 | log.msg("Copying sshx: %s" % scp_cmd) | |
401 | sshx_scp = scp_cmd.split() |
|
401 | sshx_scp = scp_cmd.split() | |
402 | d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ) |
|
402 | d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ) | |
403 | return d |
|
403 | return d | |
404 |
|
404 | |||
405 | def _ssh_engine(self, hostname, count): |
|
405 | def _ssh_engine(self, hostname, count): | |
406 | exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % ( |
|
406 | exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % ( | |
407 | hostname, self.temp_dir, |
|
407 | hostname, self.temp_dir, | |
408 | os.environ['USER'], self.engine_command |
|
408 | os.environ['USER'], self.engine_command | |
409 | ) |
|
409 | ) | |
410 | cmds = exec_engine.split() |
|
410 | cmds = exec_engine.split() | |
411 | dlist = [] |
|
411 | dlist = [] | |
412 | log.msg("about to start engines...") |
|
412 | log.msg("about to start engines...") | |
413 | for i in range(count): |
|
413 | for i in range(count): | |
414 | log.msg('Starting engines: %s' % exec_engine) |
|
414 | log.msg('Starting engines: %s' % exec_engine) | |
415 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) |
|
415 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) | |
416 | dlist.append(d) |
|
416 | dlist.append(d) | |
417 | return gatherBoth(dlist, consumeErrors=True) |
|
417 | return gatherBoth(dlist, consumeErrors=True) | |
418 |
|
418 | |||
419 | def kill(self): |
|
419 | def kill(self): | |
420 | dlist = [] |
|
420 | dlist = [] | |
421 | for host in self.engine_hosts.keys(): |
|
421 | for host in self.engine_hosts.keys(): | |
422 | d = self._killall(host) |
|
422 | d = self._killall(host) | |
423 | dlist.append(d) |
|
423 | dlist.append(d) | |
424 | return gatherBoth(dlist, consumeErrors=True) |
|
424 | return gatherBoth(dlist, consumeErrors=True) | |
425 |
|
425 | |||
426 | def _killall(self, hostname): |
|
426 | def _killall(self, hostname): | |
427 | d = self._scp_engine_killer(hostname) |
|
427 | d = self._scp_engine_killer(hostname) | |
428 | d.addCallback(lambda r: self._ssh_kill(hostname)) |
|
428 | d.addCallback(lambda r: self._ssh_kill(hostname)) | |
429 | # d.addErrback(self._exec_err) |
|
429 | # d.addErrback(self._exec_err) | |
430 | return d |
|
430 | return d | |
431 |
|
431 | |||
432 | def _scp_engine_killer(self, hostname): |
|
432 | def _scp_engine_killer(self, hostname): | |
433 | scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % ( |
|
433 | scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % ( | |
434 | self.engine_killer, |
|
434 | self.engine_killer, | |
435 | hostname, |
|
435 | hostname, | |
436 | self.temp_dir, |
|
436 | self.temp_dir, | |
437 | os.environ['USER'] |
|
437 | os.environ['USER'] | |
438 | ) |
|
438 | ) | |
439 | cmds = scp_cmd.split() |
|
439 | cmds = scp_cmd.split() | |
440 | log.msg('Copying engine_killer: %s' % scp_cmd) |
|
440 | log.msg('Copying engine_killer: %s' % scp_cmd) | |
441 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) |
|
441 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) | |
442 | return d |
|
442 | return d | |
443 |
|
443 | |||
444 | def _ssh_kill(self, hostname): |
|
444 | def _ssh_kill(self, hostname): | |
445 | kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % ( |
|
445 | kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % ( | |
446 | hostname, |
|
446 | hostname, | |
447 | self.temp_dir, |
|
447 | self.temp_dir, | |
448 | os.environ['USER'] |
|
448 | os.environ['USER'] | |
449 | ) |
|
449 | ) | |
450 | log.msg('Killing engine: %s' % kill_cmd) |
|
450 | log.msg('Killing engine: %s' % kill_cmd) | |
451 | kill_cmd = kill_cmd.split() |
|
451 | kill_cmd = kill_cmd.split() | |
452 | d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ) |
|
452 | d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ) | |
453 | return d |
|
453 | return d | |
454 |
|
454 | |||
455 | def _exec_err(self, r): |
|
455 | def _exec_err(self, r): | |
456 | log.msg(r) |
|
456 | log.msg(r) | |
457 |
|
457 | |||
458 | #----------------------------------------------------------------------------- |
|
458 | #----------------------------------------------------------------------------- | |
459 | # Main functions for the different types of clusters |
|
459 | # Main functions for the different types of clusters | |
460 | #----------------------------------------------------------------------------- |
|
460 | #----------------------------------------------------------------------------- | |
461 |
|
461 | |||
462 | # TODO: |
|
462 | # TODO: | |
463 | # The logic in these codes should be moved into classes like LocalCluster |
|
463 | # The logic in these codes should be moved into classes like LocalCluster | |
464 | # MpirunCluster, PBSCluster, etc. This would remove alot of the duplications. |
|
464 | # MpirunCluster, PBSCluster, etc. This would remove alot of the duplications. | |
465 | # The main functions should then just parse the command line arguments, create |
|
465 | # The main functions should then just parse the command line arguments, create | |
466 | # the appropriate class and call a 'start' method. |
|
466 | # the appropriate class and call a 'start' method. | |
467 |
|
467 | |||
468 | def check_security(args, cont_args): |
|
468 | def check_security(args, cont_args): | |
469 | if (not args.x or not args.y) and not have_crypto: |
|
469 | if (not args.x or not args.y) and not have_crypto: | |
470 | log.err(""" |
|
470 | log.err(""" | |
471 | OpenSSL/pyOpenSSL is not available, so we can't run in secure mode. |
|
471 | OpenSSL/pyOpenSSL is not available, so we can't run in secure mode. | |
472 | Try running ipcluster with the -xy flags: ipcluster local -xy -n 4""") |
|
472 | Try running ipcluster with the -xy flags: ipcluster local -xy -n 4""") | |
473 | reactor.stop() |
|
473 | reactor.stop() | |
474 | return False |
|
474 | return False | |
475 | if args.x: |
|
475 | if args.x: | |
476 | cont_args.append('-x') |
|
476 | cont_args.append('-x') | |
477 | if args.y: |
|
477 | if args.y: | |
478 | cont_args.append('-y') |
|
478 | cont_args.append('-y') | |
479 | return True |
|
479 | return True | |
480 |
|
480 | |||
|
481 | def check_reuse(args, cont_args): | |||
|
482 | if args.r: | |||
|
483 | cont_args.append('-r') | |||
|
484 | if args.client_port == 0 or args.engine_port == 0: | |||
|
485 | log.err(""" | |||
|
486 | To reuse FURL files, you must also set the client and engine ports using | |||
|
487 | the --client-port and --engine-port options.""") | |||
|
488 | reactor.stop() | |||
|
489 | return False | |||
|
490 | cont_args.append('--client-port=%i' % args.client_port) | |||
|
491 | cont_args.append('--engine-port=%i' % args.engine_port) | |||
|
492 | return True | |||
481 |
|
493 | |||
482 | def main_local(args): |
|
494 | def main_local(args): | |
483 | cont_args = [] |
|
495 | cont_args = [] | |
484 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
496 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) | |
485 |
|
497 | |||
486 | # Check security settings before proceeding |
|
498 | # Check security settings before proceeding | |
487 | if not check_security(args, cont_args): |
|
499 | if not check_security(args, cont_args): | |
488 | return |
|
500 | return | |
489 |
|
501 | |||
|
502 | # See if we are reusing FURL files | |||
|
503 | if not check_reuse(args, cont_args): | |||
|
504 | return | |||
|
505 | ||||
490 | cl = ControllerLauncher(extra_args=cont_args) |
|
506 | cl = ControllerLauncher(extra_args=cont_args) | |
491 | dstart = cl.start() |
|
507 | dstart = cl.start() | |
492 | def start_engines(cont_pid): |
|
508 | def start_engines(cont_pid): | |
493 | engine_args = [] |
|
509 | engine_args = [] | |
494 | engine_args.append('--logfile=%s' % \ |
|
510 | engine_args.append('--logfile=%s' % \ | |
495 | pjoin(args.logdir,'ipengine%s-' % cont_pid)) |
|
511 | pjoin(args.logdir,'ipengine%s-' % cont_pid)) | |
496 | eset = LocalEngineSet(extra_args=engine_args) |
|
512 | eset = LocalEngineSet(extra_args=engine_args) | |
497 | def shutdown(signum, frame): |
|
513 | def shutdown(signum, frame): | |
498 | log.msg('Stopping local cluster') |
|
514 | log.msg('Stopping local cluster') | |
499 | # We are still playing with the times here, but these seem |
|
515 | # We are still playing with the times here, but these seem | |
500 | # to be reliable in allowing everything to exit cleanly. |
|
516 | # to be reliable in allowing everything to exit cleanly. | |
501 | eset.interrupt_then_kill(0.5) |
|
517 | eset.interrupt_then_kill(0.5) | |
502 | cl.interrupt_then_kill(0.5) |
|
518 | cl.interrupt_then_kill(0.5) | |
503 | reactor.callLater(1.0, reactor.stop) |
|
519 | reactor.callLater(1.0, reactor.stop) | |
504 | signal.signal(signal.SIGINT,shutdown) |
|
520 | signal.signal(signal.SIGINT,shutdown) | |
505 | d = eset.start(args.n) |
|
521 | d = eset.start(args.n) | |
506 | return d |
|
522 | return d | |
507 | def delay_start(cont_pid): |
|
523 | def delay_start(cont_pid): | |
508 | # This is needed because the controller doesn't start listening |
|
524 | # This is needed because the controller doesn't start listening | |
509 | # right when it starts and the controller needs to write |
|
525 | # right when it starts and the controller needs to write | |
510 | # furl files for the engine to pick up |
|
526 | # furl files for the engine to pick up | |
511 | reactor.callLater(1.0, start_engines, cont_pid) |
|
527 | reactor.callLater(1.0, start_engines, cont_pid) | |
512 | dstart.addCallback(delay_start) |
|
528 | dstart.addCallback(delay_start) | |
513 | dstart.addErrback(lambda f: f.raiseException()) |
|
529 | dstart.addErrback(lambda f: f.raiseException()) | |
514 |
|
530 | |||
515 |
|
531 | |||
516 |
def main_mpi |
|
532 | def main_mpi(args): | |
517 | cont_args = [] |
|
533 | cont_args = [] | |
518 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
534 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) | |
519 |
|
535 | |||
520 | # Check security settings before proceeding |
|
536 | # Check security settings before proceeding | |
521 | if not check_security(args, cont_args): |
|
537 | if not check_security(args, cont_args): | |
522 | return |
|
538 | return | |
523 |
|
539 | |||
|
540 | # See if we are reusing FURL files | |||
|
541 | if not check_reuse(args, cont_args): | |||
|
542 | return | |||
|
543 | ||||
524 | cl = ControllerLauncher(extra_args=cont_args) |
|
544 | cl = ControllerLauncher(extra_args=cont_args) | |
525 | dstart = cl.start() |
|
545 | dstart = cl.start() | |
526 | def start_engines(cont_pid): |
|
546 | def start_engines(cont_pid): | |
527 |
raw_args = [ |
|
547 | raw_args = [args.cmd] | |
528 | raw_args.extend(['-n',str(args.n)]) |
|
548 | raw_args.extend(['-n',str(args.n)]) | |
529 | raw_args.append('ipengine') |
|
549 | raw_args.append('ipengine') | |
530 | raw_args.append('-l') |
|
550 | raw_args.append('-l') | |
531 | raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid)) |
|
551 | raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid)) | |
532 | if args.mpi: |
|
552 | if args.mpi: | |
533 | raw_args.append('--mpi=%s' % args.mpi) |
|
553 | raw_args.append('--mpi=%s' % args.mpi) | |
534 | eset = ProcessLauncher(raw_args) |
|
554 | eset = ProcessLauncher(raw_args) | |
535 | def shutdown(signum, frame): |
|
555 | def shutdown(signum, frame): | |
536 | log.msg('Stopping local cluster') |
|
556 | log.msg('Stopping local cluster') | |
537 | # We are still playing with the times here, but these seem |
|
557 | # We are still playing with the times here, but these seem | |
538 | # to be reliable in allowing everything to exit cleanly. |
|
558 | # to be reliable in allowing everything to exit cleanly. | |
539 | eset.interrupt_then_kill(1.0) |
|
559 | eset.interrupt_then_kill(1.0) | |
540 | cl.interrupt_then_kill(1.0) |
|
560 | cl.interrupt_then_kill(1.0) | |
541 | reactor.callLater(2.0, reactor.stop) |
|
561 | reactor.callLater(2.0, reactor.stop) | |
542 | signal.signal(signal.SIGINT,shutdown) |
|
562 | signal.signal(signal.SIGINT,shutdown) | |
543 | d = eset.start() |
|
563 | d = eset.start() | |
544 | return d |
|
564 | return d | |
545 | def delay_start(cont_pid): |
|
565 | def delay_start(cont_pid): | |
546 | # This is needed because the controller doesn't start listening |
|
566 | # This is needed because the controller doesn't start listening | |
547 | # right when it starts and the controller needs to write |
|
567 | # right when it starts and the controller needs to write | |
548 | # furl files for the engine to pick up |
|
568 | # furl files for the engine to pick up | |
549 | reactor.callLater(1.0, start_engines, cont_pid) |
|
569 | reactor.callLater(1.0, start_engines, cont_pid) | |
550 | dstart.addCallback(delay_start) |
|
570 | dstart.addCallback(delay_start) | |
551 | dstart.addErrback(lambda f: f.raiseException()) |
|
571 | dstart.addErrback(lambda f: f.raiseException()) | |
552 |
|
572 | |||
553 |
|
573 | |||
554 | def main_pbs(args): |
|
574 | def main_pbs(args): | |
555 | cont_args = [] |
|
575 | cont_args = [] | |
556 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
576 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) | |
557 |
|
577 | |||
558 | # Check security settings before proceeding |
|
578 | # Check security settings before proceeding | |
559 | if not check_security(args, cont_args): |
|
579 | if not check_security(args, cont_args): | |
560 | return |
|
580 | return | |
561 |
|
581 | |||
|
582 | # See if we are reusing FURL files | |||
|
583 | if not check_reuse(args, cont_args): | |||
|
584 | return | |||
|
585 | ||||
562 | cl = ControllerLauncher(extra_args=cont_args) |
|
586 | cl = ControllerLauncher(extra_args=cont_args) | |
563 | dstart = cl.start() |
|
587 | dstart = cl.start() | |
564 | def start_engines(r): |
|
588 | def start_engines(r): | |
565 | pbs_set = PBSEngineSet(args.pbsscript) |
|
589 | pbs_set = PBSEngineSet(args.pbsscript) | |
566 | def shutdown(signum, frame): |
|
590 | def shutdown(signum, frame): | |
567 | log.msg('Stopping pbs cluster') |
|
591 | log.msg('Stopping pbs cluster') | |
568 | d = pbs_set.kill() |
|
592 | d = pbs_set.kill() | |
569 | d.addBoth(lambda _: cl.interrupt_then_kill(1.0)) |
|
593 | d.addBoth(lambda _: cl.interrupt_then_kill(1.0)) | |
570 | d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop)) |
|
594 | d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop)) | |
571 | signal.signal(signal.SIGINT,shutdown) |
|
595 | signal.signal(signal.SIGINT,shutdown) | |
572 | d = pbs_set.start(args.n) |
|
596 | d = pbs_set.start(args.n) | |
573 | return d |
|
597 | return d | |
574 | dstart.addCallback(start_engines) |
|
598 | dstart.addCallback(start_engines) | |
575 | dstart.addErrback(lambda f: f.raiseException()) |
|
599 | dstart.addErrback(lambda f: f.raiseException()) | |
576 |
|
600 | |||
577 |
|
601 | |||
578 | def main_ssh(args): |
|
602 | def main_ssh(args): | |
579 | """Start a controller on localhost and engines using ssh. |
|
603 | """Start a controller on localhost and engines using ssh. | |
580 |
|
604 | |||
581 | Your clusterfile should look like:: |
|
605 | Your clusterfile should look like:: | |
582 |
|
606 | |||
583 | send_furl = False # True, if you want |
|
607 | send_furl = False # True, if you want | |
584 | engines = { |
|
608 | engines = { | |
585 | 'engine_host1' : engine_count, |
|
609 | 'engine_host1' : engine_count, | |
586 | 'engine_host2' : engine_count2 |
|
610 | 'engine_host2' : engine_count2 | |
587 | } |
|
611 | } | |
588 | """ |
|
612 | """ | |
589 | clusterfile = {} |
|
613 | clusterfile = {} | |
590 | execfile(args.clusterfile, clusterfile) |
|
614 | execfile(args.clusterfile, clusterfile) | |
591 | if not clusterfile.has_key('send_furl'): |
|
615 | if not clusterfile.has_key('send_furl'): | |
592 | clusterfile['send_furl'] = False |
|
616 | clusterfile['send_furl'] = False | |
593 |
|
617 | |||
594 | cont_args = [] |
|
618 | cont_args = [] | |
595 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
619 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) | |
596 |
|
620 | |||
597 | # Check security settings before proceeding |
|
621 | # Check security settings before proceeding | |
598 | if not check_security(args, cont_args): |
|
622 | if not check_security(args, cont_args): | |
599 | return |
|
623 | return | |
600 |
|
624 | |||
|
625 | # See if we are reusing FURL files | |||
|
626 | if not check_reuse(args, cont_args): | |||
|
627 | return | |||
|
628 | ||||
601 | cl = ControllerLauncher(extra_args=cont_args) |
|
629 | cl = ControllerLauncher(extra_args=cont_args) | |
602 | dstart = cl.start() |
|
630 | dstart = cl.start() | |
603 | def start_engines(cont_pid): |
|
631 | def start_engines(cont_pid): | |
604 | ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx) |
|
632 | ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx) | |
605 | def shutdown(signum, frame): |
|
633 | def shutdown(signum, frame): | |
606 | d = ssh_set.kill() |
|
634 | d = ssh_set.kill() | |
607 | # d.addErrback(log.err) |
|
|||
608 | cl.interrupt_then_kill(1.0) |
|
635 | cl.interrupt_then_kill(1.0) | |
609 | reactor.callLater(2.0, reactor.stop) |
|
636 | reactor.callLater(2.0, reactor.stop) | |
610 | signal.signal(signal.SIGINT,shutdown) |
|
637 | signal.signal(signal.SIGINT,shutdown) | |
611 | d = ssh_set.start(clusterfile['send_furl']) |
|
638 | d = ssh_set.start(clusterfile['send_furl']) | |
612 | return d |
|
639 | return d | |
613 |
|
640 | |||
614 | def delay_start(cont_pid): |
|
641 | def delay_start(cont_pid): | |
615 | reactor.callLater(1.0, start_engines, cont_pid) |
|
642 | reactor.callLater(1.0, start_engines, cont_pid) | |
616 |
|
643 | |||
617 | dstart.addCallback(delay_start) |
|
644 | dstart.addCallback(delay_start) | |
618 | dstart.addErrback(lambda f: f.raiseException()) |
|
645 | dstart.addErrback(lambda f: f.raiseException()) | |
619 |
|
646 | |||
620 |
|
647 | |||
621 | def get_args(): |
|
648 | def get_args(): | |
622 | base_parser = argparse.ArgumentParser(add_help=False) |
|
649 | base_parser = argparse.ArgumentParser(add_help=False) | |
623 | base_parser.add_argument( |
|
650 | base_parser.add_argument( | |
|
651 | '-r', | |||
|
652 | action='store_true', | |||
|
653 | dest='r', | |||
|
654 | help='try to reuse FURL files. Use with --client-port and --engine-port' | |||
|
655 | ) | |||
|
656 | base_parser.add_argument( | |||
|
657 | '--client-port', | |||
|
658 | type=int, | |||
|
659 | dest='client_port', | |||
|
660 | help='the port the controller will listen on for client connections', | |||
|
661 | default=0 | |||
|
662 | ) | |||
|
663 | base_parser.add_argument( | |||
|
664 | '--engine-port', | |||
|
665 | type=int, | |||
|
666 | dest='engine_port', | |||
|
667 | help='the port the controller will listen on for engine connections', | |||
|
668 | default=0 | |||
|
669 | ) | |||
|
670 | base_parser.add_argument( | |||
624 | '-x', |
|
671 | '-x', | |
625 | action='store_true', |
|
672 | action='store_true', | |
626 | dest='x', |
|
673 | dest='x', | |
627 | help='turn off client security' |
|
674 | help='turn off client security' | |
628 | ) |
|
675 | ) | |
629 | base_parser.add_argument( |
|
676 | base_parser.add_argument( | |
630 | '-y', |
|
677 | '-y', | |
631 | action='store_true', |
|
678 | action='store_true', | |
632 | dest='y', |
|
679 | dest='y', | |
633 | help='turn off engine security' |
|
680 | help='turn off engine security' | |
634 | ) |
|
681 | ) | |
635 | base_parser.add_argument( |
|
682 | base_parser.add_argument( | |
636 | "--logdir", |
|
683 | "--logdir", | |
637 | type=str, |
|
684 | type=str, | |
638 | dest="logdir", |
|
685 | dest="logdir", | |
639 | help="directory to put log files (default=$IPYTHONDIR/log)", |
|
686 | help="directory to put log files (default=$IPYTHONDIR/log)", | |
640 | default=pjoin(get_ipython_dir(),'log') |
|
687 | default=pjoin(get_ipython_dir(),'log') | |
641 | ) |
|
688 | ) | |
642 | base_parser.add_argument( |
|
689 | base_parser.add_argument( | |
643 | "-n", |
|
690 | "-n", | |
644 | "--num", |
|
691 | "--num", | |
645 | type=int, |
|
692 | type=int, | |
646 | dest="n", |
|
693 | dest="n", | |
647 | default=2, |
|
694 | default=2, | |
648 | help="the number of engines to start" |
|
695 | help="the number of engines to start" | |
649 | ) |
|
696 | ) | |
650 |
|
697 | |||
651 | parser = argparse.ArgumentParser( |
|
698 | parser = argparse.ArgumentParser( | |
652 | description='IPython cluster startup. This starts a controller and\ |
|
699 | description='IPython cluster startup. This starts a controller and\ | |
653 | engines using various approaches. THIS IS A TECHNOLOGY PREVIEW AND\ |
|
700 | engines using various approaches. THIS IS A TECHNOLOGY PREVIEW AND\ | |
654 | THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.' |
|
701 | THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.' | |
655 | ) |
|
702 | ) | |
656 | subparsers = parser.add_subparsers( |
|
703 | subparsers = parser.add_subparsers( | |
657 | help='available cluster types. For help, do "ipcluster TYPE --help"') |
|
704 | help='available cluster types. For help, do "ipcluster TYPE --help"') | |
658 |
|
705 | |||
659 | parser_local = subparsers.add_parser( |
|
706 | parser_local = subparsers.add_parser( | |
660 | 'local', |
|
707 | 'local', | |
661 | help='run a local cluster', |
|
708 | help='run a local cluster', | |
662 | parents=[base_parser] |
|
709 | parents=[base_parser] | |
663 | ) |
|
710 | ) | |
664 | parser_local.set_defaults(func=main_local) |
|
711 | parser_local.set_defaults(func=main_local) | |
665 |
|
712 | |||
666 | parser_mpirun = subparsers.add_parser( |
|
713 | parser_mpirun = subparsers.add_parser( | |
667 | 'mpirun', |
|
714 | 'mpirun', | |
668 | help='run a cluster using mpirun', |
|
715 | help='run a cluster using mpirun (mpiexec also works)', | |
669 | parents=[base_parser] |
|
716 | parents=[base_parser] | |
670 | ) |
|
717 | ) | |
671 | parser_mpirun.add_argument( |
|
718 | parser_mpirun.add_argument( | |
672 | "--mpi", |
|
719 | "--mpi", | |
673 | type=str, |
|
720 | type=str, | |
674 | dest="mpi", # Don't put a default here to allow no MPI support |
|
721 | dest="mpi", # Don't put a default here to allow no MPI support | |
675 | help="how to call MPI_Init (default=mpi4py)" |
|
722 | help="how to call MPI_Init (default=mpi4py)" | |
676 | ) |
|
723 | ) | |
677 | parser_mpirun.set_defaults(func=main_mpirun) |
|
724 | parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun') | |
|
725 | ||||
|
726 | parser_mpiexec = subparsers.add_parser( | |||
|
727 | 'mpiexec', | |||
|
728 | help='run a cluster using mpiexec (mpirun also works)', | |||
|
729 | parents=[base_parser] | |||
|
730 | ) | |||
|
731 | parser_mpiexec.add_argument( | |||
|
732 | "--mpi", | |||
|
733 | type=str, | |||
|
734 | dest="mpi", # Don't put a default here to allow no MPI support | |||
|
735 | help="how to call MPI_Init (default=mpi4py)" | |||
|
736 | ) | |||
|
737 | parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec') | |||
678 |
|
738 | |||
679 | parser_pbs = subparsers.add_parser( |
|
739 | parser_pbs = subparsers.add_parser( | |
680 | 'pbs', |
|
740 | 'pbs', | |
681 | help='run a pbs cluster', |
|
741 | help='run a pbs cluster', | |
682 | parents=[base_parser] |
|
742 | parents=[base_parser] | |
683 | ) |
|
743 | ) | |
684 | parser_pbs.add_argument( |
|
744 | parser_pbs.add_argument( | |
685 | '--pbs-script', |
|
745 | '--pbs-script', | |
686 | type=str, |
|
746 | type=str, | |
687 | dest='pbsscript', |
|
747 | dest='pbsscript', | |
688 | help='PBS script template', |
|
748 | help='PBS script template', | |
689 | default='pbs.template' |
|
749 | default='pbs.template' | |
690 | ) |
|
750 | ) | |
691 | parser_pbs.set_defaults(func=main_pbs) |
|
751 | parser_pbs.set_defaults(func=main_pbs) | |
692 |
|
752 | |||
693 | parser_ssh = subparsers.add_parser( |
|
753 | parser_ssh = subparsers.add_parser( | |
694 | 'ssh', |
|
754 | 'ssh', | |
695 | help='run a cluster using ssh, should have ssh-keys setup', |
|
755 | help='run a cluster using ssh, should have ssh-keys setup', | |
696 | parents=[base_parser] |
|
756 | parents=[base_parser] | |
697 | ) |
|
757 | ) | |
698 | parser_ssh.add_argument( |
|
758 | parser_ssh.add_argument( | |
699 | '--clusterfile', |
|
759 | '--clusterfile', | |
700 | type=str, |
|
760 | type=str, | |
701 | dest='clusterfile', |
|
761 | dest='clusterfile', | |
702 | help='python file describing the cluster', |
|
762 | help='python file describing the cluster', | |
703 | default='clusterfile.py', |
|
763 | default='clusterfile.py', | |
704 | ) |
|
764 | ) | |
705 | parser_ssh.add_argument( |
|
765 | parser_ssh.add_argument( | |
706 | '--sshx', |
|
766 | '--sshx', | |
707 | type=str, |
|
767 | type=str, | |
708 | dest='sshx', |
|
768 | dest='sshx', | |
709 | help='sshx launcher helper' |
|
769 | help='sshx launcher helper' | |
710 | ) |
|
770 | ) | |
711 | parser_ssh.set_defaults(func=main_ssh) |
|
771 | parser_ssh.set_defaults(func=main_ssh) | |
712 |
|
772 | |||
713 | args = parser.parse_args() |
|
773 | args = parser.parse_args() | |
714 | return args |
|
774 | return args | |
715 |
|
775 | |||
716 | def main(): |
|
776 | def main(): | |
717 | args = get_args() |
|
777 | args = get_args() | |
718 | reactor.callWhenRunning(args.func, args) |
|
778 | reactor.callWhenRunning(args.func, args) | |
719 | log.startLogging(sys.stdout) |
|
779 | log.startLogging(sys.stdout) | |
720 | reactor.run() |
|
780 | reactor.run() | |
721 |
|
781 | |||
722 | if __name__ == '__main__': |
|
782 | if __name__ == '__main__': | |
723 | main() |
|
783 | main() |
@@ -1,398 +1,401 b'' | |||||
1 | .. _changes: |
|
1 | .. _changes: | |
2 |
|
2 | |||
3 | ========== |
|
3 | ========== | |
4 | What's new |
|
4 | What's new | |
5 | ========== |
|
5 | ========== | |
6 |
|
6 | |||
7 | .. contents:: |
|
7 | .. contents:: | |
8 | .. |
|
8 | .. | |
9 | 1 Release 0.9.1 |
|
9 | 1 Release 0.9.1 | |
10 | 2 Release 0.9 |
|
10 | 2 Release 0.9 | |
11 | 2.1 New features |
|
11 | 2.1 New features | |
12 | 2.2 Bug fixes |
|
12 | 2.2 Bug fixes | |
13 | 2.3 Backwards incompatible changes |
|
13 | 2.3 Backwards incompatible changes | |
14 | 2.4 Changes merged in from IPython1 |
|
14 | 2.4 Changes merged in from IPython1 | |
15 | 2.4.1 New features |
|
15 | 2.4.1 New features | |
16 | 2.4.2 Bug fixes |
|
16 | 2.4.2 Bug fixes | |
17 | 2.4.3 Backwards incompatible changes |
|
17 | 2.4.3 Backwards incompatible changes | |
18 | 3 Release 0.8.4 |
|
18 | 3 Release 0.8.4 | |
19 | 4 Release 0.8.3 |
|
19 | 4 Release 0.8.3 | |
20 | 5 Release 0.8.2 |
|
20 | 5 Release 0.8.2 | |
21 | 6 Older releases |
|
21 | 6 Older releases | |
22 | .. |
|
22 | .. | |
23 |
|
23 | |||
24 | Release dev |
|
24 | Release dev | |
25 | =========== |
|
25 | =========== | |
26 |
|
26 | |||
27 | New features |
|
27 | New features | |
28 | ------------ |
|
28 | ------------ | |
29 |
|
29 | |||
30 | * The new ipcluster now has a fully working ssh mode that should work on |
|
30 | * The new ipcluster now has a fully working ssh mode that should work on | |
31 | Linux, Unix and OS X. Thanks to Vishal Vatsa for implementing this! |
|
31 | Linux, Unix and OS X. Thanks to Vishal Vatsa for implementing this! | |
32 |
|
32 | |||
33 | * The wonderful TextMate editor can now be used with %edit on OS X. Thanks |
|
33 | * The wonderful TextMate editor can now be used with %edit on OS X. Thanks | |
34 | to Matt Foster for this patch. |
|
34 | to Matt Foster for this patch. | |
35 |
|
35 | |||
36 | * Fully refactored :command:`ipcluster` command line program for starting |
|
36 | * Fully refactored :command:`ipcluster` command line program for starting | |
37 | IPython clusters. This new version is a complete rewrite and 1) is fully |
|
37 | IPython clusters. This new version is a complete rewrite and 1) is fully | |
38 | cross platform (we now use Twisted's process management), 2) has much |
|
38 | cross platform (we now use Twisted's process management), 2) has much | |
39 | improved performance, 3) uses subcommands for different types of clusters, |
|
39 | improved performance, 3) uses subcommands for different types of clusters, | |
40 | 4) uses argparse for parsing command line options, 5) has better support |
|
40 | 4) uses argparse for parsing command line options, 5) has better support | |
41 | for starting clusters using :command:`mpirun`, 6) has experimental support |
|
41 | for starting clusters using :command:`mpirun`, 6) has experimental support | |
42 | for starting engines using PBS. However, this new version of ipcluster |
|
42 | for starting engines using PBS. However, this new version of ipcluster | |
43 | should be considered a technology preview. We plan on changing the API |
|
43 | should be considered a technology preview. We plan on changing the API | |
44 | in significant ways before it is final. |
|
44 | in significant ways before it is final. | |
45 |
|
45 | |||
46 | * The :mod:`argparse` module has been added to :mod:`IPython.external`. |
|
46 | * The :mod:`argparse` module has been added to :mod:`IPython.external`. | |
47 |
|
47 | |||
48 | * Fully description of the security model added to the docs. |
|
48 | * Fully description of the security model added to the docs. | |
49 |
|
49 | |||
50 | * cd completer: show bookmarks if no other completions are available. |
|
50 | * cd completer: show bookmarks if no other completions are available. | |
51 |
|
51 | |||
52 | * sh profile: easy way to give 'title' to prompt: assign to variable |
|
52 | * sh profile: easy way to give 'title' to prompt: assign to variable | |
53 | '_prompt_title'. It looks like this:: |
|
53 | '_prompt_title'. It looks like this:: | |
54 |
|
54 | |||
55 | [~]|1> _prompt_title = 'sudo!' |
|
55 | [~]|1> _prompt_title = 'sudo!' | |
56 | sudo![~]|2> |
|
56 | sudo![~]|2> | |
57 |
|
57 | |||
58 | * %edit: If you do '%edit pasted_block', pasted_block |
|
58 | * %edit: If you do '%edit pasted_block', pasted_block | |
59 | variable gets updated with new data (so repeated |
|
59 | variable gets updated with new data (so repeated | |
60 | editing makes sense) |
|
60 | editing makes sense) | |
61 |
|
61 | |||
62 | Bug fixes |
|
62 | Bug fixes | |
63 | --------- |
|
63 | --------- | |
64 |
|
64 | |||
65 | * Numerous bugs on Windows with the new ipcluster have been fixed. |
|
65 | * Numerous bugs on Windows with the new ipcluster have been fixed. | |
66 |
|
66 | |||
67 | * The ipengine and ipcontroller scripts now handle missing furl files |
|
67 | * The ipengine and ipcontroller scripts now handle missing furl files | |
68 | more gracefully by giving better error messages. |
|
68 | more gracefully by giving better error messages. | |
69 |
|
69 | |||
70 | * %rehashx: Aliases no longer contain dots. python3.0 binary |
|
70 | * %rehashx: Aliases no longer contain dots. python3.0 binary | |
71 | will create alias python30. Fixes: |
|
71 | will create alias python30. Fixes: | |
72 | #259716 "commands with dots in them don't work" |
|
72 | #259716 "commands with dots in them don't work" | |
73 |
|
73 | |||
74 | * %cpaste: %cpaste -r repeats the last pasted block. |
|
74 | * %cpaste: %cpaste -r repeats the last pasted block. | |
75 | The block is assigned to pasted_block even if code |
|
75 | The block is assigned to pasted_block even if code | |
76 | raises exception. |
|
76 | raises exception. | |
77 |
|
77 | |||
|
78 | * Bug #274067 'The code in get_home_dir is broken for py2exe' was | |||
|
79 | fixed. | |||
|
80 | ||||
78 | Backwards incompatible changes |
|
81 | Backwards incompatible changes | |
79 | ------------------------------ |
|
82 | ------------------------------ | |
80 |
|
83 | |||
81 | * The controller now has a ``-r`` flag that needs to be used if you want to |
|
84 | * The controller now has a ``-r`` flag that needs to be used if you want to | |
82 | reuse existing furl files. Otherwise they are deleted (the default). |
|
85 | reuse existing furl files. Otherwise they are deleted (the default). | |
83 |
|
86 | |||
84 | * Remove ipy_leo.py. "easy_install ipython-extension" to get it. |
|
87 | * Remove ipy_leo.py. "easy_install ipython-extension" to get it. | |
85 | (done to decouple it from ipython release cycle) |
|
88 | (done to decouple it from ipython release cycle) | |
86 |
|
89 | |||
87 |
|
90 | |||
88 |
|
91 | |||
89 | Release 0.9.1 |
|
92 | Release 0.9.1 | |
90 | ============= |
|
93 | ============= | |
91 |
|
94 | |||
92 | This release was quickly made to restore compatibility with Python 2.4, which |
|
95 | This release was quickly made to restore compatibility with Python 2.4, which | |
93 | version 0.9 accidentally broke. No new features were introduced, other than |
|
96 | version 0.9 accidentally broke. No new features were introduced, other than | |
94 | some additional testing support for internal use. |
|
97 | some additional testing support for internal use. | |
95 |
|
98 | |||
96 |
|
99 | |||
97 | Release 0.9 |
|
100 | Release 0.9 | |
98 | =========== |
|
101 | =========== | |
99 |
|
102 | |||
100 | New features |
|
103 | New features | |
101 | ------------ |
|
104 | ------------ | |
102 |
|
105 | |||
103 | * All furl files and security certificates are now put in a read-only |
|
106 | * All furl files and security certificates are now put in a read-only | |
104 | directory named ~./ipython/security. |
|
107 | directory named ~./ipython/security. | |
105 |
|
108 | |||
106 | * A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that |
|
109 | * A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that | |
107 | determines the user's IPython directory in a robust manner. |
|
110 | determines the user's IPython directory in a robust manner. | |
108 |
|
111 | |||
109 | * Laurent's WX application has been given a top-level script called |
|
112 | * Laurent's WX application has been given a top-level script called | |
110 | ipython-wx, and it has received numerous fixes. We expect this code to be |
|
113 | ipython-wx, and it has received numerous fixes. We expect this code to be | |
111 | architecturally better integrated with Gael's WX 'ipython widget' over the |
|
114 | architecturally better integrated with Gael's WX 'ipython widget' over the | |
112 | next few releases. |
|
115 | next few releases. | |
113 |
|
116 | |||
114 | * The Editor synchronization work by Vivian De Smedt has been merged in. This |
|
117 | * The Editor synchronization work by Vivian De Smedt has been merged in. This | |
115 | code adds a number of new editor hooks to synchronize with editors under |
|
118 | code adds a number of new editor hooks to synchronize with editors under | |
116 | Windows. |
|
119 | Windows. | |
117 |
|
120 | |||
118 | * A new, still experimental but highly functional, WX shell by Gael Varoquaux. |
|
121 | * A new, still experimental but highly functional, WX shell by Gael Varoquaux. | |
119 | This work was sponsored by Enthought, and while it's still very new, it is |
|
122 | This work was sponsored by Enthought, and while it's still very new, it is | |
120 | based on a more cleanly organized arhictecture of the various IPython |
|
123 | based on a more cleanly organized arhictecture of the various IPython | |
121 | components. We will continue to develop this over the next few releases as a |
|
124 | components. We will continue to develop this over the next few releases as a | |
122 | model for GUI components that use IPython. |
|
125 | model for GUI components that use IPython. | |
123 |
|
126 | |||
124 | * Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework), |
|
127 | * Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework), | |
125 | authored by Barry Wark. Currently the WX and the Cocoa ones have slightly |
|
128 | authored by Barry Wark. Currently the WX and the Cocoa ones have slightly | |
126 | different internal organizations, but the whole team is working on finding |
|
129 | different internal organizations, but the whole team is working on finding | |
127 | what the right abstraction points are for a unified codebase. |
|
130 | what the right abstraction points are for a unified codebase. | |
128 |
|
131 | |||
129 | * As part of the frontend work, Barry Wark also implemented an experimental |
|
132 | * As part of the frontend work, Barry Wark also implemented an experimental | |
130 | event notification system that various ipython components can use. In the |
|
133 | event notification system that various ipython components can use. In the | |
131 | next release the implications and use patterns of this system regarding the |
|
134 | next release the implications and use patterns of this system regarding the | |
132 | various GUI options will be worked out. |
|
135 | various GUI options will be worked out. | |
133 |
|
136 | |||
134 | * IPython finally has a full test system, that can test docstrings with |
|
137 | * IPython finally has a full test system, that can test docstrings with | |
135 | IPython-specific functionality. There are still a few pieces missing for it |
|
138 | IPython-specific functionality. There are still a few pieces missing for it | |
136 | to be widely accessible to all users (so they can run the test suite at any |
|
139 | to be widely accessible to all users (so they can run the test suite at any | |
137 | time and report problems), but it now works for the developers. We are |
|
140 | time and report problems), but it now works for the developers. We are | |
138 | working hard on continuing to improve it, as this was probably IPython's |
|
141 | working hard on continuing to improve it, as this was probably IPython's | |
139 | major Achilles heel (the lack of proper test coverage made it effectively |
|
142 | major Achilles heel (the lack of proper test coverage made it effectively | |
140 | impossible to do large-scale refactoring). The full test suite can now |
|
143 | impossible to do large-scale refactoring). The full test suite can now | |
141 | be run using the :command:`iptest` command line program. |
|
144 | be run using the :command:`iptest` command line program. | |
142 |
|
145 | |||
143 | * The notion of a task has been completely reworked. An `ITask` interface has |
|
146 | * The notion of a task has been completely reworked. An `ITask` interface has | |
144 | been created. This interface defines the methods that tasks need to |
|
147 | been created. This interface defines the methods that tasks need to | |
145 | implement. These methods are now responsible for things like submitting |
|
148 | implement. These methods are now responsible for things like submitting | |
146 | tasks and processing results. There are two basic task types: |
|
149 | tasks and processing results. There are two basic task types: | |
147 | :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but |
|
150 | :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but | |
148 | renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on |
|
151 | renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on | |
149 | a function. |
|
152 | a function. | |
150 |
|
153 | |||
151 | * A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to |
|
154 | * A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to | |
152 | standardize the idea of a `map` method. This interface has a single `map` |
|
155 | standardize the idea of a `map` method. This interface has a single `map` | |
153 | method that has the same syntax as the built-in `map`. We have also defined |
|
156 | method that has the same syntax as the built-in `map`. We have also defined | |
154 | a `mapper` factory interface that creates objects that implement |
|
157 | a `mapper` factory interface that creates objects that implement | |
155 | :class:`IPython.kernel.mapper.IMapper` for different controllers. Both the |
|
158 | :class:`IPython.kernel.mapper.IMapper` for different controllers. Both the | |
156 | multiengine and task controller now have mapping capabilties. |
|
159 | multiengine and task controller now have mapping capabilties. | |
157 |
|
160 | |||
158 | * The parallel function capabilities have been reworks. The major changes are |
|
161 | * The parallel function capabilities have been reworks. The major changes are | |
159 | that i) there is now an `@parallel` magic that creates parallel functions, |
|
162 | that i) there is now an `@parallel` magic that creates parallel functions, | |
160 | ii) the syntax for mulitple variable follows that of `map`, iii) both the |
|
163 | ii) the syntax for mulitple variable follows that of `map`, iii) both the | |
161 | multiengine and task controller now have a parallel function implementation. |
|
164 | multiengine and task controller now have a parallel function implementation. | |
162 |
|
165 | |||
163 | * All of the parallel computing capabilities from `ipython1-dev` have been |
|
166 | * All of the parallel computing capabilities from `ipython1-dev` have been | |
164 | merged into IPython proper. This resulted in the following new subpackages: |
|
167 | merged into IPython proper. This resulted in the following new subpackages: | |
165 | :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`, |
|
168 | :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`, | |
166 | :mod:`IPython.tools` and :mod:`IPython.testing`. |
|
169 | :mod:`IPython.tools` and :mod:`IPython.testing`. | |
167 |
|
170 | |||
168 | * As part of merging in the `ipython1-dev` stuff, the `setup.py` script and |
|
171 | * As part of merging in the `ipython1-dev` stuff, the `setup.py` script and | |
169 | friends have been completely refactored. Now we are checking for |
|
172 | friends have been completely refactored. Now we are checking for | |
170 | dependencies using the approach that matplotlib uses. |
|
173 | dependencies using the approach that matplotlib uses. | |
171 |
|
174 | |||
172 | * The documentation has been completely reorganized to accept the |
|
175 | * The documentation has been completely reorganized to accept the | |
173 | documentation from `ipython1-dev`. |
|
176 | documentation from `ipython1-dev`. | |
174 |
|
177 | |||
175 | * We have switched to using Foolscap for all of our network protocols in |
|
178 | * We have switched to using Foolscap for all of our network protocols in | |
176 | :mod:`IPython.kernel`. This gives us secure connections that are both |
|
179 | :mod:`IPython.kernel`. This gives us secure connections that are both | |
177 | encrypted and authenticated. |
|
180 | encrypted and authenticated. | |
178 |
|
181 | |||
179 | * We have a brand new `COPYING.txt` files that describes the IPython license |
|
182 | * We have a brand new `COPYING.txt` files that describes the IPython license | |
180 | and copyright. The biggest change is that we are putting "The IPython |
|
183 | and copyright. The biggest change is that we are putting "The IPython | |
181 | Development Team" as the copyright holder. We give more details about |
|
184 | Development Team" as the copyright holder. We give more details about | |
182 | exactly what this means in this file. All developer should read this and use |
|
185 | exactly what this means in this file. All developer should read this and use | |
183 | the new banner in all IPython source code files. |
|
186 | the new banner in all IPython source code files. | |
184 |
|
187 | |||
185 | * sh profile: ./foo runs foo as system command, no need to do !./foo anymore |
|
188 | * sh profile: ./foo runs foo as system command, no need to do !./foo anymore | |
186 |
|
189 | |||
187 | * String lists now support ``sort(field, nums = True)`` method (to easily sort |
|
190 | * String lists now support ``sort(field, nums = True)`` method (to easily sort | |
188 | system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``. |
|
191 | system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``. | |
189 |
|
192 | |||
190 | * '%cpaste foo' now assigns the pasted block as string list, instead of string |
|
193 | * '%cpaste foo' now assigns the pasted block as string list, instead of string | |
191 |
|
194 | |||
192 | * The ipcluster script now run by default with no security. This is done |
|
195 | * The ipcluster script now run by default with no security. This is done | |
193 | because the main usage of the script is for starting things on localhost. |
|
196 | because the main usage of the script is for starting things on localhost. | |
194 | Eventually when ipcluster is able to start things on other hosts, we will put |
|
197 | Eventually when ipcluster is able to start things on other hosts, we will put | |
195 | security back. |
|
198 | security back. | |
196 |
|
199 | |||
197 | * 'cd --foo' searches directory history for string foo, and jumps to that dir. |
|
200 | * 'cd --foo' searches directory history for string foo, and jumps to that dir. | |
198 | Last part of dir name is checked first. If no matches for that are found, |
|
201 | Last part of dir name is checked first. If no matches for that are found, | |
199 | look at the whole path. |
|
202 | look at the whole path. | |
200 |
|
203 | |||
201 |
|
204 | |||
202 | Bug fixes |
|
205 | Bug fixes | |
203 | --------- |
|
206 | --------- | |
204 |
|
207 | |||
205 | * The Windows installer has been fixed. Now all IPython scripts have ``.bat`` |
|
208 | * The Windows installer has been fixed. Now all IPython scripts have ``.bat`` | |
206 | versions created. Also, the Start Menu shortcuts have been updated. |
|
209 | versions created. Also, the Start Menu shortcuts have been updated. | |
207 |
|
210 | |||
208 | * The colors escapes in the multiengine client are now turned off on win32 as |
|
211 | * The colors escapes in the multiengine client are now turned off on win32 as | |
209 | they don't print correctly. |
|
212 | they don't print correctly. | |
210 |
|
213 | |||
211 | * The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing |
|
214 | * The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing | |
212 | mpi_import_statement incorrectly, which was leading the engine to crash when |
|
215 | mpi_import_statement incorrectly, which was leading the engine to crash when | |
213 | mpi was enabled. |
|
216 | mpi was enabled. | |
214 |
|
217 | |||
215 | * A few subpackages had missing ``__init__.py`` files. |
|
218 | * A few subpackages had missing ``__init__.py`` files. | |
216 |
|
219 | |||
217 | * The documentation is only created if Sphinx is found. Previously, the |
|
220 | * The documentation is only created if Sphinx is found. Previously, the | |
218 | ``setup.py`` script would fail if it was missing. |
|
221 | ``setup.py`` script would fail if it was missing. | |
219 |
|
222 | |||
220 | * Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as |
|
223 | * Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as | |
221 | it caused problems on certain platforms. |
|
224 | it caused problems on certain platforms. | |
222 |
|
225 | |||
223 |
|
226 | |||
224 | Backwards incompatible changes |
|
227 | Backwards incompatible changes | |
225 | ------------------------------ |
|
228 | ------------------------------ | |
226 |
|
229 | |||
227 | * The ``clusterfile`` options of the :command:`ipcluster` command has been |
|
230 | * The ``clusterfile`` options of the :command:`ipcluster` command has been | |
228 | removed as it was not working and it will be replaced soon by something much |
|
231 | removed as it was not working and it will be replaced soon by something much | |
229 | more robust. |
|
232 | more robust. | |
230 |
|
233 | |||
231 | * The :mod:`IPython.kernel` configuration now properly find the user's |
|
234 | * The :mod:`IPython.kernel` configuration now properly find the user's | |
232 | IPython directory. |
|
235 | IPython directory. | |
233 |
|
236 | |||
234 | * In ipapi, the :func:`make_user_ns` function has been replaced with |
|
237 | * In ipapi, the :func:`make_user_ns` function has been replaced with | |
235 | :func:`make_user_namespaces`, to support dict subclasses in namespace |
|
238 | :func:`make_user_namespaces`, to support dict subclasses in namespace | |
236 | creation. |
|
239 | creation. | |
237 |
|
240 | |||
238 | * :class:`IPython.kernel.client.Task` has been renamed |
|
241 | * :class:`IPython.kernel.client.Task` has been renamed | |
239 | :class:`IPython.kernel.client.StringTask` to make way for new task types. |
|
242 | :class:`IPython.kernel.client.StringTask` to make way for new task types. | |
240 |
|
243 | |||
241 | * The keyword argument `style` has been renamed `dist` in `scatter`, `gather` |
|
244 | * The keyword argument `style` has been renamed `dist` in `scatter`, `gather` | |
242 | and `map`. |
|
245 | and `map`. | |
243 |
|
246 | |||
244 | * Renamed the values that the rename `dist` keyword argument can have from |
|
247 | * Renamed the values that the rename `dist` keyword argument can have from | |
245 | `'basic'` to `'b'`. |
|
248 | `'basic'` to `'b'`. | |
246 |
|
249 | |||
247 | * IPython has a larger set of dependencies if you want all of its capabilities. |
|
250 | * IPython has a larger set of dependencies if you want all of its capabilities. | |
248 | See the `setup.py` script for details. |
|
251 | See the `setup.py` script for details. | |
249 |
|
252 | |||
250 | * The constructors for :class:`IPython.kernel.client.MultiEngineClient` and |
|
253 | * The constructors for :class:`IPython.kernel.client.MultiEngineClient` and | |
251 | :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple. |
|
254 | :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple. | |
252 | Instead they take the filename of a file that contains the FURL for that |
|
255 | Instead they take the filename of a file that contains the FURL for that | |
253 | client. If the FURL file is in your IPYTHONDIR, it will be found automatically |
|
256 | client. If the FURL file is in your IPYTHONDIR, it will be found automatically | |
254 | and the constructor can be left empty. |
|
257 | and the constructor can be left empty. | |
255 |
|
258 | |||
256 | * The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created |
|
259 | * The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created | |
257 | using the factory functions :func:`get_multiengine_client` and |
|
260 | using the factory functions :func:`get_multiengine_client` and | |
258 | :func:`get_task_client`. These return a `Deferred` to the actual client. |
|
261 | :func:`get_task_client`. These return a `Deferred` to the actual client. | |
259 |
|
262 | |||
260 | * The command line options to `ipcontroller` and `ipengine` have changed to |
|
263 | * The command line options to `ipcontroller` and `ipengine` have changed to | |
261 | reflect the new Foolscap network protocol and the FURL files. Please see the |
|
264 | reflect the new Foolscap network protocol and the FURL files. Please see the | |
262 | help for these scripts for details. |
|
265 | help for these scripts for details. | |
263 |
|
266 | |||
264 | * The configuration files for the kernel have changed because of the Foolscap |
|
267 | * The configuration files for the kernel have changed because of the Foolscap | |
265 | stuff. If you were using custom config files before, you should delete them |
|
268 | stuff. If you were using custom config files before, you should delete them | |
266 | and regenerate new ones. |
|
269 | and regenerate new ones. | |
267 |
|
270 | |||
268 | Changes merged in from IPython1 |
|
271 | Changes merged in from IPython1 | |
269 | ------------------------------- |
|
272 | ------------------------------- | |
270 |
|
273 | |||
271 | New features |
|
274 | New features | |
272 | ............ |
|
275 | ............ | |
273 |
|
276 | |||
274 | * Much improved ``setup.py`` and ``setupegg.py`` scripts. Because Twisted and |
|
277 | * Much improved ``setup.py`` and ``setupegg.py`` scripts. Because Twisted and | |
275 | zope.interface are now easy installable, we can declare them as dependencies |
|
278 | zope.interface are now easy installable, we can declare them as dependencies | |
276 | in our setupegg.py script. |
|
279 | in our setupegg.py script. | |
277 |
|
280 | |||
278 | * IPython is now compatible with Twisted 2.5.0 and 8.x. |
|
281 | * IPython is now compatible with Twisted 2.5.0 and 8.x. | |
279 |
|
282 | |||
280 | * Added a new example of how to use :mod:`ipython1.kernel.asynclient`. |
|
283 | * Added a new example of how to use :mod:`ipython1.kernel.asynclient`. | |
281 |
|
284 | |||
282 | * Initial draft of a process daemon in :mod:`ipython1.daemon`. This has not |
|
285 | * Initial draft of a process daemon in :mod:`ipython1.daemon`. This has not | |
283 | been merged into IPython and is still in `ipython1-dev`. |
|
286 | been merged into IPython and is still in `ipython1-dev`. | |
284 |
|
287 | |||
285 | * The ``TaskController`` now has methods for getting the queue status. |
|
288 | * The ``TaskController`` now has methods for getting the queue status. | |
286 |
|
289 | |||
287 | * The ``TaskResult`` objects not have information about how long the task |
|
290 | * The ``TaskResult`` objects not have information about how long the task | |
288 | took to run. |
|
291 | took to run. | |
289 |
|
292 | |||
290 | * We are attaching additional attributes to exceptions ``(_ipython_*)`` that |
|
293 | * We are attaching additional attributes to exceptions ``(_ipython_*)`` that | |
291 | we use to carry additional info around. |
|
294 | we use to carry additional info around. | |
292 |
|
295 | |||
293 | * New top-level module :mod:`asyncclient` that has asynchronous versions (that |
|
296 | * New top-level module :mod:`asyncclient` that has asynchronous versions (that | |
294 | return deferreds) of the client classes. This is designed to users who want |
|
297 | return deferreds) of the client classes. This is designed to users who want | |
295 | to run their own Twisted reactor. |
|
298 | to run their own Twisted reactor. | |
296 |
|
299 | |||
297 | * All the clients in :mod:`client` are now based on Twisted. This is done by |
|
300 | * All the clients in :mod:`client` are now based on Twisted. This is done by | |
298 | running the Twisted reactor in a separate thread and using the |
|
301 | running the Twisted reactor in a separate thread and using the | |
299 | :func:`blockingCallFromThread` function that is in recent versions of Twisted. |
|
302 | :func:`blockingCallFromThread` function that is in recent versions of Twisted. | |
300 |
|
303 | |||
301 | * Functions can now be pushed/pulled to/from engines using |
|
304 | * Functions can now be pushed/pulled to/from engines using | |
302 | :meth:`MultiEngineClient.push_function` and |
|
305 | :meth:`MultiEngineClient.push_function` and | |
303 | :meth:`MultiEngineClient.pull_function`. |
|
306 | :meth:`MultiEngineClient.pull_function`. | |
304 |
|
307 | |||
305 | * Gather/scatter are now implemented in the client to reduce the work load |
|
308 | * Gather/scatter are now implemented in the client to reduce the work load | |
306 | of the controller and improve performance. |
|
309 | of the controller and improve performance. | |
307 |
|
310 | |||
308 | * Complete rewrite of the IPython docuementation. All of the documentation |
|
311 | * Complete rewrite of the IPython docuementation. All of the documentation | |
309 | from the IPython website has been moved into docs/source as restructured |
|
312 | from the IPython website has been moved into docs/source as restructured | |
310 | text documents. PDF and HTML documentation are being generated using |
|
313 | text documents. PDF and HTML documentation are being generated using | |
311 | Sphinx. |
|
314 | Sphinx. | |
312 |
|
315 | |||
313 | * New developer oriented documentation: development guidelines and roadmap. |
|
316 | * New developer oriented documentation: development guidelines and roadmap. | |
314 |
|
317 | |||
315 | * Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` |
|
318 | * Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` | |
316 | file that is organized by release and is meant to provide something more |
|
319 | file that is organized by release and is meant to provide something more | |
317 | relevant for users. |
|
320 | relevant for users. | |
318 |
|
321 | |||
319 | Bug fixes |
|
322 | Bug fixes | |
320 | ......... |
|
323 | ......... | |
321 |
|
324 | |||
322 | * Created a proper ``MANIFEST.in`` file to create source distributions. |
|
325 | * Created a proper ``MANIFEST.in`` file to create source distributions. | |
323 |
|
326 | |||
324 | * Fixed a bug in the ``MultiEngine`` interface. Previously, multi-engine |
|
327 | * Fixed a bug in the ``MultiEngine`` interface. Previously, multi-engine | |
325 | actions were being collected with a :class:`DeferredList` with |
|
328 | actions were being collected with a :class:`DeferredList` with | |
326 | ``fireononeerrback=1``. This meant that methods were returning |
|
329 | ``fireononeerrback=1``. This meant that methods were returning | |
327 | before all engines had given their results. This was causing extremely odd |
|
330 | before all engines had given their results. This was causing extremely odd | |
328 | bugs in certain cases. To fix this problem, we have 1) set |
|
331 | bugs in certain cases. To fix this problem, we have 1) set | |
329 | ``fireononeerrback=0`` to make sure all results (or exceptions) are in |
|
332 | ``fireononeerrback=0`` to make sure all results (or exceptions) are in | |
330 | before returning and 2) introduced a :exc:`CompositeError` exception |
|
333 | before returning and 2) introduced a :exc:`CompositeError` exception | |
331 | that wraps all of the engine exceptions. This is a huge change as it means |
|
334 | that wraps all of the engine exceptions. This is a huge change as it means | |
332 | that users will have to catch :exc:`CompositeError` rather than the actual |
|
335 | that users will have to catch :exc:`CompositeError` rather than the actual | |
333 | exception. |
|
336 | exception. | |
334 |
|
337 | |||
335 | Backwards incompatible changes |
|
338 | Backwards incompatible changes | |
336 | .............................. |
|
339 | .............................. | |
337 |
|
340 | |||
338 | * All names have been renamed to conform to the lowercase_with_underscore |
|
341 | * All names have been renamed to conform to the lowercase_with_underscore | |
339 | convention. This will require users to change references to all names like |
|
342 | convention. This will require users to change references to all names like | |
340 | ``queueStatus`` to ``queue_status``. |
|
343 | ``queueStatus`` to ``queue_status``. | |
341 |
|
344 | |||
342 | * Previously, methods like :meth:`MultiEngineClient.push` and |
|
345 | * Previously, methods like :meth:`MultiEngineClient.push` and | |
343 | :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``. This was |
|
346 | :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``. This was | |
344 | becoming a problem as we weren't able to introduce new keyword arguments into |
|
347 | becoming a problem as we weren't able to introduce new keyword arguments into | |
345 | the API. Now these methods simple take a dict or sequence. This has also |
|
348 | the API. Now these methods simple take a dict or sequence. This has also | |
346 | allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and |
|
349 | allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and | |
347 | :meth:`pullAll`. These things are now handled with the ``targets`` keyword |
|
350 | :meth:`pullAll`. These things are now handled with the ``targets`` keyword | |
348 | argument that defaults to ``'all'``. |
|
351 | argument that defaults to ``'all'``. | |
349 |
|
352 | |||
350 | * The :attr:`MultiEngineClient.magicTargets` has been renamed to |
|
353 | * The :attr:`MultiEngineClient.magicTargets` has been renamed to | |
351 | :attr:`MultiEngineClient.targets`. |
|
354 | :attr:`MultiEngineClient.targets`. | |
352 |
|
355 | |||
353 | * All methods in the MultiEngine interface now accept the optional keyword |
|
356 | * All methods in the MultiEngine interface now accept the optional keyword | |
354 | argument ``block``. |
|
357 | argument ``block``. | |
355 |
|
358 | |||
356 | * Renamed :class:`RemoteController` to :class:`MultiEngineClient` and |
|
359 | * Renamed :class:`RemoteController` to :class:`MultiEngineClient` and | |
357 | :class:`TaskController` to :class:`TaskClient`. |
|
360 | :class:`TaskController` to :class:`TaskClient`. | |
358 |
|
361 | |||
359 | * Renamed the top-level module from :mod:`api` to :mod:`client`. |
|
362 | * Renamed the top-level module from :mod:`api` to :mod:`client`. | |
360 |
|
363 | |||
361 | * Most methods in the multiengine interface now raise a :exc:`CompositeError` |
|
364 | * Most methods in the multiengine interface now raise a :exc:`CompositeError` | |
362 | exception that wraps the user's exceptions, rather than just raising the raw |
|
365 | exception that wraps the user's exceptions, rather than just raising the raw | |
363 | user's exception. |
|
366 | user's exception. | |
364 |
|
367 | |||
365 | * Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` |
|
368 | * Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` | |
366 | and ``pull``. |
|
369 | and ``pull``. | |
367 |
|
370 | |||
368 |
|
371 | |||
369 | Release 0.8.4 |
|
372 | Release 0.8.4 | |
370 | ============= |
|
373 | ============= | |
371 |
|
374 | |||
372 | This was a quick release to fix an unfortunate bug that slipped into the 0.8.3 |
|
375 | This was a quick release to fix an unfortunate bug that slipped into the 0.8.3 | |
373 | release. The ``--twisted`` option was disabled, as it turned out to be broken |
|
376 | release. The ``--twisted`` option was disabled, as it turned out to be broken | |
374 | across several platforms. |
|
377 | across several platforms. | |
375 |
|
378 | |||
376 |
|
379 | |||
377 | Release 0.8.3 |
|
380 | Release 0.8.3 | |
378 | ============= |
|
381 | ============= | |
379 |
|
382 | |||
380 | * pydb is now disabled by default (due to %run -d problems). You can enable |
|
383 | * pydb is now disabled by default (due to %run -d problems). You can enable | |
381 | it by passing -pydb command line argument to IPython. Note that setting |
|
384 | it by passing -pydb command line argument to IPython. Note that setting | |
382 | it in config file won't work. |
|
385 | it in config file won't work. | |
383 |
|
386 | |||
384 |
|
387 | |||
385 | Release 0.8.2 |
|
388 | Release 0.8.2 | |
386 | ============= |
|
389 | ============= | |
387 |
|
390 | |||
388 | * %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory |
|
391 | * %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory | |
389 | and jumps to /foo. The current behaviour is closer to the documented |
|
392 | and jumps to /foo. The current behaviour is closer to the documented | |
390 | behaviour, and should not trip anyone. |
|
393 | behaviour, and should not trip anyone. | |
391 |
|
394 | |||
392 |
|
395 | |||
393 | Older releases |
|
396 | Older releases | |
394 | ============== |
|
397 | ============== | |
395 |
|
398 | |||
396 | Changes in earlier releases of IPython are described in the older file |
|
399 | Changes in earlier releases of IPython are described in the older file | |
397 | ``ChangeLog``. Please refer to this document for details. |
|
400 | ``ChangeLog``. Please refer to this document for details. | |
398 |
|
401 |
@@ -1,157 +1,157 b'' | |||||
1 | .. _parallelmpi: |
|
1 | .. _parallelmpi: | |
2 |
|
2 | |||
3 | ======================= |
|
3 | ======================= | |
4 | Using MPI with IPython |
|
4 | Using MPI with IPython | |
5 | ======================= |
|
5 | ======================= | |
6 |
|
6 | |||
7 | Often, a parallel algorithm will require moving data between the engines. One way of accomplishing this is by doing a pull and then a push using the multiengine client. However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination. |
|
7 | Often, a parallel algorithm will require moving data between the engines. One way of accomplishing this is by doing a pull and then a push using the multiengine client. However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination. | |
8 |
|
8 | |||
9 | A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_. IPython's parallel computing architecture has been designed from the ground up to integrate with MPI. This document describes how to use MPI with IPython. |
|
9 | A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_. IPython's parallel computing architecture has been designed from the ground up to integrate with MPI. This document describes how to use MPI with IPython. | |
10 |
|
10 | |||
11 | Additional installation requirements |
|
11 | Additional installation requirements | |
12 | ==================================== |
|
12 | ==================================== | |
13 |
|
13 | |||
14 | If you want to use MPI with IPython, you will need to install: |
|
14 | If you want to use MPI with IPython, you will need to install: | |
15 |
|
15 | |||
16 | * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH. |
|
16 | * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH. | |
17 | * The mpi4py [mpi4py]_ package. |
|
17 | * The mpi4py [mpi4py]_ package. | |
18 |
|
18 | |||
19 | .. note:: |
|
19 | .. note:: | |
20 |
|
20 | |||
21 | The mpi4py package is not a strict requirement. However, you need to |
|
21 | The mpi4py package is not a strict requirement. However, you need to | |
22 | have *some* way of calling MPI from Python. You also need some way of |
|
22 | have *some* way of calling MPI from Python. You also need some way of | |
23 | making sure that :func:`MPI_Init` is called when the IPython engines start |
|
23 | making sure that :func:`MPI_Init` is called when the IPython engines start | |
24 | up. There are a number of ways of doing this and a good number of |
|
24 | up. There are a number of ways of doing this and a good number of | |
25 | associated subtleties. We highly recommend just using mpi4py as it |
|
25 | associated subtleties. We highly recommend just using mpi4py as it | |
26 | takes care of most of these problems. If you want to do something |
|
26 | takes care of most of these problems. If you want to do something | |
27 | different, let us know and we can help you get started. |
|
27 | different, let us know and we can help you get started. | |
28 |
|
28 | |||
29 | Starting the engines with MPI enabled |
|
29 | Starting the engines with MPI enabled | |
30 | ===================================== |
|
30 | ===================================== | |
31 |
|
31 | |||
32 | To use code that calls MPI, there are typically two things that MPI requires. |
|
32 | To use code that calls MPI, there are typically two things that MPI requires. | |
33 |
|
33 | |||
34 | 1. The process that wants to call MPI must be started using |
|
34 | 1. The process that wants to call MPI must be started using | |
35 |
:command:`mpi |
|
35 | :command:`mpiexec` or a batch system (like PBS) that has MPI support. | |
36 | 2. Once the process starts, it must call :func:`MPI_Init`. |
|
36 | 2. Once the process starts, it must call :func:`MPI_Init`. | |
37 |
|
37 | |||
38 | There are a couple of ways that you can start the IPython engines and get these things to happen. |
|
38 | There are a couple of ways that you can start the IPython engines and get these things to happen. | |
39 |
|
39 | |||
40 |
Automatic starting using :command:`mpi |
|
40 | Automatic starting using :command:`mpiexec` and :command:`ipcluster` | |
41 | ------------------------------------------------------------------- |
|
41 | ------------------------------------------------------------------- | |
42 |
|
42 | |||
43 |
The easiest approach is to use the `mpi |
|
43 | The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`:: | |
44 |
|
44 | |||
45 |
$ ipcluster mpi |
|
45 | $ ipcluster mpiexec -n 4 | |
46 |
|
46 | |||
47 | This approach is best as interrupting :command:`ipcluster` will automatically |
|
47 | This approach is best as interrupting :command:`ipcluster` will automatically | |
48 | stop and clean up the controller and engines. |
|
48 | stop and clean up the controller and engines. | |
49 |
|
49 | |||
50 |
Manual starting using :command:`mpi |
|
50 | Manual starting using :command:`mpiexec` | |
51 | --------------------------------------- |
|
51 | --------------------------------------- | |
52 |
|
52 | |||
53 |
If you want to start the IPython engines using the :command:`mpi |
|
53 | If you want to start the IPython engines using the :command:`mpiexec`, just do:: | |
54 |
|
54 | |||
55 |
$ mpi |
|
55 | $ mpiexec -n 4 ipengine --mpi=mpi4py | |
56 |
|
56 | |||
57 | This requires that you already have a controller running and that the FURL |
|
57 | This requires that you already have a controller running and that the FURL | |
58 | files for the engines are in place. We also have built in support for |
|
58 | files for the engines are in place. We also have built in support for | |
59 | PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by |
|
59 | PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by | |
60 | starting the engines with:: |
|
60 | starting the engines with:: | |
61 |
|
61 | |||
62 |
mpi |
|
62 | mpiexec -n 4 ipengine --mpi=pytrilinos | |
63 |
|
63 | |||
64 | Automatic starting using PBS and :command:`ipcluster` |
|
64 | Automatic starting using PBS and :command:`ipcluster` | |
65 | ----------------------------------------------------- |
|
65 | ----------------------------------------------------- | |
66 |
|
66 | |||
67 | The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`. |
|
67 | The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`. | |
68 |
|
68 | |||
69 | Actually using MPI |
|
69 | Actually using MPI | |
70 | ================== |
|
70 | ================== | |
71 |
|
71 | |||
72 | Once the engines are running with MPI enabled, you are ready to go. You can now call any code that uses MPI in the IPython engines. And, all of this can be done interactively. Here we show a simple example that uses mpi4py [mpi4py]_. |
|
72 | Once the engines are running with MPI enabled, you are ready to go. You can now call any code that uses MPI in the IPython engines. And, all of this can be done interactively. Here we show a simple example that uses mpi4py [mpi4py]_. | |
73 |
|
73 | |||
74 | First, lets define a simply function that uses MPI to calculate the sum of a distributed array. Save the following text in a file called :file:`psum.py`: |
|
74 | First, lets define a simply function that uses MPI to calculate the sum of a distributed array. Save the following text in a file called :file:`psum.py`: | |
75 |
|
75 | |||
76 | .. sourcecode:: python |
|
76 | .. sourcecode:: python | |
77 |
|
77 | |||
78 | from mpi4py import MPI |
|
78 | from mpi4py import MPI | |
79 | import numpy as np |
|
79 | import numpy as np | |
80 |
|
80 | |||
81 | def psum(a): |
|
81 | def psum(a): | |
82 | s = np.sum(a) |
|
82 | s = np.sum(a) | |
83 | return MPI.COMM_WORLD.Allreduce(s,MPI.SUM) |
|
83 | return MPI.COMM_WORLD.Allreduce(s,MPI.SUM) | |
84 |
|
84 | |||
85 | Now, start an IPython cluster in the same directory as :file:`psum.py`:: |
|
85 | Now, start an IPython cluster in the same directory as :file:`psum.py`:: | |
86 |
|
86 | |||
87 |
$ ipcluster mpi |
|
87 | $ ipcluster mpiexec -n 4 | |
88 |
|
88 | |||
89 | Finally, connect to the cluster and use this function interactively. In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function: |
|
89 | Finally, connect to the cluster and use this function interactively. In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function: | |
90 |
|
90 | |||
91 | .. sourcecode:: ipython |
|
91 | .. sourcecode:: ipython | |
92 |
|
92 | |||
93 | In [1]: from IPython.kernel import client |
|
93 | In [1]: from IPython.kernel import client | |
94 |
|
94 | |||
95 | In [2]: mec = client.MultiEngineClient() |
|
95 | In [2]: mec = client.MultiEngineClient() | |
96 |
|
96 | |||
97 | In [3]: mec.activate() |
|
97 | In [3]: mec.activate() | |
98 |
|
98 | |||
99 | In [4]: px import numpy as np |
|
99 | In [4]: px import numpy as np | |
100 | Parallel execution on engines: all |
|
100 | Parallel execution on engines: all | |
101 | Out[4]: |
|
101 | Out[4]: | |
102 | <Results List> |
|
102 | <Results List> | |
103 | [0] In [13]: import numpy as np |
|
103 | [0] In [13]: import numpy as np | |
104 | [1] In [13]: import numpy as np |
|
104 | [1] In [13]: import numpy as np | |
105 | [2] In [13]: import numpy as np |
|
105 | [2] In [13]: import numpy as np | |
106 | [3] In [13]: import numpy as np |
|
106 | [3] In [13]: import numpy as np | |
107 |
|
107 | |||
108 | In [6]: px a = np.random.rand(100) |
|
108 | In [6]: px a = np.random.rand(100) | |
109 | Parallel execution on engines: all |
|
109 | Parallel execution on engines: all | |
110 | Out[6]: |
|
110 | Out[6]: | |
111 | <Results List> |
|
111 | <Results List> | |
112 | [0] In [15]: a = np.random.rand(100) |
|
112 | [0] In [15]: a = np.random.rand(100) | |
113 | [1] In [15]: a = np.random.rand(100) |
|
113 | [1] In [15]: a = np.random.rand(100) | |
114 | [2] In [15]: a = np.random.rand(100) |
|
114 | [2] In [15]: a = np.random.rand(100) | |
115 | [3] In [15]: a = np.random.rand(100) |
|
115 | [3] In [15]: a = np.random.rand(100) | |
116 |
|
116 | |||
117 | In [7]: px from psum import psum |
|
117 | In [7]: px from psum import psum | |
118 | Parallel execution on engines: all |
|
118 | Parallel execution on engines: all | |
119 | Out[7]: |
|
119 | Out[7]: | |
120 | <Results List> |
|
120 | <Results List> | |
121 | [0] In [16]: from psum import psum |
|
121 | [0] In [16]: from psum import psum | |
122 | [1] In [16]: from psum import psum |
|
122 | [1] In [16]: from psum import psum | |
123 | [2] In [16]: from psum import psum |
|
123 | [2] In [16]: from psum import psum | |
124 | [3] In [16]: from psum import psum |
|
124 | [3] In [16]: from psum import psum | |
125 |
|
125 | |||
126 | In [8]: px s = psum(a) |
|
126 | In [8]: px s = psum(a) | |
127 | Parallel execution on engines: all |
|
127 | Parallel execution on engines: all | |
128 | Out[8]: |
|
128 | Out[8]: | |
129 | <Results List> |
|
129 | <Results List> | |
130 | [0] In [17]: s = psum(a) |
|
130 | [0] In [17]: s = psum(a) | |
131 | [1] In [17]: s = psum(a) |
|
131 | [1] In [17]: s = psum(a) | |
132 | [2] In [17]: s = psum(a) |
|
132 | [2] In [17]: s = psum(a) | |
133 | [3] In [17]: s = psum(a) |
|
133 | [3] In [17]: s = psum(a) | |
134 |
|
134 | |||
135 | In [9]: px print s |
|
135 | In [9]: px print s | |
136 | Parallel execution on engines: all |
|
136 | Parallel execution on engines: all | |
137 | Out[9]: |
|
137 | Out[9]: | |
138 | <Results List> |
|
138 | <Results List> | |
139 | [0] In [18]: print s |
|
139 | [0] In [18]: print s | |
140 | [0] Out[18]: 187.451545803 |
|
140 | [0] Out[18]: 187.451545803 | |
141 |
|
141 | |||
142 | [1] In [18]: print s |
|
142 | [1] In [18]: print s | |
143 | [1] Out[18]: 187.451545803 |
|
143 | [1] Out[18]: 187.451545803 | |
144 |
|
144 | |||
145 | [2] In [18]: print s |
|
145 | [2] In [18]: print s | |
146 | [2] Out[18]: 187.451545803 |
|
146 | [2] Out[18]: 187.451545803 | |
147 |
|
147 | |||
148 | [3] In [18]: print s |
|
148 | [3] In [18]: print s | |
149 | [3] Out[18]: 187.451545803 |
|
149 | [3] Out[18]: 187.451545803 | |
150 |
|
150 | |||
151 | Any Python code that makes calls to MPI can be used in this manner, including |
|
151 | Any Python code that makes calls to MPI can be used in this manner, including | |
152 | compiled C, C++ and Fortran libraries that have been exposed to Python. |
|
152 | compiled C, C++ and Fortran libraries that have been exposed to Python. | |
153 |
|
153 | |||
154 | .. [MPI] Message Passing Interface. http://www-unix.mcs.anl.gov/mpi/ |
|
154 | .. [MPI] Message Passing Interface. http://www-unix.mcs.anl.gov/mpi/ | |
155 | .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/ |
|
155 | .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/ | |
156 | .. [OpenMPI] Open MPI. http://www.open-mpi.org/ |
|
156 | .. [OpenMPI] Open MPI. http://www.open-mpi.org/ | |
157 | .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/ No newline at end of file |
|
157 | .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/ |
@@ -1,324 +1,336 b'' | |||||
1 | .. _parallel_process: |
|
1 | .. _parallel_process: | |
2 |
|
2 | |||
3 | =========================================== |
|
3 | =========================================== | |
4 | Starting the IPython controller and engines |
|
4 | Starting the IPython controller and engines | |
5 | =========================================== |
|
5 | =========================================== | |
6 |
|
6 | |||
7 | To use IPython for parallel computing, you need to start one instance of |
|
7 | To use IPython for parallel computing, you need to start one instance of | |
8 | the controller and one or more instances of the engine. The controller |
|
8 | the controller and one or more instances of the engine. The controller | |
9 | and each engine can run on different machines or on the same machine. |
|
9 | and each engine can run on different machines or on the same machine. | |
10 | Because of this, there are many different possibilities. |
|
10 | Because of this, there are many different possibilities. | |
11 |
|
11 | |||
12 | Broadly speaking, there are two ways of going about starting a controller and engines: |
|
12 | Broadly speaking, there are two ways of going about starting a controller and engines: | |
13 |
|
13 | |||
14 | * In an automated manner using the :command:`ipcluster` command. |
|
14 | * In an automated manner using the :command:`ipcluster` command. | |
15 | * In a more manual way using the :command:`ipcontroller` and |
|
15 | * In a more manual way using the :command:`ipcontroller` and | |
16 | :command:`ipengine` commands. |
|
16 | :command:`ipengine` commands. | |
17 |
|
17 | |||
18 | This document describes both of these methods. We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases. |
|
18 | This document describes both of these methods. We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases. | |
19 |
|
19 | |||
20 | General considerations |
|
20 | General considerations | |
21 | ====================== |
|
21 | ====================== | |
22 |
|
22 | |||
23 | Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines. These things come up no matter which method you use to start your IPython cluster. |
|
23 | Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines. These things come up no matter which method you use to start your IPython cluster. | |
24 |
|
24 | |||
25 | Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``. The following steps are then required: |
|
25 | Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``. The following steps are then required: | |
26 |
|
26 | |||
27 | 1. Start the controller on ``host0`` by running :command:`ipcontroller` on |
|
27 | 1. Start the controller on ``host0`` by running :command:`ipcontroller` on | |
28 | ``host0``. |
|
28 | ``host0``. | |
29 | 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the |
|
29 | 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the | |
30 | controller from ``host0`` to hosts ``host1``-``hostn``. |
|
30 | controller from ``host0`` to hosts ``host1``-``hostn``. | |
31 | 3. Start the engines on hosts ``host1``-``hostn`` by running |
|
31 | 3. Start the engines on hosts ``host1``-``hostn`` by running | |
32 | :command:`ipengine`. This command has to be told where the FURL file |
|
32 | :command:`ipengine`. This command has to be told where the FURL file | |
33 | (:file:`ipcontroller-engine.furl`) is located. |
|
33 | (:file:`ipcontroller-engine.furl`) is located. | |
34 |
|
34 | |||
35 | At this point, the controller and engines will be connected. By default, the |
|
35 | At this point, the controller and engines will be connected. By default, the | |
36 | FURL files created by the controller are put into the |
|
36 | FURL files created by the controller are put into the | |
37 | :file:`~/.ipython/security` directory. If the engines share a filesystem with |
|
37 | :file:`~/.ipython/security` directory. If the engines share a filesystem with | |
38 | the controller, step 2 can be skipped as the engines will automatically look |
|
38 | the controller, step 2 can be skipped as the engines will automatically look | |
39 | at that location. |
|
39 | at that location. | |
40 |
|
40 | |||
41 | The final step required required to actually use the running controller from a |
|
41 | The final step required required to actually use the running controller from a | |
42 | client is to move the FURL files :file:`ipcontroller-mec.furl` and |
|
42 | client is to move the FURL files :file:`ipcontroller-mec.furl` and | |
43 | :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will |
|
43 | :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will | |
44 | be run. If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically. Otherwise, the full path to them has to be passed to the client's constructor. |
|
44 | be run. If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically. Otherwise, the full path to them has to be passed to the client's constructor. | |
45 |
|
45 | |||
46 | Using :command:`ipcluster` |
|
46 | Using :command:`ipcluster` | |
47 | ========================== |
|
47 | ========================== | |
48 |
|
48 | |||
49 | The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations: |
|
49 | The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations: | |
50 |
|
50 | |||
51 | 1. When the controller and engines are all run on localhost. This is useful |
|
51 | 1. When the controller and engines are all run on localhost. This is useful | |
52 | for testing or running on a multicore computer. |
|
52 | for testing or running on a multicore computer. | |
53 | 2. When engines are started using the :command:`mpirun` command that comes |
|
53 | 2. When engines are started using the :command:`mpirun` command that comes | |
54 | with most MPI [MPI]_ implementations |
|
54 | with most MPI [MPI]_ implementations | |
55 | 3. When engines are started using the PBS [PBS]_ batch system. |
|
55 | 3. When engines are started using the PBS [PBS]_ batch system. | |
56 | 4. When the controller is started on localhost and the engines are started on |
|
56 | 4. When the controller is started on localhost and the engines are started on | |
57 | remote nodes using :command:`ssh`. |
|
57 | remote nodes using :command:`ssh`. | |
58 |
|
58 | |||
59 | .. note:: |
|
59 | .. note:: | |
60 |
|
60 | |||
61 | It is also possible for advanced users to add support to |
|
61 | It is also possible for advanced users to add support to | |
62 | :command:`ipcluster` for starting controllers and engines using other |
|
62 | :command:`ipcluster` for starting controllers and engines using other | |
63 | methods (like Sun's Grid Engine for example). |
|
63 | methods (like Sun's Grid Engine for example). | |
64 |
|
64 | |||
65 | .. note:: |
|
65 | .. note:: | |
66 |
|
66 | |||
67 | Currently :command:`ipcluster` requires that the |
|
67 | Currently :command:`ipcluster` requires that the | |
68 | :file:`~/.ipython/security` directory live on a shared filesystem that is |
|
68 | :file:`~/.ipython/security` directory live on a shared filesystem that is | |
69 | seen by both the controller and engines. If you don't have a shared file |
|
69 | seen by both the controller and engines. If you don't have a shared file | |
70 | system you will need to use :command:`ipcontroller` and |
|
70 | system you will need to use :command:`ipcontroller` and | |
71 | :command:`ipengine` directly. This constraint can be relaxed if you are |
|
71 | :command:`ipengine` directly. This constraint can be relaxed if you are | |
72 | using the :command:`ssh` method to start the cluster. |
|
72 | using the :command:`ssh` method to start the cluster. | |
73 |
|
73 | |||
74 | Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller` |
|
74 | Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller` | |
75 | and :command:`ipengine` to perform the steps described above. |
|
75 | and :command:`ipengine` to perform the steps described above. | |
76 |
|
76 | |||
77 | Using :command:`ipcluster` in local mode |
|
77 | Using :command:`ipcluster` in local mode | |
78 | ---------------------------------------- |
|
78 | ---------------------------------------- | |
79 |
|
79 | |||
80 | To start one controller and 4 engines on localhost, just do:: |
|
80 | To start one controller and 4 engines on localhost, just do:: | |
81 |
|
81 | |||
82 | $ ipcluster local -n 4 |
|
82 | $ ipcluster local -n 4 | |
83 |
|
83 | |||
84 | To see other command line options for the local mode, do:: |
|
84 | To see other command line options for the local mode, do:: | |
85 |
|
85 | |||
86 | $ ipcluster local -h |
|
86 | $ ipcluster local -h | |
87 |
|
87 | |||
88 | Using :command:`ipcluster` in mpirun mode |
|
88 | Using :command:`ipcluster` in mpiexec/mpirun mode | |
89 | ----------------------------------------- |
|
89 | ------------------------------------------------- | |
90 |
|
90 | |||
91 | The mpirun mode is useful if you: |
|
91 | The mpiexec/mpirun mode is useful if you: | |
92 |
|
92 | |||
93 | 1. Have MPI installed. |
|
93 | 1. Have MPI installed. | |
94 |
2. Your systems are configured to use the :command:`mpi |
|
94 | 2. Your systems are configured to use the :command:`mpiexec` or | |
95 | processes. |
|
95 | :command:`mpirun` commands to start MPI processes. | |
|
96 | ||||
|
97 | .. note:: | |||
|
98 | ||||
|
99 | The preferred command to use is :command:`mpiexec`. However, we also | |||
|
100 | support :command:`mpirun` for backwards compatibility. The underlying | |||
|
101 | logic used is exactly the same, the only difference being the name of the | |||
|
102 | command line program that is called. | |||
96 |
|
103 | |||
97 | If these are satisfied, you can start an IPython cluster using:: |
|
104 | If these are satisfied, you can start an IPython cluster using:: | |
98 |
|
105 | |||
99 |
$ ipcluster mpi |
|
106 | $ ipcluster mpiexec -n 4 | |
100 |
|
107 | |||
101 | This does the following: |
|
108 | This does the following: | |
102 |
|
109 | |||
103 | 1. Starts the IPython controller on current host. |
|
110 | 1. Starts the IPython controller on current host. | |
104 |
2. Uses :command:`mpi |
|
111 | 2. Uses :command:`mpiexec` to start 4 engines. | |
105 |
|
112 | |||
106 | On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting. The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option:: |
|
113 | On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting. The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option:: | |
107 |
|
114 | |||
108 |
$ ipcluster mpi |
|
115 | $ ipcluster mpiexec -n 4 --mpi=mpi4py | |
109 |
|
116 | |||
110 | Unfortunately, even this won't work for some MPI implementations. If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time. Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use. However, this custom Python executable approach will not work with :command:`ipcluster` currently. |
|
117 | Unfortunately, even this won't work for some MPI implementations. If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time. Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use. However, this custom Python executable approach will not work with :command:`ipcluster` currently. | |
111 |
|
118 | |||
112 | Additional command line options for this mode can be found by doing:: |
|
119 | Additional command line options for this mode can be found by doing:: | |
113 |
|
120 | |||
114 |
$ ipcluster mpi |
|
121 | $ ipcluster mpiexec -h | |
115 |
|
122 | |||
116 | More details on using MPI with IPython can be found :ref:`here <parallelmpi>`. |
|
123 | More details on using MPI with IPython can be found :ref:`here <parallelmpi>`. | |
117 |
|
124 | |||
118 |
|
125 | |||
119 | Using :command:`ipcluster` in PBS mode |
|
126 | Using :command:`ipcluster` in PBS mode | |
120 | -------------------------------------- |
|
127 | -------------------------------------- | |
121 |
|
128 | |||
122 | The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To use this mode, you first need to create a PBS script template that will be used to start the engines. Here is a sample PBS script template: |
|
129 | The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To use this mode, you first need to create a PBS script template that will be used to start the engines. Here is a sample PBS script template: | |
123 |
|
130 | |||
124 | .. sourcecode:: bash |
|
131 | .. sourcecode:: bash | |
125 |
|
132 | |||
126 | #PBS -N ipython |
|
133 | #PBS -N ipython | |
127 | #PBS -j oe |
|
134 | #PBS -j oe | |
128 | #PBS -l walltime=00:10:00 |
|
135 | #PBS -l walltime=00:10:00 | |
129 | #PBS -l nodes=${n/4}:ppn=4 |
|
136 | #PBS -l nodes=${n/4}:ppn=4 | |
130 | #PBS -q parallel |
|
137 | #PBS -q parallel | |
131 |
|
138 | |||
132 | cd $$PBS_O_WORKDIR |
|
139 | cd $$PBS_O_WORKDIR | |
133 | export PATH=$$HOME/usr/local/bin |
|
140 | export PATH=$$HOME/usr/local/bin | |
134 | export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages |
|
141 | export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages | |
135 | /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine |
|
142 | /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine | |
136 |
|
143 | |||
137 | There are a few important points about this template: |
|
144 | There are a few important points about this template: | |
138 |
|
145 | |||
139 | 1. This template will be rendered at runtime using IPython's :mod:`Itpl` |
|
146 | 1. This template will be rendered at runtime using IPython's :mod:`Itpl` | |
140 | template engine. |
|
147 | template engine. | |
141 |
|
148 | |||
142 | 2. Instead of putting in the actual number of engines, use the notation |
|
149 | 2. Instead of putting in the actual number of engines, use the notation | |
143 | ``${n}`` to indicate the number of engines to be started. You can also uses |
|
150 | ``${n}`` to indicate the number of engines to be started. You can also uses | |
144 | expressions like ``${n/4}`` in the template to indicate the number of |
|
151 | expressions like ``${n/4}`` in the template to indicate the number of | |
145 | nodes. |
|
152 | nodes. | |
146 |
|
153 | |||
147 | 3. Because ``$`` is a special character used by the template engine, you must |
|
154 | 3. Because ``$`` is a special character used by the template engine, you must | |
148 | escape any ``$`` by using ``$$``. This is important when referring to |
|
155 | escape any ``$`` by using ``$$``. This is important when referring to | |
149 | environment variables in the template. |
|
156 | environment variables in the template. | |
150 |
|
157 | |||
151 | 4. Any options to :command:`ipengine` should be given in the batch script |
|
158 | 4. Any options to :command:`ipengine` should be given in the batch script | |
152 | template. |
|
159 | template. | |
153 |
|
160 | |||
154 | 5. Depending on the configuration of you system, you may have to set |
|
161 | 5. Depending on the configuration of you system, you may have to set | |
155 | environment variables in the script template. |
|
162 | environment variables in the script template. | |
156 |
|
163 | |||
157 | Once you have created such a script, save it with a name like :file:`pbs.template`. Now you are ready to start your job:: |
|
164 | Once you have created such a script, save it with a name like :file:`pbs.template`. Now you are ready to start your job:: | |
158 |
|
165 | |||
159 | $ ipcluster pbs -n 128 --pbs-script=pbs.template |
|
166 | $ ipcluster pbs -n 128 --pbs-script=pbs.template | |
160 |
|
167 | |||
161 | Additional command line options for this mode can be found by doing:: |
|
168 | Additional command line options for this mode can be found by doing:: | |
162 |
|
169 | |||
163 | $ ipcluster pbs -h |
|
170 | $ ipcluster pbs -h | |
164 |
|
171 | |||
165 | Using :command:`ipcluster` in SSH mode |
|
172 | Using :command:`ipcluster` in SSH mode | |
166 | -------------------------------------- |
|
173 | -------------------------------------- | |
167 |
|
174 | |||
168 | The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote |
|
175 | The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote | |
169 | nodes and the :command:`ipcontroller` on localhost. |
|
176 | nodes and the :command:`ipcontroller` on localhost. | |
170 |
|
177 | |||
171 | When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins. |
|
178 | When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins. | |
172 |
|
179 | |||
173 | To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile": |
|
180 | To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile": | |
174 |
|
181 | |||
175 | .. sourcecode:: python |
|
182 | .. sourcecode:: python | |
176 |
|
183 | |||
177 | send_furl = True |
|
184 | send_furl = True | |
178 | engines = { 'host1.example.com' : 2, |
|
185 | engines = { 'host1.example.com' : 2, | |
179 | 'host2.example.com' : 5, |
|
186 | 'host2.example.com' : 5, | |
180 | 'host3.example.com' : 1, |
|
187 | 'host3.example.com' : 1, | |
181 | 'host4.example.com' : 8 } |
|
188 | 'host4.example.com' : 8 } | |
182 |
|
189 | |||
183 | Since this is a regular python file usual python syntax applies. Things to note: |
|
190 | Since this is a regular python file usual python syntax applies. Things to note: | |
184 |
|
191 | |||
185 | * The `engines` dict, where the keys is the host we want to run engines on and |
|
192 | * The `engines` dict, where the keys is the host we want to run engines on and | |
186 | the value is the number of engines to run on that host. |
|
193 | the value is the number of engines to run on that host. | |
187 | * send_furl can either be `True` or `False`, if `True` it will copy over the |
|
194 | * send_furl can either be `True` or `False`, if `True` it will copy over the | |
188 | furl needed for :command:`ipengine` to each host. |
|
195 | furl needed for :command:`ipengine` to each host. | |
189 |
|
196 | |||
190 | The ``--clusterfile`` command line option lets you specify the file to use for |
|
197 | The ``--clusterfile`` command line option lets you specify the file to use for | |
191 | the cluster definition. Once you have your cluster file and you can |
|
198 | the cluster definition. Once you have your cluster file and you can | |
192 | :command:`ssh` into the remote hosts with out an password you are ready to |
|
199 | :command:`ssh` into the remote hosts with out an password you are ready to | |
193 | start your cluster like so: |
|
200 | start your cluster like so: | |
194 |
|
201 | |||
195 | .. sourcecode:: bash |
|
202 | .. sourcecode:: bash | |
196 |
|
203 | |||
197 | $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py |
|
204 | $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py | |
198 |
|
205 | |||
199 |
|
206 | |||
200 | Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts: |
|
207 | Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts: | |
201 |
|
208 | |||
202 | * sshx.sh |
|
209 | * sshx.sh | |
203 | * engine_killer.sh |
|
210 | * engine_killer.sh | |
204 |
|
211 | |||
205 | Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there. On most Unix, Linux and OS X systems this is /tmp. |
|
212 | Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there. On most Unix, Linux and OS X systems this is /tmp. | |
206 |
|
213 | |||
207 | The default sshx.sh is the following: |
|
214 | The default sshx.sh is the following: | |
208 |
|
215 | |||
209 | .. sourcecode:: bash |
|
216 | .. sourcecode:: bash | |
210 |
|
217 | |||
211 | #!/bin/sh |
|
218 | #!/bin/sh | |
212 | "$@" &> /dev/null & |
|
219 | "$@" &> /dev/null & | |
213 | echo $! |
|
220 | echo $! | |
214 |
|
221 | |||
215 | If you want to use a custom sshx.sh script you need to use the ``--sshx`` |
|
222 | If you want to use a custom sshx.sh script you need to use the ``--sshx`` | |
216 | option and specify the file to use. Using a custom sshx.sh file could be |
|
223 | option and specify the file to use. Using a custom sshx.sh file could be | |
217 | helpful when you need to setup the environment on the remote host before |
|
224 | helpful when you need to setup the environment on the remote host before | |
218 | executing :command:`ipengine`. |
|
225 | executing :command:`ipengine`. | |
219 |
|
226 | |||
220 | For a detailed options list: |
|
227 | For a detailed options list: | |
221 |
|
228 | |||
222 | .. sourcecode:: bash |
|
229 | .. sourcecode:: bash | |
223 |
|
230 | |||
224 | $ ipcluster ssh -h |
|
231 | $ ipcluster ssh -h | |
225 |
|
232 | |||
226 | Current limitations of the SSH mode of :command:`ipcluster` are: |
|
233 | Current limitations of the SSH mode of :command:`ipcluster` are: | |
227 |
|
234 | |||
228 | * Untested on Windows. Would require a working :command:`ssh` on Windows. |
|
235 | * Untested on Windows. Would require a working :command:`ssh` on Windows. | |
229 | Also, we are using shell scripts to setup and execute commands on remote |
|
236 | Also, we are using shell scripts to setup and execute commands on remote | |
230 | hosts. |
|
237 | hosts. | |
231 | * :command:`ipcontroller` is started on localhost, with no option to start it |
|
238 | * :command:`ipcontroller` is started on localhost, with no option to start it | |
232 | on a remote node. |
|
239 | on a remote node. | |
233 |
|
240 | |||
234 | Using the :command:`ipcontroller` and :command:`ipengine` commands |
|
241 | Using the :command:`ipcontroller` and :command:`ipengine` commands | |
235 | ================================================================== |
|
242 | ================================================================== | |
236 |
|
243 | |||
237 | It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines. This approach gives you full control over all aspects of the startup process. |
|
244 | It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines. This approach gives you full control over all aspects of the startup process. | |
238 |
|
245 | |||
239 | Starting the controller and engine on your local machine |
|
246 | Starting the controller and engine on your local machine | |
240 | -------------------------------------------------------- |
|
247 | -------------------------------------------------------- | |
241 |
|
248 | |||
242 | To use :command:`ipcontroller` and :command:`ipengine` to start things on your |
|
249 | To use :command:`ipcontroller` and :command:`ipengine` to start things on your | |
243 | local machine, do the following. |
|
250 | local machine, do the following. | |
244 |
|
251 | |||
245 | First start the controller:: |
|
252 | First start the controller:: | |
246 |
|
253 | |||
247 | $ ipcontroller |
|
254 | $ ipcontroller | |
248 |
|
255 | |||
249 | Next, start however many instances of the engine you want using (repeatedly) the command:: |
|
256 | Next, start however many instances of the engine you want using (repeatedly) the command:: | |
250 |
|
257 | |||
251 | $ ipengine |
|
258 | $ ipengine | |
252 |
|
259 | |||
253 | The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython. |
|
260 | The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython. | |
254 |
|
261 | |||
255 | .. warning:: |
|
262 | .. warning:: | |
256 |
|
263 | |||
257 | The order of the above operations is very important. You *must* |
|
264 | The order of the above operations is very important. You *must* | |
258 | start the controller before the engines, since the engines connect |
|
265 | start the controller before the engines, since the engines connect | |
259 | to the controller as they get started. |
|
266 | to the controller as they get started. | |
260 |
|
267 | |||
261 | .. note:: |
|
268 | .. note:: | |
262 |
|
269 | |||
263 | On some platforms (OS X), to put the controller and engine into the |
|
270 | On some platforms (OS X), to put the controller and engine into the | |
264 | background you may need to give these commands in the form ``(ipcontroller |
|
271 | background you may need to give these commands in the form ``(ipcontroller | |
265 | &)`` and ``(ipengine &)`` (with the parentheses) for them to work |
|
272 | &)`` and ``(ipengine &)`` (with the parentheses) for them to work | |
266 | properly. |
|
273 | properly. | |
267 |
|
274 | |||
268 | Starting the controller and engines on different hosts |
|
275 | Starting the controller and engines on different hosts | |
269 | ------------------------------------------------------ |
|
276 | ------------------------------------------------------ | |
270 |
|
277 | |||
271 | When the controller and engines are running on different hosts, things are |
|
278 | When the controller and engines are running on different hosts, things are | |
272 | slightly more complicated, but the underlying ideas are the same: |
|
279 | slightly more complicated, but the underlying ideas are the same: | |
273 |
|
280 | |||
274 | 1. Start the controller on a host using :command:`ipcontroller`. |
|
281 | 1. Start the controller on a host using :command:`ipcontroller`. | |
275 | 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run. |
|
282 | 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run. | |
276 | 3. Use :command:`ipengine` on the engine's hosts to start the engines. |
|
283 | 3. Use :command:`ipengine` on the engine's hosts to start the engines. | |
277 |
|
284 | |||
278 | The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located. There are two ways you can do this: |
|
285 | The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located. There are two ways you can do this: | |
279 |
|
286 | |||
280 | * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security` |
|
287 | * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security` | |
281 | directory on the engine's host, where it will be found automatically. |
|
288 | directory on the engine's host, where it will be found automatically. | |
282 | * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file`` |
|
289 | * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file`` | |
283 | flag. |
|
290 | flag. | |
284 |
|
291 | |||
285 | The ``--furl-file`` flag works like this:: |
|
292 | The ``--furl-file`` flag works like this:: | |
286 |
|
293 | |||
287 | $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl |
|
294 | $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl | |
288 |
|
295 | |||
289 | .. note:: |
|
296 | .. note:: | |
290 |
|
297 | |||
291 | If the controller's and engine's hosts all have a shared file system |
|
298 | If the controller's and engine's hosts all have a shared file system | |
292 | (:file:`~./ipython/security` is the same on all of them), then things |
|
299 | (:file:`~./ipython/security` is the same on all of them), then things | |
293 | will just work! |
|
300 | will just work! | |
294 |
|
301 | |||
295 | Make FURL files persistent |
|
302 | Make FURL files persistent | |
296 | --------------------------- |
|
303 | --------------------------- | |
297 |
|
304 | |||
298 | At fist glance it may seem that that managing the FURL files is a bit annoying. Going back to the house and key analogy, copying the FURL around each time you start the controller is like having to make a new key every time you want to unlock the door and enter your house. As with your house, you want to be able to create the key (or FURL file) once, and then simply use it at any point in the future. |
|
305 | At fist glance it may seem that that managing the FURL files is a bit annoying. Going back to the house and key analogy, copying the FURL around each time you start the controller is like having to make a new key every time you want to unlock the door and enter your house. As with your house, you want to be able to create the key (or FURL file) once, and then simply use it at any point in the future. | |
299 |
|
306 | |||
300 | This is possible. The only thing you have to do is decide what ports the controller will listen on for the engines and clients. This is done as follows:: |
|
307 | This is possible. The only thing you have to do is decide what ports the controller will listen on for the engines and clients. This is done as follows:: | |
301 |
|
308 | |||
302 | $ ipcontroller -r --client-port=10101 --engine-port=10102 |
|
309 | $ ipcontroller -r --client-port=10101 --engine-port=10102 | |
303 |
|
310 | |||
|
311 | These options also work with all of the various modes of | |||
|
312 | :command:`ipcluster`:: | |||
|
313 | ||||
|
314 | $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102 | |||
|
315 | ||||
304 | Then, just copy the furl files over the first time and you are set. You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports. |
|
316 | Then, just copy the furl files over the first time and you are set. You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports. | |
305 |
|
317 | |||
306 | .. note:: |
|
318 | .. note:: | |
307 |
|
319 | |||
308 | You may ask the question: what ports does the controller listen on if you |
|
320 | You may ask the question: what ports does the controller listen on if you | |
309 | don't tell is to use specific ones? The default is to use high random port |
|
321 | don't tell is to use specific ones? The default is to use high random port | |
310 | numbers. We do this for two reasons: i) to increase security through |
|
322 | numbers. We do this for two reasons: i) to increase security through | |
311 | obscurity and ii) to multiple controllers on a given host to start and |
|
323 | obscurity and ii) to multiple controllers on a given host to start and | |
312 | automatically use different ports. |
|
324 | automatically use different ports. | |
313 |
|
325 | |||
314 | Log files |
|
326 | Log files | |
315 | --------- |
|
327 | --------- | |
316 |
|
328 | |||
317 | All of the components of IPython have log files associated with them. |
|
329 | All of the components of IPython have log files associated with them. | |
318 | These log files can be extremely useful in debugging problems with |
|
330 | These log files can be extremely useful in debugging problems with | |
319 | IPython and can be found in the directory :file:`~/.ipython/log`. Sending |
|
331 | IPython and can be found in the directory :file:`~/.ipython/log`. Sending | |
320 | the log files to us will often help us to debug any problems. |
|
332 | the log files to us will often help us to debug any problems. | |
321 |
|
333 | |||
322 |
|
334 | |||
323 | .. [PBS] Portable Batch System. http://www.openpbs.org/ |
|
335 | .. [PBS] Portable Batch System. http://www.openpbs.org/ | |
324 | .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent |
|
336 | .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent |
General Comments 0
You need to be logged in to leave comments.
Login now