##// END OF EJS Templates
remove `source` key from display_data
MinRK -
Show More
@@ -1,812 +1,812 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Top-level display functions for displaying object in different formats.
2 """Top-level display functions for displaying object in different formats.
3
3
4 Authors:
4 Authors:
5
5
6 * Brian Granger
6 * Brian Granger
7 """
7 """
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2013 The IPython Development Team
10 # Copyright (C) 2013 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 from __future__ import print_function
20 from __future__ import print_function
21
21
22 import os
22 import os
23 import struct
23 import struct
24
24
25 from IPython.core.formatters import _safe_get_formatter_method
25 from IPython.core.formatters import _safe_get_formatter_method
26 from IPython.utils.py3compat import (string_types, cast_bytes_py2, cast_unicode,
26 from IPython.utils.py3compat import (string_types, cast_bytes_py2, cast_unicode,
27 unicode_type)
27 unicode_type)
28 from IPython.testing.skipdoctest import skip_doctest
28 from IPython.testing.skipdoctest import skip_doctest
29 from .displaypub import publish_display_data
29 from .displaypub import publish_display_data
30
30
31 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
32 # utility functions
32 # utility functions
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34
34
35 def _safe_exists(path):
35 def _safe_exists(path):
36 """Check path, but don't let exceptions raise"""
36 """Check path, but don't let exceptions raise"""
37 try:
37 try:
38 return os.path.exists(path)
38 return os.path.exists(path)
39 except Exception:
39 except Exception:
40 return False
40 return False
41
41
42 def _merge(d1, d2):
42 def _merge(d1, d2):
43 """Like update, but merges sub-dicts instead of clobbering at the top level.
43 """Like update, but merges sub-dicts instead of clobbering at the top level.
44
44
45 Updates d1 in-place
45 Updates d1 in-place
46 """
46 """
47
47
48 if not isinstance(d2, dict) or not isinstance(d1, dict):
48 if not isinstance(d2, dict) or not isinstance(d1, dict):
49 return d2
49 return d2
50 for key, value in d2.items():
50 for key, value in d2.items():
51 d1[key] = _merge(d1.get(key), value)
51 d1[key] = _merge(d1.get(key), value)
52 return d1
52 return d1
53
53
54 def _display_mimetype(mimetype, objs, raw=False, metadata=None):
54 def _display_mimetype(mimetype, objs, raw=False, metadata=None):
55 """internal implementation of all display_foo methods
55 """internal implementation of all display_foo methods
56
56
57 Parameters
57 Parameters
58 ----------
58 ----------
59 mimetype : str
59 mimetype : str
60 The mimetype to be published (e.g. 'image/png')
60 The mimetype to be published (e.g. 'image/png')
61 objs : tuple of objects
61 objs : tuple of objects
62 The Python objects to display, or if raw=True raw text data to
62 The Python objects to display, or if raw=True raw text data to
63 display.
63 display.
64 raw : bool
64 raw : bool
65 Are the data objects raw data or Python objects that need to be
65 Are the data objects raw data or Python objects that need to be
66 formatted before display? [default: False]
66 formatted before display? [default: False]
67 metadata : dict (optional)
67 metadata : dict (optional)
68 Metadata to be associated with the specific mimetype output.
68 Metadata to be associated with the specific mimetype output.
69 """
69 """
70 if metadata:
70 if metadata:
71 metadata = {mimetype: metadata}
71 metadata = {mimetype: metadata}
72 if raw:
72 if raw:
73 # turn list of pngdata into list of { 'image/png': pngdata }
73 # turn list of pngdata into list of { 'image/png': pngdata }
74 objs = [ {mimetype: obj} for obj in objs ]
74 objs = [ {mimetype: obj} for obj in objs ]
75 display(*objs, raw=raw, metadata=metadata, include=[mimetype])
75 display(*objs, raw=raw, metadata=metadata, include=[mimetype])
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Main functions
78 # Main functions
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 def display(*objs, **kwargs):
81 def display(*objs, **kwargs):
82 """Display a Python object in all frontends.
82 """Display a Python object in all frontends.
83
83
84 By default all representations will be computed and sent to the frontends.
84 By default all representations will be computed and sent to the frontends.
85 Frontends can decide which representation is used and how.
85 Frontends can decide which representation is used and how.
86
86
87 Parameters
87 Parameters
88 ----------
88 ----------
89 objs : tuple of objects
89 objs : tuple of objects
90 The Python objects to display.
90 The Python objects to display.
91 raw : bool, optional
91 raw : bool, optional
92 Are the objects to be displayed already mimetype-keyed dicts of raw display data,
92 Are the objects to be displayed already mimetype-keyed dicts of raw display data,
93 or Python objects that need to be formatted before display? [default: False]
93 or Python objects that need to be formatted before display? [default: False]
94 include : list or tuple, optional
94 include : list or tuple, optional
95 A list of format type strings (MIME types) to include in the
95 A list of format type strings (MIME types) to include in the
96 format data dict. If this is set *only* the format types included
96 format data dict. If this is set *only* the format types included
97 in this list will be computed.
97 in this list will be computed.
98 exclude : list or tuple, optional
98 exclude : list or tuple, optional
99 A list of format type strings (MIME types) to exclude in the format
99 A list of format type strings (MIME types) to exclude in the format
100 data dict. If this is set all format types will be computed,
100 data dict. If this is set all format types will be computed,
101 except for those included in this argument.
101 except for those included in this argument.
102 metadata : dict, optional
102 metadata : dict, optional
103 A dictionary of metadata to associate with the output.
103 A dictionary of metadata to associate with the output.
104 mime-type keys in this dictionary will be associated with the individual
104 mime-type keys in this dictionary will be associated with the individual
105 representation formats, if they exist.
105 representation formats, if they exist.
106 """
106 """
107 raw = kwargs.get('raw', False)
107 raw = kwargs.get('raw', False)
108 include = kwargs.get('include')
108 include = kwargs.get('include')
109 exclude = kwargs.get('exclude')
109 exclude = kwargs.get('exclude')
110 metadata = kwargs.get('metadata')
110 metadata = kwargs.get('metadata')
111
111
112 from IPython.core.interactiveshell import InteractiveShell
112 from IPython.core.interactiveshell import InteractiveShell
113
113
114 if not raw:
114 if not raw:
115 format = InteractiveShell.instance().display_formatter.format
115 format = InteractiveShell.instance().display_formatter.format
116
116
117 for obj in objs:
117 for obj in objs:
118
118
119 # If _ipython_display_ is defined, use that to display this object.
119 # If _ipython_display_ is defined, use that to display this object.
120 display_method = _safe_get_formatter_method(obj, '_ipython_display_')
120 display_method = _safe_get_formatter_method(obj, '_ipython_display_')
121 if display_method is not None:
121 if display_method is not None:
122 try:
122 try:
123 display_method(**kwargs)
123 display_method(**kwargs)
124 except NotImplementedError:
124 except NotImplementedError:
125 pass
125 pass
126 else:
126 else:
127 continue
127 continue
128 if raw:
128 if raw:
129 publish_display_data('display', obj, metadata)
129 publish_display_data(data=obj, metadata=metadata)
130 else:
130 else:
131 format_dict, md_dict = format(obj, include=include, exclude=exclude)
131 format_dict, md_dict = format(obj, include=include, exclude=exclude)
132 if metadata:
132 if metadata:
133 # kwarg-specified metadata gets precedence
133 # kwarg-specified metadata gets precedence
134 _merge(md_dict, metadata)
134 _merge(md_dict, metadata)
135 publish_display_data('display', format_dict, md_dict)
135 publish_display_data(data=format_dict, metadata=md_dict)
136
136
137
137
138 def display_pretty(*objs, **kwargs):
138 def display_pretty(*objs, **kwargs):
139 """Display the pretty (default) representation of an object.
139 """Display the pretty (default) representation of an object.
140
140
141 Parameters
141 Parameters
142 ----------
142 ----------
143 objs : tuple of objects
143 objs : tuple of objects
144 The Python objects to display, or if raw=True raw text data to
144 The Python objects to display, or if raw=True raw text data to
145 display.
145 display.
146 raw : bool
146 raw : bool
147 Are the data objects raw data or Python objects that need to be
147 Are the data objects raw data or Python objects that need to be
148 formatted before display? [default: False]
148 formatted before display? [default: False]
149 metadata : dict (optional)
149 metadata : dict (optional)
150 Metadata to be associated with the specific mimetype output.
150 Metadata to be associated with the specific mimetype output.
151 """
151 """
152 _display_mimetype('text/plain', objs, **kwargs)
152 _display_mimetype('text/plain', objs, **kwargs)
153
153
154
154
155 def display_html(*objs, **kwargs):
155 def display_html(*objs, **kwargs):
156 """Display the HTML representation of an object.
156 """Display the HTML representation of an object.
157
157
158 Parameters
158 Parameters
159 ----------
159 ----------
160 objs : tuple of objects
160 objs : tuple of objects
161 The Python objects to display, or if raw=True raw HTML data to
161 The Python objects to display, or if raw=True raw HTML data to
162 display.
162 display.
163 raw : bool
163 raw : bool
164 Are the data objects raw data or Python objects that need to be
164 Are the data objects raw data or Python objects that need to be
165 formatted before display? [default: False]
165 formatted before display? [default: False]
166 metadata : dict (optional)
166 metadata : dict (optional)
167 Metadata to be associated with the specific mimetype output.
167 Metadata to be associated with the specific mimetype output.
168 """
168 """
169 _display_mimetype('text/html', objs, **kwargs)
169 _display_mimetype('text/html', objs, **kwargs)
170
170
171
171
172 def display_markdown(*objs, **kwargs):
172 def display_markdown(*objs, **kwargs):
173 """Displays the Markdown representation of an object.
173 """Displays the Markdown representation of an object.
174
174
175 Parameters
175 Parameters
176 ----------
176 ----------
177 objs : tuple of objects
177 objs : tuple of objects
178 The Python objects to display, or if raw=True raw markdown data to
178 The Python objects to display, or if raw=True raw markdown data to
179 display.
179 display.
180 raw : bool
180 raw : bool
181 Are the data objects raw data or Python objects that need to be
181 Are the data objects raw data or Python objects that need to be
182 formatted before display? [default: False]
182 formatted before display? [default: False]
183 metadata : dict (optional)
183 metadata : dict (optional)
184 Metadata to be associated with the specific mimetype output.
184 Metadata to be associated with the specific mimetype output.
185 """
185 """
186
186
187 _display_mimetype('text/markdown', objs, **kwargs)
187 _display_mimetype('text/markdown', objs, **kwargs)
188
188
189
189
190 def display_svg(*objs, **kwargs):
190 def display_svg(*objs, **kwargs):
191 """Display the SVG representation of an object.
191 """Display the SVG representation of an object.
192
192
193 Parameters
193 Parameters
194 ----------
194 ----------
195 objs : tuple of objects
195 objs : tuple of objects
196 The Python objects to display, or if raw=True raw svg data to
196 The Python objects to display, or if raw=True raw svg data to
197 display.
197 display.
198 raw : bool
198 raw : bool
199 Are the data objects raw data or Python objects that need to be
199 Are the data objects raw data or Python objects that need to be
200 formatted before display? [default: False]
200 formatted before display? [default: False]
201 metadata : dict (optional)
201 metadata : dict (optional)
202 Metadata to be associated with the specific mimetype output.
202 Metadata to be associated with the specific mimetype output.
203 """
203 """
204 _display_mimetype('image/svg+xml', objs, **kwargs)
204 _display_mimetype('image/svg+xml', objs, **kwargs)
205
205
206
206
207 def display_png(*objs, **kwargs):
207 def display_png(*objs, **kwargs):
208 """Display the PNG representation of an object.
208 """Display the PNG representation of an object.
209
209
210 Parameters
210 Parameters
211 ----------
211 ----------
212 objs : tuple of objects
212 objs : tuple of objects
213 The Python objects to display, or if raw=True raw png data to
213 The Python objects to display, or if raw=True raw png data to
214 display.
214 display.
215 raw : bool
215 raw : bool
216 Are the data objects raw data or Python objects that need to be
216 Are the data objects raw data or Python objects that need to be
217 formatted before display? [default: False]
217 formatted before display? [default: False]
218 metadata : dict (optional)
218 metadata : dict (optional)
219 Metadata to be associated with the specific mimetype output.
219 Metadata to be associated with the specific mimetype output.
220 """
220 """
221 _display_mimetype('image/png', objs, **kwargs)
221 _display_mimetype('image/png', objs, **kwargs)
222
222
223
223
224 def display_jpeg(*objs, **kwargs):
224 def display_jpeg(*objs, **kwargs):
225 """Display the JPEG representation of an object.
225 """Display the JPEG representation of an object.
226
226
227 Parameters
227 Parameters
228 ----------
228 ----------
229 objs : tuple of objects
229 objs : tuple of objects
230 The Python objects to display, or if raw=True raw JPEG data to
230 The Python objects to display, or if raw=True raw JPEG data to
231 display.
231 display.
232 raw : bool
232 raw : bool
233 Are the data objects raw data or Python objects that need to be
233 Are the data objects raw data or Python objects that need to be
234 formatted before display? [default: False]
234 formatted before display? [default: False]
235 metadata : dict (optional)
235 metadata : dict (optional)
236 Metadata to be associated with the specific mimetype output.
236 Metadata to be associated with the specific mimetype output.
237 """
237 """
238 _display_mimetype('image/jpeg', objs, **kwargs)
238 _display_mimetype('image/jpeg', objs, **kwargs)
239
239
240
240
241 def display_latex(*objs, **kwargs):
241 def display_latex(*objs, **kwargs):
242 """Display the LaTeX representation of an object.
242 """Display the LaTeX representation of an object.
243
243
244 Parameters
244 Parameters
245 ----------
245 ----------
246 objs : tuple of objects
246 objs : tuple of objects
247 The Python objects to display, or if raw=True raw latex data to
247 The Python objects to display, or if raw=True raw latex data to
248 display.
248 display.
249 raw : bool
249 raw : bool
250 Are the data objects raw data or Python objects that need to be
250 Are the data objects raw data or Python objects that need to be
251 formatted before display? [default: False]
251 formatted before display? [default: False]
252 metadata : dict (optional)
252 metadata : dict (optional)
253 Metadata to be associated with the specific mimetype output.
253 Metadata to be associated with the specific mimetype output.
254 """
254 """
255 _display_mimetype('text/latex', objs, **kwargs)
255 _display_mimetype('text/latex', objs, **kwargs)
256
256
257
257
258 def display_json(*objs, **kwargs):
258 def display_json(*objs, **kwargs):
259 """Display the JSON representation of an object.
259 """Display the JSON representation of an object.
260
260
261 Note that not many frontends support displaying JSON.
261 Note that not many frontends support displaying JSON.
262
262
263 Parameters
263 Parameters
264 ----------
264 ----------
265 objs : tuple of objects
265 objs : tuple of objects
266 The Python objects to display, or if raw=True raw json data to
266 The Python objects to display, or if raw=True raw json data to
267 display.
267 display.
268 raw : bool
268 raw : bool
269 Are the data objects raw data or Python objects that need to be
269 Are the data objects raw data or Python objects that need to be
270 formatted before display? [default: False]
270 formatted before display? [default: False]
271 metadata : dict (optional)
271 metadata : dict (optional)
272 Metadata to be associated with the specific mimetype output.
272 Metadata to be associated with the specific mimetype output.
273 """
273 """
274 _display_mimetype('application/json', objs, **kwargs)
274 _display_mimetype('application/json', objs, **kwargs)
275
275
276
276
277 def display_javascript(*objs, **kwargs):
277 def display_javascript(*objs, **kwargs):
278 """Display the Javascript representation of an object.
278 """Display the Javascript representation of an object.
279
279
280 Parameters
280 Parameters
281 ----------
281 ----------
282 objs : tuple of objects
282 objs : tuple of objects
283 The Python objects to display, or if raw=True raw javascript data to
283 The Python objects to display, or if raw=True raw javascript data to
284 display.
284 display.
285 raw : bool
285 raw : bool
286 Are the data objects raw data or Python objects that need to be
286 Are the data objects raw data or Python objects that need to be
287 formatted before display? [default: False]
287 formatted before display? [default: False]
288 metadata : dict (optional)
288 metadata : dict (optional)
289 Metadata to be associated with the specific mimetype output.
289 Metadata to be associated with the specific mimetype output.
290 """
290 """
291 _display_mimetype('application/javascript', objs, **kwargs)
291 _display_mimetype('application/javascript', objs, **kwargs)
292
292
293
293
294 def display_pdf(*objs, **kwargs):
294 def display_pdf(*objs, **kwargs):
295 """Display the PDF representation of an object.
295 """Display the PDF representation of an object.
296
296
297 Parameters
297 Parameters
298 ----------
298 ----------
299 objs : tuple of objects
299 objs : tuple of objects
300 The Python objects to display, or if raw=True raw javascript data to
300 The Python objects to display, or if raw=True raw javascript data to
301 display.
301 display.
302 raw : bool
302 raw : bool
303 Are the data objects raw data or Python objects that need to be
303 Are the data objects raw data or Python objects that need to be
304 formatted before display? [default: False]
304 formatted before display? [default: False]
305 metadata : dict (optional)
305 metadata : dict (optional)
306 Metadata to be associated with the specific mimetype output.
306 Metadata to be associated with the specific mimetype output.
307 """
307 """
308 _display_mimetype('application/pdf', objs, **kwargs)
308 _display_mimetype('application/pdf', objs, **kwargs)
309
309
310
310
311 #-----------------------------------------------------------------------------
311 #-----------------------------------------------------------------------------
312 # Smart classes
312 # Smart classes
313 #-----------------------------------------------------------------------------
313 #-----------------------------------------------------------------------------
314
314
315
315
316 class DisplayObject(object):
316 class DisplayObject(object):
317 """An object that wraps data to be displayed."""
317 """An object that wraps data to be displayed."""
318
318
319 _read_flags = 'r'
319 _read_flags = 'r'
320 _show_mem_addr = False
320 _show_mem_addr = False
321
321
322 def __init__(self, data=None, url=None, filename=None):
322 def __init__(self, data=None, url=None, filename=None):
323 """Create a display object given raw data.
323 """Create a display object given raw data.
324
324
325 When this object is returned by an expression or passed to the
325 When this object is returned by an expression or passed to the
326 display function, it will result in the data being displayed
326 display function, it will result in the data being displayed
327 in the frontend. The MIME type of the data should match the
327 in the frontend. The MIME type of the data should match the
328 subclasses used, so the Png subclass should be used for 'image/png'
328 subclasses used, so the Png subclass should be used for 'image/png'
329 data. If the data is a URL, the data will first be downloaded
329 data. If the data is a URL, the data will first be downloaded
330 and then displayed. If
330 and then displayed. If
331
331
332 Parameters
332 Parameters
333 ----------
333 ----------
334 data : unicode, str or bytes
334 data : unicode, str or bytes
335 The raw data or a URL or file to load the data from
335 The raw data or a URL or file to load the data from
336 url : unicode
336 url : unicode
337 A URL to download the data from.
337 A URL to download the data from.
338 filename : unicode
338 filename : unicode
339 Path to a local file to load the data from.
339 Path to a local file to load the data from.
340 """
340 """
341 if data is not None and isinstance(data, string_types):
341 if data is not None and isinstance(data, string_types):
342 if data.startswith('http') and url is None:
342 if data.startswith('http') and url is None:
343 url = data
343 url = data
344 filename = None
344 filename = None
345 data = None
345 data = None
346 elif _safe_exists(data) and filename is None:
346 elif _safe_exists(data) and filename is None:
347 url = None
347 url = None
348 filename = data
348 filename = data
349 data = None
349 data = None
350
350
351 self.data = data
351 self.data = data
352 self.url = url
352 self.url = url
353 self.filename = None if filename is None else unicode_type(filename)
353 self.filename = None if filename is None else unicode_type(filename)
354
354
355 self.reload()
355 self.reload()
356 self._check_data()
356 self._check_data()
357
357
358 def __repr__(self):
358 def __repr__(self):
359 if not self._show_mem_addr:
359 if not self._show_mem_addr:
360 cls = self.__class__
360 cls = self.__class__
361 r = "<%s.%s object>" % (cls.__module__, cls.__name__)
361 r = "<%s.%s object>" % (cls.__module__, cls.__name__)
362 else:
362 else:
363 r = super(DisplayObject, self).__repr__()
363 r = super(DisplayObject, self).__repr__()
364 return r
364 return r
365
365
366 def _check_data(self):
366 def _check_data(self):
367 """Override in subclasses if there's something to check."""
367 """Override in subclasses if there's something to check."""
368 pass
368 pass
369
369
370 def reload(self):
370 def reload(self):
371 """Reload the raw data from file or URL."""
371 """Reload the raw data from file or URL."""
372 if self.filename is not None:
372 if self.filename is not None:
373 with open(self.filename, self._read_flags) as f:
373 with open(self.filename, self._read_flags) as f:
374 self.data = f.read()
374 self.data = f.read()
375 elif self.url is not None:
375 elif self.url is not None:
376 try:
376 try:
377 try:
377 try:
378 from urllib.request import urlopen # Py3
378 from urllib.request import urlopen # Py3
379 except ImportError:
379 except ImportError:
380 from urllib2 import urlopen
380 from urllib2 import urlopen
381 response = urlopen(self.url)
381 response = urlopen(self.url)
382 self.data = response.read()
382 self.data = response.read()
383 # extract encoding from header, if there is one:
383 # extract encoding from header, if there is one:
384 encoding = None
384 encoding = None
385 for sub in response.headers['content-type'].split(';'):
385 for sub in response.headers['content-type'].split(';'):
386 sub = sub.strip()
386 sub = sub.strip()
387 if sub.startswith('charset'):
387 if sub.startswith('charset'):
388 encoding = sub.split('=')[-1].strip()
388 encoding = sub.split('=')[-1].strip()
389 break
389 break
390 # decode data, if an encoding was specified
390 # decode data, if an encoding was specified
391 if encoding:
391 if encoding:
392 self.data = self.data.decode(encoding, 'replace')
392 self.data = self.data.decode(encoding, 'replace')
393 except:
393 except:
394 self.data = None
394 self.data = None
395
395
396 class TextDisplayObject(DisplayObject):
396 class TextDisplayObject(DisplayObject):
397 """Validate that display data is text"""
397 """Validate that display data is text"""
398 def _check_data(self):
398 def _check_data(self):
399 if self.data is not None and not isinstance(self.data, string_types):
399 if self.data is not None and not isinstance(self.data, string_types):
400 raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data))
400 raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data))
401
401
402 class Pretty(TextDisplayObject):
402 class Pretty(TextDisplayObject):
403
403
404 def _repr_pretty_(self):
404 def _repr_pretty_(self):
405 return self.data
405 return self.data
406
406
407
407
408 class HTML(TextDisplayObject):
408 class HTML(TextDisplayObject):
409
409
410 def _repr_html_(self):
410 def _repr_html_(self):
411 return self.data
411 return self.data
412
412
413 def __html__(self):
413 def __html__(self):
414 """
414 """
415 This method exists to inform other HTML-using modules (e.g. Markupsafe,
415 This method exists to inform other HTML-using modules (e.g. Markupsafe,
416 htmltag, etc) that this object is HTML and does not need things like
416 htmltag, etc) that this object is HTML and does not need things like
417 special characters (<>&) escaped.
417 special characters (<>&) escaped.
418 """
418 """
419 return self._repr_html_()
419 return self._repr_html_()
420
420
421
421
422 class Markdown(TextDisplayObject):
422 class Markdown(TextDisplayObject):
423
423
424 def _repr_markdown_(self):
424 def _repr_markdown_(self):
425 return self.data
425 return self.data
426
426
427
427
428 class Math(TextDisplayObject):
428 class Math(TextDisplayObject):
429
429
430 def _repr_latex_(self):
430 def _repr_latex_(self):
431 s = self.data.strip('$')
431 s = self.data.strip('$')
432 return "$$%s$$" % s
432 return "$$%s$$" % s
433
433
434
434
435 class Latex(TextDisplayObject):
435 class Latex(TextDisplayObject):
436
436
437 def _repr_latex_(self):
437 def _repr_latex_(self):
438 return self.data
438 return self.data
439
439
440
440
441 class SVG(DisplayObject):
441 class SVG(DisplayObject):
442
442
443 # wrap data in a property, which extracts the <svg> tag, discarding
443 # wrap data in a property, which extracts the <svg> tag, discarding
444 # document headers
444 # document headers
445 _data = None
445 _data = None
446
446
447 @property
447 @property
448 def data(self):
448 def data(self):
449 return self._data
449 return self._data
450
450
451 @data.setter
451 @data.setter
452 def data(self, svg):
452 def data(self, svg):
453 if svg is None:
453 if svg is None:
454 self._data = None
454 self._data = None
455 return
455 return
456 # parse into dom object
456 # parse into dom object
457 from xml.dom import minidom
457 from xml.dom import minidom
458 svg = cast_bytes_py2(svg)
458 svg = cast_bytes_py2(svg)
459 x = minidom.parseString(svg)
459 x = minidom.parseString(svg)
460 # get svg tag (should be 1)
460 # get svg tag (should be 1)
461 found_svg = x.getElementsByTagName('svg')
461 found_svg = x.getElementsByTagName('svg')
462 if found_svg:
462 if found_svg:
463 svg = found_svg[0].toxml()
463 svg = found_svg[0].toxml()
464 else:
464 else:
465 # fallback on the input, trust the user
465 # fallback on the input, trust the user
466 # but this is probably an error.
466 # but this is probably an error.
467 pass
467 pass
468 svg = cast_unicode(svg)
468 svg = cast_unicode(svg)
469 self._data = svg
469 self._data = svg
470
470
471 def _repr_svg_(self):
471 def _repr_svg_(self):
472 return self.data
472 return self.data
473
473
474
474
475 class JSON(TextDisplayObject):
475 class JSON(TextDisplayObject):
476
476
477 def _repr_json_(self):
477 def _repr_json_(self):
478 return self.data
478 return self.data
479
479
480 css_t = """$("head").append($("<link/>").attr({
480 css_t = """$("head").append($("<link/>").attr({
481 rel: "stylesheet",
481 rel: "stylesheet",
482 type: "text/css",
482 type: "text/css",
483 href: "%s"
483 href: "%s"
484 }));
484 }));
485 """
485 """
486
486
487 lib_t1 = """$.getScript("%s", function () {
487 lib_t1 = """$.getScript("%s", function () {
488 """
488 """
489 lib_t2 = """});
489 lib_t2 = """});
490 """
490 """
491
491
492 class Javascript(TextDisplayObject):
492 class Javascript(TextDisplayObject):
493
493
494 def __init__(self, data=None, url=None, filename=None, lib=None, css=None):
494 def __init__(self, data=None, url=None, filename=None, lib=None, css=None):
495 """Create a Javascript display object given raw data.
495 """Create a Javascript display object given raw data.
496
496
497 When this object is returned by an expression or passed to the
497 When this object is returned by an expression or passed to the
498 display function, it will result in the data being displayed
498 display function, it will result in the data being displayed
499 in the frontend. If the data is a URL, the data will first be
499 in the frontend. If the data is a URL, the data will first be
500 downloaded and then displayed.
500 downloaded and then displayed.
501
501
502 In the Notebook, the containing element will be available as `element`,
502 In the Notebook, the containing element will be available as `element`,
503 and jQuery will be available. The output area starts hidden, so if
503 and jQuery will be available. The output area starts hidden, so if
504 the js appends content to `element` that should be visible, then
504 the js appends content to `element` that should be visible, then
505 it must call `container.show()` to unhide the area.
505 it must call `container.show()` to unhide the area.
506
506
507 Parameters
507 Parameters
508 ----------
508 ----------
509 data : unicode, str or bytes
509 data : unicode, str or bytes
510 The Javascript source code or a URL to download it from.
510 The Javascript source code or a URL to download it from.
511 url : unicode
511 url : unicode
512 A URL to download the data from.
512 A URL to download the data from.
513 filename : unicode
513 filename : unicode
514 Path to a local file to load the data from.
514 Path to a local file to load the data from.
515 lib : list or str
515 lib : list or str
516 A sequence of Javascript library URLs to load asynchronously before
516 A sequence of Javascript library URLs to load asynchronously before
517 running the source code. The full URLs of the libraries should
517 running the source code. The full URLs of the libraries should
518 be given. A single Javascript library URL can also be given as a
518 be given. A single Javascript library URL can also be given as a
519 string.
519 string.
520 css: : list or str
520 css: : list or str
521 A sequence of css files to load before running the source code.
521 A sequence of css files to load before running the source code.
522 The full URLs of the css files should be given. A single css URL
522 The full URLs of the css files should be given. A single css URL
523 can also be given as a string.
523 can also be given as a string.
524 """
524 """
525 if isinstance(lib, string_types):
525 if isinstance(lib, string_types):
526 lib = [lib]
526 lib = [lib]
527 elif lib is None:
527 elif lib is None:
528 lib = []
528 lib = []
529 if isinstance(css, string_types):
529 if isinstance(css, string_types):
530 css = [css]
530 css = [css]
531 elif css is None:
531 elif css is None:
532 css = []
532 css = []
533 if not isinstance(lib, (list,tuple)):
533 if not isinstance(lib, (list,tuple)):
534 raise TypeError('expected sequence, got: %r' % lib)
534 raise TypeError('expected sequence, got: %r' % lib)
535 if not isinstance(css, (list,tuple)):
535 if not isinstance(css, (list,tuple)):
536 raise TypeError('expected sequence, got: %r' % css)
536 raise TypeError('expected sequence, got: %r' % css)
537 self.lib = lib
537 self.lib = lib
538 self.css = css
538 self.css = css
539 super(Javascript, self).__init__(data=data, url=url, filename=filename)
539 super(Javascript, self).__init__(data=data, url=url, filename=filename)
540
540
541 def _repr_javascript_(self):
541 def _repr_javascript_(self):
542 r = ''
542 r = ''
543 for c in self.css:
543 for c in self.css:
544 r += css_t % c
544 r += css_t % c
545 for l in self.lib:
545 for l in self.lib:
546 r += lib_t1 % l
546 r += lib_t1 % l
547 r += self.data
547 r += self.data
548 r += lib_t2*len(self.lib)
548 r += lib_t2*len(self.lib)
549 return r
549 return r
550
550
551 # constants for identifying png/jpeg data
551 # constants for identifying png/jpeg data
552 _PNG = b'\x89PNG\r\n\x1a\n'
552 _PNG = b'\x89PNG\r\n\x1a\n'
553 _JPEG = b'\xff\xd8'
553 _JPEG = b'\xff\xd8'
554
554
555 def _pngxy(data):
555 def _pngxy(data):
556 """read the (width, height) from a PNG header"""
556 """read the (width, height) from a PNG header"""
557 ihdr = data.index(b'IHDR')
557 ihdr = data.index(b'IHDR')
558 # next 8 bytes are width/height
558 # next 8 bytes are width/height
559 w4h4 = data[ihdr+4:ihdr+12]
559 w4h4 = data[ihdr+4:ihdr+12]
560 return struct.unpack('>ii', w4h4)
560 return struct.unpack('>ii', w4h4)
561
561
562 def _jpegxy(data):
562 def _jpegxy(data):
563 """read the (width, height) from a JPEG header"""
563 """read the (width, height) from a JPEG header"""
564 # adapted from http://www.64lines.com/jpeg-width-height
564 # adapted from http://www.64lines.com/jpeg-width-height
565
565
566 idx = 4
566 idx = 4
567 while True:
567 while True:
568 block_size = struct.unpack('>H', data[idx:idx+2])[0]
568 block_size = struct.unpack('>H', data[idx:idx+2])[0]
569 idx = idx + block_size
569 idx = idx + block_size
570 if data[idx:idx+2] == b'\xFF\xC0':
570 if data[idx:idx+2] == b'\xFF\xC0':
571 # found Start of Frame
571 # found Start of Frame
572 iSOF = idx
572 iSOF = idx
573 break
573 break
574 else:
574 else:
575 # read another block
575 # read another block
576 idx += 2
576 idx += 2
577
577
578 h, w = struct.unpack('>HH', data[iSOF+5:iSOF+9])
578 h, w = struct.unpack('>HH', data[iSOF+5:iSOF+9])
579 return w, h
579 return w, h
580
580
581 class Image(DisplayObject):
581 class Image(DisplayObject):
582
582
583 _read_flags = 'rb'
583 _read_flags = 'rb'
584 _FMT_JPEG = u'jpeg'
584 _FMT_JPEG = u'jpeg'
585 _FMT_PNG = u'png'
585 _FMT_PNG = u'png'
586 _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG]
586 _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG]
587
587
588 def __init__(self, data=None, url=None, filename=None, format=u'png', embed=None, width=None, height=None, retina=False):
588 def __init__(self, data=None, url=None, filename=None, format=u'png', embed=None, width=None, height=None, retina=False):
589 """Create a PNG/JPEG image object given raw data.
589 """Create a PNG/JPEG image object given raw data.
590
590
591 When this object is returned by an input cell or passed to the
591 When this object is returned by an input cell or passed to the
592 display function, it will result in the image being displayed
592 display function, it will result in the image being displayed
593 in the frontend.
593 in the frontend.
594
594
595 Parameters
595 Parameters
596 ----------
596 ----------
597 data : unicode, str or bytes
597 data : unicode, str or bytes
598 The raw image data or a URL or filename to load the data from.
598 The raw image data or a URL or filename to load the data from.
599 This always results in embedded image data.
599 This always results in embedded image data.
600 url : unicode
600 url : unicode
601 A URL to download the data from. If you specify `url=`,
601 A URL to download the data from. If you specify `url=`,
602 the image data will not be embedded unless you also specify `embed=True`.
602 the image data will not be embedded unless you also specify `embed=True`.
603 filename : unicode
603 filename : unicode
604 Path to a local file to load the data from.
604 Path to a local file to load the data from.
605 Images from a file are always embedded.
605 Images from a file are always embedded.
606 format : unicode
606 format : unicode
607 The format of the image data (png/jpeg/jpg). If a filename or URL is given
607 The format of the image data (png/jpeg/jpg). If a filename or URL is given
608 for format will be inferred from the filename extension.
608 for format will be inferred from the filename extension.
609 embed : bool
609 embed : bool
610 Should the image data be embedded using a data URI (True) or be
610 Should the image data be embedded using a data URI (True) or be
611 loaded using an <img> tag. Set this to True if you want the image
611 loaded using an <img> tag. Set this to True if you want the image
612 to be viewable later with no internet connection in the notebook.
612 to be viewable later with no internet connection in the notebook.
613
613
614 Default is `True`, unless the keyword argument `url` is set, then
614 Default is `True`, unless the keyword argument `url` is set, then
615 default value is `False`.
615 default value is `False`.
616
616
617 Note that QtConsole is not able to display images if `embed` is set to `False`
617 Note that QtConsole is not able to display images if `embed` is set to `False`
618 width : int
618 width : int
619 Width to which to constrain the image in html
619 Width to which to constrain the image in html
620 height : int
620 height : int
621 Height to which to constrain the image in html
621 Height to which to constrain the image in html
622 retina : bool
622 retina : bool
623 Automatically set the width and height to half of the measured
623 Automatically set the width and height to half of the measured
624 width and height.
624 width and height.
625 This only works for embedded images because it reads the width/height
625 This only works for embedded images because it reads the width/height
626 from image data.
626 from image data.
627 For non-embedded images, you can just set the desired display width
627 For non-embedded images, you can just set the desired display width
628 and height directly.
628 and height directly.
629
629
630 Examples
630 Examples
631 --------
631 --------
632 # embedded image data, works in qtconsole and notebook
632 # embedded image data, works in qtconsole and notebook
633 # when passed positionally, the first arg can be any of raw image data,
633 # when passed positionally, the first arg can be any of raw image data,
634 # a URL, or a filename from which to load image data.
634 # a URL, or a filename from which to load image data.
635 # The result is always embedding image data for inline images.
635 # The result is always embedding image data for inline images.
636 Image('http://www.google.fr/images/srpr/logo3w.png')
636 Image('http://www.google.fr/images/srpr/logo3w.png')
637 Image('/path/to/image.jpg')
637 Image('/path/to/image.jpg')
638 Image(b'RAW_PNG_DATA...')
638 Image(b'RAW_PNG_DATA...')
639
639
640 # Specifying Image(url=...) does not embed the image data,
640 # Specifying Image(url=...) does not embed the image data,
641 # it only generates `<img>` tag with a link to the source.
641 # it only generates `<img>` tag with a link to the source.
642 # This will not work in the qtconsole or offline.
642 # This will not work in the qtconsole or offline.
643 Image(url='http://www.google.fr/images/srpr/logo3w.png')
643 Image(url='http://www.google.fr/images/srpr/logo3w.png')
644
644
645 """
645 """
646 if filename is not None:
646 if filename is not None:
647 ext = self._find_ext(filename)
647 ext = self._find_ext(filename)
648 elif url is not None:
648 elif url is not None:
649 ext = self._find_ext(url)
649 ext = self._find_ext(url)
650 elif data is None:
650 elif data is None:
651 raise ValueError("No image data found. Expecting filename, url, or data.")
651 raise ValueError("No image data found. Expecting filename, url, or data.")
652 elif isinstance(data, string_types) and (
652 elif isinstance(data, string_types) and (
653 data.startswith('http') or _safe_exists(data)
653 data.startswith('http') or _safe_exists(data)
654 ):
654 ):
655 ext = self._find_ext(data)
655 ext = self._find_ext(data)
656 else:
656 else:
657 ext = None
657 ext = None
658
658
659 if ext is not None:
659 if ext is not None:
660 format = ext.lower()
660 format = ext.lower()
661 if ext == u'jpg' or ext == u'jpeg':
661 if ext == u'jpg' or ext == u'jpeg':
662 format = self._FMT_JPEG
662 format = self._FMT_JPEG
663 if ext == u'png':
663 if ext == u'png':
664 format = self._FMT_PNG
664 format = self._FMT_PNG
665 elif isinstance(data, bytes) and format == 'png':
665 elif isinstance(data, bytes) and format == 'png':
666 # infer image type from image data header,
666 # infer image type from image data header,
667 # only if format might not have been specified.
667 # only if format might not have been specified.
668 if data[:2] == _JPEG:
668 if data[:2] == _JPEG:
669 format = 'jpeg'
669 format = 'jpeg'
670
670
671 self.format = unicode_type(format).lower()
671 self.format = unicode_type(format).lower()
672 self.embed = embed if embed is not None else (url is None)
672 self.embed = embed if embed is not None else (url is None)
673
673
674 if self.embed and self.format not in self._ACCEPTABLE_EMBEDDINGS:
674 if self.embed and self.format not in self._ACCEPTABLE_EMBEDDINGS:
675 raise ValueError("Cannot embed the '%s' image format" % (self.format))
675 raise ValueError("Cannot embed the '%s' image format" % (self.format))
676 self.width = width
676 self.width = width
677 self.height = height
677 self.height = height
678 self.retina = retina
678 self.retina = retina
679 super(Image, self).__init__(data=data, url=url, filename=filename)
679 super(Image, self).__init__(data=data, url=url, filename=filename)
680
680
681 if retina:
681 if retina:
682 self._retina_shape()
682 self._retina_shape()
683
683
684 def _retina_shape(self):
684 def _retina_shape(self):
685 """load pixel-doubled width and height from image data"""
685 """load pixel-doubled width and height from image data"""
686 if not self.embed:
686 if not self.embed:
687 return
687 return
688 if self.format == 'png':
688 if self.format == 'png':
689 w, h = _pngxy(self.data)
689 w, h = _pngxy(self.data)
690 elif self.format == 'jpeg':
690 elif self.format == 'jpeg':
691 w, h = _jpegxy(self.data)
691 w, h = _jpegxy(self.data)
692 else:
692 else:
693 # retina only supports png
693 # retina only supports png
694 return
694 return
695 self.width = w // 2
695 self.width = w // 2
696 self.height = h // 2
696 self.height = h // 2
697
697
698 def reload(self):
698 def reload(self):
699 """Reload the raw data from file or URL."""
699 """Reload the raw data from file or URL."""
700 if self.embed:
700 if self.embed:
701 super(Image,self).reload()
701 super(Image,self).reload()
702 if self.retina:
702 if self.retina:
703 self._retina_shape()
703 self._retina_shape()
704
704
705 def _repr_html_(self):
705 def _repr_html_(self):
706 if not self.embed:
706 if not self.embed:
707 width = height = ''
707 width = height = ''
708 if self.width:
708 if self.width:
709 width = ' width="%d"' % self.width
709 width = ' width="%d"' % self.width
710 if self.height:
710 if self.height:
711 height = ' height="%d"' % self.height
711 height = ' height="%d"' % self.height
712 return u'<img src="%s"%s%s/>' % (self.url, width, height)
712 return u'<img src="%s"%s%s/>' % (self.url, width, height)
713
713
714 def _data_and_metadata(self):
714 def _data_and_metadata(self):
715 """shortcut for returning metadata with shape information, if defined"""
715 """shortcut for returning metadata with shape information, if defined"""
716 md = {}
716 md = {}
717 if self.width:
717 if self.width:
718 md['width'] = self.width
718 md['width'] = self.width
719 if self.height:
719 if self.height:
720 md['height'] = self.height
720 md['height'] = self.height
721 if md:
721 if md:
722 return self.data, md
722 return self.data, md
723 else:
723 else:
724 return self.data
724 return self.data
725
725
726 def _repr_png_(self):
726 def _repr_png_(self):
727 if self.embed and self.format == u'png':
727 if self.embed and self.format == u'png':
728 return self._data_and_metadata()
728 return self._data_and_metadata()
729
729
730 def _repr_jpeg_(self):
730 def _repr_jpeg_(self):
731 if self.embed and (self.format == u'jpeg' or self.format == u'jpg'):
731 if self.embed and (self.format == u'jpeg' or self.format == u'jpg'):
732 return self._data_and_metadata()
732 return self._data_and_metadata()
733
733
734 def _find_ext(self, s):
734 def _find_ext(self, s):
735 return unicode_type(s.split('.')[-1].lower())
735 return unicode_type(s.split('.')[-1].lower())
736
736
737
737
738 def clear_output(wait=False):
738 def clear_output(wait=False):
739 """Clear the output of the current cell receiving output.
739 """Clear the output of the current cell receiving output.
740
740
741 Parameters
741 Parameters
742 ----------
742 ----------
743 wait : bool [default: false]
743 wait : bool [default: false]
744 Wait to clear the output until new output is available to replace it."""
744 Wait to clear the output until new output is available to replace it."""
745 from IPython.core.interactiveshell import InteractiveShell
745 from IPython.core.interactiveshell import InteractiveShell
746 if InteractiveShell.initialized():
746 if InteractiveShell.initialized():
747 InteractiveShell.instance().display_pub.clear_output(wait)
747 InteractiveShell.instance().display_pub.clear_output(wait)
748 else:
748 else:
749 from IPython.utils import io
749 from IPython.utils import io
750 print('\033[2K\r', file=io.stdout, end='')
750 print('\033[2K\r', file=io.stdout, end='')
751 io.stdout.flush()
751 io.stdout.flush()
752 print('\033[2K\r', file=io.stderr, end='')
752 print('\033[2K\r', file=io.stderr, end='')
753 io.stderr.flush()
753 io.stderr.flush()
754
754
755
755
756 @skip_doctest
756 @skip_doctest
757 def set_matplotlib_formats(*formats, **kwargs):
757 def set_matplotlib_formats(*formats, **kwargs):
758 """Select figure formats for the inline backend. Optionally pass quality for JPEG.
758 """Select figure formats for the inline backend. Optionally pass quality for JPEG.
759
759
760 For example, this enables PNG and JPEG output with a JPEG quality of 90%::
760 For example, this enables PNG and JPEG output with a JPEG quality of 90%::
761
761
762 In [1]: set_matplotlib_formats('png', 'jpeg', quality=90)
762 In [1]: set_matplotlib_formats('png', 'jpeg', quality=90)
763
763
764 To set this in your config files use the following::
764 To set this in your config files use the following::
765
765
766 c.InlineBackend.figure_formats = {'png', 'jpeg'}
766 c.InlineBackend.figure_formats = {'png', 'jpeg'}
767 c.InlineBackend.print_figure_kwargs.update({'quality' : 90})
767 c.InlineBackend.print_figure_kwargs.update({'quality' : 90})
768
768
769 Parameters
769 Parameters
770 ----------
770 ----------
771 *formats : strs
771 *formats : strs
772 One or more figure formats to enable: 'png', 'retina', 'jpeg', 'svg', 'pdf'.
772 One or more figure formats to enable: 'png', 'retina', 'jpeg', 'svg', 'pdf'.
773 **kwargs :
773 **kwargs :
774 Keyword args will be relayed to ``figure.canvas.print_figure``.
774 Keyword args will be relayed to ``figure.canvas.print_figure``.
775 """
775 """
776 from IPython.core.interactiveshell import InteractiveShell
776 from IPython.core.interactiveshell import InteractiveShell
777 from IPython.core.pylabtools import select_figure_formats
777 from IPython.core.pylabtools import select_figure_formats
778 from IPython.kernel.zmq.pylab.config import InlineBackend
778 from IPython.kernel.zmq.pylab.config import InlineBackend
779 # build kwargs, starting with InlineBackend config
779 # build kwargs, starting with InlineBackend config
780 kw = {}
780 kw = {}
781 cfg = InlineBackend.instance()
781 cfg = InlineBackend.instance()
782 kw.update(cfg.print_figure_kwargs)
782 kw.update(cfg.print_figure_kwargs)
783 kw.update(**kwargs)
783 kw.update(**kwargs)
784 shell = InteractiveShell.instance()
784 shell = InteractiveShell.instance()
785 select_figure_formats(shell, formats, **kw)
785 select_figure_formats(shell, formats, **kw)
786
786
787 @skip_doctest
787 @skip_doctest
788 def set_matplotlib_close(close=True):
788 def set_matplotlib_close(close=True):
789 """Set whether the inline backend closes all figures automatically or not.
789 """Set whether the inline backend closes all figures automatically or not.
790
790
791 By default, the inline backend used in the IPython Notebook will close all
791 By default, the inline backend used in the IPython Notebook will close all
792 matplotlib figures automatically after each cell is run. This means that
792 matplotlib figures automatically after each cell is run. This means that
793 plots in different cells won't interfere. Sometimes, you may want to make
793 plots in different cells won't interfere. Sometimes, you may want to make
794 a plot in one cell and then refine it in later cells. This can be accomplished
794 a plot in one cell and then refine it in later cells. This can be accomplished
795 by::
795 by::
796
796
797 In [1]: set_matplotlib_close(False)
797 In [1]: set_matplotlib_close(False)
798
798
799 To set this in your config files use the following::
799 To set this in your config files use the following::
800
800
801 c.InlineBackend.close_figures = False
801 c.InlineBackend.close_figures = False
802
802
803 Parameters
803 Parameters
804 ----------
804 ----------
805 close : bool
805 close : bool
806 Should all matplotlib figures be automatically closed after each cell is
806 Should all matplotlib figures be automatically closed after each cell is
807 run?
807 run?
808 """
808 """
809 from IPython.kernel.zmq.pylab.config import InlineBackend
809 from IPython.kernel.zmq.pylab.config import InlineBackend
810 cfg = InlineBackend.instance()
810 cfg = InlineBackend.instance()
811 cfg.close_figures = close
811 cfg.close_figures = close
812
812
@@ -1,179 +1,159 b''
1 """An interface for publishing rich data to frontends.
1 """An interface for publishing rich data to frontends.
2
2
3 There are two components of the display system:
3 There are two components of the display system:
4
4
5 * Display formatters, which take a Python object and compute the
5 * Display formatters, which take a Python object and compute the
6 representation of the object in various formats (text, HTML, SVG, etc.).
6 representation of the object in various formats (text, HTML, SVG, etc.).
7 * The display publisher that is used to send the representation data to the
7 * The display publisher that is used to send the representation data to the
8 various frontends.
8 various frontends.
9
9
10 This module defines the logic display publishing. The display publisher uses
10 This module defines the logic display publishing. The display publisher uses
11 the ``display_data`` message type that is defined in the IPython messaging
11 the ``display_data`` message type that is defined in the IPython messaging
12 spec.
12 spec.
13
14 Authors:
15
16 * Brian Granger
17 """
13 """
18
14
19 #-----------------------------------------------------------------------------
15 # Copyright (c) IPython Development Team.
20 # Copyright (C) 2008-2011 The IPython Development Team
16 # Distributed under the terms of the Modified BSD License.
21 #
22 # Distributed under the terms of the BSD License. The full license is in
23 # the file COPYING, distributed as part of this software.
24 #-----------------------------------------------------------------------------
25
26 #-----------------------------------------------------------------------------
27 # Imports
28 #-----------------------------------------------------------------------------
29
17
30 from __future__ import print_function
18 from __future__ import print_function
31
19
32 from IPython.config.configurable import Configurable
20 from IPython.config.configurable import Configurable
33 from IPython.utils import io
21 from IPython.utils import io
34 from IPython.utils.py3compat import string_types
22 from IPython.utils.py3compat import string_types
35 from IPython.utils.traitlets import List
23 from IPython.utils.traitlets import List
36
24
37 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
38 # Main payload class
26 # Main payload class
39 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
40
28
41 class DisplayPublisher(Configurable):
29 class DisplayPublisher(Configurable):
42 """A traited class that publishes display data to frontends.
30 """A traited class that publishes display data to frontends.
43
31
44 Instances of this class are created by the main IPython object and should
32 Instances of this class are created by the main IPython object and should
45 be accessed there.
33 be accessed there.
46 """
34 """
47
35
48 def _validate_data(self, source, data, metadata=None):
36 def _validate_data(self, data, metadata=None):
49 """Validate the display data.
37 """Validate the display data.
50
38
51 Parameters
39 Parameters
52 ----------
40 ----------
53 source : str
54 The fully dotted name of the callable that created the data, like
55 :func:`foo.bar.my_formatter`.
56 data : dict
41 data : dict
57 The formata data dictionary.
42 The formata data dictionary.
58 metadata : dict
43 metadata : dict
59 Any metadata for the data.
44 Any metadata for the data.
60 """
45 """
61
46
62 if not isinstance(source, string_types):
63 raise TypeError('source must be a str, got: %r' % source)
64 if not isinstance(data, dict):
47 if not isinstance(data, dict):
65 raise TypeError('data must be a dict, got: %r' % data)
48 raise TypeError('data must be a dict, got: %r' % data)
66 if metadata is not None:
49 if metadata is not None:
67 if not isinstance(metadata, dict):
50 if not isinstance(metadata, dict):
68 raise TypeError('metadata must be a dict, got: %r' % data)
51 raise TypeError('metadata must be a dict, got: %r' % data)
69
52
70 def publish(self, source, data, metadata=None):
53 def publish(self, data, metadata=None, source=None):
71 """Publish data and metadata to all frontends.
54 """Publish data and metadata to all frontends.
72
55
73 See the ``display_data`` message in the messaging documentation for
56 See the ``display_data`` message in the messaging documentation for
74 more details about this message type.
57 more details about this message type.
75
58
76 The following MIME types are currently implemented:
59 The following MIME types are currently implemented:
77
60
78 * text/plain
61 * text/plain
79 * text/html
62 * text/html
80 * text/markdown
63 * text/markdown
81 * text/latex
64 * text/latex
82 * application/json
65 * application/json
83 * application/javascript
66 * application/javascript
84 * image/png
67 * image/png
85 * image/jpeg
68 * image/jpeg
86 * image/svg+xml
69 * image/svg+xml
87
70
88 Parameters
71 Parameters
89 ----------
72 ----------
90 source : str
91 A string that give the function or method that created the data,
92 such as 'IPython.core.page'.
93 data : dict
73 data : dict
94 A dictionary having keys that are valid MIME types (like
74 A dictionary having keys that are valid MIME types (like
95 'text/plain' or 'image/svg+xml') and values that are the data for
75 'text/plain' or 'image/svg+xml') and values that are the data for
96 that MIME type. The data itself must be a JSON'able data
76 that MIME type. The data itself must be a JSON'able data
97 structure. Minimally all data should have the 'text/plain' data,
77 structure. Minimally all data should have the 'text/plain' data,
98 which can be displayed by all frontends. If more than the plain
78 which can be displayed by all frontends. If more than the plain
99 text is given, it is up to the frontend to decide which
79 text is given, it is up to the frontend to decide which
100 representation to use.
80 representation to use.
101 metadata : dict
81 metadata : dict
102 A dictionary for metadata related to the data. This can contain
82 A dictionary for metadata related to the data. This can contain
103 arbitrary key, value pairs that frontends can use to interpret
83 arbitrary key, value pairs that frontends can use to interpret
104 the data. Metadata specific to each mime-type can be specified
84 the data. Metadata specific to each mime-type can be specified
105 in the metadata dict with the same mime-type keys as
85 in the metadata dict with the same mime-type keys as
106 the data itself.
86 the data itself.
87 source : str, deprecated
88 Unused.
107 """
89 """
108
90
109 # The default is to simply write the plain text data using io.stdout.
91 # The default is to simply write the plain text data using io.stdout.
110 if 'text/plain' in data:
92 if 'text/plain' in data:
111 print(data['text/plain'], file=io.stdout)
93 print(data['text/plain'], file=io.stdout)
112
94
113 def clear_output(self, wait=False):
95 def clear_output(self, wait=False):
114 """Clear the output of the cell receiving output."""
96 """Clear the output of the cell receiving output."""
115 print('\033[2K\r', file=io.stdout, end='')
97 print('\033[2K\r', file=io.stdout, end='')
116 io.stdout.flush()
98 io.stdout.flush()
117 print('\033[2K\r', file=io.stderr, end='')
99 print('\033[2K\r', file=io.stderr, end='')
118 io.stderr.flush()
100 io.stderr.flush()
119
101
120
102
121 class CapturingDisplayPublisher(DisplayPublisher):
103 class CapturingDisplayPublisher(DisplayPublisher):
122 """A DisplayPublisher that stores"""
104 """A DisplayPublisher that stores"""
123 outputs = List()
105 outputs = List()
124
106
125 def publish(self, source, data, metadata=None):
107 def publish(self, data, metadata=None, source=None):
126 self.outputs.append((source, data, metadata))
108 self.outputs.append((data, metadata))
127
109
128 def clear_output(self, wait=False):
110 def clear_output(self, wait=False):
129 super(CapturingDisplayPublisher, self).clear_output(wait)
111 super(CapturingDisplayPublisher, self).clear_output(wait)
130
112
131 # empty the list, *do not* reassign a new list
113 # empty the list, *do not* reassign a new list
132 del self.outputs[:]
114 del self.outputs[:]
133
115
134
116
135 def publish_display_data(source, data, metadata=None):
117 def publish_display_data(data, metadata=None, source=None):
136 """Publish data and metadata to all frontends.
118 """Publish data and metadata to all frontends.
137
119
138 See the ``display_data`` message in the messaging documentation for
120 See the ``display_data`` message in the messaging documentation for
139 more details about this message type.
121 more details about this message type.
140
122
141 The following MIME types are currently implemented:
123 The following MIME types are currently implemented:
142
124
143 * text/plain
125 * text/plain
144 * text/html
126 * text/html
145 * text/markdown
127 * text/markdown
146 * text/latex
128 * text/latex
147 * application/json
129 * application/json
148 * application/javascript
130 * application/javascript
149 * image/png
131 * image/png
150 * image/jpeg
132 * image/jpeg
151 * image/svg+xml
133 * image/svg+xml
152
134
153 Parameters
135 Parameters
154 ----------
136 ----------
155 source : str
156 A string that give the function or method that created the data,
157 such as 'IPython.core.page'.
158 data : dict
137 data : dict
159 A dictionary having keys that are valid MIME types (like
138 A dictionary having keys that are valid MIME types (like
160 'text/plain' or 'image/svg+xml') and values that are the data for
139 'text/plain' or 'image/svg+xml') and values that are the data for
161 that MIME type. The data itself must be a JSON'able data
140 that MIME type. The data itself must be a JSON'able data
162 structure. Minimally all data should have the 'text/plain' data,
141 structure. Minimally all data should have the 'text/plain' data,
163 which can be displayed by all frontends. If more than the plain
142 which can be displayed by all frontends. If more than the plain
164 text is given, it is up to the frontend to decide which
143 text is given, it is up to the frontend to decide which
165 representation to use.
144 representation to use.
166 metadata : dict
145 metadata : dict
167 A dictionary for metadata related to the data. This can contain
146 A dictionary for metadata related to the data. This can contain
168 arbitrary key, value pairs that frontends can use to interpret
147 arbitrary key, value pairs that frontends can use to interpret
169 the data. mime-type keys matching those in data can be used
148 the data. mime-type keys matching those in data can be used
170 to specify metadata about particular representations.
149 to specify metadata about particular representations.
150 source : str, deprecated
151 Unused.
171 """
152 """
172 from IPython.core.interactiveshell import InteractiveShell
153 from IPython.core.interactiveshell import InteractiveShell
173 InteractiveShell.instance().display_pub.publish(
154 InteractiveShell.instance().display_pub.publish(
174 source,
155 data=data,
175 data,
156 metadata=metadata,
176 metadata
177 )
157 )
178
158
179
159
@@ -1,401 +1,401 b''
1 """Test suite for our zeromq-based message specification."""
1 """Test suite for our zeromq-based message specification."""
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 import re
6 import re
7 from distutils.version import LooseVersion as V
7 from distutils.version import LooseVersion as V
8 from subprocess import PIPE
8 from subprocess import PIPE
9 try:
9 try:
10 from queue import Empty # Py 3
10 from queue import Empty # Py 3
11 except ImportError:
11 except ImportError:
12 from Queue import Empty # Py 2
12 from Queue import Empty # Py 2
13
13
14 import nose.tools as nt
14 import nose.tools as nt
15
15
16 from IPython.kernel import KernelManager
16 from IPython.kernel import KernelManager
17
17
18 from IPython.utils.traitlets import (
18 from IPython.utils.traitlets import (
19 HasTraits, TraitError, Bool, Unicode, Dict, Integer, List, Enum, Any,
19 HasTraits, TraitError, Bool, Unicode, Dict, Integer, List, Enum, Any,
20 )
20 )
21 from IPython.utils.py3compat import string_types, iteritems
21 from IPython.utils.py3compat import string_types, iteritems
22
22
23 from .utils import TIMEOUT, start_global_kernel, flush_channels, execute
23 from .utils import TIMEOUT, start_global_kernel, flush_channels, execute
24
24
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Globals
26 # Globals
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 KC = None
28 KC = None
29
29
30 def setup():
30 def setup():
31 global KC
31 global KC
32 KC = start_global_kernel()
32 KC = start_global_kernel()
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Message Spec References
35 # Message Spec References
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37
37
38 class Reference(HasTraits):
38 class Reference(HasTraits):
39
39
40 """
40 """
41 Base class for message spec specification testing.
41 Base class for message spec specification testing.
42
42
43 This class is the core of the message specification test. The
43 This class is the core of the message specification test. The
44 idea is that child classes implement trait attributes for each
44 idea is that child classes implement trait attributes for each
45 message keys, so that message keys can be tested against these
45 message keys, so that message keys can be tested against these
46 traits using :meth:`check` method.
46 traits using :meth:`check` method.
47
47
48 """
48 """
49
49
50 def check(self, d):
50 def check(self, d):
51 """validate a dict against our traits"""
51 """validate a dict against our traits"""
52 for key in self.trait_names():
52 for key in self.trait_names():
53 nt.assert_in(key, d)
53 nt.assert_in(key, d)
54 # FIXME: always allow None, probably not a good idea
54 # FIXME: always allow None, probably not a good idea
55 if d[key] is None:
55 if d[key] is None:
56 continue
56 continue
57 try:
57 try:
58 setattr(self, key, d[key])
58 setattr(self, key, d[key])
59 except TraitError as e:
59 except TraitError as e:
60 assert False, str(e)
60 assert False, str(e)
61
61
62 class Version(Unicode):
62 class Version(Unicode):
63 def validate(self, obj, value):
63 def validate(self, obj, value):
64 min_version = self.default_value
64 min_version = self.default_value
65 if V(value) < V(min_version):
65 if V(value) < V(min_version):
66 raise TraitError("bad version: %s < %s" % (value, min_version))
66 raise TraitError("bad version: %s < %s" % (value, min_version))
67
67
68 class RMessage(Reference):
68 class RMessage(Reference):
69 msg_id = Unicode()
69 msg_id = Unicode()
70 msg_type = Unicode()
70 msg_type = Unicode()
71 header = Dict()
71 header = Dict()
72 parent_header = Dict()
72 parent_header = Dict()
73 content = Dict()
73 content = Dict()
74
74
75 def check(self, d):
75 def check(self, d):
76 super(RMessage, self).check(d)
76 super(RMessage, self).check(d)
77 RHeader().check(self.header)
77 RHeader().check(self.header)
78 if self.parent_header:
78 if self.parent_header:
79 RHeader().check(self.parent_header)
79 RHeader().check(self.parent_header)
80
80
81 class RHeader(Reference):
81 class RHeader(Reference):
82 msg_id = Unicode()
82 msg_id = Unicode()
83 msg_type = Unicode()
83 msg_type = Unicode()
84 session = Unicode()
84 session = Unicode()
85 username = Unicode()
85 username = Unicode()
86 version = Version('5.0')
86 version = Version('5.0')
87
87
88 mime_pat = re.compile(r'\w+/\w+')
88 mime_pat = re.compile(r'\w+/\w+')
89
89
90 class MimeBundle(Reference):
90 class MimeBundle(Reference):
91 metadata = Dict()
91 metadata = Dict()
92 data = Dict()
92 data = Dict()
93 def _data_changed(self, name, old, new):
93 def _data_changed(self, name, old, new):
94 for k,v in iteritems(new):
94 for k,v in iteritems(new):
95 assert mime_pat.match(k)
95 assert mime_pat.match(k)
96 nt.assert_is_instance(v, string_types)
96 nt.assert_is_instance(v, string_types)
97
97
98 # shell replies
98 # shell replies
99
99
100 class ExecuteReply(Reference):
100 class ExecuteReply(Reference):
101 execution_count = Integer()
101 execution_count = Integer()
102 status = Enum((u'ok', u'error'))
102 status = Enum((u'ok', u'error'))
103
103
104 def check(self, d):
104 def check(self, d):
105 Reference.check(self, d)
105 Reference.check(self, d)
106 if d['status'] == 'ok':
106 if d['status'] == 'ok':
107 ExecuteReplyOkay().check(d)
107 ExecuteReplyOkay().check(d)
108 elif d['status'] == 'error':
108 elif d['status'] == 'error':
109 ExecuteReplyError().check(d)
109 ExecuteReplyError().check(d)
110
110
111
111
112 class ExecuteReplyOkay(Reference):
112 class ExecuteReplyOkay(Reference):
113 payload = List(Dict)
113 payload = List(Dict)
114 user_expressions = Dict()
114 user_expressions = Dict()
115
115
116
116
117 class ExecuteReplyError(Reference):
117 class ExecuteReplyError(Reference):
118 ename = Unicode()
118 ename = Unicode()
119 evalue = Unicode()
119 evalue = Unicode()
120 traceback = List(Unicode)
120 traceback = List(Unicode)
121
121
122
122
123 class OInfoReply(MimeBundle):
123 class OInfoReply(MimeBundle):
124 name = Unicode()
124 name = Unicode()
125 found = Bool()
125 found = Bool()
126
126
127
127
128 class ArgSpec(Reference):
128 class ArgSpec(Reference):
129 args = List(Unicode)
129 args = List(Unicode)
130 varargs = Unicode()
130 varargs = Unicode()
131 varkw = Unicode()
131 varkw = Unicode()
132 defaults = List()
132 defaults = List()
133
133
134
134
135 class Status(Reference):
135 class Status(Reference):
136 execution_state = Enum((u'busy', u'idle', u'starting'))
136 execution_state = Enum((u'busy', u'idle', u'starting'))
137
137
138
138
139 class CompleteReply(Reference):
139 class CompleteReply(Reference):
140 matches = List(Unicode)
140 matches = List(Unicode)
141
141
142
142
143 class KernelInfoReply(Reference):
143 class KernelInfoReply(Reference):
144 protocol_version = Version('5.0')
144 protocol_version = Version('5.0')
145 implementation = Unicode('ipython')
145 implementation = Unicode('ipython')
146 implementation_version = Version('2.1')
146 implementation_version = Version('2.1')
147 language_version = Version('2.7')
147 language_version = Version('2.7')
148 language = Unicode('python')
148 language = Unicode('python')
149 banner = Unicode()
149 banner = Unicode()
150
150
151
151
152 # IOPub messages
152 # IOPub messages
153
153
154 class ExecuteInput(Reference):
154 class ExecuteInput(Reference):
155 code = Unicode()
155 code = Unicode()
156 execution_count = Integer()
156 execution_count = Integer()
157
157
158
158
159 Error = ExecuteReplyError
159 Error = ExecuteReplyError
160
160
161
161
162 class Stream(Reference):
162 class Stream(Reference):
163 name = Enum((u'stdout', u'stderr'))
163 name = Enum((u'stdout', u'stderr'))
164 data = Unicode()
164 data = Unicode()
165
165
166
166
167 class DisplayData(MimeBundle):
167 class DisplayData(MimeBundle):
168 source = Unicode()
168 pass
169
169
170
170
171 class ExecuteResult(MimeBundle):
171 class ExecuteResult(MimeBundle):
172 execution_count = Integer()
172 execution_count = Integer()
173
173
174
174
175 references = {
175 references = {
176 'execute_reply' : ExecuteReply(),
176 'execute_reply' : ExecuteReply(),
177 'object_info_reply' : OInfoReply(),
177 'object_info_reply' : OInfoReply(),
178 'status' : Status(),
178 'status' : Status(),
179 'complete_reply' : CompleteReply(),
179 'complete_reply' : CompleteReply(),
180 'kernel_info_reply': KernelInfoReply(),
180 'kernel_info_reply': KernelInfoReply(),
181 'execute_input' : ExecuteInput(),
181 'execute_input' : ExecuteInput(),
182 'execute_result' : ExecuteResult(),
182 'execute_result' : ExecuteResult(),
183 'error' : Error(),
183 'error' : Error(),
184 'stream' : Stream(),
184 'stream' : Stream(),
185 'display_data' : DisplayData(),
185 'display_data' : DisplayData(),
186 'header' : RHeader(),
186 'header' : RHeader(),
187 }
187 }
188 """
188 """
189 Specifications of `content` part of the reply messages.
189 Specifications of `content` part of the reply messages.
190 """
190 """
191
191
192
192
193 def validate_message(msg, msg_type=None, parent=None):
193 def validate_message(msg, msg_type=None, parent=None):
194 """validate a message
194 """validate a message
195
195
196 This is a generator, and must be iterated through to actually
196 This is a generator, and must be iterated through to actually
197 trigger each test.
197 trigger each test.
198
198
199 If msg_type and/or parent are given, the msg_type and/or parent msg_id
199 If msg_type and/or parent are given, the msg_type and/or parent msg_id
200 are compared with the given values.
200 are compared with the given values.
201 """
201 """
202 RMessage().check(msg)
202 RMessage().check(msg)
203 if msg_type:
203 if msg_type:
204 nt.assert_equal(msg['msg_type'], msg_type)
204 nt.assert_equal(msg['msg_type'], msg_type)
205 if parent:
205 if parent:
206 nt.assert_equal(msg['parent_header']['msg_id'], parent)
206 nt.assert_equal(msg['parent_header']['msg_id'], parent)
207 content = msg['content']
207 content = msg['content']
208 ref = references[msg['msg_type']]
208 ref = references[msg['msg_type']]
209 ref.check(content)
209 ref.check(content)
210
210
211
211
212 #-----------------------------------------------------------------------------
212 #-----------------------------------------------------------------------------
213 # Tests
213 # Tests
214 #-----------------------------------------------------------------------------
214 #-----------------------------------------------------------------------------
215
215
216 # Shell channel
216 # Shell channel
217
217
218 def test_execute():
218 def test_execute():
219 flush_channels()
219 flush_channels()
220
220
221 msg_id = KC.execute(code='x=1')
221 msg_id = KC.execute(code='x=1')
222 reply = KC.get_shell_msg(timeout=TIMEOUT)
222 reply = KC.get_shell_msg(timeout=TIMEOUT)
223 validate_message(reply, 'execute_reply', msg_id)
223 validate_message(reply, 'execute_reply', msg_id)
224
224
225
225
226 def test_execute_silent():
226 def test_execute_silent():
227 flush_channels()
227 flush_channels()
228 msg_id, reply = execute(code='x=1', silent=True)
228 msg_id, reply = execute(code='x=1', silent=True)
229
229
230 # flush status=idle
230 # flush status=idle
231 status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
231 status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
232 validate_message(status, 'status', msg_id)
232 validate_message(status, 'status', msg_id)
233 nt.assert_equal(status['content']['execution_state'], 'idle')
233 nt.assert_equal(status['content']['execution_state'], 'idle')
234
234
235 nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
235 nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
236 count = reply['execution_count']
236 count = reply['execution_count']
237
237
238 msg_id, reply = execute(code='x=2', silent=True)
238 msg_id, reply = execute(code='x=2', silent=True)
239
239
240 # flush status=idle
240 # flush status=idle
241 status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
241 status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
242 validate_message(status, 'status', msg_id)
242 validate_message(status, 'status', msg_id)
243 nt.assert_equal(status['content']['execution_state'], 'idle')
243 nt.assert_equal(status['content']['execution_state'], 'idle')
244
244
245 nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
245 nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
246 count_2 = reply['execution_count']
246 count_2 = reply['execution_count']
247 nt.assert_equal(count_2, count)
247 nt.assert_equal(count_2, count)
248
248
249
249
250 def test_execute_error():
250 def test_execute_error():
251 flush_channels()
251 flush_channels()
252
252
253 msg_id, reply = execute(code='1/0')
253 msg_id, reply = execute(code='1/0')
254 nt.assert_equal(reply['status'], 'error')
254 nt.assert_equal(reply['status'], 'error')
255 nt.assert_equal(reply['ename'], 'ZeroDivisionError')
255 nt.assert_equal(reply['ename'], 'ZeroDivisionError')
256
256
257 error = KC.iopub_channel.get_msg(timeout=TIMEOUT)
257 error = KC.iopub_channel.get_msg(timeout=TIMEOUT)
258 validate_message(error, 'error', msg_id)
258 validate_message(error, 'error', msg_id)
259
259
260
260
261 def test_execute_inc():
261 def test_execute_inc():
262 """execute request should increment execution_count"""
262 """execute request should increment execution_count"""
263 flush_channels()
263 flush_channels()
264
264
265 msg_id, reply = execute(code='x=1')
265 msg_id, reply = execute(code='x=1')
266 count = reply['execution_count']
266 count = reply['execution_count']
267
267
268 flush_channels()
268 flush_channels()
269
269
270 msg_id, reply = execute(code='x=2')
270 msg_id, reply = execute(code='x=2')
271 count_2 = reply['execution_count']
271 count_2 = reply['execution_count']
272 nt.assert_equal(count_2, count+1)
272 nt.assert_equal(count_2, count+1)
273
273
274
274
275 def test_user_expressions():
275 def test_user_expressions():
276 flush_channels()
276 flush_channels()
277
277
278 msg_id, reply = execute(code='x=1', user_expressions=dict(foo='x+1'))
278 msg_id, reply = execute(code='x=1', user_expressions=dict(foo='x+1'))
279 user_expressions = reply['user_expressions']
279 user_expressions = reply['user_expressions']
280 nt.assert_equal(user_expressions, {u'foo': {
280 nt.assert_equal(user_expressions, {u'foo': {
281 u'status': u'ok',
281 u'status': u'ok',
282 u'data': {u'text/plain': u'2'},
282 u'data': {u'text/plain': u'2'},
283 u'metadata': {},
283 u'metadata': {},
284 }})
284 }})
285
285
286
286
287 def test_user_expressions_fail():
287 def test_user_expressions_fail():
288 flush_channels()
288 flush_channels()
289
289
290 msg_id, reply = execute(code='x=0', user_expressions=dict(foo='nosuchname'))
290 msg_id, reply = execute(code='x=0', user_expressions=dict(foo='nosuchname'))
291 user_expressions = reply['user_expressions']
291 user_expressions = reply['user_expressions']
292 foo = user_expressions['foo']
292 foo = user_expressions['foo']
293 nt.assert_equal(foo['status'], 'error')
293 nt.assert_equal(foo['status'], 'error')
294 nt.assert_equal(foo['ename'], 'NameError')
294 nt.assert_equal(foo['ename'], 'NameError')
295
295
296
296
297 def test_oinfo():
297 def test_oinfo():
298 flush_channels()
298 flush_channels()
299
299
300 msg_id = KC.object_info('a')
300 msg_id = KC.object_info('a')
301 reply = KC.get_shell_msg(timeout=TIMEOUT)
301 reply = KC.get_shell_msg(timeout=TIMEOUT)
302 validate_message(reply, 'object_info_reply', msg_id)
302 validate_message(reply, 'object_info_reply', msg_id)
303
303
304
304
305 def test_oinfo_found():
305 def test_oinfo_found():
306 flush_channels()
306 flush_channels()
307
307
308 msg_id, reply = execute(code='a=5')
308 msg_id, reply = execute(code='a=5')
309
309
310 msg_id = KC.object_info('a')
310 msg_id = KC.object_info('a')
311 reply = KC.get_shell_msg(timeout=TIMEOUT)
311 reply = KC.get_shell_msg(timeout=TIMEOUT)
312 validate_message(reply, 'object_info_reply', msg_id)
312 validate_message(reply, 'object_info_reply', msg_id)
313 content = reply['content']
313 content = reply['content']
314 assert content['found']
314 assert content['found']
315 nt.assert_equal(content['name'], 'a')
315 nt.assert_equal(content['name'], 'a')
316 text = content['data']['text/plain']
316 text = content['data']['text/plain']
317 nt.assert_in('Type:', text)
317 nt.assert_in('Type:', text)
318 nt.assert_in('Docstring:', text)
318 nt.assert_in('Docstring:', text)
319
319
320
320
321 def test_oinfo_detail():
321 def test_oinfo_detail():
322 flush_channels()
322 flush_channels()
323
323
324 msg_id, reply = execute(code='ip=get_ipython()')
324 msg_id, reply = execute(code='ip=get_ipython()')
325
325
326 msg_id = KC.object_info('ip.object_inspect', cursor_pos=10, detail_level=1)
326 msg_id = KC.object_info('ip.object_inspect', cursor_pos=10, detail_level=1)
327 reply = KC.get_shell_msg(timeout=TIMEOUT)
327 reply = KC.get_shell_msg(timeout=TIMEOUT)
328 validate_message(reply, 'object_info_reply', msg_id)
328 validate_message(reply, 'object_info_reply', msg_id)
329 content = reply['content']
329 content = reply['content']
330 assert content['found']
330 assert content['found']
331 nt.assert_equal(content['name'], 'ip.object_inspect')
331 nt.assert_equal(content['name'], 'ip.object_inspect')
332 text = content['data']['text/plain']
332 text = content['data']['text/plain']
333 nt.assert_in('Definition:', text)
333 nt.assert_in('Definition:', text)
334 nt.assert_in('Source:', text)
334 nt.assert_in('Source:', text)
335
335
336
336
337 def test_oinfo_not_found():
337 def test_oinfo_not_found():
338 flush_channels()
338 flush_channels()
339
339
340 msg_id = KC.object_info('dne')
340 msg_id = KC.object_info('dne')
341 reply = KC.get_shell_msg(timeout=TIMEOUT)
341 reply = KC.get_shell_msg(timeout=TIMEOUT)
342 validate_message(reply, 'object_info_reply', msg_id)
342 validate_message(reply, 'object_info_reply', msg_id)
343 content = reply['content']
343 content = reply['content']
344 nt.assert_false(content['found'])
344 nt.assert_false(content['found'])
345
345
346
346
347 def test_complete():
347 def test_complete():
348 flush_channels()
348 flush_channels()
349
349
350 msg_id, reply = execute(code="alpha = albert = 5")
350 msg_id, reply = execute(code="alpha = albert = 5")
351
351
352 msg_id = KC.complete('al', 2)
352 msg_id = KC.complete('al', 2)
353 reply = KC.get_shell_msg(timeout=TIMEOUT)
353 reply = KC.get_shell_msg(timeout=TIMEOUT)
354 validate_message(reply, 'complete_reply', msg_id)
354 validate_message(reply, 'complete_reply', msg_id)
355 matches = reply['content']['matches']
355 matches = reply['content']['matches']
356 for name in ('alpha', 'albert'):
356 for name in ('alpha', 'albert'):
357 nt.assert_in(name, matches)
357 nt.assert_in(name, matches)
358
358
359
359
360 def test_kernel_info_request():
360 def test_kernel_info_request():
361 flush_channels()
361 flush_channels()
362
362
363 msg_id = KC.kernel_info()
363 msg_id = KC.kernel_info()
364 reply = KC.get_shell_msg(timeout=TIMEOUT)
364 reply = KC.get_shell_msg(timeout=TIMEOUT)
365 validate_message(reply, 'kernel_info_reply', msg_id)
365 validate_message(reply, 'kernel_info_reply', msg_id)
366
366
367
367
368 def test_single_payload():
368 def test_single_payload():
369 flush_channels()
369 flush_channels()
370 msg_id, reply = execute(code="for i in range(3):\n"+
370 msg_id, reply = execute(code="for i in range(3):\n"+
371 " x=range?\n")
371 " x=range?\n")
372 payload = reply['payload']
372 payload = reply['payload']
373 next_input_pls = [pl for pl in payload if pl["source"] == "set_next_input"]
373 next_input_pls = [pl for pl in payload if pl["source"] == "set_next_input"]
374 nt.assert_equal(len(next_input_pls), 1)
374 nt.assert_equal(len(next_input_pls), 1)
375
375
376
376
377 # IOPub channel
377 # IOPub channel
378
378
379
379
380 def test_stream():
380 def test_stream():
381 flush_channels()
381 flush_channels()
382
382
383 msg_id, reply = execute("print('hi')")
383 msg_id, reply = execute("print('hi')")
384
384
385 stdout = KC.iopub_channel.get_msg(timeout=TIMEOUT)
385 stdout = KC.iopub_channel.get_msg(timeout=TIMEOUT)
386 validate_message(stdout, 'stream', msg_id)
386 validate_message(stdout, 'stream', msg_id)
387 content = stdout['content']
387 content = stdout['content']
388 nt.assert_equal(content['name'], u'stdout')
388 nt.assert_equal(content['name'], u'stdout')
389 nt.assert_equal(content['data'], u'hi\n')
389 nt.assert_equal(content['data'], u'hi\n')
390
390
391
391
392 def test_display_data():
392 def test_display_data():
393 flush_channels()
393 flush_channels()
394
394
395 msg_id, reply = execute("from IPython.core.display import display; display(1)")
395 msg_id, reply = execute("from IPython.core.display import display; display(1)")
396
396
397 display = KC.iopub_channel.get_msg(timeout=TIMEOUT)
397 display = KC.iopub_channel.get_msg(timeout=TIMEOUT)
398 validate_message(display, 'display_data', parent=msg_id)
398 validate_message(display, 'display_data', parent=msg_id)
399 data = display['content']['data']
399 data = display['content']['data']
400 nt.assert_equal(data['text/plain'], u'1')
400 nt.assert_equal(data['text/plain'], u'1')
401
401
@@ -1,573 +1,572 b''
1 """A ZMQ-based subclass of InteractiveShell.
1 """A ZMQ-based subclass of InteractiveShell.
2
2
3 This code is meant to ease the refactoring of the base InteractiveShell into
3 This code is meant to ease the refactoring of the base InteractiveShell into
4 something with a cleaner architecture for 2-process use, without actually
4 something with a cleaner architecture for 2-process use, without actually
5 breaking InteractiveShell itself. So we're doing something a bit ugly, where
5 breaking InteractiveShell itself. So we're doing something a bit ugly, where
6 we subclass and override what we want to fix. Once this is working well, we
6 we subclass and override what we want to fix. Once this is working well, we
7 can go back to the base class and refactor the code for a cleaner inheritance
7 can go back to the base class and refactor the code for a cleaner inheritance
8 implementation that doesn't rely on so much monkeypatching.
8 implementation that doesn't rely on so much monkeypatching.
9
9
10 But this lets us maintain a fully working IPython as we develop the new
10 But this lets us maintain a fully working IPython as we develop the new
11 machinery. This should thus be thought of as scaffolding.
11 machinery. This should thus be thought of as scaffolding.
12 """
12 """
13
13
14 # Copyright (c) IPython Development Team.
14 # Copyright (c) IPython Development Team.
15 # Distributed under the terms of the Modified BSD License.
15 # Distributed under the terms of the Modified BSD License.
16
16
17 from __future__ import print_function
17 from __future__ import print_function
18
18
19 import os
19 import os
20 import sys
20 import sys
21 import time
21 import time
22
22
23 from zmq.eventloop import ioloop
23 from zmq.eventloop import ioloop
24
24
25 from IPython.core.interactiveshell import (
25 from IPython.core.interactiveshell import (
26 InteractiveShell, InteractiveShellABC
26 InteractiveShell, InteractiveShellABC
27 )
27 )
28 from IPython.core import page
28 from IPython.core import page
29 from IPython.core.autocall import ZMQExitAutocall
29 from IPython.core.autocall import ZMQExitAutocall
30 from IPython.core.displaypub import DisplayPublisher
30 from IPython.core.displaypub import DisplayPublisher
31 from IPython.core.error import UsageError
31 from IPython.core.error import UsageError
32 from IPython.core.magics import MacroToEdit, CodeMagics
32 from IPython.core.magics import MacroToEdit, CodeMagics
33 from IPython.core.magic import magics_class, line_magic, Magics
33 from IPython.core.magic import magics_class, line_magic, Magics
34 from IPython.core.payloadpage import install_payload_page
34 from IPython.core.payloadpage import install_payload_page
35 from IPython.core.usage import default_gui_banner
35 from IPython.core.usage import default_gui_banner
36 from IPython.display import display, Javascript
36 from IPython.display import display, Javascript
37 from IPython.kernel.inprocess.socket import SocketABC
37 from IPython.kernel.inprocess.socket import SocketABC
38 from IPython.kernel import (
38 from IPython.kernel import (
39 get_connection_file, get_connection_info, connect_qtconsole
39 get_connection_file, get_connection_info, connect_qtconsole
40 )
40 )
41 from IPython.testing.skipdoctest import skip_doctest
41 from IPython.testing.skipdoctest import skip_doctest
42 from IPython.utils import openpy
42 from IPython.utils import openpy
43 from IPython.utils.jsonutil import json_clean, encode_images
43 from IPython.utils.jsonutil import json_clean, encode_images
44 from IPython.utils.process import arg_split
44 from IPython.utils.process import arg_split
45 from IPython.utils import py3compat
45 from IPython.utils import py3compat
46 from IPython.utils.py3compat import unicode_type
46 from IPython.utils.py3compat import unicode_type
47 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes, Any
47 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes, Any
48 from IPython.utils.warn import error
48 from IPython.utils.warn import error
49 from IPython.kernel.zmq.displayhook import ZMQShellDisplayHook
49 from IPython.kernel.zmq.displayhook import ZMQShellDisplayHook
50 from IPython.kernel.zmq.datapub import ZMQDataPublisher
50 from IPython.kernel.zmq.datapub import ZMQDataPublisher
51 from IPython.kernel.zmq.session import extract_header
51 from IPython.kernel.zmq.session import extract_header
52 from IPython.kernel.comm import CommManager
52 from IPython.kernel.comm import CommManager
53 from .session import Session
53 from .session import Session
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Functions and classes
56 # Functions and classes
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 class ZMQDisplayPublisher(DisplayPublisher):
59 class ZMQDisplayPublisher(DisplayPublisher):
60 """A display publisher that publishes data using a ZeroMQ PUB socket."""
60 """A display publisher that publishes data using a ZeroMQ PUB socket."""
61
61
62 session = Instance(Session)
62 session = Instance(Session)
63 pub_socket = Instance(SocketABC)
63 pub_socket = Instance(SocketABC)
64 parent_header = Dict({})
64 parent_header = Dict({})
65 topic = CBytes(b'display_data')
65 topic = CBytes(b'display_data')
66
66
67 def set_parent(self, parent):
67 def set_parent(self, parent):
68 """Set the parent for outbound messages."""
68 """Set the parent for outbound messages."""
69 self.parent_header = extract_header(parent)
69 self.parent_header = extract_header(parent)
70
70
71 def _flush_streams(self):
71 def _flush_streams(self):
72 """flush IO Streams prior to display"""
72 """flush IO Streams prior to display"""
73 sys.stdout.flush()
73 sys.stdout.flush()
74 sys.stderr.flush()
74 sys.stderr.flush()
75
75
76 def publish(self, source, data, metadata=None):
76 def publish(self, data, metadata=None, source=None):
77 self._flush_streams()
77 self._flush_streams()
78 if metadata is None:
78 if metadata is None:
79 metadata = {}
79 metadata = {}
80 self._validate_data(source, data, metadata)
80 self._validate_data(data, metadata)
81 content = {}
81 content = {}
82 content['source'] = source
83 content['data'] = encode_images(data)
82 content['data'] = encode_images(data)
84 content['metadata'] = metadata
83 content['metadata'] = metadata
85 self.session.send(
84 self.session.send(
86 self.pub_socket, u'display_data', json_clean(content),
85 self.pub_socket, u'display_data', json_clean(content),
87 parent=self.parent_header, ident=self.topic,
86 parent=self.parent_header, ident=self.topic,
88 )
87 )
89
88
90 def clear_output(self, wait=False):
89 def clear_output(self, wait=False):
91 content = dict(wait=wait)
90 content = dict(wait=wait)
92 self._flush_streams()
91 self._flush_streams()
93 self.session.send(
92 self.session.send(
94 self.pub_socket, u'clear_output', content,
93 self.pub_socket, u'clear_output', content,
95 parent=self.parent_header, ident=self.topic,
94 parent=self.parent_header, ident=self.topic,
96 )
95 )
97
96
98 @magics_class
97 @magics_class
99 class KernelMagics(Magics):
98 class KernelMagics(Magics):
100 #------------------------------------------------------------------------
99 #------------------------------------------------------------------------
101 # Magic overrides
100 # Magic overrides
102 #------------------------------------------------------------------------
101 #------------------------------------------------------------------------
103 # Once the base class stops inheriting from magic, this code needs to be
102 # Once the base class stops inheriting from magic, this code needs to be
104 # moved into a separate machinery as well. For now, at least isolate here
103 # moved into a separate machinery as well. For now, at least isolate here
105 # the magics which this class needs to implement differently from the base
104 # the magics which this class needs to implement differently from the base
106 # class, or that are unique to it.
105 # class, or that are unique to it.
107
106
108 @line_magic
107 @line_magic
109 def doctest_mode(self, parameter_s=''):
108 def doctest_mode(self, parameter_s=''):
110 """Toggle doctest mode on and off.
109 """Toggle doctest mode on and off.
111
110
112 This mode is intended to make IPython behave as much as possible like a
111 This mode is intended to make IPython behave as much as possible like a
113 plain Python shell, from the perspective of how its prompts, exceptions
112 plain Python shell, from the perspective of how its prompts, exceptions
114 and output look. This makes it easy to copy and paste parts of a
113 and output look. This makes it easy to copy and paste parts of a
115 session into doctests. It does so by:
114 session into doctests. It does so by:
116
115
117 - Changing the prompts to the classic ``>>>`` ones.
116 - Changing the prompts to the classic ``>>>`` ones.
118 - Changing the exception reporting mode to 'Plain'.
117 - Changing the exception reporting mode to 'Plain'.
119 - Disabling pretty-printing of output.
118 - Disabling pretty-printing of output.
120
119
121 Note that IPython also supports the pasting of code snippets that have
120 Note that IPython also supports the pasting of code snippets that have
122 leading '>>>' and '...' prompts in them. This means that you can paste
121 leading '>>>' and '...' prompts in them. This means that you can paste
123 doctests from files or docstrings (even if they have leading
122 doctests from files or docstrings (even if they have leading
124 whitespace), and the code will execute correctly. You can then use
123 whitespace), and the code will execute correctly. You can then use
125 '%history -t' to see the translated history; this will give you the
124 '%history -t' to see the translated history; this will give you the
126 input after removal of all the leading prompts and whitespace, which
125 input after removal of all the leading prompts and whitespace, which
127 can be pasted back into an editor.
126 can be pasted back into an editor.
128
127
129 With these features, you can switch into this mode easily whenever you
128 With these features, you can switch into this mode easily whenever you
130 need to do testing and changes to doctests, without having to leave
129 need to do testing and changes to doctests, without having to leave
131 your existing IPython session.
130 your existing IPython session.
132 """
131 """
133
132
134 from IPython.utils.ipstruct import Struct
133 from IPython.utils.ipstruct import Struct
135
134
136 # Shorthands
135 # Shorthands
137 shell = self.shell
136 shell = self.shell
138 disp_formatter = self.shell.display_formatter
137 disp_formatter = self.shell.display_formatter
139 ptformatter = disp_formatter.formatters['text/plain']
138 ptformatter = disp_formatter.formatters['text/plain']
140 # dstore is a data store kept in the instance metadata bag to track any
139 # dstore is a data store kept in the instance metadata bag to track any
141 # changes we make, so we can undo them later.
140 # changes we make, so we can undo them later.
142 dstore = shell.meta.setdefault('doctest_mode', Struct())
141 dstore = shell.meta.setdefault('doctest_mode', Struct())
143 save_dstore = dstore.setdefault
142 save_dstore = dstore.setdefault
144
143
145 # save a few values we'll need to recover later
144 # save a few values we'll need to recover later
146 mode = save_dstore('mode', False)
145 mode = save_dstore('mode', False)
147 save_dstore('rc_pprint', ptformatter.pprint)
146 save_dstore('rc_pprint', ptformatter.pprint)
148 save_dstore('rc_active_types',disp_formatter.active_types)
147 save_dstore('rc_active_types',disp_formatter.active_types)
149 save_dstore('xmode', shell.InteractiveTB.mode)
148 save_dstore('xmode', shell.InteractiveTB.mode)
150
149
151 if mode == False:
150 if mode == False:
152 # turn on
151 # turn on
153 ptformatter.pprint = False
152 ptformatter.pprint = False
154 disp_formatter.active_types = ['text/plain']
153 disp_formatter.active_types = ['text/plain']
155 shell.magic('xmode Plain')
154 shell.magic('xmode Plain')
156 else:
155 else:
157 # turn off
156 # turn off
158 ptformatter.pprint = dstore.rc_pprint
157 ptformatter.pprint = dstore.rc_pprint
159 disp_formatter.active_types = dstore.rc_active_types
158 disp_formatter.active_types = dstore.rc_active_types
160 shell.magic("xmode " + dstore.xmode)
159 shell.magic("xmode " + dstore.xmode)
161
160
162 # Store new mode and inform on console
161 # Store new mode and inform on console
163 dstore.mode = bool(1-int(mode))
162 dstore.mode = bool(1-int(mode))
164 mode_label = ['OFF','ON'][dstore.mode]
163 mode_label = ['OFF','ON'][dstore.mode]
165 print('Doctest mode is:', mode_label)
164 print('Doctest mode is:', mode_label)
166
165
167 # Send the payload back so that clients can modify their prompt display
166 # Send the payload back so that clients can modify their prompt display
168 payload = dict(
167 payload = dict(
169 source='doctest_mode',
168 source='doctest_mode',
170 mode=dstore.mode)
169 mode=dstore.mode)
171 shell.payload_manager.write_payload(payload)
170 shell.payload_manager.write_payload(payload)
172
171
173
172
174 _find_edit_target = CodeMagics._find_edit_target
173 _find_edit_target = CodeMagics._find_edit_target
175
174
176 @skip_doctest
175 @skip_doctest
177 @line_magic
176 @line_magic
178 def edit(self, parameter_s='', last_call=['','']):
177 def edit(self, parameter_s='', last_call=['','']):
179 """Bring up an editor and execute the resulting code.
178 """Bring up an editor and execute the resulting code.
180
179
181 Usage:
180 Usage:
182 %edit [options] [args]
181 %edit [options] [args]
183
182
184 %edit runs an external text editor. You will need to set the command for
183 %edit runs an external text editor. You will need to set the command for
185 this editor via the ``TerminalInteractiveShell.editor`` option in your
184 this editor via the ``TerminalInteractiveShell.editor`` option in your
186 configuration file before it will work.
185 configuration file before it will work.
187
186
188 This command allows you to conveniently edit multi-line code right in
187 This command allows you to conveniently edit multi-line code right in
189 your IPython session.
188 your IPython session.
190
189
191 If called without arguments, %edit opens up an empty editor with a
190 If called without arguments, %edit opens up an empty editor with a
192 temporary file and will execute the contents of this file when you
191 temporary file and will execute the contents of this file when you
193 close it (don't forget to save it!).
192 close it (don't forget to save it!).
194
193
195 Options:
194 Options:
196
195
197 -n <number>
196 -n <number>
198 Open the editor at a specified line number. By default, the IPython
197 Open the editor at a specified line number. By default, the IPython
199 editor hook uses the unix syntax 'editor +N filename', but you can
198 editor hook uses the unix syntax 'editor +N filename', but you can
200 configure this by providing your own modified hook if your favorite
199 configure this by providing your own modified hook if your favorite
201 editor supports line-number specifications with a different syntax.
200 editor supports line-number specifications with a different syntax.
202
201
203 -p
202 -p
204 Call the editor with the same data as the previous time it was used,
203 Call the editor with the same data as the previous time it was used,
205 regardless of how long ago (in your current session) it was.
204 regardless of how long ago (in your current session) it was.
206
205
207 -r
206 -r
208 Use 'raw' input. This option only applies to input taken from the
207 Use 'raw' input. This option only applies to input taken from the
209 user's history. By default, the 'processed' history is used, so that
208 user's history. By default, the 'processed' history is used, so that
210 magics are loaded in their transformed version to valid Python. If
209 magics are loaded in their transformed version to valid Python. If
211 this option is given, the raw input as typed as the command line is
210 this option is given, the raw input as typed as the command line is
212 used instead. When you exit the editor, it will be executed by
211 used instead. When you exit the editor, it will be executed by
213 IPython's own processor.
212 IPython's own processor.
214
213
215 Arguments:
214 Arguments:
216
215
217 If arguments are given, the following possibilites exist:
216 If arguments are given, the following possibilites exist:
218
217
219 - The arguments are numbers or pairs of colon-separated numbers (like
218 - The arguments are numbers or pairs of colon-separated numbers (like
220 1 4:8 9). These are interpreted as lines of previous input to be
219 1 4:8 9). These are interpreted as lines of previous input to be
221 loaded into the editor. The syntax is the same of the %macro command.
220 loaded into the editor. The syntax is the same of the %macro command.
222
221
223 - If the argument doesn't start with a number, it is evaluated as a
222 - If the argument doesn't start with a number, it is evaluated as a
224 variable and its contents loaded into the editor. You can thus edit
223 variable and its contents loaded into the editor. You can thus edit
225 any string which contains python code (including the result of
224 any string which contains python code (including the result of
226 previous edits).
225 previous edits).
227
226
228 - If the argument is the name of an object (other than a string),
227 - If the argument is the name of an object (other than a string),
229 IPython will try to locate the file where it was defined and open the
228 IPython will try to locate the file where it was defined and open the
230 editor at the point where it is defined. You can use ``%edit function``
229 editor at the point where it is defined. You can use ``%edit function``
231 to load an editor exactly at the point where 'function' is defined,
230 to load an editor exactly at the point where 'function' is defined,
232 edit it and have the file be executed automatically.
231 edit it and have the file be executed automatically.
233
232
234 If the object is a macro (see %macro for details), this opens up your
233 If the object is a macro (see %macro for details), this opens up your
235 specified editor with a temporary file containing the macro's data.
234 specified editor with a temporary file containing the macro's data.
236 Upon exit, the macro is reloaded with the contents of the file.
235 Upon exit, the macro is reloaded with the contents of the file.
237
236
238 Note: opening at an exact line is only supported under Unix, and some
237 Note: opening at an exact line is only supported under Unix, and some
239 editors (like kedit and gedit up to Gnome 2.8) do not understand the
238 editors (like kedit and gedit up to Gnome 2.8) do not understand the
240 '+NUMBER' parameter necessary for this feature. Good editors like
239 '+NUMBER' parameter necessary for this feature. Good editors like
241 (X)Emacs, vi, jed, pico and joe all do.
240 (X)Emacs, vi, jed, pico and joe all do.
242
241
243 - If the argument is not found as a variable, IPython will look for a
242 - If the argument is not found as a variable, IPython will look for a
244 file with that name (adding .py if necessary) and load it into the
243 file with that name (adding .py if necessary) and load it into the
245 editor. It will execute its contents with execfile() when you exit,
244 editor. It will execute its contents with execfile() when you exit,
246 loading any code in the file into your interactive namespace.
245 loading any code in the file into your interactive namespace.
247
246
248 Unlike in the terminal, this is designed to use a GUI editor, and we do
247 Unlike in the terminal, this is designed to use a GUI editor, and we do
249 not know when it has closed. So the file you edit will not be
248 not know when it has closed. So the file you edit will not be
250 automatically executed or printed.
249 automatically executed or printed.
251
250
252 Note that %edit is also available through the alias %ed.
251 Note that %edit is also available through the alias %ed.
253 """
252 """
254
253
255 opts,args = self.parse_options(parameter_s,'prn:')
254 opts,args = self.parse_options(parameter_s,'prn:')
256
255
257 try:
256 try:
258 filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
257 filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
259 except MacroToEdit as e:
258 except MacroToEdit as e:
260 # TODO: Implement macro editing over 2 processes.
259 # TODO: Implement macro editing over 2 processes.
261 print("Macro editing not yet implemented in 2-process model.")
260 print("Macro editing not yet implemented in 2-process model.")
262 return
261 return
263
262
264 # Make sure we send to the client an absolute path, in case the working
263 # Make sure we send to the client an absolute path, in case the working
265 # directory of client and kernel don't match
264 # directory of client and kernel don't match
266 filename = os.path.abspath(filename)
265 filename = os.path.abspath(filename)
267
266
268 payload = {
267 payload = {
269 'source' : 'edit_magic',
268 'source' : 'edit_magic',
270 'filename' : filename,
269 'filename' : filename,
271 'line_number' : lineno
270 'line_number' : lineno
272 }
271 }
273 self.shell.payload_manager.write_payload(payload)
272 self.shell.payload_manager.write_payload(payload)
274
273
275 # A few magics that are adapted to the specifics of using pexpect and a
274 # A few magics that are adapted to the specifics of using pexpect and a
276 # remote terminal
275 # remote terminal
277
276
278 @line_magic
277 @line_magic
279 def clear(self, arg_s):
278 def clear(self, arg_s):
280 """Clear the terminal."""
279 """Clear the terminal."""
281 if os.name == 'posix':
280 if os.name == 'posix':
282 self.shell.system("clear")
281 self.shell.system("clear")
283 else:
282 else:
284 self.shell.system("cls")
283 self.shell.system("cls")
285
284
286 if os.name == 'nt':
285 if os.name == 'nt':
287 # This is the usual name in windows
286 # This is the usual name in windows
288 cls = line_magic('cls')(clear)
287 cls = line_magic('cls')(clear)
289
288
290 # Terminal pagers won't work over pexpect, but we do have our own pager
289 # Terminal pagers won't work over pexpect, but we do have our own pager
291
290
292 @line_magic
291 @line_magic
293 def less(self, arg_s):
292 def less(self, arg_s):
294 """Show a file through the pager.
293 """Show a file through the pager.
295
294
296 Files ending in .py are syntax-highlighted."""
295 Files ending in .py are syntax-highlighted."""
297 if not arg_s:
296 if not arg_s:
298 raise UsageError('Missing filename.')
297 raise UsageError('Missing filename.')
299
298
300 cont = open(arg_s).read()
299 cont = open(arg_s).read()
301 if arg_s.endswith('.py'):
300 if arg_s.endswith('.py'):
302 cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
301 cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
303 else:
302 else:
304 cont = open(arg_s).read()
303 cont = open(arg_s).read()
305 page.page(cont)
304 page.page(cont)
306
305
307 more = line_magic('more')(less)
306 more = line_magic('more')(less)
308
307
309 # Man calls a pager, so we also need to redefine it
308 # Man calls a pager, so we also need to redefine it
310 if os.name == 'posix':
309 if os.name == 'posix':
311 @line_magic
310 @line_magic
312 def man(self, arg_s):
311 def man(self, arg_s):
313 """Find the man page for the given command and display in pager."""
312 """Find the man page for the given command and display in pager."""
314 page.page(self.shell.getoutput('man %s | col -b' % arg_s,
313 page.page(self.shell.getoutput('man %s | col -b' % arg_s,
315 split=False))
314 split=False))
316
315
317 @line_magic
316 @line_magic
318 def connect_info(self, arg_s):
317 def connect_info(self, arg_s):
319 """Print information for connecting other clients to this kernel
318 """Print information for connecting other clients to this kernel
320
319
321 It will print the contents of this session's connection file, as well as
320 It will print the contents of this session's connection file, as well as
322 shortcuts for local clients.
321 shortcuts for local clients.
323
322
324 In the simplest case, when called from the most recently launched kernel,
323 In the simplest case, when called from the most recently launched kernel,
325 secondary clients can be connected, simply with:
324 secondary clients can be connected, simply with:
326
325
327 $> ipython <app> --existing
326 $> ipython <app> --existing
328
327
329 """
328 """
330
329
331 from IPython.core.application import BaseIPythonApplication as BaseIPApp
330 from IPython.core.application import BaseIPythonApplication as BaseIPApp
332
331
333 if BaseIPApp.initialized():
332 if BaseIPApp.initialized():
334 app = BaseIPApp.instance()
333 app = BaseIPApp.instance()
335 security_dir = app.profile_dir.security_dir
334 security_dir = app.profile_dir.security_dir
336 profile = app.profile
335 profile = app.profile
337 else:
336 else:
338 profile = 'default'
337 profile = 'default'
339 security_dir = ''
338 security_dir = ''
340
339
341 try:
340 try:
342 connection_file = get_connection_file()
341 connection_file = get_connection_file()
343 info = get_connection_info(unpack=False)
342 info = get_connection_info(unpack=False)
344 except Exception as e:
343 except Exception as e:
345 error("Could not get connection info: %r" % e)
344 error("Could not get connection info: %r" % e)
346 return
345 return
347
346
348 # add profile flag for non-default profile
347 # add profile flag for non-default profile
349 profile_flag = "--profile %s" % profile if profile != 'default' else ""
348 profile_flag = "--profile %s" % profile if profile != 'default' else ""
350
349
351 # if it's in the security dir, truncate to basename
350 # if it's in the security dir, truncate to basename
352 if security_dir == os.path.dirname(connection_file):
351 if security_dir == os.path.dirname(connection_file):
353 connection_file = os.path.basename(connection_file)
352 connection_file = os.path.basename(connection_file)
354
353
355
354
356 print (info + '\n')
355 print (info + '\n')
357 print ("Paste the above JSON into a file, and connect with:\n"
356 print ("Paste the above JSON into a file, and connect with:\n"
358 " $> ipython <app> --existing <file>\n"
357 " $> ipython <app> --existing <file>\n"
359 "or, if you are local, you can connect with just:\n"
358 "or, if you are local, you can connect with just:\n"
360 " $> ipython <app> --existing {0} {1}\n"
359 " $> ipython <app> --existing {0} {1}\n"
361 "or even just:\n"
360 "or even just:\n"
362 " $> ipython <app> --existing {1}\n"
361 " $> ipython <app> --existing {1}\n"
363 "if this is the most recent IPython session you have started.".format(
362 "if this is the most recent IPython session you have started.".format(
364 connection_file, profile_flag
363 connection_file, profile_flag
365 )
364 )
366 )
365 )
367
366
368 @line_magic
367 @line_magic
369 def qtconsole(self, arg_s):
368 def qtconsole(self, arg_s):
370 """Open a qtconsole connected to this kernel.
369 """Open a qtconsole connected to this kernel.
371
370
372 Useful for connecting a qtconsole to running notebooks, for better
371 Useful for connecting a qtconsole to running notebooks, for better
373 debugging.
372 debugging.
374 """
373 """
375
374
376 # %qtconsole should imply bind_kernel for engines:
375 # %qtconsole should imply bind_kernel for engines:
377 try:
376 try:
378 from IPython.parallel import bind_kernel
377 from IPython.parallel import bind_kernel
379 except ImportError:
378 except ImportError:
380 # technically possible, because parallel has higher pyzmq min-version
379 # technically possible, because parallel has higher pyzmq min-version
381 pass
380 pass
382 else:
381 else:
383 bind_kernel()
382 bind_kernel()
384
383
385 try:
384 try:
386 p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
385 p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
387 except Exception as e:
386 except Exception as e:
388 error("Could not start qtconsole: %r" % e)
387 error("Could not start qtconsole: %r" % e)
389 return
388 return
390
389
391 @line_magic
390 @line_magic
392 def autosave(self, arg_s):
391 def autosave(self, arg_s):
393 """Set the autosave interval in the notebook (in seconds).
392 """Set the autosave interval in the notebook (in seconds).
394
393
395 The default value is 120, or two minutes.
394 The default value is 120, or two minutes.
396 ``%autosave 0`` will disable autosave.
395 ``%autosave 0`` will disable autosave.
397
396
398 This magic only has an effect when called from the notebook interface.
397 This magic only has an effect when called from the notebook interface.
399 It has no effect when called in a startup file.
398 It has no effect when called in a startup file.
400 """
399 """
401
400
402 try:
401 try:
403 interval = int(arg_s)
402 interval = int(arg_s)
404 except ValueError:
403 except ValueError:
405 raise UsageError("%%autosave requires an integer, got %r" % arg_s)
404 raise UsageError("%%autosave requires an integer, got %r" % arg_s)
406
405
407 # javascript wants milliseconds
406 # javascript wants milliseconds
408 milliseconds = 1000 * interval
407 milliseconds = 1000 * interval
409 display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds),
408 display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds),
410 include=['application/javascript']
409 include=['application/javascript']
411 )
410 )
412 if interval:
411 if interval:
413 print("Autosaving every %i seconds" % interval)
412 print("Autosaving every %i seconds" % interval)
414 else:
413 else:
415 print("Autosave disabled")
414 print("Autosave disabled")
416
415
417
416
418 class ZMQInteractiveShell(InteractiveShell):
417 class ZMQInteractiveShell(InteractiveShell):
419 """A subclass of InteractiveShell for ZMQ."""
418 """A subclass of InteractiveShell for ZMQ."""
420
419
421 displayhook_class = Type(ZMQShellDisplayHook)
420 displayhook_class = Type(ZMQShellDisplayHook)
422 display_pub_class = Type(ZMQDisplayPublisher)
421 display_pub_class = Type(ZMQDisplayPublisher)
423 data_pub_class = Type(ZMQDataPublisher)
422 data_pub_class = Type(ZMQDataPublisher)
424 kernel = Any()
423 kernel = Any()
425 parent_header = Any()
424 parent_header = Any()
426
425
427 def _banner1_default(self):
426 def _banner1_default(self):
428 return default_gui_banner
427 return default_gui_banner
429
428
430 # Override the traitlet in the parent class, because there's no point using
429 # Override the traitlet in the parent class, because there's no point using
431 # readline for the kernel. Can be removed when the readline code is moved
430 # readline for the kernel. Can be removed when the readline code is moved
432 # to the terminal frontend.
431 # to the terminal frontend.
433 colors_force = CBool(True)
432 colors_force = CBool(True)
434 readline_use = CBool(False)
433 readline_use = CBool(False)
435 # autoindent has no meaning in a zmqshell, and attempting to enable it
434 # autoindent has no meaning in a zmqshell, and attempting to enable it
436 # will print a warning in the absence of readline.
435 # will print a warning in the absence of readline.
437 autoindent = CBool(False)
436 autoindent = CBool(False)
438
437
439 exiter = Instance(ZMQExitAutocall)
438 exiter = Instance(ZMQExitAutocall)
440 def _exiter_default(self):
439 def _exiter_default(self):
441 return ZMQExitAutocall(self)
440 return ZMQExitAutocall(self)
442
441
443 def _exit_now_changed(self, name, old, new):
442 def _exit_now_changed(self, name, old, new):
444 """stop eventloop when exit_now fires"""
443 """stop eventloop when exit_now fires"""
445 if new:
444 if new:
446 loop = ioloop.IOLoop.instance()
445 loop = ioloop.IOLoop.instance()
447 loop.add_timeout(time.time()+0.1, loop.stop)
446 loop.add_timeout(time.time()+0.1, loop.stop)
448
447
449 keepkernel_on_exit = None
448 keepkernel_on_exit = None
450
449
451 # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
450 # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
452 # interactive input being read; we provide event loop support in ipkernel
451 # interactive input being read; we provide event loop support in ipkernel
453 @staticmethod
452 @staticmethod
454 def enable_gui(gui):
453 def enable_gui(gui):
455 from .eventloops import enable_gui as real_enable_gui
454 from .eventloops import enable_gui as real_enable_gui
456 try:
455 try:
457 real_enable_gui(gui)
456 real_enable_gui(gui)
458 except ValueError as e:
457 except ValueError as e:
459 raise UsageError("%s" % e)
458 raise UsageError("%s" % e)
460
459
461 def init_environment(self):
460 def init_environment(self):
462 """Configure the user's environment.
461 """Configure the user's environment.
463
462
464 """
463 """
465 env = os.environ
464 env = os.environ
466 # These two ensure 'ls' produces nice coloring on BSD-derived systems
465 # These two ensure 'ls' produces nice coloring on BSD-derived systems
467 env['TERM'] = 'xterm-color'
466 env['TERM'] = 'xterm-color'
468 env['CLICOLOR'] = '1'
467 env['CLICOLOR'] = '1'
469 # Since normal pagers don't work at all (over pexpect we don't have
468 # Since normal pagers don't work at all (over pexpect we don't have
470 # single-key control of the subprocess), try to disable paging in
469 # single-key control of the subprocess), try to disable paging in
471 # subprocesses as much as possible.
470 # subprocesses as much as possible.
472 env['PAGER'] = 'cat'
471 env['PAGER'] = 'cat'
473 env['GIT_PAGER'] = 'cat'
472 env['GIT_PAGER'] = 'cat'
474
473
475 # And install the payload version of page.
474 # And install the payload version of page.
476 install_payload_page()
475 install_payload_page()
477
476
478 def auto_rewrite_input(self, cmd):
477 def auto_rewrite_input(self, cmd):
479 """Called to show the auto-rewritten input for autocall and friends.
478 """Called to show the auto-rewritten input for autocall and friends.
480
479
481 FIXME: this payload is currently not correctly processed by the
480 FIXME: this payload is currently not correctly processed by the
482 frontend.
481 frontend.
483 """
482 """
484 new = self.prompt_manager.render('rewrite') + cmd
483 new = self.prompt_manager.render('rewrite') + cmd
485 payload = dict(
484 payload = dict(
486 source='auto_rewrite_input',
485 source='auto_rewrite_input',
487 transformed_input=new,
486 transformed_input=new,
488 )
487 )
489 self.payload_manager.write_payload(payload)
488 self.payload_manager.write_payload(payload)
490
489
491 def ask_exit(self):
490 def ask_exit(self):
492 """Engage the exit actions."""
491 """Engage the exit actions."""
493 self.exit_now = True
492 self.exit_now = True
494 payload = dict(
493 payload = dict(
495 source='ask_exit',
494 source='ask_exit',
496 exit=True,
495 exit=True,
497 keepkernel=self.keepkernel_on_exit,
496 keepkernel=self.keepkernel_on_exit,
498 )
497 )
499 self.payload_manager.write_payload(payload)
498 self.payload_manager.write_payload(payload)
500
499
501 def _showtraceback(self, etype, evalue, stb):
500 def _showtraceback(self, etype, evalue, stb):
502 # try to preserve ordering of tracebacks and print statements
501 # try to preserve ordering of tracebacks and print statements
503 sys.stdout.flush()
502 sys.stdout.flush()
504 sys.stderr.flush()
503 sys.stderr.flush()
505
504
506 exc_content = {
505 exc_content = {
507 u'traceback' : stb,
506 u'traceback' : stb,
508 u'ename' : unicode_type(etype.__name__),
507 u'ename' : unicode_type(etype.__name__),
509 u'evalue' : py3compat.safe_unicode(evalue),
508 u'evalue' : py3compat.safe_unicode(evalue),
510 }
509 }
511
510
512 dh = self.displayhook
511 dh = self.displayhook
513 # Send exception info over pub socket for other clients than the caller
512 # Send exception info over pub socket for other clients than the caller
514 # to pick up
513 # to pick up
515 topic = None
514 topic = None
516 if dh.topic:
515 if dh.topic:
517 topic = dh.topic.replace(b'execute_result', b'error')
516 topic = dh.topic.replace(b'execute_result', b'error')
518
517
519 exc_msg = dh.session.send(dh.pub_socket, u'error', json_clean(exc_content), dh.parent_header, ident=topic)
518 exc_msg = dh.session.send(dh.pub_socket, u'error', json_clean(exc_content), dh.parent_header, ident=topic)
520
519
521 # FIXME - Hack: store exception info in shell object. Right now, the
520 # FIXME - Hack: store exception info in shell object. Right now, the
522 # caller is reading this info after the fact, we need to fix this logic
521 # caller is reading this info after the fact, we need to fix this logic
523 # to remove this hack. Even uglier, we need to store the error status
522 # to remove this hack. Even uglier, we need to store the error status
524 # here, because in the main loop, the logic that sets it is being
523 # here, because in the main loop, the logic that sets it is being
525 # skipped because runlines swallows the exceptions.
524 # skipped because runlines swallows the exceptions.
526 exc_content[u'status'] = u'error'
525 exc_content[u'status'] = u'error'
527 self._reply_content = exc_content
526 self._reply_content = exc_content
528 # /FIXME
527 # /FIXME
529
528
530 return exc_content
529 return exc_content
531
530
532 def set_next_input(self, text):
531 def set_next_input(self, text):
533 """Send the specified text to the frontend to be presented at the next
532 """Send the specified text to the frontend to be presented at the next
534 input cell."""
533 input cell."""
535 payload = dict(
534 payload = dict(
536 source='set_next_input',
535 source='set_next_input',
537 text=text
536 text=text
538 )
537 )
539 self.payload_manager.write_payload(payload)
538 self.payload_manager.write_payload(payload)
540
539
541 def set_parent(self, parent):
540 def set_parent(self, parent):
542 """Set the parent header for associating output with its triggering input"""
541 """Set the parent header for associating output with its triggering input"""
543 self.parent_header = parent
542 self.parent_header = parent
544 self.displayhook.set_parent(parent)
543 self.displayhook.set_parent(parent)
545 self.display_pub.set_parent(parent)
544 self.display_pub.set_parent(parent)
546 self.data_pub.set_parent(parent)
545 self.data_pub.set_parent(parent)
547 try:
546 try:
548 sys.stdout.set_parent(parent)
547 sys.stdout.set_parent(parent)
549 except AttributeError:
548 except AttributeError:
550 pass
549 pass
551 try:
550 try:
552 sys.stderr.set_parent(parent)
551 sys.stderr.set_parent(parent)
553 except AttributeError:
552 except AttributeError:
554 pass
553 pass
555
554
556 def get_parent(self):
555 def get_parent(self):
557 return self.parent_header
556 return self.parent_header
558
557
559 #-------------------------------------------------------------------------
558 #-------------------------------------------------------------------------
560 # Things related to magics
559 # Things related to magics
561 #-------------------------------------------------------------------------
560 #-------------------------------------------------------------------------
562
561
563 def init_magics(self):
562 def init_magics(self):
564 super(ZMQInteractiveShell, self).init_magics()
563 super(ZMQInteractiveShell, self).init_magics()
565 self.register_magics(KernelMagics)
564 self.register_magics(KernelMagics)
566 self.magics_manager.register_alias('ed', 'edit')
565 self.magics_manager.register_alias('ed', 'edit')
567
566
568 def init_comms(self):
567 def init_comms(self):
569 self.comm_manager = CommManager(shell=self, parent=self)
568 self.comm_manager = CommManager(shell=self, parent=self)
570 self.configurables.append(self.comm_manager)
569 self.configurables.append(self.comm_manager)
571
570
572
571
573 InteractiveShellABC.register(ZMQInteractiveShell)
572 InteractiveShellABC.register(ZMQInteractiveShell)
@@ -1,1863 +1,1863 b''
1 """A semi-synchronous Client for IPython parallel"""
1 """A semi-synchronous Client for IPython parallel"""
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 from __future__ import print_function
6 from __future__ import print_function
7
7
8 import os
8 import os
9 import json
9 import json
10 import sys
10 import sys
11 from threading import Thread, Event
11 from threading import Thread, Event
12 import time
12 import time
13 import warnings
13 import warnings
14 from datetime import datetime
14 from datetime import datetime
15 from getpass import getpass
15 from getpass import getpass
16 from pprint import pprint
16 from pprint import pprint
17
17
18 pjoin = os.path.join
18 pjoin = os.path.join
19
19
20 import zmq
20 import zmq
21
21
22 from IPython.config.configurable import MultipleInstanceError
22 from IPython.config.configurable import MultipleInstanceError
23 from IPython.core.application import BaseIPythonApplication
23 from IPython.core.application import BaseIPythonApplication
24 from IPython.core.profiledir import ProfileDir, ProfileDirError
24 from IPython.core.profiledir import ProfileDir, ProfileDirError
25
25
26 from IPython.utils.capture import RichOutput
26 from IPython.utils.capture import RichOutput
27 from IPython.utils.coloransi import TermColors
27 from IPython.utils.coloransi import TermColors
28 from IPython.utils.jsonutil import rekey, extract_dates, parse_date
28 from IPython.utils.jsonutil import rekey, extract_dates, parse_date
29 from IPython.utils.localinterfaces import localhost, is_local_ip
29 from IPython.utils.localinterfaces import localhost, is_local_ip
30 from IPython.utils.path import get_ipython_dir
30 from IPython.utils.path import get_ipython_dir
31 from IPython.utils.py3compat import cast_bytes, string_types, xrange, iteritems
31 from IPython.utils.py3compat import cast_bytes, string_types, xrange, iteritems
32 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
32 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
33 Dict, List, Bool, Set, Any)
33 Dict, List, Bool, Set, Any)
34 from IPython.external.decorator import decorator
34 from IPython.external.decorator import decorator
35 from IPython.external.ssh import tunnel
35 from IPython.external.ssh import tunnel
36
36
37 from IPython.parallel import Reference
37 from IPython.parallel import Reference
38 from IPython.parallel import error
38 from IPython.parallel import error
39 from IPython.parallel import util
39 from IPython.parallel import util
40
40
41 from IPython.kernel.zmq.session import Session, Message
41 from IPython.kernel.zmq.session import Session, Message
42 from IPython.kernel.zmq import serialize
42 from IPython.kernel.zmq import serialize
43
43
44 from .asyncresult import AsyncResult, AsyncHubResult
44 from .asyncresult import AsyncResult, AsyncHubResult
45 from .view import DirectView, LoadBalancedView
45 from .view import DirectView, LoadBalancedView
46
46
47 #--------------------------------------------------------------------------
47 #--------------------------------------------------------------------------
48 # Decorators for Client methods
48 # Decorators for Client methods
49 #--------------------------------------------------------------------------
49 #--------------------------------------------------------------------------
50
50
51 @decorator
51 @decorator
52 def spin_first(f, self, *args, **kwargs):
52 def spin_first(f, self, *args, **kwargs):
53 """Call spin() to sync state prior to calling the method."""
53 """Call spin() to sync state prior to calling the method."""
54 self.spin()
54 self.spin()
55 return f(self, *args, **kwargs)
55 return f(self, *args, **kwargs)
56
56
57
57
58 #--------------------------------------------------------------------------
58 #--------------------------------------------------------------------------
59 # Classes
59 # Classes
60 #--------------------------------------------------------------------------
60 #--------------------------------------------------------------------------
61
61
62
62
63 class ExecuteReply(RichOutput):
63 class ExecuteReply(RichOutput):
64 """wrapper for finished Execute results"""
64 """wrapper for finished Execute results"""
65 def __init__(self, msg_id, content, metadata):
65 def __init__(self, msg_id, content, metadata):
66 self.msg_id = msg_id
66 self.msg_id = msg_id
67 self._content = content
67 self._content = content
68 self.execution_count = content['execution_count']
68 self.execution_count = content['execution_count']
69 self.metadata = metadata
69 self.metadata = metadata
70
70
71 # RichOutput overrides
71 # RichOutput overrides
72
72
73 @property
73 @property
74 def source(self):
74 def source(self):
75 execute_result = self.metadata['execute_result']
75 execute_result = self.metadata['execute_result']
76 if execute_result:
76 if execute_result:
77 return execute_result.get('source', '')
77 return execute_result.get('source', '')
78
78
79 @property
79 @property
80 def data(self):
80 def data(self):
81 execute_result = self.metadata['execute_result']
81 execute_result = self.metadata['execute_result']
82 if execute_result:
82 if execute_result:
83 return execute_result.get('data', {})
83 return execute_result.get('data', {})
84
84
85 @property
85 @property
86 def _metadata(self):
86 def _metadata(self):
87 execute_result = self.metadata['execute_result']
87 execute_result = self.metadata['execute_result']
88 if execute_result:
88 if execute_result:
89 return execute_result.get('metadata', {})
89 return execute_result.get('metadata', {})
90
90
91 def display(self):
91 def display(self):
92 from IPython.display import publish_display_data
92 from IPython.display import publish_display_data
93 publish_display_data(self.source, self.data, self.metadata)
93 publish_display_data(self.data, self.metadata)
94
94
95 def _repr_mime_(self, mime):
95 def _repr_mime_(self, mime):
96 if mime not in self.data:
96 if mime not in self.data:
97 return
97 return
98 data = self.data[mime]
98 data = self.data[mime]
99 if mime in self._metadata:
99 if mime in self._metadata:
100 return data, self._metadata[mime]
100 return data, self._metadata[mime]
101 else:
101 else:
102 return data
102 return data
103
103
104 def __getitem__(self, key):
104 def __getitem__(self, key):
105 return self.metadata[key]
105 return self.metadata[key]
106
106
107 def __getattr__(self, key):
107 def __getattr__(self, key):
108 if key not in self.metadata:
108 if key not in self.metadata:
109 raise AttributeError(key)
109 raise AttributeError(key)
110 return self.metadata[key]
110 return self.metadata[key]
111
111
112 def __repr__(self):
112 def __repr__(self):
113 execute_result = self.metadata['execute_result'] or {'data':{}}
113 execute_result = self.metadata['execute_result'] or {'data':{}}
114 text_out = execute_result['data'].get('text/plain', '')
114 text_out = execute_result['data'].get('text/plain', '')
115 if len(text_out) > 32:
115 if len(text_out) > 32:
116 text_out = text_out[:29] + '...'
116 text_out = text_out[:29] + '...'
117
117
118 return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
118 return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
119
119
120 def _repr_pretty_(self, p, cycle):
120 def _repr_pretty_(self, p, cycle):
121 execute_result = self.metadata['execute_result'] or {'data':{}}
121 execute_result = self.metadata['execute_result'] or {'data':{}}
122 text_out = execute_result['data'].get('text/plain', '')
122 text_out = execute_result['data'].get('text/plain', '')
123
123
124 if not text_out:
124 if not text_out:
125 return
125 return
126
126
127 try:
127 try:
128 ip = get_ipython()
128 ip = get_ipython()
129 except NameError:
129 except NameError:
130 colors = "NoColor"
130 colors = "NoColor"
131 else:
131 else:
132 colors = ip.colors
132 colors = ip.colors
133
133
134 if colors == "NoColor":
134 if colors == "NoColor":
135 out = normal = ""
135 out = normal = ""
136 else:
136 else:
137 out = TermColors.Red
137 out = TermColors.Red
138 normal = TermColors.Normal
138 normal = TermColors.Normal
139
139
140 if '\n' in text_out and not text_out.startswith('\n'):
140 if '\n' in text_out and not text_out.startswith('\n'):
141 # add newline for multiline reprs
141 # add newline for multiline reprs
142 text_out = '\n' + text_out
142 text_out = '\n' + text_out
143
143
144 p.text(
144 p.text(
145 out + u'Out[%i:%i]: ' % (
145 out + u'Out[%i:%i]: ' % (
146 self.metadata['engine_id'], self.execution_count
146 self.metadata['engine_id'], self.execution_count
147 ) + normal + text_out
147 ) + normal + text_out
148 )
148 )
149
149
150
150
151 class Metadata(dict):
151 class Metadata(dict):
152 """Subclass of dict for initializing metadata values.
152 """Subclass of dict for initializing metadata values.
153
153
154 Attribute access works on keys.
154 Attribute access works on keys.
155
155
156 These objects have a strict set of keys - errors will raise if you try
156 These objects have a strict set of keys - errors will raise if you try
157 to add new keys.
157 to add new keys.
158 """
158 """
159 def __init__(self, *args, **kwargs):
159 def __init__(self, *args, **kwargs):
160 dict.__init__(self)
160 dict.__init__(self)
161 md = {'msg_id' : None,
161 md = {'msg_id' : None,
162 'submitted' : None,
162 'submitted' : None,
163 'started' : None,
163 'started' : None,
164 'completed' : None,
164 'completed' : None,
165 'received' : None,
165 'received' : None,
166 'engine_uuid' : None,
166 'engine_uuid' : None,
167 'engine_id' : None,
167 'engine_id' : None,
168 'follow' : None,
168 'follow' : None,
169 'after' : None,
169 'after' : None,
170 'status' : None,
170 'status' : None,
171
171
172 'execute_input' : None,
172 'execute_input' : None,
173 'execute_result' : None,
173 'execute_result' : None,
174 'error' : None,
174 'error' : None,
175 'stdout' : '',
175 'stdout' : '',
176 'stderr' : '',
176 'stderr' : '',
177 'outputs' : [],
177 'outputs' : [],
178 'data': {},
178 'data': {},
179 'outputs_ready' : False,
179 'outputs_ready' : False,
180 }
180 }
181 self.update(md)
181 self.update(md)
182 self.update(dict(*args, **kwargs))
182 self.update(dict(*args, **kwargs))
183
183
184 def __getattr__(self, key):
184 def __getattr__(self, key):
185 """getattr aliased to getitem"""
185 """getattr aliased to getitem"""
186 if key in self:
186 if key in self:
187 return self[key]
187 return self[key]
188 else:
188 else:
189 raise AttributeError(key)
189 raise AttributeError(key)
190
190
191 def __setattr__(self, key, value):
191 def __setattr__(self, key, value):
192 """setattr aliased to setitem, with strict"""
192 """setattr aliased to setitem, with strict"""
193 if key in self:
193 if key in self:
194 self[key] = value
194 self[key] = value
195 else:
195 else:
196 raise AttributeError(key)
196 raise AttributeError(key)
197
197
198 def __setitem__(self, key, value):
198 def __setitem__(self, key, value):
199 """strict static key enforcement"""
199 """strict static key enforcement"""
200 if key in self:
200 if key in self:
201 dict.__setitem__(self, key, value)
201 dict.__setitem__(self, key, value)
202 else:
202 else:
203 raise KeyError(key)
203 raise KeyError(key)
204
204
205
205
206 class Client(HasTraits):
206 class Client(HasTraits):
207 """A semi-synchronous client to the IPython ZMQ cluster
207 """A semi-synchronous client to the IPython ZMQ cluster
208
208
209 Parameters
209 Parameters
210 ----------
210 ----------
211
211
212 url_file : str/unicode; path to ipcontroller-client.json
212 url_file : str/unicode; path to ipcontroller-client.json
213 This JSON file should contain all the information needed to connect to a cluster,
213 This JSON file should contain all the information needed to connect to a cluster,
214 and is likely the only argument needed.
214 and is likely the only argument needed.
215 Connection information for the Hub's registration. If a json connector
215 Connection information for the Hub's registration. If a json connector
216 file is given, then likely no further configuration is necessary.
216 file is given, then likely no further configuration is necessary.
217 [Default: use profile]
217 [Default: use profile]
218 profile : bytes
218 profile : bytes
219 The name of the Cluster profile to be used to find connector information.
219 The name of the Cluster profile to be used to find connector information.
220 If run from an IPython application, the default profile will be the same
220 If run from an IPython application, the default profile will be the same
221 as the running application, otherwise it will be 'default'.
221 as the running application, otherwise it will be 'default'.
222 cluster_id : str
222 cluster_id : str
223 String id to added to runtime files, to prevent name collisions when using
223 String id to added to runtime files, to prevent name collisions when using
224 multiple clusters with a single profile simultaneously.
224 multiple clusters with a single profile simultaneously.
225 When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
225 When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
226 Since this is text inserted into filenames, typical recommendations apply:
226 Since this is text inserted into filenames, typical recommendations apply:
227 Simple character strings are ideal, and spaces are not recommended (but
227 Simple character strings are ideal, and spaces are not recommended (but
228 should generally work)
228 should generally work)
229 context : zmq.Context
229 context : zmq.Context
230 Pass an existing zmq.Context instance, otherwise the client will create its own.
230 Pass an existing zmq.Context instance, otherwise the client will create its own.
231 debug : bool
231 debug : bool
232 flag for lots of message printing for debug purposes
232 flag for lots of message printing for debug purposes
233 timeout : int/float
233 timeout : int/float
234 time (in seconds) to wait for connection replies from the Hub
234 time (in seconds) to wait for connection replies from the Hub
235 [Default: 10]
235 [Default: 10]
236
236
237 #-------------- session related args ----------------
237 #-------------- session related args ----------------
238
238
239 config : Config object
239 config : Config object
240 If specified, this will be relayed to the Session for configuration
240 If specified, this will be relayed to the Session for configuration
241 username : str
241 username : str
242 set username for the session object
242 set username for the session object
243
243
244 #-------------- ssh related args ----------------
244 #-------------- ssh related args ----------------
245 # These are args for configuring the ssh tunnel to be used
245 # These are args for configuring the ssh tunnel to be used
246 # credentials are used to forward connections over ssh to the Controller
246 # credentials are used to forward connections over ssh to the Controller
247 # Note that the ip given in `addr` needs to be relative to sshserver
247 # Note that the ip given in `addr` needs to be relative to sshserver
248 # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
248 # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
249 # and set sshserver as the same machine the Controller is on. However,
249 # and set sshserver as the same machine the Controller is on. However,
250 # the only requirement is that sshserver is able to see the Controller
250 # the only requirement is that sshserver is able to see the Controller
251 # (i.e. is within the same trusted network).
251 # (i.e. is within the same trusted network).
252
252
253 sshserver : str
253 sshserver : str
254 A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port'
254 A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port'
255 If keyfile or password is specified, and this is not, it will default to
255 If keyfile or password is specified, and this is not, it will default to
256 the ip given in addr.
256 the ip given in addr.
257 sshkey : str; path to ssh private key file
257 sshkey : str; path to ssh private key file
258 This specifies a key to be used in ssh login, default None.
258 This specifies a key to be used in ssh login, default None.
259 Regular default ssh keys will be used without specifying this argument.
259 Regular default ssh keys will be used without specifying this argument.
260 password : str
260 password : str
261 Your ssh password to sshserver. Note that if this is left None,
261 Your ssh password to sshserver. Note that if this is left None,
262 you will be prompted for it if passwordless key based login is unavailable.
262 you will be prompted for it if passwordless key based login is unavailable.
263 paramiko : bool
263 paramiko : bool
264 flag for whether to use paramiko instead of shell ssh for tunneling.
264 flag for whether to use paramiko instead of shell ssh for tunneling.
265 [default: True on win32, False else]
265 [default: True on win32, False else]
266
266
267
267
268 Attributes
268 Attributes
269 ----------
269 ----------
270
270
271 ids : list of int engine IDs
271 ids : list of int engine IDs
272 requesting the ids attribute always synchronizes
272 requesting the ids attribute always synchronizes
273 the registration state. To request ids without synchronization,
273 the registration state. To request ids without synchronization,
274 use semi-private _ids attributes.
274 use semi-private _ids attributes.
275
275
276 history : list of msg_ids
276 history : list of msg_ids
277 a list of msg_ids, keeping track of all the execution
277 a list of msg_ids, keeping track of all the execution
278 messages you have submitted in order.
278 messages you have submitted in order.
279
279
280 outstanding : set of msg_ids
280 outstanding : set of msg_ids
281 a set of msg_ids that have been submitted, but whose
281 a set of msg_ids that have been submitted, but whose
282 results have not yet been received.
282 results have not yet been received.
283
283
284 results : dict
284 results : dict
285 a dict of all our results, keyed by msg_id
285 a dict of all our results, keyed by msg_id
286
286
287 block : bool
287 block : bool
288 determines default behavior when block not specified
288 determines default behavior when block not specified
289 in execution methods
289 in execution methods
290
290
291 Methods
291 Methods
292 -------
292 -------
293
293
294 spin
294 spin
295 flushes incoming results and registration state changes
295 flushes incoming results and registration state changes
296 control methods spin, and requesting `ids` also ensures up to date
296 control methods spin, and requesting `ids` also ensures up to date
297
297
298 wait
298 wait
299 wait on one or more msg_ids
299 wait on one or more msg_ids
300
300
301 execution methods
301 execution methods
302 apply
302 apply
303 legacy: execute, run
303 legacy: execute, run
304
304
305 data movement
305 data movement
306 push, pull, scatter, gather
306 push, pull, scatter, gather
307
307
308 query methods
308 query methods
309 queue_status, get_result, purge, result_status
309 queue_status, get_result, purge, result_status
310
310
311 control methods
311 control methods
312 abort, shutdown
312 abort, shutdown
313
313
314 """
314 """
315
315
316
316
317 block = Bool(False)
317 block = Bool(False)
318 outstanding = Set()
318 outstanding = Set()
319 results = Instance('collections.defaultdict', (dict,))
319 results = Instance('collections.defaultdict', (dict,))
320 metadata = Instance('collections.defaultdict', (Metadata,))
320 metadata = Instance('collections.defaultdict', (Metadata,))
321 history = List()
321 history = List()
322 debug = Bool(False)
322 debug = Bool(False)
323 _spin_thread = Any()
323 _spin_thread = Any()
324 _stop_spinning = Any()
324 _stop_spinning = Any()
325
325
326 profile=Unicode()
326 profile=Unicode()
327 def _profile_default(self):
327 def _profile_default(self):
328 if BaseIPythonApplication.initialized():
328 if BaseIPythonApplication.initialized():
329 # an IPython app *might* be running, try to get its profile
329 # an IPython app *might* be running, try to get its profile
330 try:
330 try:
331 return BaseIPythonApplication.instance().profile
331 return BaseIPythonApplication.instance().profile
332 except (AttributeError, MultipleInstanceError):
332 except (AttributeError, MultipleInstanceError):
333 # could be a *different* subclass of config.Application,
333 # could be a *different* subclass of config.Application,
334 # which would raise one of these two errors.
334 # which would raise one of these two errors.
335 return u'default'
335 return u'default'
336 else:
336 else:
337 return u'default'
337 return u'default'
338
338
339
339
340 _outstanding_dict = Instance('collections.defaultdict', (set,))
340 _outstanding_dict = Instance('collections.defaultdict', (set,))
341 _ids = List()
341 _ids = List()
342 _connected=Bool(False)
342 _connected=Bool(False)
343 _ssh=Bool(False)
343 _ssh=Bool(False)
344 _context = Instance('zmq.Context')
344 _context = Instance('zmq.Context')
345 _config = Dict()
345 _config = Dict()
346 _engines=Instance(util.ReverseDict, (), {})
346 _engines=Instance(util.ReverseDict, (), {})
347 # _hub_socket=Instance('zmq.Socket')
347 # _hub_socket=Instance('zmq.Socket')
348 _query_socket=Instance('zmq.Socket')
348 _query_socket=Instance('zmq.Socket')
349 _control_socket=Instance('zmq.Socket')
349 _control_socket=Instance('zmq.Socket')
350 _iopub_socket=Instance('zmq.Socket')
350 _iopub_socket=Instance('zmq.Socket')
351 _notification_socket=Instance('zmq.Socket')
351 _notification_socket=Instance('zmq.Socket')
352 _mux_socket=Instance('zmq.Socket')
352 _mux_socket=Instance('zmq.Socket')
353 _task_socket=Instance('zmq.Socket')
353 _task_socket=Instance('zmq.Socket')
354 _task_scheme=Unicode()
354 _task_scheme=Unicode()
355 _closed = False
355 _closed = False
356 _ignored_control_replies=Integer(0)
356 _ignored_control_replies=Integer(0)
357 _ignored_hub_replies=Integer(0)
357 _ignored_hub_replies=Integer(0)
358
358
359 def __new__(self, *args, **kw):
359 def __new__(self, *args, **kw):
360 # don't raise on positional args
360 # don't raise on positional args
361 return HasTraits.__new__(self, **kw)
361 return HasTraits.__new__(self, **kw)
362
362
363 def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
363 def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
364 context=None, debug=False,
364 context=None, debug=False,
365 sshserver=None, sshkey=None, password=None, paramiko=None,
365 sshserver=None, sshkey=None, password=None, paramiko=None,
366 timeout=10, cluster_id=None, **extra_args
366 timeout=10, cluster_id=None, **extra_args
367 ):
367 ):
368 if profile:
368 if profile:
369 super(Client, self).__init__(debug=debug, profile=profile)
369 super(Client, self).__init__(debug=debug, profile=profile)
370 else:
370 else:
371 super(Client, self).__init__(debug=debug)
371 super(Client, self).__init__(debug=debug)
372 if context is None:
372 if context is None:
373 context = zmq.Context.instance()
373 context = zmq.Context.instance()
374 self._context = context
374 self._context = context
375 self._stop_spinning = Event()
375 self._stop_spinning = Event()
376
376
377 if 'url_or_file' in extra_args:
377 if 'url_or_file' in extra_args:
378 url_file = extra_args['url_or_file']
378 url_file = extra_args['url_or_file']
379 warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
379 warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
380
380
381 if url_file and util.is_url(url_file):
381 if url_file and util.is_url(url_file):
382 raise ValueError("single urls cannot be specified, url-files must be used.")
382 raise ValueError("single urls cannot be specified, url-files must be used.")
383
383
384 self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
384 self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
385
385
386 if self._cd is not None:
386 if self._cd is not None:
387 if url_file is None:
387 if url_file is None:
388 if not cluster_id:
388 if not cluster_id:
389 client_json = 'ipcontroller-client.json'
389 client_json = 'ipcontroller-client.json'
390 else:
390 else:
391 client_json = 'ipcontroller-%s-client.json' % cluster_id
391 client_json = 'ipcontroller-%s-client.json' % cluster_id
392 url_file = pjoin(self._cd.security_dir, client_json)
392 url_file = pjoin(self._cd.security_dir, client_json)
393 if url_file is None:
393 if url_file is None:
394 raise ValueError(
394 raise ValueError(
395 "I can't find enough information to connect to a hub!"
395 "I can't find enough information to connect to a hub!"
396 " Please specify at least one of url_file or profile."
396 " Please specify at least one of url_file or profile."
397 )
397 )
398
398
399 with open(url_file) as f:
399 with open(url_file) as f:
400 cfg = json.load(f)
400 cfg = json.load(f)
401
401
402 self._task_scheme = cfg['task_scheme']
402 self._task_scheme = cfg['task_scheme']
403
403
404 # sync defaults from args, json:
404 # sync defaults from args, json:
405 if sshserver:
405 if sshserver:
406 cfg['ssh'] = sshserver
406 cfg['ssh'] = sshserver
407
407
408 location = cfg.setdefault('location', None)
408 location = cfg.setdefault('location', None)
409
409
410 proto,addr = cfg['interface'].split('://')
410 proto,addr = cfg['interface'].split('://')
411 addr = util.disambiguate_ip_address(addr, location)
411 addr = util.disambiguate_ip_address(addr, location)
412 cfg['interface'] = "%s://%s" % (proto, addr)
412 cfg['interface'] = "%s://%s" % (proto, addr)
413
413
414 # turn interface,port into full urls:
414 # turn interface,port into full urls:
415 for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
415 for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
416 cfg[key] = cfg['interface'] + ':%i' % cfg[key]
416 cfg[key] = cfg['interface'] + ':%i' % cfg[key]
417
417
418 url = cfg['registration']
418 url = cfg['registration']
419
419
420 if location is not None and addr == localhost():
420 if location is not None and addr == localhost():
421 # location specified, and connection is expected to be local
421 # location specified, and connection is expected to be local
422 if not is_local_ip(location) and not sshserver:
422 if not is_local_ip(location) and not sshserver:
423 # load ssh from JSON *only* if the controller is not on
423 # load ssh from JSON *only* if the controller is not on
424 # this machine
424 # this machine
425 sshserver=cfg['ssh']
425 sshserver=cfg['ssh']
426 if not is_local_ip(location) and not sshserver:
426 if not is_local_ip(location) and not sshserver:
427 # warn if no ssh specified, but SSH is probably needed
427 # warn if no ssh specified, but SSH is probably needed
428 # This is only a warning, because the most likely cause
428 # This is only a warning, because the most likely cause
429 # is a local Controller on a laptop whose IP is dynamic
429 # is a local Controller on a laptop whose IP is dynamic
430 warnings.warn("""
430 warnings.warn("""
431 Controller appears to be listening on localhost, but not on this machine.
431 Controller appears to be listening on localhost, but not on this machine.
432 If this is true, you should specify Client(...,sshserver='you@%s')
432 If this is true, you should specify Client(...,sshserver='you@%s')
433 or instruct your controller to listen on an external IP."""%location,
433 or instruct your controller to listen on an external IP."""%location,
434 RuntimeWarning)
434 RuntimeWarning)
435 elif not sshserver:
435 elif not sshserver:
436 # otherwise sync with cfg
436 # otherwise sync with cfg
437 sshserver = cfg['ssh']
437 sshserver = cfg['ssh']
438
438
439 self._config = cfg
439 self._config = cfg
440
440
441 self._ssh = bool(sshserver or sshkey or password)
441 self._ssh = bool(sshserver or sshkey or password)
442 if self._ssh and sshserver is None:
442 if self._ssh and sshserver is None:
443 # default to ssh via localhost
443 # default to ssh via localhost
444 sshserver = addr
444 sshserver = addr
445 if self._ssh and password is None:
445 if self._ssh and password is None:
446 if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
446 if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
447 password=False
447 password=False
448 else:
448 else:
449 password = getpass("SSH Password for %s: "%sshserver)
449 password = getpass("SSH Password for %s: "%sshserver)
450 ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
450 ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
451
451
452 # configure and construct the session
452 # configure and construct the session
453 try:
453 try:
454 extra_args['packer'] = cfg['pack']
454 extra_args['packer'] = cfg['pack']
455 extra_args['unpacker'] = cfg['unpack']
455 extra_args['unpacker'] = cfg['unpack']
456 extra_args['key'] = cast_bytes(cfg['key'])
456 extra_args['key'] = cast_bytes(cfg['key'])
457 extra_args['signature_scheme'] = cfg['signature_scheme']
457 extra_args['signature_scheme'] = cfg['signature_scheme']
458 except KeyError as exc:
458 except KeyError as exc:
459 msg = '\n'.join([
459 msg = '\n'.join([
460 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
460 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
461 "If you are reusing connection files, remove them and start ipcontroller again."
461 "If you are reusing connection files, remove them and start ipcontroller again."
462 ])
462 ])
463 raise ValueError(msg.format(exc.message))
463 raise ValueError(msg.format(exc.message))
464
464
465 self.session = Session(**extra_args)
465 self.session = Session(**extra_args)
466
466
467 self._query_socket = self._context.socket(zmq.DEALER)
467 self._query_socket = self._context.socket(zmq.DEALER)
468
468
469 if self._ssh:
469 if self._ssh:
470 tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
470 tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
471 else:
471 else:
472 self._query_socket.connect(cfg['registration'])
472 self._query_socket.connect(cfg['registration'])
473
473
474 self.session.debug = self.debug
474 self.session.debug = self.debug
475
475
476 self._notification_handlers = {'registration_notification' : self._register_engine,
476 self._notification_handlers = {'registration_notification' : self._register_engine,
477 'unregistration_notification' : self._unregister_engine,
477 'unregistration_notification' : self._unregister_engine,
478 'shutdown_notification' : lambda msg: self.close(),
478 'shutdown_notification' : lambda msg: self.close(),
479 }
479 }
480 self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
480 self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
481 'apply_reply' : self._handle_apply_reply}
481 'apply_reply' : self._handle_apply_reply}
482
482
483 try:
483 try:
484 self._connect(sshserver, ssh_kwargs, timeout)
484 self._connect(sshserver, ssh_kwargs, timeout)
485 except:
485 except:
486 self.close(linger=0)
486 self.close(linger=0)
487 raise
487 raise
488
488
489 # last step: setup magics, if we are in IPython:
489 # last step: setup magics, if we are in IPython:
490
490
491 try:
491 try:
492 ip = get_ipython()
492 ip = get_ipython()
493 except NameError:
493 except NameError:
494 return
494 return
495 else:
495 else:
496 if 'px' not in ip.magics_manager.magics:
496 if 'px' not in ip.magics_manager.magics:
497 # in IPython but we are the first Client.
497 # in IPython but we are the first Client.
498 # activate a default view for parallel magics.
498 # activate a default view for parallel magics.
499 self.activate()
499 self.activate()
500
500
501 def __del__(self):
501 def __del__(self):
502 """cleanup sockets, but _not_ context."""
502 """cleanup sockets, but _not_ context."""
503 self.close()
503 self.close()
504
504
505 def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
505 def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
506 if ipython_dir is None:
506 if ipython_dir is None:
507 ipython_dir = get_ipython_dir()
507 ipython_dir = get_ipython_dir()
508 if profile_dir is not None:
508 if profile_dir is not None:
509 try:
509 try:
510 self._cd = ProfileDir.find_profile_dir(profile_dir)
510 self._cd = ProfileDir.find_profile_dir(profile_dir)
511 return
511 return
512 except ProfileDirError:
512 except ProfileDirError:
513 pass
513 pass
514 elif profile is not None:
514 elif profile is not None:
515 try:
515 try:
516 self._cd = ProfileDir.find_profile_dir_by_name(
516 self._cd = ProfileDir.find_profile_dir_by_name(
517 ipython_dir, profile)
517 ipython_dir, profile)
518 return
518 return
519 except ProfileDirError:
519 except ProfileDirError:
520 pass
520 pass
521 self._cd = None
521 self._cd = None
522
522
523 def _update_engines(self, engines):
523 def _update_engines(self, engines):
524 """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
524 """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
525 for k,v in iteritems(engines):
525 for k,v in iteritems(engines):
526 eid = int(k)
526 eid = int(k)
527 if eid not in self._engines:
527 if eid not in self._engines:
528 self._ids.append(eid)
528 self._ids.append(eid)
529 self._engines[eid] = v
529 self._engines[eid] = v
530 self._ids = sorted(self._ids)
530 self._ids = sorted(self._ids)
531 if sorted(self._engines.keys()) != list(range(len(self._engines))) and \
531 if sorted(self._engines.keys()) != list(range(len(self._engines))) and \
532 self._task_scheme == 'pure' and self._task_socket:
532 self._task_scheme == 'pure' and self._task_socket:
533 self._stop_scheduling_tasks()
533 self._stop_scheduling_tasks()
534
534
535 def _stop_scheduling_tasks(self):
535 def _stop_scheduling_tasks(self):
536 """Stop scheduling tasks because an engine has been unregistered
536 """Stop scheduling tasks because an engine has been unregistered
537 from a pure ZMQ scheduler.
537 from a pure ZMQ scheduler.
538 """
538 """
539 self._task_socket.close()
539 self._task_socket.close()
540 self._task_socket = None
540 self._task_socket = None
541 msg = "An engine has been unregistered, and we are using pure " +\
541 msg = "An engine has been unregistered, and we are using pure " +\
542 "ZMQ task scheduling. Task farming will be disabled."
542 "ZMQ task scheduling. Task farming will be disabled."
543 if self.outstanding:
543 if self.outstanding:
544 msg += " If you were running tasks when this happened, " +\
544 msg += " If you were running tasks when this happened, " +\
545 "some `outstanding` msg_ids may never resolve."
545 "some `outstanding` msg_ids may never resolve."
546 warnings.warn(msg, RuntimeWarning)
546 warnings.warn(msg, RuntimeWarning)
547
547
548 def _build_targets(self, targets):
548 def _build_targets(self, targets):
549 """Turn valid target IDs or 'all' into two lists:
549 """Turn valid target IDs or 'all' into two lists:
550 (int_ids, uuids).
550 (int_ids, uuids).
551 """
551 """
552 if not self._ids:
552 if not self._ids:
553 # flush notification socket if no engines yet, just in case
553 # flush notification socket if no engines yet, just in case
554 if not self.ids:
554 if not self.ids:
555 raise error.NoEnginesRegistered("Can't build targets without any engines")
555 raise error.NoEnginesRegistered("Can't build targets without any engines")
556
556
557 if targets is None:
557 if targets is None:
558 targets = self._ids
558 targets = self._ids
559 elif isinstance(targets, string_types):
559 elif isinstance(targets, string_types):
560 if targets.lower() == 'all':
560 if targets.lower() == 'all':
561 targets = self._ids
561 targets = self._ids
562 else:
562 else:
563 raise TypeError("%r not valid str target, must be 'all'"%(targets))
563 raise TypeError("%r not valid str target, must be 'all'"%(targets))
564 elif isinstance(targets, int):
564 elif isinstance(targets, int):
565 if targets < 0:
565 if targets < 0:
566 targets = self.ids[targets]
566 targets = self.ids[targets]
567 if targets not in self._ids:
567 if targets not in self._ids:
568 raise IndexError("No such engine: %i"%targets)
568 raise IndexError("No such engine: %i"%targets)
569 targets = [targets]
569 targets = [targets]
570
570
571 if isinstance(targets, slice):
571 if isinstance(targets, slice):
572 indices = list(range(len(self._ids))[targets])
572 indices = list(range(len(self._ids))[targets])
573 ids = self.ids
573 ids = self.ids
574 targets = [ ids[i] for i in indices ]
574 targets = [ ids[i] for i in indices ]
575
575
576 if not isinstance(targets, (tuple, list, xrange)):
576 if not isinstance(targets, (tuple, list, xrange)):
577 raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
577 raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
578
578
579 return [cast_bytes(self._engines[t]) for t in targets], list(targets)
579 return [cast_bytes(self._engines[t]) for t in targets], list(targets)
580
580
581 def _connect(self, sshserver, ssh_kwargs, timeout):
581 def _connect(self, sshserver, ssh_kwargs, timeout):
582 """setup all our socket connections to the cluster. This is called from
582 """setup all our socket connections to the cluster. This is called from
583 __init__."""
583 __init__."""
584
584
585 # Maybe allow reconnecting?
585 # Maybe allow reconnecting?
586 if self._connected:
586 if self._connected:
587 return
587 return
588 self._connected=True
588 self._connected=True
589
589
590 def connect_socket(s, url):
590 def connect_socket(s, url):
591 if self._ssh:
591 if self._ssh:
592 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
592 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
593 else:
593 else:
594 return s.connect(url)
594 return s.connect(url)
595
595
596 self.session.send(self._query_socket, 'connection_request')
596 self.session.send(self._query_socket, 'connection_request')
597 # use Poller because zmq.select has wrong units in pyzmq 2.1.7
597 # use Poller because zmq.select has wrong units in pyzmq 2.1.7
598 poller = zmq.Poller()
598 poller = zmq.Poller()
599 poller.register(self._query_socket, zmq.POLLIN)
599 poller.register(self._query_socket, zmq.POLLIN)
600 # poll expects milliseconds, timeout is seconds
600 # poll expects milliseconds, timeout is seconds
601 evts = poller.poll(timeout*1000)
601 evts = poller.poll(timeout*1000)
602 if not evts:
602 if not evts:
603 raise error.TimeoutError("Hub connection request timed out")
603 raise error.TimeoutError("Hub connection request timed out")
604 idents,msg = self.session.recv(self._query_socket,mode=0)
604 idents,msg = self.session.recv(self._query_socket,mode=0)
605 if self.debug:
605 if self.debug:
606 pprint(msg)
606 pprint(msg)
607 content = msg['content']
607 content = msg['content']
608 # self._config['registration'] = dict(content)
608 # self._config['registration'] = dict(content)
609 cfg = self._config
609 cfg = self._config
610 if content['status'] == 'ok':
610 if content['status'] == 'ok':
611 self._mux_socket = self._context.socket(zmq.DEALER)
611 self._mux_socket = self._context.socket(zmq.DEALER)
612 connect_socket(self._mux_socket, cfg['mux'])
612 connect_socket(self._mux_socket, cfg['mux'])
613
613
614 self._task_socket = self._context.socket(zmq.DEALER)
614 self._task_socket = self._context.socket(zmq.DEALER)
615 connect_socket(self._task_socket, cfg['task'])
615 connect_socket(self._task_socket, cfg['task'])
616
616
617 self._notification_socket = self._context.socket(zmq.SUB)
617 self._notification_socket = self._context.socket(zmq.SUB)
618 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
618 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
619 connect_socket(self._notification_socket, cfg['notification'])
619 connect_socket(self._notification_socket, cfg['notification'])
620
620
621 self._control_socket = self._context.socket(zmq.DEALER)
621 self._control_socket = self._context.socket(zmq.DEALER)
622 connect_socket(self._control_socket, cfg['control'])
622 connect_socket(self._control_socket, cfg['control'])
623
623
624 self._iopub_socket = self._context.socket(zmq.SUB)
624 self._iopub_socket = self._context.socket(zmq.SUB)
625 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
625 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
626 connect_socket(self._iopub_socket, cfg['iopub'])
626 connect_socket(self._iopub_socket, cfg['iopub'])
627
627
628 self._update_engines(dict(content['engines']))
628 self._update_engines(dict(content['engines']))
629 else:
629 else:
630 self._connected = False
630 self._connected = False
631 raise Exception("Failed to connect!")
631 raise Exception("Failed to connect!")
632
632
633 #--------------------------------------------------------------------------
633 #--------------------------------------------------------------------------
634 # handlers and callbacks for incoming messages
634 # handlers and callbacks for incoming messages
635 #--------------------------------------------------------------------------
635 #--------------------------------------------------------------------------
636
636
637 def _unwrap_exception(self, content):
637 def _unwrap_exception(self, content):
638 """unwrap exception, and remap engine_id to int."""
638 """unwrap exception, and remap engine_id to int."""
639 e = error.unwrap_exception(content)
639 e = error.unwrap_exception(content)
640 # print e.traceback
640 # print e.traceback
641 if e.engine_info:
641 if e.engine_info:
642 e_uuid = e.engine_info['engine_uuid']
642 e_uuid = e.engine_info['engine_uuid']
643 eid = self._engines[e_uuid]
643 eid = self._engines[e_uuid]
644 e.engine_info['engine_id'] = eid
644 e.engine_info['engine_id'] = eid
645 return e
645 return e
646
646
647 def _extract_metadata(self, msg):
647 def _extract_metadata(self, msg):
648 header = msg['header']
648 header = msg['header']
649 parent = msg['parent_header']
649 parent = msg['parent_header']
650 msg_meta = msg['metadata']
650 msg_meta = msg['metadata']
651 content = msg['content']
651 content = msg['content']
652 md = {'msg_id' : parent['msg_id'],
652 md = {'msg_id' : parent['msg_id'],
653 'received' : datetime.now(),
653 'received' : datetime.now(),
654 'engine_uuid' : msg_meta.get('engine', None),
654 'engine_uuid' : msg_meta.get('engine', None),
655 'follow' : msg_meta.get('follow', []),
655 'follow' : msg_meta.get('follow', []),
656 'after' : msg_meta.get('after', []),
656 'after' : msg_meta.get('after', []),
657 'status' : content['status'],
657 'status' : content['status'],
658 }
658 }
659
659
660 if md['engine_uuid'] is not None:
660 if md['engine_uuid'] is not None:
661 md['engine_id'] = self._engines.get(md['engine_uuid'], None)
661 md['engine_id'] = self._engines.get(md['engine_uuid'], None)
662
662
663 if 'date' in parent:
663 if 'date' in parent:
664 md['submitted'] = parent['date']
664 md['submitted'] = parent['date']
665 if 'started' in msg_meta:
665 if 'started' in msg_meta:
666 md['started'] = parse_date(msg_meta['started'])
666 md['started'] = parse_date(msg_meta['started'])
667 if 'date' in header:
667 if 'date' in header:
668 md['completed'] = header['date']
668 md['completed'] = header['date']
669 return md
669 return md
670
670
671 def _register_engine(self, msg):
671 def _register_engine(self, msg):
672 """Register a new engine, and update our connection info."""
672 """Register a new engine, and update our connection info."""
673 content = msg['content']
673 content = msg['content']
674 eid = content['id']
674 eid = content['id']
675 d = {eid : content['uuid']}
675 d = {eid : content['uuid']}
676 self._update_engines(d)
676 self._update_engines(d)
677
677
678 def _unregister_engine(self, msg):
678 def _unregister_engine(self, msg):
679 """Unregister an engine that has died."""
679 """Unregister an engine that has died."""
680 content = msg['content']
680 content = msg['content']
681 eid = int(content['id'])
681 eid = int(content['id'])
682 if eid in self._ids:
682 if eid in self._ids:
683 self._ids.remove(eid)
683 self._ids.remove(eid)
684 uuid = self._engines.pop(eid)
684 uuid = self._engines.pop(eid)
685
685
686 self._handle_stranded_msgs(eid, uuid)
686 self._handle_stranded_msgs(eid, uuid)
687
687
688 if self._task_socket and self._task_scheme == 'pure':
688 if self._task_socket and self._task_scheme == 'pure':
689 self._stop_scheduling_tasks()
689 self._stop_scheduling_tasks()
690
690
691 def _handle_stranded_msgs(self, eid, uuid):
691 def _handle_stranded_msgs(self, eid, uuid):
692 """Handle messages known to be on an engine when the engine unregisters.
692 """Handle messages known to be on an engine when the engine unregisters.
693
693
694 It is possible that this will fire prematurely - that is, an engine will
694 It is possible that this will fire prematurely - that is, an engine will
695 go down after completing a result, and the client will be notified
695 go down after completing a result, and the client will be notified
696 of the unregistration and later receive the successful result.
696 of the unregistration and later receive the successful result.
697 """
697 """
698
698
699 outstanding = self._outstanding_dict[uuid]
699 outstanding = self._outstanding_dict[uuid]
700
700
701 for msg_id in list(outstanding):
701 for msg_id in list(outstanding):
702 if msg_id in self.results:
702 if msg_id in self.results:
703 # we already
703 # we already
704 continue
704 continue
705 try:
705 try:
706 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
706 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
707 except:
707 except:
708 content = error.wrap_exception()
708 content = error.wrap_exception()
709 # build a fake message:
709 # build a fake message:
710 msg = self.session.msg('apply_reply', content=content)
710 msg = self.session.msg('apply_reply', content=content)
711 msg['parent_header']['msg_id'] = msg_id
711 msg['parent_header']['msg_id'] = msg_id
712 msg['metadata']['engine'] = uuid
712 msg['metadata']['engine'] = uuid
713 self._handle_apply_reply(msg)
713 self._handle_apply_reply(msg)
714
714
715 def _handle_execute_reply(self, msg):
715 def _handle_execute_reply(self, msg):
716 """Save the reply to an execute_request into our results.
716 """Save the reply to an execute_request into our results.
717
717
718 execute messages are never actually used. apply is used instead.
718 execute messages are never actually used. apply is used instead.
719 """
719 """
720
720
721 parent = msg['parent_header']
721 parent = msg['parent_header']
722 msg_id = parent['msg_id']
722 msg_id = parent['msg_id']
723 if msg_id not in self.outstanding:
723 if msg_id not in self.outstanding:
724 if msg_id in self.history:
724 if msg_id in self.history:
725 print("got stale result: %s"%msg_id)
725 print("got stale result: %s"%msg_id)
726 else:
726 else:
727 print("got unknown result: %s"%msg_id)
727 print("got unknown result: %s"%msg_id)
728 else:
728 else:
729 self.outstanding.remove(msg_id)
729 self.outstanding.remove(msg_id)
730
730
731 content = msg['content']
731 content = msg['content']
732 header = msg['header']
732 header = msg['header']
733
733
734 # construct metadata:
734 # construct metadata:
735 md = self.metadata[msg_id]
735 md = self.metadata[msg_id]
736 md.update(self._extract_metadata(msg))
736 md.update(self._extract_metadata(msg))
737 # is this redundant?
737 # is this redundant?
738 self.metadata[msg_id] = md
738 self.metadata[msg_id] = md
739
739
740 e_outstanding = self._outstanding_dict[md['engine_uuid']]
740 e_outstanding = self._outstanding_dict[md['engine_uuid']]
741 if msg_id in e_outstanding:
741 if msg_id in e_outstanding:
742 e_outstanding.remove(msg_id)
742 e_outstanding.remove(msg_id)
743
743
744 # construct result:
744 # construct result:
745 if content['status'] == 'ok':
745 if content['status'] == 'ok':
746 self.results[msg_id] = ExecuteReply(msg_id, content, md)
746 self.results[msg_id] = ExecuteReply(msg_id, content, md)
747 elif content['status'] == 'aborted':
747 elif content['status'] == 'aborted':
748 self.results[msg_id] = error.TaskAborted(msg_id)
748 self.results[msg_id] = error.TaskAborted(msg_id)
749 elif content['status'] == 'resubmitted':
749 elif content['status'] == 'resubmitted':
750 # TODO: handle resubmission
750 # TODO: handle resubmission
751 pass
751 pass
752 else:
752 else:
753 self.results[msg_id] = self._unwrap_exception(content)
753 self.results[msg_id] = self._unwrap_exception(content)
754
754
755 def _handle_apply_reply(self, msg):
755 def _handle_apply_reply(self, msg):
756 """Save the reply to an apply_request into our results."""
756 """Save the reply to an apply_request into our results."""
757 parent = msg['parent_header']
757 parent = msg['parent_header']
758 msg_id = parent['msg_id']
758 msg_id = parent['msg_id']
759 if msg_id not in self.outstanding:
759 if msg_id not in self.outstanding:
760 if msg_id in self.history:
760 if msg_id in self.history:
761 print("got stale result: %s"%msg_id)
761 print("got stale result: %s"%msg_id)
762 print(self.results[msg_id])
762 print(self.results[msg_id])
763 print(msg)
763 print(msg)
764 else:
764 else:
765 print("got unknown result: %s"%msg_id)
765 print("got unknown result: %s"%msg_id)
766 else:
766 else:
767 self.outstanding.remove(msg_id)
767 self.outstanding.remove(msg_id)
768 content = msg['content']
768 content = msg['content']
769 header = msg['header']
769 header = msg['header']
770
770
771 # construct metadata:
771 # construct metadata:
772 md = self.metadata[msg_id]
772 md = self.metadata[msg_id]
773 md.update(self._extract_metadata(msg))
773 md.update(self._extract_metadata(msg))
774 # is this redundant?
774 # is this redundant?
775 self.metadata[msg_id] = md
775 self.metadata[msg_id] = md
776
776
777 e_outstanding = self._outstanding_dict[md['engine_uuid']]
777 e_outstanding = self._outstanding_dict[md['engine_uuid']]
778 if msg_id in e_outstanding:
778 if msg_id in e_outstanding:
779 e_outstanding.remove(msg_id)
779 e_outstanding.remove(msg_id)
780
780
781 # construct result:
781 # construct result:
782 if content['status'] == 'ok':
782 if content['status'] == 'ok':
783 self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
783 self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
784 elif content['status'] == 'aborted':
784 elif content['status'] == 'aborted':
785 self.results[msg_id] = error.TaskAborted(msg_id)
785 self.results[msg_id] = error.TaskAborted(msg_id)
786 elif content['status'] == 'resubmitted':
786 elif content['status'] == 'resubmitted':
787 # TODO: handle resubmission
787 # TODO: handle resubmission
788 pass
788 pass
789 else:
789 else:
790 self.results[msg_id] = self._unwrap_exception(content)
790 self.results[msg_id] = self._unwrap_exception(content)
791
791
792 def _flush_notifications(self):
792 def _flush_notifications(self):
793 """Flush notifications of engine registrations waiting
793 """Flush notifications of engine registrations waiting
794 in ZMQ queue."""
794 in ZMQ queue."""
795 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
795 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
796 while msg is not None:
796 while msg is not None:
797 if self.debug:
797 if self.debug:
798 pprint(msg)
798 pprint(msg)
799 msg_type = msg['header']['msg_type']
799 msg_type = msg['header']['msg_type']
800 handler = self._notification_handlers.get(msg_type, None)
800 handler = self._notification_handlers.get(msg_type, None)
801 if handler is None:
801 if handler is None:
802 raise Exception("Unhandled message type: %s" % msg_type)
802 raise Exception("Unhandled message type: %s" % msg_type)
803 else:
803 else:
804 handler(msg)
804 handler(msg)
805 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
805 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
806
806
807 def _flush_results(self, sock):
807 def _flush_results(self, sock):
808 """Flush task or queue results waiting in ZMQ queue."""
808 """Flush task or queue results waiting in ZMQ queue."""
809 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
809 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
810 while msg is not None:
810 while msg is not None:
811 if self.debug:
811 if self.debug:
812 pprint(msg)
812 pprint(msg)
813 msg_type = msg['header']['msg_type']
813 msg_type = msg['header']['msg_type']
814 handler = self._queue_handlers.get(msg_type, None)
814 handler = self._queue_handlers.get(msg_type, None)
815 if handler is None:
815 if handler is None:
816 raise Exception("Unhandled message type: %s" % msg_type)
816 raise Exception("Unhandled message type: %s" % msg_type)
817 else:
817 else:
818 handler(msg)
818 handler(msg)
819 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
819 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
820
820
821 def _flush_control(self, sock):
821 def _flush_control(self, sock):
822 """Flush replies from the control channel waiting
822 """Flush replies from the control channel waiting
823 in the ZMQ queue.
823 in the ZMQ queue.
824
824
825 Currently: ignore them."""
825 Currently: ignore them."""
826 if self._ignored_control_replies <= 0:
826 if self._ignored_control_replies <= 0:
827 return
827 return
828 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
828 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
829 while msg is not None:
829 while msg is not None:
830 self._ignored_control_replies -= 1
830 self._ignored_control_replies -= 1
831 if self.debug:
831 if self.debug:
832 pprint(msg)
832 pprint(msg)
833 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
833 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
834
834
835 def _flush_ignored_control(self):
835 def _flush_ignored_control(self):
836 """flush ignored control replies"""
836 """flush ignored control replies"""
837 while self._ignored_control_replies > 0:
837 while self._ignored_control_replies > 0:
838 self.session.recv(self._control_socket)
838 self.session.recv(self._control_socket)
839 self._ignored_control_replies -= 1
839 self._ignored_control_replies -= 1
840
840
841 def _flush_ignored_hub_replies(self):
841 def _flush_ignored_hub_replies(self):
842 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
842 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
843 while msg is not None:
843 while msg is not None:
844 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
844 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
845
845
846 def _flush_iopub(self, sock):
846 def _flush_iopub(self, sock):
847 """Flush replies from the iopub channel waiting
847 """Flush replies from the iopub channel waiting
848 in the ZMQ queue.
848 in the ZMQ queue.
849 """
849 """
850 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
850 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
851 while msg is not None:
851 while msg is not None:
852 if self.debug:
852 if self.debug:
853 pprint(msg)
853 pprint(msg)
854 parent = msg['parent_header']
854 parent = msg['parent_header']
855 # ignore IOPub messages with no parent.
855 # ignore IOPub messages with no parent.
856 # Caused by print statements or warnings from before the first execution.
856 # Caused by print statements or warnings from before the first execution.
857 if not parent:
857 if not parent:
858 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
858 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
859 continue
859 continue
860 msg_id = parent['msg_id']
860 msg_id = parent['msg_id']
861 content = msg['content']
861 content = msg['content']
862 header = msg['header']
862 header = msg['header']
863 msg_type = msg['header']['msg_type']
863 msg_type = msg['header']['msg_type']
864
864
865 # init metadata:
865 # init metadata:
866 md = self.metadata[msg_id]
866 md = self.metadata[msg_id]
867
867
868 if msg_type == 'stream':
868 if msg_type == 'stream':
869 name = content['name']
869 name = content['name']
870 s = md[name] or ''
870 s = md[name] or ''
871 md[name] = s + content['data']
871 md[name] = s + content['data']
872 elif msg_type == 'error':
872 elif msg_type == 'error':
873 md.update({'error' : self._unwrap_exception(content)})
873 md.update({'error' : self._unwrap_exception(content)})
874 elif msg_type == 'execute_input':
874 elif msg_type == 'execute_input':
875 md.update({'execute_input' : content['code']})
875 md.update({'execute_input' : content['code']})
876 elif msg_type == 'display_data':
876 elif msg_type == 'display_data':
877 md['outputs'].append(content)
877 md['outputs'].append(content)
878 elif msg_type == 'execute_result':
878 elif msg_type == 'execute_result':
879 md['execute_result'] = content
879 md['execute_result'] = content
880 elif msg_type == 'data_message':
880 elif msg_type == 'data_message':
881 data, remainder = serialize.unserialize_object(msg['buffers'])
881 data, remainder = serialize.unserialize_object(msg['buffers'])
882 md['data'].update(data)
882 md['data'].update(data)
883 elif msg_type == 'status':
883 elif msg_type == 'status':
884 # idle message comes after all outputs
884 # idle message comes after all outputs
885 if content['execution_state'] == 'idle':
885 if content['execution_state'] == 'idle':
886 md['outputs_ready'] = True
886 md['outputs_ready'] = True
887 else:
887 else:
888 # unhandled msg_type (status, etc.)
888 # unhandled msg_type (status, etc.)
889 pass
889 pass
890
890
891 # reduntant?
891 # reduntant?
892 self.metadata[msg_id] = md
892 self.metadata[msg_id] = md
893
893
894 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
894 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
895
895
896 #--------------------------------------------------------------------------
896 #--------------------------------------------------------------------------
897 # len, getitem
897 # len, getitem
898 #--------------------------------------------------------------------------
898 #--------------------------------------------------------------------------
899
899
900 def __len__(self):
900 def __len__(self):
901 """len(client) returns # of engines."""
901 """len(client) returns # of engines."""
902 return len(self.ids)
902 return len(self.ids)
903
903
904 def __getitem__(self, key):
904 def __getitem__(self, key):
905 """index access returns DirectView multiplexer objects
905 """index access returns DirectView multiplexer objects
906
906
907 Must be int, slice, or list/tuple/xrange of ints"""
907 Must be int, slice, or list/tuple/xrange of ints"""
908 if not isinstance(key, (int, slice, tuple, list, xrange)):
908 if not isinstance(key, (int, slice, tuple, list, xrange)):
909 raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
909 raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
910 else:
910 else:
911 return self.direct_view(key)
911 return self.direct_view(key)
912
912
913 def __iter__(self):
913 def __iter__(self):
914 """Since we define getitem, Client is iterable
914 """Since we define getitem, Client is iterable
915
915
916 but unless we also define __iter__, it won't work correctly unless engine IDs
916 but unless we also define __iter__, it won't work correctly unless engine IDs
917 start at zero and are continuous.
917 start at zero and are continuous.
918 """
918 """
919 for eid in self.ids:
919 for eid in self.ids:
920 yield self.direct_view(eid)
920 yield self.direct_view(eid)
921
921
922 #--------------------------------------------------------------------------
922 #--------------------------------------------------------------------------
923 # Begin public methods
923 # Begin public methods
924 #--------------------------------------------------------------------------
924 #--------------------------------------------------------------------------
925
925
926 @property
926 @property
927 def ids(self):
927 def ids(self):
928 """Always up-to-date ids property."""
928 """Always up-to-date ids property."""
929 self._flush_notifications()
929 self._flush_notifications()
930 # always copy:
930 # always copy:
931 return list(self._ids)
931 return list(self._ids)
932
932
933 def activate(self, targets='all', suffix=''):
933 def activate(self, targets='all', suffix=''):
934 """Create a DirectView and register it with IPython magics
934 """Create a DirectView and register it with IPython magics
935
935
936 Defines the magics `%px, %autopx, %pxresult, %%px`
936 Defines the magics `%px, %autopx, %pxresult, %%px`
937
937
938 Parameters
938 Parameters
939 ----------
939 ----------
940
940
941 targets: int, list of ints, or 'all'
941 targets: int, list of ints, or 'all'
942 The engines on which the view's magics will run
942 The engines on which the view's magics will run
943 suffix: str [default: '']
943 suffix: str [default: '']
944 The suffix, if any, for the magics. This allows you to have
944 The suffix, if any, for the magics. This allows you to have
945 multiple views associated with parallel magics at the same time.
945 multiple views associated with parallel magics at the same time.
946
946
947 e.g. ``rc.activate(targets=0, suffix='0')`` will give you
947 e.g. ``rc.activate(targets=0, suffix='0')`` will give you
948 the magics ``%px0``, ``%pxresult0``, etc. for running magics just
948 the magics ``%px0``, ``%pxresult0``, etc. for running magics just
949 on engine 0.
949 on engine 0.
950 """
950 """
951 view = self.direct_view(targets)
951 view = self.direct_view(targets)
952 view.block = True
952 view.block = True
953 view.activate(suffix)
953 view.activate(suffix)
954 return view
954 return view
955
955
956 def close(self, linger=None):
956 def close(self, linger=None):
957 """Close my zmq Sockets
957 """Close my zmq Sockets
958
958
959 If `linger`, set the zmq LINGER socket option,
959 If `linger`, set the zmq LINGER socket option,
960 which allows discarding of messages.
960 which allows discarding of messages.
961 """
961 """
962 if self._closed:
962 if self._closed:
963 return
963 return
964 self.stop_spin_thread()
964 self.stop_spin_thread()
965 snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
965 snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
966 for name in snames:
966 for name in snames:
967 socket = getattr(self, name)
967 socket = getattr(self, name)
968 if socket is not None and not socket.closed:
968 if socket is not None and not socket.closed:
969 if linger is not None:
969 if linger is not None:
970 socket.close(linger=linger)
970 socket.close(linger=linger)
971 else:
971 else:
972 socket.close()
972 socket.close()
973 self._closed = True
973 self._closed = True
974
974
975 def _spin_every(self, interval=1):
975 def _spin_every(self, interval=1):
976 """target func for use in spin_thread"""
976 """target func for use in spin_thread"""
977 while True:
977 while True:
978 if self._stop_spinning.is_set():
978 if self._stop_spinning.is_set():
979 return
979 return
980 time.sleep(interval)
980 time.sleep(interval)
981 self.spin()
981 self.spin()
982
982
983 def spin_thread(self, interval=1):
983 def spin_thread(self, interval=1):
984 """call Client.spin() in a background thread on some regular interval
984 """call Client.spin() in a background thread on some regular interval
985
985
986 This helps ensure that messages don't pile up too much in the zmq queue
986 This helps ensure that messages don't pile up too much in the zmq queue
987 while you are working on other things, or just leaving an idle terminal.
987 while you are working on other things, or just leaving an idle terminal.
988
988
989 It also helps limit potential padding of the `received` timestamp
989 It also helps limit potential padding of the `received` timestamp
990 on AsyncResult objects, used for timings.
990 on AsyncResult objects, used for timings.
991
991
992 Parameters
992 Parameters
993 ----------
993 ----------
994
994
995 interval : float, optional
995 interval : float, optional
996 The interval on which to spin the client in the background thread
996 The interval on which to spin the client in the background thread
997 (simply passed to time.sleep).
997 (simply passed to time.sleep).
998
998
999 Notes
999 Notes
1000 -----
1000 -----
1001
1001
1002 For precision timing, you may want to use this method to put a bound
1002 For precision timing, you may want to use this method to put a bound
1003 on the jitter (in seconds) in `received` timestamps used
1003 on the jitter (in seconds) in `received` timestamps used
1004 in AsyncResult.wall_time.
1004 in AsyncResult.wall_time.
1005
1005
1006 """
1006 """
1007 if self._spin_thread is not None:
1007 if self._spin_thread is not None:
1008 self.stop_spin_thread()
1008 self.stop_spin_thread()
1009 self._stop_spinning.clear()
1009 self._stop_spinning.clear()
1010 self._spin_thread = Thread(target=self._spin_every, args=(interval,))
1010 self._spin_thread = Thread(target=self._spin_every, args=(interval,))
1011 self._spin_thread.daemon = True
1011 self._spin_thread.daemon = True
1012 self._spin_thread.start()
1012 self._spin_thread.start()
1013
1013
1014 def stop_spin_thread(self):
1014 def stop_spin_thread(self):
1015 """stop background spin_thread, if any"""
1015 """stop background spin_thread, if any"""
1016 if self._spin_thread is not None:
1016 if self._spin_thread is not None:
1017 self._stop_spinning.set()
1017 self._stop_spinning.set()
1018 self._spin_thread.join()
1018 self._spin_thread.join()
1019 self._spin_thread = None
1019 self._spin_thread = None
1020
1020
1021 def spin(self):
1021 def spin(self):
1022 """Flush any registration notifications and execution results
1022 """Flush any registration notifications and execution results
1023 waiting in the ZMQ queue.
1023 waiting in the ZMQ queue.
1024 """
1024 """
1025 if self._notification_socket:
1025 if self._notification_socket:
1026 self._flush_notifications()
1026 self._flush_notifications()
1027 if self._iopub_socket:
1027 if self._iopub_socket:
1028 self._flush_iopub(self._iopub_socket)
1028 self._flush_iopub(self._iopub_socket)
1029 if self._mux_socket:
1029 if self._mux_socket:
1030 self._flush_results(self._mux_socket)
1030 self._flush_results(self._mux_socket)
1031 if self._task_socket:
1031 if self._task_socket:
1032 self._flush_results(self._task_socket)
1032 self._flush_results(self._task_socket)
1033 if self._control_socket:
1033 if self._control_socket:
1034 self._flush_control(self._control_socket)
1034 self._flush_control(self._control_socket)
1035 if self._query_socket:
1035 if self._query_socket:
1036 self._flush_ignored_hub_replies()
1036 self._flush_ignored_hub_replies()
1037
1037
1038 def wait(self, jobs=None, timeout=-1):
1038 def wait(self, jobs=None, timeout=-1):
1039 """waits on one or more `jobs`, for up to `timeout` seconds.
1039 """waits on one or more `jobs`, for up to `timeout` seconds.
1040
1040
1041 Parameters
1041 Parameters
1042 ----------
1042 ----------
1043
1043
1044 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
1044 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
1045 ints are indices to self.history
1045 ints are indices to self.history
1046 strs are msg_ids
1046 strs are msg_ids
1047 default: wait on all outstanding messages
1047 default: wait on all outstanding messages
1048 timeout : float
1048 timeout : float
1049 a time in seconds, after which to give up.
1049 a time in seconds, after which to give up.
1050 default is -1, which means no timeout
1050 default is -1, which means no timeout
1051
1051
1052 Returns
1052 Returns
1053 -------
1053 -------
1054
1054
1055 True : when all msg_ids are done
1055 True : when all msg_ids are done
1056 False : timeout reached, some msg_ids still outstanding
1056 False : timeout reached, some msg_ids still outstanding
1057 """
1057 """
1058 tic = time.time()
1058 tic = time.time()
1059 if jobs is None:
1059 if jobs is None:
1060 theids = self.outstanding
1060 theids = self.outstanding
1061 else:
1061 else:
1062 if isinstance(jobs, string_types + (int, AsyncResult)):
1062 if isinstance(jobs, string_types + (int, AsyncResult)):
1063 jobs = [jobs]
1063 jobs = [jobs]
1064 theids = set()
1064 theids = set()
1065 for job in jobs:
1065 for job in jobs:
1066 if isinstance(job, int):
1066 if isinstance(job, int):
1067 # index access
1067 # index access
1068 job = self.history[job]
1068 job = self.history[job]
1069 elif isinstance(job, AsyncResult):
1069 elif isinstance(job, AsyncResult):
1070 theids.update(job.msg_ids)
1070 theids.update(job.msg_ids)
1071 continue
1071 continue
1072 theids.add(job)
1072 theids.add(job)
1073 if not theids.intersection(self.outstanding):
1073 if not theids.intersection(self.outstanding):
1074 return True
1074 return True
1075 self.spin()
1075 self.spin()
1076 while theids.intersection(self.outstanding):
1076 while theids.intersection(self.outstanding):
1077 if timeout >= 0 and ( time.time()-tic ) > timeout:
1077 if timeout >= 0 and ( time.time()-tic ) > timeout:
1078 break
1078 break
1079 time.sleep(1e-3)
1079 time.sleep(1e-3)
1080 self.spin()
1080 self.spin()
1081 return len(theids.intersection(self.outstanding)) == 0
1081 return len(theids.intersection(self.outstanding)) == 0
1082
1082
1083 #--------------------------------------------------------------------------
1083 #--------------------------------------------------------------------------
1084 # Control methods
1084 # Control methods
1085 #--------------------------------------------------------------------------
1085 #--------------------------------------------------------------------------
1086
1086
1087 @spin_first
1087 @spin_first
1088 def clear(self, targets=None, block=None):
1088 def clear(self, targets=None, block=None):
1089 """Clear the namespace in target(s)."""
1089 """Clear the namespace in target(s)."""
1090 block = self.block if block is None else block
1090 block = self.block if block is None else block
1091 targets = self._build_targets(targets)[0]
1091 targets = self._build_targets(targets)[0]
1092 for t in targets:
1092 for t in targets:
1093 self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
1093 self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
1094 error = False
1094 error = False
1095 if block:
1095 if block:
1096 self._flush_ignored_control()
1096 self._flush_ignored_control()
1097 for i in range(len(targets)):
1097 for i in range(len(targets)):
1098 idents,msg = self.session.recv(self._control_socket,0)
1098 idents,msg = self.session.recv(self._control_socket,0)
1099 if self.debug:
1099 if self.debug:
1100 pprint(msg)
1100 pprint(msg)
1101 if msg['content']['status'] != 'ok':
1101 if msg['content']['status'] != 'ok':
1102 error = self._unwrap_exception(msg['content'])
1102 error = self._unwrap_exception(msg['content'])
1103 else:
1103 else:
1104 self._ignored_control_replies += len(targets)
1104 self._ignored_control_replies += len(targets)
1105 if error:
1105 if error:
1106 raise error
1106 raise error
1107
1107
1108
1108
1109 @spin_first
1109 @spin_first
1110 def abort(self, jobs=None, targets=None, block=None):
1110 def abort(self, jobs=None, targets=None, block=None):
1111 """Abort specific jobs from the execution queues of target(s).
1111 """Abort specific jobs from the execution queues of target(s).
1112
1112
1113 This is a mechanism to prevent jobs that have already been submitted
1113 This is a mechanism to prevent jobs that have already been submitted
1114 from executing.
1114 from executing.
1115
1115
1116 Parameters
1116 Parameters
1117 ----------
1117 ----------
1118
1118
1119 jobs : msg_id, list of msg_ids, or AsyncResult
1119 jobs : msg_id, list of msg_ids, or AsyncResult
1120 The jobs to be aborted
1120 The jobs to be aborted
1121
1121
1122 If unspecified/None: abort all outstanding jobs.
1122 If unspecified/None: abort all outstanding jobs.
1123
1123
1124 """
1124 """
1125 block = self.block if block is None else block
1125 block = self.block if block is None else block
1126 jobs = jobs if jobs is not None else list(self.outstanding)
1126 jobs = jobs if jobs is not None else list(self.outstanding)
1127 targets = self._build_targets(targets)[0]
1127 targets = self._build_targets(targets)[0]
1128
1128
1129 msg_ids = []
1129 msg_ids = []
1130 if isinstance(jobs, string_types + (AsyncResult,)):
1130 if isinstance(jobs, string_types + (AsyncResult,)):
1131 jobs = [jobs]
1131 jobs = [jobs]
1132 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1132 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1133 if bad_ids:
1133 if bad_ids:
1134 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1134 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1135 for j in jobs:
1135 for j in jobs:
1136 if isinstance(j, AsyncResult):
1136 if isinstance(j, AsyncResult):
1137 msg_ids.extend(j.msg_ids)
1137 msg_ids.extend(j.msg_ids)
1138 else:
1138 else:
1139 msg_ids.append(j)
1139 msg_ids.append(j)
1140 content = dict(msg_ids=msg_ids)
1140 content = dict(msg_ids=msg_ids)
1141 for t in targets:
1141 for t in targets:
1142 self.session.send(self._control_socket, 'abort_request',
1142 self.session.send(self._control_socket, 'abort_request',
1143 content=content, ident=t)
1143 content=content, ident=t)
1144 error = False
1144 error = False
1145 if block:
1145 if block:
1146 self._flush_ignored_control()
1146 self._flush_ignored_control()
1147 for i in range(len(targets)):
1147 for i in range(len(targets)):
1148 idents,msg = self.session.recv(self._control_socket,0)
1148 idents,msg = self.session.recv(self._control_socket,0)
1149 if self.debug:
1149 if self.debug:
1150 pprint(msg)
1150 pprint(msg)
1151 if msg['content']['status'] != 'ok':
1151 if msg['content']['status'] != 'ok':
1152 error = self._unwrap_exception(msg['content'])
1152 error = self._unwrap_exception(msg['content'])
1153 else:
1153 else:
1154 self._ignored_control_replies += len(targets)
1154 self._ignored_control_replies += len(targets)
1155 if error:
1155 if error:
1156 raise error
1156 raise error
1157
1157
1158 @spin_first
1158 @spin_first
1159 def shutdown(self, targets='all', restart=False, hub=False, block=None):
1159 def shutdown(self, targets='all', restart=False, hub=False, block=None):
1160 """Terminates one or more engine processes, optionally including the hub.
1160 """Terminates one or more engine processes, optionally including the hub.
1161
1161
1162 Parameters
1162 Parameters
1163 ----------
1163 ----------
1164
1164
1165 targets: list of ints or 'all' [default: all]
1165 targets: list of ints or 'all' [default: all]
1166 Which engines to shutdown.
1166 Which engines to shutdown.
1167 hub: bool [default: False]
1167 hub: bool [default: False]
1168 Whether to include the Hub. hub=True implies targets='all'.
1168 Whether to include the Hub. hub=True implies targets='all'.
1169 block: bool [default: self.block]
1169 block: bool [default: self.block]
1170 Whether to wait for clean shutdown replies or not.
1170 Whether to wait for clean shutdown replies or not.
1171 restart: bool [default: False]
1171 restart: bool [default: False]
1172 NOT IMPLEMENTED
1172 NOT IMPLEMENTED
1173 whether to restart engines after shutting them down.
1173 whether to restart engines after shutting them down.
1174 """
1174 """
1175 from IPython.parallel.error import NoEnginesRegistered
1175 from IPython.parallel.error import NoEnginesRegistered
1176 if restart:
1176 if restart:
1177 raise NotImplementedError("Engine restart is not yet implemented")
1177 raise NotImplementedError("Engine restart is not yet implemented")
1178
1178
1179 block = self.block if block is None else block
1179 block = self.block if block is None else block
1180 if hub:
1180 if hub:
1181 targets = 'all'
1181 targets = 'all'
1182 try:
1182 try:
1183 targets = self._build_targets(targets)[0]
1183 targets = self._build_targets(targets)[0]
1184 except NoEnginesRegistered:
1184 except NoEnginesRegistered:
1185 targets = []
1185 targets = []
1186 for t in targets:
1186 for t in targets:
1187 self.session.send(self._control_socket, 'shutdown_request',
1187 self.session.send(self._control_socket, 'shutdown_request',
1188 content={'restart':restart},ident=t)
1188 content={'restart':restart},ident=t)
1189 error = False
1189 error = False
1190 if block or hub:
1190 if block or hub:
1191 self._flush_ignored_control()
1191 self._flush_ignored_control()
1192 for i in range(len(targets)):
1192 for i in range(len(targets)):
1193 idents,msg = self.session.recv(self._control_socket, 0)
1193 idents,msg = self.session.recv(self._control_socket, 0)
1194 if self.debug:
1194 if self.debug:
1195 pprint(msg)
1195 pprint(msg)
1196 if msg['content']['status'] != 'ok':
1196 if msg['content']['status'] != 'ok':
1197 error = self._unwrap_exception(msg['content'])
1197 error = self._unwrap_exception(msg['content'])
1198 else:
1198 else:
1199 self._ignored_control_replies += len(targets)
1199 self._ignored_control_replies += len(targets)
1200
1200
1201 if hub:
1201 if hub:
1202 time.sleep(0.25)
1202 time.sleep(0.25)
1203 self.session.send(self._query_socket, 'shutdown_request')
1203 self.session.send(self._query_socket, 'shutdown_request')
1204 idents,msg = self.session.recv(self._query_socket, 0)
1204 idents,msg = self.session.recv(self._query_socket, 0)
1205 if self.debug:
1205 if self.debug:
1206 pprint(msg)
1206 pprint(msg)
1207 if msg['content']['status'] != 'ok':
1207 if msg['content']['status'] != 'ok':
1208 error = self._unwrap_exception(msg['content'])
1208 error = self._unwrap_exception(msg['content'])
1209
1209
1210 if error:
1210 if error:
1211 raise error
1211 raise error
1212
1212
1213 #--------------------------------------------------------------------------
1213 #--------------------------------------------------------------------------
1214 # Execution related methods
1214 # Execution related methods
1215 #--------------------------------------------------------------------------
1215 #--------------------------------------------------------------------------
1216
1216
1217 def _maybe_raise(self, result):
1217 def _maybe_raise(self, result):
1218 """wrapper for maybe raising an exception if apply failed."""
1218 """wrapper for maybe raising an exception if apply failed."""
1219 if isinstance(result, error.RemoteError):
1219 if isinstance(result, error.RemoteError):
1220 raise result
1220 raise result
1221
1221
1222 return result
1222 return result
1223
1223
1224 def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
1224 def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
1225 ident=None):
1225 ident=None):
1226 """construct and send an apply message via a socket.
1226 """construct and send an apply message via a socket.
1227
1227
1228 This is the principal method with which all engine execution is performed by views.
1228 This is the principal method with which all engine execution is performed by views.
1229 """
1229 """
1230
1230
1231 if self._closed:
1231 if self._closed:
1232 raise RuntimeError("Client cannot be used after its sockets have been closed")
1232 raise RuntimeError("Client cannot be used after its sockets have been closed")
1233
1233
1234 # defaults:
1234 # defaults:
1235 args = args if args is not None else []
1235 args = args if args is not None else []
1236 kwargs = kwargs if kwargs is not None else {}
1236 kwargs = kwargs if kwargs is not None else {}
1237 metadata = metadata if metadata is not None else {}
1237 metadata = metadata if metadata is not None else {}
1238
1238
1239 # validate arguments
1239 # validate arguments
1240 if not callable(f) and not isinstance(f, Reference):
1240 if not callable(f) and not isinstance(f, Reference):
1241 raise TypeError("f must be callable, not %s"%type(f))
1241 raise TypeError("f must be callable, not %s"%type(f))
1242 if not isinstance(args, (tuple, list)):
1242 if not isinstance(args, (tuple, list)):
1243 raise TypeError("args must be tuple or list, not %s"%type(args))
1243 raise TypeError("args must be tuple or list, not %s"%type(args))
1244 if not isinstance(kwargs, dict):
1244 if not isinstance(kwargs, dict):
1245 raise TypeError("kwargs must be dict, not %s"%type(kwargs))
1245 raise TypeError("kwargs must be dict, not %s"%type(kwargs))
1246 if not isinstance(metadata, dict):
1246 if not isinstance(metadata, dict):
1247 raise TypeError("metadata must be dict, not %s"%type(metadata))
1247 raise TypeError("metadata must be dict, not %s"%type(metadata))
1248
1248
1249 bufs = serialize.pack_apply_message(f, args, kwargs,
1249 bufs = serialize.pack_apply_message(f, args, kwargs,
1250 buffer_threshold=self.session.buffer_threshold,
1250 buffer_threshold=self.session.buffer_threshold,
1251 item_threshold=self.session.item_threshold,
1251 item_threshold=self.session.item_threshold,
1252 )
1252 )
1253
1253
1254 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
1254 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
1255 metadata=metadata, track=track)
1255 metadata=metadata, track=track)
1256
1256
1257 msg_id = msg['header']['msg_id']
1257 msg_id = msg['header']['msg_id']
1258 self.outstanding.add(msg_id)
1258 self.outstanding.add(msg_id)
1259 if ident:
1259 if ident:
1260 # possibly routed to a specific engine
1260 # possibly routed to a specific engine
1261 if isinstance(ident, list):
1261 if isinstance(ident, list):
1262 ident = ident[-1]
1262 ident = ident[-1]
1263 if ident in self._engines.values():
1263 if ident in self._engines.values():
1264 # save for later, in case of engine death
1264 # save for later, in case of engine death
1265 self._outstanding_dict[ident].add(msg_id)
1265 self._outstanding_dict[ident].add(msg_id)
1266 self.history.append(msg_id)
1266 self.history.append(msg_id)
1267 self.metadata[msg_id]['submitted'] = datetime.now()
1267 self.metadata[msg_id]['submitted'] = datetime.now()
1268
1268
1269 return msg
1269 return msg
1270
1270
1271 def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
1271 def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
1272 """construct and send an execute request via a socket.
1272 """construct and send an execute request via a socket.
1273
1273
1274 """
1274 """
1275
1275
1276 if self._closed:
1276 if self._closed:
1277 raise RuntimeError("Client cannot be used after its sockets have been closed")
1277 raise RuntimeError("Client cannot be used after its sockets have been closed")
1278
1278
1279 # defaults:
1279 # defaults:
1280 metadata = metadata if metadata is not None else {}
1280 metadata = metadata if metadata is not None else {}
1281
1281
1282 # validate arguments
1282 # validate arguments
1283 if not isinstance(code, string_types):
1283 if not isinstance(code, string_types):
1284 raise TypeError("code must be text, not %s" % type(code))
1284 raise TypeError("code must be text, not %s" % type(code))
1285 if not isinstance(metadata, dict):
1285 if not isinstance(metadata, dict):
1286 raise TypeError("metadata must be dict, not %s" % type(metadata))
1286 raise TypeError("metadata must be dict, not %s" % type(metadata))
1287
1287
1288 content = dict(code=code, silent=bool(silent), user_expressions={})
1288 content = dict(code=code, silent=bool(silent), user_expressions={})
1289
1289
1290
1290
1291 msg = self.session.send(socket, "execute_request", content=content, ident=ident,
1291 msg = self.session.send(socket, "execute_request", content=content, ident=ident,
1292 metadata=metadata)
1292 metadata=metadata)
1293
1293
1294 msg_id = msg['header']['msg_id']
1294 msg_id = msg['header']['msg_id']
1295 self.outstanding.add(msg_id)
1295 self.outstanding.add(msg_id)
1296 if ident:
1296 if ident:
1297 # possibly routed to a specific engine
1297 # possibly routed to a specific engine
1298 if isinstance(ident, list):
1298 if isinstance(ident, list):
1299 ident = ident[-1]
1299 ident = ident[-1]
1300 if ident in self._engines.values():
1300 if ident in self._engines.values():
1301 # save for later, in case of engine death
1301 # save for later, in case of engine death
1302 self._outstanding_dict[ident].add(msg_id)
1302 self._outstanding_dict[ident].add(msg_id)
1303 self.history.append(msg_id)
1303 self.history.append(msg_id)
1304 self.metadata[msg_id]['submitted'] = datetime.now()
1304 self.metadata[msg_id]['submitted'] = datetime.now()
1305
1305
1306 return msg
1306 return msg
1307
1307
1308 #--------------------------------------------------------------------------
1308 #--------------------------------------------------------------------------
1309 # construct a View object
1309 # construct a View object
1310 #--------------------------------------------------------------------------
1310 #--------------------------------------------------------------------------
1311
1311
1312 def load_balanced_view(self, targets=None):
1312 def load_balanced_view(self, targets=None):
1313 """construct a DirectView object.
1313 """construct a DirectView object.
1314
1314
1315 If no arguments are specified, create a LoadBalancedView
1315 If no arguments are specified, create a LoadBalancedView
1316 using all engines.
1316 using all engines.
1317
1317
1318 Parameters
1318 Parameters
1319 ----------
1319 ----------
1320
1320
1321 targets: list,slice,int,etc. [default: use all engines]
1321 targets: list,slice,int,etc. [default: use all engines]
1322 The subset of engines across which to load-balance
1322 The subset of engines across which to load-balance
1323 """
1323 """
1324 if targets == 'all':
1324 if targets == 'all':
1325 targets = None
1325 targets = None
1326 if targets is not None:
1326 if targets is not None:
1327 targets = self._build_targets(targets)[1]
1327 targets = self._build_targets(targets)[1]
1328 return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
1328 return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
1329
1329
1330 def direct_view(self, targets='all'):
1330 def direct_view(self, targets='all'):
1331 """construct a DirectView object.
1331 """construct a DirectView object.
1332
1332
1333 If no targets are specified, create a DirectView using all engines.
1333 If no targets are specified, create a DirectView using all engines.
1334
1334
1335 rc.direct_view('all') is distinguished from rc[:] in that 'all' will
1335 rc.direct_view('all') is distinguished from rc[:] in that 'all' will
1336 evaluate the target engines at each execution, whereas rc[:] will connect to
1336 evaluate the target engines at each execution, whereas rc[:] will connect to
1337 all *current* engines, and that list will not change.
1337 all *current* engines, and that list will not change.
1338
1338
1339 That is, 'all' will always use all engines, whereas rc[:] will not use
1339 That is, 'all' will always use all engines, whereas rc[:] will not use
1340 engines added after the DirectView is constructed.
1340 engines added after the DirectView is constructed.
1341
1341
1342 Parameters
1342 Parameters
1343 ----------
1343 ----------
1344
1344
1345 targets: list,slice,int,etc. [default: use all engines]
1345 targets: list,slice,int,etc. [default: use all engines]
1346 The engines to use for the View
1346 The engines to use for the View
1347 """
1347 """
1348 single = isinstance(targets, int)
1348 single = isinstance(targets, int)
1349 # allow 'all' to be lazily evaluated at each execution
1349 # allow 'all' to be lazily evaluated at each execution
1350 if targets != 'all':
1350 if targets != 'all':
1351 targets = self._build_targets(targets)[1]
1351 targets = self._build_targets(targets)[1]
1352 if single:
1352 if single:
1353 targets = targets[0]
1353 targets = targets[0]
1354 return DirectView(client=self, socket=self._mux_socket, targets=targets)
1354 return DirectView(client=self, socket=self._mux_socket, targets=targets)
1355
1355
1356 #--------------------------------------------------------------------------
1356 #--------------------------------------------------------------------------
1357 # Query methods
1357 # Query methods
1358 #--------------------------------------------------------------------------
1358 #--------------------------------------------------------------------------
1359
1359
1360 @spin_first
1360 @spin_first
1361 def get_result(self, indices_or_msg_ids=None, block=None):
1361 def get_result(self, indices_or_msg_ids=None, block=None):
1362 """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
1362 """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
1363
1363
1364 If the client already has the results, no request to the Hub will be made.
1364 If the client already has the results, no request to the Hub will be made.
1365
1365
1366 This is a convenient way to construct AsyncResult objects, which are wrappers
1366 This is a convenient way to construct AsyncResult objects, which are wrappers
1367 that include metadata about execution, and allow for awaiting results that
1367 that include metadata about execution, and allow for awaiting results that
1368 were not submitted by this Client.
1368 were not submitted by this Client.
1369
1369
1370 It can also be a convenient way to retrieve the metadata associated with
1370 It can also be a convenient way to retrieve the metadata associated with
1371 blocking execution, since it always retrieves
1371 blocking execution, since it always retrieves
1372
1372
1373 Examples
1373 Examples
1374 --------
1374 --------
1375 ::
1375 ::
1376
1376
1377 In [10]: r = client.apply()
1377 In [10]: r = client.apply()
1378
1378
1379 Parameters
1379 Parameters
1380 ----------
1380 ----------
1381
1381
1382 indices_or_msg_ids : integer history index, str msg_id, or list of either
1382 indices_or_msg_ids : integer history index, str msg_id, or list of either
1383 The indices or msg_ids of indices to be retrieved
1383 The indices or msg_ids of indices to be retrieved
1384
1384
1385 block : bool
1385 block : bool
1386 Whether to wait for the result to be done
1386 Whether to wait for the result to be done
1387
1387
1388 Returns
1388 Returns
1389 -------
1389 -------
1390
1390
1391 AsyncResult
1391 AsyncResult
1392 A single AsyncResult object will always be returned.
1392 A single AsyncResult object will always be returned.
1393
1393
1394 AsyncHubResult
1394 AsyncHubResult
1395 A subclass of AsyncResult that retrieves results from the Hub
1395 A subclass of AsyncResult that retrieves results from the Hub
1396
1396
1397 """
1397 """
1398 block = self.block if block is None else block
1398 block = self.block if block is None else block
1399 if indices_or_msg_ids is None:
1399 if indices_or_msg_ids is None:
1400 indices_or_msg_ids = -1
1400 indices_or_msg_ids = -1
1401
1401
1402 single_result = False
1402 single_result = False
1403 if not isinstance(indices_or_msg_ids, (list,tuple)):
1403 if not isinstance(indices_or_msg_ids, (list,tuple)):
1404 indices_or_msg_ids = [indices_or_msg_ids]
1404 indices_or_msg_ids = [indices_or_msg_ids]
1405 single_result = True
1405 single_result = True
1406
1406
1407 theids = []
1407 theids = []
1408 for id in indices_or_msg_ids:
1408 for id in indices_or_msg_ids:
1409 if isinstance(id, int):
1409 if isinstance(id, int):
1410 id = self.history[id]
1410 id = self.history[id]
1411 if not isinstance(id, string_types):
1411 if not isinstance(id, string_types):
1412 raise TypeError("indices must be str or int, not %r"%id)
1412 raise TypeError("indices must be str or int, not %r"%id)
1413 theids.append(id)
1413 theids.append(id)
1414
1414
1415 local_ids = [msg_id for msg_id in theids if (msg_id in self.outstanding or msg_id in self.results)]
1415 local_ids = [msg_id for msg_id in theids if (msg_id in self.outstanding or msg_id in self.results)]
1416 remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]
1416 remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]
1417
1417
1418 # given single msg_id initially, get_result shot get the result itself,
1418 # given single msg_id initially, get_result shot get the result itself,
1419 # not a length-one list
1419 # not a length-one list
1420 if single_result:
1420 if single_result:
1421 theids = theids[0]
1421 theids = theids[0]
1422
1422
1423 if remote_ids:
1423 if remote_ids:
1424 ar = AsyncHubResult(self, msg_ids=theids)
1424 ar = AsyncHubResult(self, msg_ids=theids)
1425 else:
1425 else:
1426 ar = AsyncResult(self, msg_ids=theids)
1426 ar = AsyncResult(self, msg_ids=theids)
1427
1427
1428 if block:
1428 if block:
1429 ar.wait()
1429 ar.wait()
1430
1430
1431 return ar
1431 return ar
1432
1432
1433 @spin_first
1433 @spin_first
1434 def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
1434 def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
1435 """Resubmit one or more tasks.
1435 """Resubmit one or more tasks.
1436
1436
1437 in-flight tasks may not be resubmitted.
1437 in-flight tasks may not be resubmitted.
1438
1438
1439 Parameters
1439 Parameters
1440 ----------
1440 ----------
1441
1441
1442 indices_or_msg_ids : integer history index, str msg_id, or list of either
1442 indices_or_msg_ids : integer history index, str msg_id, or list of either
1443 The indices or msg_ids of indices to be retrieved
1443 The indices or msg_ids of indices to be retrieved
1444
1444
1445 block : bool
1445 block : bool
1446 Whether to wait for the result to be done
1446 Whether to wait for the result to be done
1447
1447
1448 Returns
1448 Returns
1449 -------
1449 -------
1450
1450
1451 AsyncHubResult
1451 AsyncHubResult
1452 A subclass of AsyncResult that retrieves results from the Hub
1452 A subclass of AsyncResult that retrieves results from the Hub
1453
1453
1454 """
1454 """
1455 block = self.block if block is None else block
1455 block = self.block if block is None else block
1456 if indices_or_msg_ids is None:
1456 if indices_or_msg_ids is None:
1457 indices_or_msg_ids = -1
1457 indices_or_msg_ids = -1
1458
1458
1459 if not isinstance(indices_or_msg_ids, (list,tuple)):
1459 if not isinstance(indices_or_msg_ids, (list,tuple)):
1460 indices_or_msg_ids = [indices_or_msg_ids]
1460 indices_or_msg_ids = [indices_or_msg_ids]
1461
1461
1462 theids = []
1462 theids = []
1463 for id in indices_or_msg_ids:
1463 for id in indices_or_msg_ids:
1464 if isinstance(id, int):
1464 if isinstance(id, int):
1465 id = self.history[id]
1465 id = self.history[id]
1466 if not isinstance(id, string_types):
1466 if not isinstance(id, string_types):
1467 raise TypeError("indices must be str or int, not %r"%id)
1467 raise TypeError("indices must be str or int, not %r"%id)
1468 theids.append(id)
1468 theids.append(id)
1469
1469
1470 content = dict(msg_ids = theids)
1470 content = dict(msg_ids = theids)
1471
1471
1472 self.session.send(self._query_socket, 'resubmit_request', content)
1472 self.session.send(self._query_socket, 'resubmit_request', content)
1473
1473
1474 zmq.select([self._query_socket], [], [])
1474 zmq.select([self._query_socket], [], [])
1475 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1475 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1476 if self.debug:
1476 if self.debug:
1477 pprint(msg)
1477 pprint(msg)
1478 content = msg['content']
1478 content = msg['content']
1479 if content['status'] != 'ok':
1479 if content['status'] != 'ok':
1480 raise self._unwrap_exception(content)
1480 raise self._unwrap_exception(content)
1481 mapping = content['resubmitted']
1481 mapping = content['resubmitted']
1482 new_ids = [ mapping[msg_id] for msg_id in theids ]
1482 new_ids = [ mapping[msg_id] for msg_id in theids ]
1483
1483
1484 ar = AsyncHubResult(self, msg_ids=new_ids)
1484 ar = AsyncHubResult(self, msg_ids=new_ids)
1485
1485
1486 if block:
1486 if block:
1487 ar.wait()
1487 ar.wait()
1488
1488
1489 return ar
1489 return ar
1490
1490
1491 @spin_first
1491 @spin_first
1492 def result_status(self, msg_ids, status_only=True):
1492 def result_status(self, msg_ids, status_only=True):
1493 """Check on the status of the result(s) of the apply request with `msg_ids`.
1493 """Check on the status of the result(s) of the apply request with `msg_ids`.
1494
1494
1495 If status_only is False, then the actual results will be retrieved, else
1495 If status_only is False, then the actual results will be retrieved, else
1496 only the status of the results will be checked.
1496 only the status of the results will be checked.
1497
1497
1498 Parameters
1498 Parameters
1499 ----------
1499 ----------
1500
1500
1501 msg_ids : list of msg_ids
1501 msg_ids : list of msg_ids
1502 if int:
1502 if int:
1503 Passed as index to self.history for convenience.
1503 Passed as index to self.history for convenience.
1504 status_only : bool (default: True)
1504 status_only : bool (default: True)
1505 if False:
1505 if False:
1506 Retrieve the actual results of completed tasks.
1506 Retrieve the actual results of completed tasks.
1507
1507
1508 Returns
1508 Returns
1509 -------
1509 -------
1510
1510
1511 results : dict
1511 results : dict
1512 There will always be the keys 'pending' and 'completed', which will
1512 There will always be the keys 'pending' and 'completed', which will
1513 be lists of msg_ids that are incomplete or complete. If `status_only`
1513 be lists of msg_ids that are incomplete or complete. If `status_only`
1514 is False, then completed results will be keyed by their `msg_id`.
1514 is False, then completed results will be keyed by their `msg_id`.
1515 """
1515 """
1516 if not isinstance(msg_ids, (list,tuple)):
1516 if not isinstance(msg_ids, (list,tuple)):
1517 msg_ids = [msg_ids]
1517 msg_ids = [msg_ids]
1518
1518
1519 theids = []
1519 theids = []
1520 for msg_id in msg_ids:
1520 for msg_id in msg_ids:
1521 if isinstance(msg_id, int):
1521 if isinstance(msg_id, int):
1522 msg_id = self.history[msg_id]
1522 msg_id = self.history[msg_id]
1523 if not isinstance(msg_id, string_types):
1523 if not isinstance(msg_id, string_types):
1524 raise TypeError("msg_ids must be str, not %r"%msg_id)
1524 raise TypeError("msg_ids must be str, not %r"%msg_id)
1525 theids.append(msg_id)
1525 theids.append(msg_id)
1526
1526
1527 completed = []
1527 completed = []
1528 local_results = {}
1528 local_results = {}
1529
1529
1530 # comment this block out to temporarily disable local shortcut:
1530 # comment this block out to temporarily disable local shortcut:
1531 for msg_id in theids:
1531 for msg_id in theids:
1532 if msg_id in self.results:
1532 if msg_id in self.results:
1533 completed.append(msg_id)
1533 completed.append(msg_id)
1534 local_results[msg_id] = self.results[msg_id]
1534 local_results[msg_id] = self.results[msg_id]
1535 theids.remove(msg_id)
1535 theids.remove(msg_id)
1536
1536
1537 if theids: # some not locally cached
1537 if theids: # some not locally cached
1538 content = dict(msg_ids=theids, status_only=status_only)
1538 content = dict(msg_ids=theids, status_only=status_only)
1539 msg = self.session.send(self._query_socket, "result_request", content=content)
1539 msg = self.session.send(self._query_socket, "result_request", content=content)
1540 zmq.select([self._query_socket], [], [])
1540 zmq.select([self._query_socket], [], [])
1541 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1541 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1542 if self.debug:
1542 if self.debug:
1543 pprint(msg)
1543 pprint(msg)
1544 content = msg['content']
1544 content = msg['content']
1545 if content['status'] != 'ok':
1545 if content['status'] != 'ok':
1546 raise self._unwrap_exception(content)
1546 raise self._unwrap_exception(content)
1547 buffers = msg['buffers']
1547 buffers = msg['buffers']
1548 else:
1548 else:
1549 content = dict(completed=[],pending=[])
1549 content = dict(completed=[],pending=[])
1550
1550
1551 content['completed'].extend(completed)
1551 content['completed'].extend(completed)
1552
1552
1553 if status_only:
1553 if status_only:
1554 return content
1554 return content
1555
1555
1556 failures = []
1556 failures = []
1557 # load cached results into result:
1557 # load cached results into result:
1558 content.update(local_results)
1558 content.update(local_results)
1559
1559
1560 # update cache with results:
1560 # update cache with results:
1561 for msg_id in sorted(theids):
1561 for msg_id in sorted(theids):
1562 if msg_id in content['completed']:
1562 if msg_id in content['completed']:
1563 rec = content[msg_id]
1563 rec = content[msg_id]
1564 parent = extract_dates(rec['header'])
1564 parent = extract_dates(rec['header'])
1565 header = extract_dates(rec['result_header'])
1565 header = extract_dates(rec['result_header'])
1566 rcontent = rec['result_content']
1566 rcontent = rec['result_content']
1567 iodict = rec['io']
1567 iodict = rec['io']
1568 if isinstance(rcontent, str):
1568 if isinstance(rcontent, str):
1569 rcontent = self.session.unpack(rcontent)
1569 rcontent = self.session.unpack(rcontent)
1570
1570
1571 md = self.metadata[msg_id]
1571 md = self.metadata[msg_id]
1572 md_msg = dict(
1572 md_msg = dict(
1573 content=rcontent,
1573 content=rcontent,
1574 parent_header=parent,
1574 parent_header=parent,
1575 header=header,
1575 header=header,
1576 metadata=rec['result_metadata'],
1576 metadata=rec['result_metadata'],
1577 )
1577 )
1578 md.update(self._extract_metadata(md_msg))
1578 md.update(self._extract_metadata(md_msg))
1579 if rec.get('received'):
1579 if rec.get('received'):
1580 md['received'] = parse_date(rec['received'])
1580 md['received'] = parse_date(rec['received'])
1581 md.update(iodict)
1581 md.update(iodict)
1582
1582
1583 if rcontent['status'] == 'ok':
1583 if rcontent['status'] == 'ok':
1584 if header['msg_type'] == 'apply_reply':
1584 if header['msg_type'] == 'apply_reply':
1585 res,buffers = serialize.unserialize_object(buffers)
1585 res,buffers = serialize.unserialize_object(buffers)
1586 elif header['msg_type'] == 'execute_reply':
1586 elif header['msg_type'] == 'execute_reply':
1587 res = ExecuteReply(msg_id, rcontent, md)
1587 res = ExecuteReply(msg_id, rcontent, md)
1588 else:
1588 else:
1589 raise KeyError("unhandled msg type: %r" % header['msg_type'])
1589 raise KeyError("unhandled msg type: %r" % header['msg_type'])
1590 else:
1590 else:
1591 res = self._unwrap_exception(rcontent)
1591 res = self._unwrap_exception(rcontent)
1592 failures.append(res)
1592 failures.append(res)
1593
1593
1594 self.results[msg_id] = res
1594 self.results[msg_id] = res
1595 content[msg_id] = res
1595 content[msg_id] = res
1596
1596
1597 if len(theids) == 1 and failures:
1597 if len(theids) == 1 and failures:
1598 raise failures[0]
1598 raise failures[0]
1599
1599
1600 error.collect_exceptions(failures, "result_status")
1600 error.collect_exceptions(failures, "result_status")
1601 return content
1601 return content
1602
1602
1603 @spin_first
1603 @spin_first
1604 def queue_status(self, targets='all', verbose=False):
1604 def queue_status(self, targets='all', verbose=False):
1605 """Fetch the status of engine queues.
1605 """Fetch the status of engine queues.
1606
1606
1607 Parameters
1607 Parameters
1608 ----------
1608 ----------
1609
1609
1610 targets : int/str/list of ints/strs
1610 targets : int/str/list of ints/strs
1611 the engines whose states are to be queried.
1611 the engines whose states are to be queried.
1612 default : all
1612 default : all
1613 verbose : bool
1613 verbose : bool
1614 Whether to return lengths only, or lists of ids for each element
1614 Whether to return lengths only, or lists of ids for each element
1615 """
1615 """
1616 if targets == 'all':
1616 if targets == 'all':
1617 # allow 'all' to be evaluated on the engine
1617 # allow 'all' to be evaluated on the engine
1618 engine_ids = None
1618 engine_ids = None
1619 else:
1619 else:
1620 engine_ids = self._build_targets(targets)[1]
1620 engine_ids = self._build_targets(targets)[1]
1621 content = dict(targets=engine_ids, verbose=verbose)
1621 content = dict(targets=engine_ids, verbose=verbose)
1622 self.session.send(self._query_socket, "queue_request", content=content)
1622 self.session.send(self._query_socket, "queue_request", content=content)
1623 idents,msg = self.session.recv(self._query_socket, 0)
1623 idents,msg = self.session.recv(self._query_socket, 0)
1624 if self.debug:
1624 if self.debug:
1625 pprint(msg)
1625 pprint(msg)
1626 content = msg['content']
1626 content = msg['content']
1627 status = content.pop('status')
1627 status = content.pop('status')
1628 if status != 'ok':
1628 if status != 'ok':
1629 raise self._unwrap_exception(content)
1629 raise self._unwrap_exception(content)
1630 content = rekey(content)
1630 content = rekey(content)
1631 if isinstance(targets, int):
1631 if isinstance(targets, int):
1632 return content[targets]
1632 return content[targets]
1633 else:
1633 else:
1634 return content
1634 return content
1635
1635
1636 def _build_msgids_from_target(self, targets=None):
1636 def _build_msgids_from_target(self, targets=None):
1637 """Build a list of msg_ids from the list of engine targets"""
1637 """Build a list of msg_ids from the list of engine targets"""
1638 if not targets: # needed as _build_targets otherwise uses all engines
1638 if not targets: # needed as _build_targets otherwise uses all engines
1639 return []
1639 return []
1640 target_ids = self._build_targets(targets)[0]
1640 target_ids = self._build_targets(targets)[0]
1641 return [md_id for md_id in self.metadata if self.metadata[md_id]["engine_uuid"] in target_ids]
1641 return [md_id for md_id in self.metadata if self.metadata[md_id]["engine_uuid"] in target_ids]
1642
1642
1643 def _build_msgids_from_jobs(self, jobs=None):
1643 def _build_msgids_from_jobs(self, jobs=None):
1644 """Build a list of msg_ids from "jobs" """
1644 """Build a list of msg_ids from "jobs" """
1645 if not jobs:
1645 if not jobs:
1646 return []
1646 return []
1647 msg_ids = []
1647 msg_ids = []
1648 if isinstance(jobs, string_types + (AsyncResult,)):
1648 if isinstance(jobs, string_types + (AsyncResult,)):
1649 jobs = [jobs]
1649 jobs = [jobs]
1650 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1650 bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
1651 if bad_ids:
1651 if bad_ids:
1652 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1652 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1653 for j in jobs:
1653 for j in jobs:
1654 if isinstance(j, AsyncResult):
1654 if isinstance(j, AsyncResult):
1655 msg_ids.extend(j.msg_ids)
1655 msg_ids.extend(j.msg_ids)
1656 else:
1656 else:
1657 msg_ids.append(j)
1657 msg_ids.append(j)
1658 return msg_ids
1658 return msg_ids
1659
1659
1660 def purge_local_results(self, jobs=[], targets=[]):
1660 def purge_local_results(self, jobs=[], targets=[]):
1661 """Clears the client caches of results and their metadata.
1661 """Clears the client caches of results and their metadata.
1662
1662
1663 Individual results can be purged by msg_id, or the entire
1663 Individual results can be purged by msg_id, or the entire
1664 history of specific targets can be purged.
1664 history of specific targets can be purged.
1665
1665
1666 Use `purge_local_results('all')` to scrub everything from the Clients's
1666 Use `purge_local_results('all')` to scrub everything from the Clients's
1667 results and metadata caches.
1667 results and metadata caches.
1668
1668
1669 After this call all `AsyncResults` are invalid and should be discarded.
1669 After this call all `AsyncResults` are invalid and should be discarded.
1670
1670
1671 If you must "reget" the results, you can still do so by using
1671 If you must "reget" the results, you can still do so by using
1672 `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
1672 `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
1673 redownload the results from the hub if they are still available
1673 redownload the results from the hub if they are still available
1674 (i.e `client.purge_hub_results(...)` has not been called.
1674 (i.e `client.purge_hub_results(...)` has not been called.
1675
1675
1676 Parameters
1676 Parameters
1677 ----------
1677 ----------
1678
1678
1679 jobs : str or list of str or AsyncResult objects
1679 jobs : str or list of str or AsyncResult objects
1680 the msg_ids whose results should be purged.
1680 the msg_ids whose results should be purged.
1681 targets : int/list of ints
1681 targets : int/list of ints
1682 The engines, by integer ID, whose entire result histories are to be purged.
1682 The engines, by integer ID, whose entire result histories are to be purged.
1683
1683
1684 Raises
1684 Raises
1685 ------
1685 ------
1686
1686
1687 RuntimeError : if any of the tasks to be purged are still outstanding.
1687 RuntimeError : if any of the tasks to be purged are still outstanding.
1688
1688
1689 """
1689 """
1690 if not targets and not jobs:
1690 if not targets and not jobs:
1691 raise ValueError("Must specify at least one of `targets` and `jobs`")
1691 raise ValueError("Must specify at least one of `targets` and `jobs`")
1692
1692
1693 if jobs == 'all':
1693 if jobs == 'all':
1694 if self.outstanding:
1694 if self.outstanding:
1695 raise RuntimeError("Can't purge outstanding tasks: %s" % self.outstanding)
1695 raise RuntimeError("Can't purge outstanding tasks: %s" % self.outstanding)
1696 self.results.clear()
1696 self.results.clear()
1697 self.metadata.clear()
1697 self.metadata.clear()
1698 else:
1698 else:
1699 msg_ids = set()
1699 msg_ids = set()
1700 msg_ids.update(self._build_msgids_from_target(targets))
1700 msg_ids.update(self._build_msgids_from_target(targets))
1701 msg_ids.update(self._build_msgids_from_jobs(jobs))
1701 msg_ids.update(self._build_msgids_from_jobs(jobs))
1702 still_outstanding = self.outstanding.intersection(msg_ids)
1702 still_outstanding = self.outstanding.intersection(msg_ids)
1703 if still_outstanding:
1703 if still_outstanding:
1704 raise RuntimeError("Can't purge outstanding tasks: %s" % still_outstanding)
1704 raise RuntimeError("Can't purge outstanding tasks: %s" % still_outstanding)
1705 for mid in msg_ids:
1705 for mid in msg_ids:
1706 self.results.pop(mid)
1706 self.results.pop(mid)
1707 self.metadata.pop(mid)
1707 self.metadata.pop(mid)
1708
1708
1709
1709
1710 @spin_first
1710 @spin_first
1711 def purge_hub_results(self, jobs=[], targets=[]):
1711 def purge_hub_results(self, jobs=[], targets=[]):
1712 """Tell the Hub to forget results.
1712 """Tell the Hub to forget results.
1713
1713
1714 Individual results can be purged by msg_id, or the entire
1714 Individual results can be purged by msg_id, or the entire
1715 history of specific targets can be purged.
1715 history of specific targets can be purged.
1716
1716
1717 Use `purge_results('all')` to scrub everything from the Hub's db.
1717 Use `purge_results('all')` to scrub everything from the Hub's db.
1718
1718
1719 Parameters
1719 Parameters
1720 ----------
1720 ----------
1721
1721
1722 jobs : str or list of str or AsyncResult objects
1722 jobs : str or list of str or AsyncResult objects
1723 the msg_ids whose results should be forgotten.
1723 the msg_ids whose results should be forgotten.
1724 targets : int/str/list of ints/strs
1724 targets : int/str/list of ints/strs
1725 The targets, by int_id, whose entire history is to be purged.
1725 The targets, by int_id, whose entire history is to be purged.
1726
1726
1727 default : None
1727 default : None
1728 """
1728 """
1729 if not targets and not jobs:
1729 if not targets and not jobs:
1730 raise ValueError("Must specify at least one of `targets` and `jobs`")
1730 raise ValueError("Must specify at least one of `targets` and `jobs`")
1731 if targets:
1731 if targets:
1732 targets = self._build_targets(targets)[1]
1732 targets = self._build_targets(targets)[1]
1733
1733
1734 # construct msg_ids from jobs
1734 # construct msg_ids from jobs
1735 if jobs == 'all':
1735 if jobs == 'all':
1736 msg_ids = jobs
1736 msg_ids = jobs
1737 else:
1737 else:
1738 msg_ids = self._build_msgids_from_jobs(jobs)
1738 msg_ids = self._build_msgids_from_jobs(jobs)
1739
1739
1740 content = dict(engine_ids=targets, msg_ids=msg_ids)
1740 content = dict(engine_ids=targets, msg_ids=msg_ids)
1741 self.session.send(self._query_socket, "purge_request", content=content)
1741 self.session.send(self._query_socket, "purge_request", content=content)
1742 idents, msg = self.session.recv(self._query_socket, 0)
1742 idents, msg = self.session.recv(self._query_socket, 0)
1743 if self.debug:
1743 if self.debug:
1744 pprint(msg)
1744 pprint(msg)
1745 content = msg['content']
1745 content = msg['content']
1746 if content['status'] != 'ok':
1746 if content['status'] != 'ok':
1747 raise self._unwrap_exception(content)
1747 raise self._unwrap_exception(content)
1748
1748
1749 def purge_results(self, jobs=[], targets=[]):
1749 def purge_results(self, jobs=[], targets=[]):
1750 """Clears the cached results from both the hub and the local client
1750 """Clears the cached results from both the hub and the local client
1751
1751
1752 Individual results can be purged by msg_id, or the entire
1752 Individual results can be purged by msg_id, or the entire
1753 history of specific targets can be purged.
1753 history of specific targets can be purged.
1754
1754
1755 Use `purge_results('all')` to scrub every cached result from both the Hub's and
1755 Use `purge_results('all')` to scrub every cached result from both the Hub's and
1756 the Client's db.
1756 the Client's db.
1757
1757
1758 Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with
1758 Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with
1759 the same arguments.
1759 the same arguments.
1760
1760
1761 Parameters
1761 Parameters
1762 ----------
1762 ----------
1763
1763
1764 jobs : str or list of str or AsyncResult objects
1764 jobs : str or list of str or AsyncResult objects
1765 the msg_ids whose results should be forgotten.
1765 the msg_ids whose results should be forgotten.
1766 targets : int/str/list of ints/strs
1766 targets : int/str/list of ints/strs
1767 The targets, by int_id, whose entire history is to be purged.
1767 The targets, by int_id, whose entire history is to be purged.
1768
1768
1769 default : None
1769 default : None
1770 """
1770 """
1771 self.purge_local_results(jobs=jobs, targets=targets)
1771 self.purge_local_results(jobs=jobs, targets=targets)
1772 self.purge_hub_results(jobs=jobs, targets=targets)
1772 self.purge_hub_results(jobs=jobs, targets=targets)
1773
1773
1774 def purge_everything(self):
1774 def purge_everything(self):
1775 """Clears all content from previous Tasks from both the hub and the local client
1775 """Clears all content from previous Tasks from both the hub and the local client
1776
1776
1777 In addition to calling `purge_results("all")` it also deletes the history and
1777 In addition to calling `purge_results("all")` it also deletes the history and
1778 other bookkeeping lists.
1778 other bookkeeping lists.
1779 """
1779 """
1780 self.purge_results("all")
1780 self.purge_results("all")
1781 self.history = []
1781 self.history = []
1782 self.session.digest_history.clear()
1782 self.session.digest_history.clear()
1783
1783
1784 @spin_first
1784 @spin_first
1785 def hub_history(self):
1785 def hub_history(self):
1786 """Get the Hub's history
1786 """Get the Hub's history
1787
1787
1788 Just like the Client, the Hub has a history, which is a list of msg_ids.
1788 Just like the Client, the Hub has a history, which is a list of msg_ids.
1789 This will contain the history of all clients, and, depending on configuration,
1789 This will contain the history of all clients, and, depending on configuration,
1790 may contain history across multiple cluster sessions.
1790 may contain history across multiple cluster sessions.
1791
1791
1792 Any msg_id returned here is a valid argument to `get_result`.
1792 Any msg_id returned here is a valid argument to `get_result`.
1793
1793
1794 Returns
1794 Returns
1795 -------
1795 -------
1796
1796
1797 msg_ids : list of strs
1797 msg_ids : list of strs
1798 list of all msg_ids, ordered by task submission time.
1798 list of all msg_ids, ordered by task submission time.
1799 """
1799 """
1800
1800
1801 self.session.send(self._query_socket, "history_request", content={})
1801 self.session.send(self._query_socket, "history_request", content={})
1802 idents, msg = self.session.recv(self._query_socket, 0)
1802 idents, msg = self.session.recv(self._query_socket, 0)
1803
1803
1804 if self.debug:
1804 if self.debug:
1805 pprint(msg)
1805 pprint(msg)
1806 content = msg['content']
1806 content = msg['content']
1807 if content['status'] != 'ok':
1807 if content['status'] != 'ok':
1808 raise self._unwrap_exception(content)
1808 raise self._unwrap_exception(content)
1809 else:
1809 else:
1810 return content['history']
1810 return content['history']
1811
1811
1812 @spin_first
1812 @spin_first
1813 def db_query(self, query, keys=None):
1813 def db_query(self, query, keys=None):
1814 """Query the Hub's TaskRecord database
1814 """Query the Hub's TaskRecord database
1815
1815
1816 This will return a list of task record dicts that match `query`
1816 This will return a list of task record dicts that match `query`
1817
1817
1818 Parameters
1818 Parameters
1819 ----------
1819 ----------
1820
1820
1821 query : mongodb query dict
1821 query : mongodb query dict
1822 The search dict. See mongodb query docs for details.
1822 The search dict. See mongodb query docs for details.
1823 keys : list of strs [optional]
1823 keys : list of strs [optional]
1824 The subset of keys to be returned. The default is to fetch everything but buffers.
1824 The subset of keys to be returned. The default is to fetch everything but buffers.
1825 'msg_id' will *always* be included.
1825 'msg_id' will *always* be included.
1826 """
1826 """
1827 if isinstance(keys, string_types):
1827 if isinstance(keys, string_types):
1828 keys = [keys]
1828 keys = [keys]
1829 content = dict(query=query, keys=keys)
1829 content = dict(query=query, keys=keys)
1830 self.session.send(self._query_socket, "db_request", content=content)
1830 self.session.send(self._query_socket, "db_request", content=content)
1831 idents, msg = self.session.recv(self._query_socket, 0)
1831 idents, msg = self.session.recv(self._query_socket, 0)
1832 if self.debug:
1832 if self.debug:
1833 pprint(msg)
1833 pprint(msg)
1834 content = msg['content']
1834 content = msg['content']
1835 if content['status'] != 'ok':
1835 if content['status'] != 'ok':
1836 raise self._unwrap_exception(content)
1836 raise self._unwrap_exception(content)
1837
1837
1838 records = content['records']
1838 records = content['records']
1839
1839
1840 buffer_lens = content['buffer_lens']
1840 buffer_lens = content['buffer_lens']
1841 result_buffer_lens = content['result_buffer_lens']
1841 result_buffer_lens = content['result_buffer_lens']
1842 buffers = msg['buffers']
1842 buffers = msg['buffers']
1843 has_bufs = buffer_lens is not None
1843 has_bufs = buffer_lens is not None
1844 has_rbufs = result_buffer_lens is not None
1844 has_rbufs = result_buffer_lens is not None
1845 for i,rec in enumerate(records):
1845 for i,rec in enumerate(records):
1846 # unpack datetime objects
1846 # unpack datetime objects
1847 for hkey in ('header', 'result_header'):
1847 for hkey in ('header', 'result_header'):
1848 if hkey in rec:
1848 if hkey in rec:
1849 rec[hkey] = extract_dates(rec[hkey])
1849 rec[hkey] = extract_dates(rec[hkey])
1850 for dtkey in ('submitted', 'started', 'completed', 'received'):
1850 for dtkey in ('submitted', 'started', 'completed', 'received'):
1851 if dtkey in rec:
1851 if dtkey in rec:
1852 rec[dtkey] = parse_date(rec[dtkey])
1852 rec[dtkey] = parse_date(rec[dtkey])
1853 # relink buffers
1853 # relink buffers
1854 if has_bufs:
1854 if has_bufs:
1855 blen = buffer_lens[i]
1855 blen = buffer_lens[i]
1856 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
1856 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
1857 if has_rbufs:
1857 if has_rbufs:
1858 blen = result_buffer_lens[i]
1858 blen = result_buffer_lens[i]
1859 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
1859 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
1860
1860
1861 return records
1861 return records
1862
1862
1863 __all__ = [ 'Client' ]
1863 __all__ = [ 'Client' ]
@@ -1,179 +1,169 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """IO capturing utilities."""
3 IO capturing utilities.
4 """
5
3
6 #-----------------------------------------------------------------------------
4 # Copyright (c) IPython Development Team.
7 # Copyright (C) 2013 The IPython Development Team
5 # Distributed under the terms of the Modified BSD License.
8 #
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
12 from __future__ import print_function, absolute_import
13
6
14 #-----------------------------------------------------------------------------
7 from __future__ import print_function, absolute_import
15 # Imports
16 #-----------------------------------------------------------------------------
17
8
18 import sys
9 import sys
19
10
20 from IPython.utils.py3compat import PY3
11 from IPython.utils.py3compat import PY3
21
12
22 if PY3:
13 if PY3:
23 from io import StringIO
14 from io import StringIO
24 else:
15 else:
25 from StringIO import StringIO
16 from StringIO import StringIO
26
17
27 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
28 # Classes and functions
19 # Classes and functions
29 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
30
21
31
22
32 class RichOutput(object):
23 class RichOutput(object):
33 def __init__(self, source="", data=None, metadata=None):
24 def __init__(self, data=None, metadata=None):
34 self.source = source
35 self.data = data or {}
25 self.data = data or {}
36 self.metadata = metadata or {}
26 self.metadata = metadata or {}
37
27
38 def display(self):
28 def display(self):
39 from IPython.display import publish_display_data
29 from IPython.display import publish_display_data
40 publish_display_data(self.source, self.data, self.metadata)
30 publish_display_data(data=self.data, metadata=self.metadata)
41
31
42 def _repr_mime_(self, mime):
32 def _repr_mime_(self, mime):
43 if mime not in self.data:
33 if mime not in self.data:
44 return
34 return
45 data = self.data[mime]
35 data = self.data[mime]
46 if mime in self.metadata:
36 if mime in self.metadata:
47 return data, self.metadata[mime]
37 return data, self.metadata[mime]
48 else:
38 else:
49 return data
39 return data
50
40
51 def _repr_html_(self):
41 def _repr_html_(self):
52 return self._repr_mime_("text/html")
42 return self._repr_mime_("text/html")
53
43
54 def _repr_latex_(self):
44 def _repr_latex_(self):
55 return self._repr_mime_("text/latex")
45 return self._repr_mime_("text/latex")
56
46
57 def _repr_json_(self):
47 def _repr_json_(self):
58 return self._repr_mime_("application/json")
48 return self._repr_mime_("application/json")
59
49
60 def _repr_javascript_(self):
50 def _repr_javascript_(self):
61 return self._repr_mime_("application/javascript")
51 return self._repr_mime_("application/javascript")
62
52
63 def _repr_png_(self):
53 def _repr_png_(self):
64 return self._repr_mime_("image/png")
54 return self._repr_mime_("image/png")
65
55
66 def _repr_jpeg_(self):
56 def _repr_jpeg_(self):
67 return self._repr_mime_("image/jpeg")
57 return self._repr_mime_("image/jpeg")
68
58
69 def _repr_svg_(self):
59 def _repr_svg_(self):
70 return self._repr_mime_("image/svg+xml")
60 return self._repr_mime_("image/svg+xml")
71
61
72
62
73 class CapturedIO(object):
63 class CapturedIO(object):
74 """Simple object for containing captured stdout/err and rich display StringIO objects
64 """Simple object for containing captured stdout/err and rich display StringIO objects
75
65
76 Each instance `c` has three attributes:
66 Each instance `c` has three attributes:
77
67
78 - ``c.stdout`` : standard output as a string
68 - ``c.stdout`` : standard output as a string
79 - ``c.stderr`` : standard error as a string
69 - ``c.stderr`` : standard error as a string
80 - ``c.outputs``: a list of rich display outputs
70 - ``c.outputs``: a list of rich display outputs
81
71
82 Additionally, there's a ``c.show()`` method which will print all of the
72 Additionally, there's a ``c.show()`` method which will print all of the
83 above in the same order, and can be invoked simply via ``c()``.
73 above in the same order, and can be invoked simply via ``c()``.
84 """
74 """
85
75
86 def __init__(self, stdout, stderr, outputs=None):
76 def __init__(self, stdout, stderr, outputs=None):
87 self._stdout = stdout
77 self._stdout = stdout
88 self._stderr = stderr
78 self._stderr = stderr
89 if outputs is None:
79 if outputs is None:
90 outputs = []
80 outputs = []
91 self._outputs = outputs
81 self._outputs = outputs
92
82
93 def __str__(self):
83 def __str__(self):
94 return self.stdout
84 return self.stdout
95
85
96 @property
86 @property
97 def stdout(self):
87 def stdout(self):
98 "Captured standard output"
88 "Captured standard output"
99 if not self._stdout:
89 if not self._stdout:
100 return ''
90 return ''
101 return self._stdout.getvalue()
91 return self._stdout.getvalue()
102
92
103 @property
93 @property
104 def stderr(self):
94 def stderr(self):
105 "Captured standard error"
95 "Captured standard error"
106 if not self._stderr:
96 if not self._stderr:
107 return ''
97 return ''
108 return self._stderr.getvalue()
98 return self._stderr.getvalue()
109
99
110 @property
100 @property
111 def outputs(self):
101 def outputs(self):
112 """A list of the captured rich display outputs, if any.
102 """A list of the captured rich display outputs, if any.
113
103
114 If you have a CapturedIO object ``c``, these can be displayed in IPython
104 If you have a CapturedIO object ``c``, these can be displayed in IPython
115 using::
105 using::
116
106
117 from IPython.display import display
107 from IPython.display import display
118 for o in c.outputs:
108 for o in c.outputs:
119 display(o)
109 display(o)
120 """
110 """
121 return [ RichOutput(s, d, md) for s, d, md in self._outputs ]
111 return [ RichOutput(d, md) for d, md in self._outputs ]
122
112
123 def show(self):
113 def show(self):
124 """write my output to sys.stdout/err as appropriate"""
114 """write my output to sys.stdout/err as appropriate"""
125 sys.stdout.write(self.stdout)
115 sys.stdout.write(self.stdout)
126 sys.stderr.write(self.stderr)
116 sys.stderr.write(self.stderr)
127 sys.stdout.flush()
117 sys.stdout.flush()
128 sys.stderr.flush()
118 sys.stderr.flush()
129 for source, data, metadata in self._outputs:
119 for data, metadata in self._outputs:
130 RichOutput(source, data, metadata).display()
120 RichOutput(data, metadata).display()
131
121
132 __call__ = show
122 __call__ = show
133
123
134
124
135 class capture_output(object):
125 class capture_output(object):
136 """context manager for capturing stdout/err"""
126 """context manager for capturing stdout/err"""
137 stdout = True
127 stdout = True
138 stderr = True
128 stderr = True
139 display = True
129 display = True
140
130
141 def __init__(self, stdout=True, stderr=True, display=True):
131 def __init__(self, stdout=True, stderr=True, display=True):
142 self.stdout = stdout
132 self.stdout = stdout
143 self.stderr = stderr
133 self.stderr = stderr
144 self.display = display
134 self.display = display
145 self.shell = None
135 self.shell = None
146
136
147 def __enter__(self):
137 def __enter__(self):
148 from IPython.core.getipython import get_ipython
138 from IPython.core.getipython import get_ipython
149 from IPython.core.displaypub import CapturingDisplayPublisher
139 from IPython.core.displaypub import CapturingDisplayPublisher
150
140
151 self.sys_stdout = sys.stdout
141 self.sys_stdout = sys.stdout
152 self.sys_stderr = sys.stderr
142 self.sys_stderr = sys.stderr
153
143
154 if self.display:
144 if self.display:
155 self.shell = get_ipython()
145 self.shell = get_ipython()
156 if self.shell is None:
146 if self.shell is None:
157 self.save_display_pub = None
147 self.save_display_pub = None
158 self.display = False
148 self.display = False
159
149
160 stdout = stderr = outputs = None
150 stdout = stderr = outputs = None
161 if self.stdout:
151 if self.stdout:
162 stdout = sys.stdout = StringIO()
152 stdout = sys.stdout = StringIO()
163 if self.stderr:
153 if self.stderr:
164 stderr = sys.stderr = StringIO()
154 stderr = sys.stderr = StringIO()
165 if self.display:
155 if self.display:
166 self.save_display_pub = self.shell.display_pub
156 self.save_display_pub = self.shell.display_pub
167 self.shell.display_pub = CapturingDisplayPublisher()
157 self.shell.display_pub = CapturingDisplayPublisher()
168 outputs = self.shell.display_pub.outputs
158 outputs = self.shell.display_pub.outputs
169
159
170
160
171 return CapturedIO(stdout, stderr, outputs)
161 return CapturedIO(stdout, stderr, outputs)
172
162
173 def __exit__(self, exc_type, exc_value, traceback):
163 def __exit__(self, exc_type, exc_value, traceback):
174 sys.stdout = self.sys_stdout
164 sys.stdout = self.sys_stdout
175 sys.stderr = self.sys_stderr
165 sys.stderr = self.sys_stderr
176 if self.display and self.shell:
166 if self.display and self.shell:
177 self.shell.display_pub = self.save_display_pub
167 self.shell.display_pub = self.save_display_pub
178
168
179
169
@@ -1,161 +1,160 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """Tests for IPython.utils.capture"""
2 """Tests for IPython.utils.capture"""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2013 The IPython Development Team
5 # Copyright (C) 2013 The IPython Development Team
6 #
6 #
7 # Distributed under the terms of the BSD License. The full license is in
7 # Distributed under the terms of the BSD License. The full license is in
8 # the file COPYING, distributed as part of this software.
8 # the file COPYING, distributed as part of this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 from __future__ import print_function
15 from __future__ import print_function
16
16
17 import sys
17 import sys
18
18
19 import nose.tools as nt
19 import nose.tools as nt
20
20
21 from IPython.utils import capture
21 from IPython.utils import capture
22
22
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24 # Globals
24 # Globals
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 _mime_map = dict(
27 _mime_map = dict(
28 _repr_png_="image/png",
28 _repr_png_="image/png",
29 _repr_jpeg_="image/jpeg",
29 _repr_jpeg_="image/jpeg",
30 _repr_svg_="image/svg+xml",
30 _repr_svg_="image/svg+xml",
31 _repr_html_="text/html",
31 _repr_html_="text/html",
32 _repr_json_="application/json",
32 _repr_json_="application/json",
33 _repr_javascript_="application/javascript",
33 _repr_javascript_="application/javascript",
34 )
34 )
35
35
36 basic_data = {
36 basic_data = {
37 'image/png' : b'binarydata',
37 'image/png' : b'binarydata',
38 'text/html' : "<b>bold</b>",
38 'text/html' : "<b>bold</b>",
39 }
39 }
40 basic_metadata = {
40 basic_metadata = {
41 'image/png' : {
41 'image/png' : {
42 'width' : 10,
42 'width' : 10,
43 'height' : 20,
43 'height' : 20,
44 },
44 },
45 }
45 }
46
46
47 full_data = {
47 full_data = {
48 'image/png' : b'binarydata',
48 'image/png' : b'binarydata',
49 'image/jpeg' : b'binarydata',
49 'image/jpeg' : b'binarydata',
50 'image/svg+xml' : "<svg>",
50 'image/svg+xml' : "<svg>",
51 'text/html' : "<b>bold</b>",
51 'text/html' : "<b>bold</b>",
52 'application/javascript' : "alert();",
52 'application/javascript' : "alert();",
53 'application/json' : "{}",
53 'application/json' : "{}",
54 }
54 }
55 full_metadata = {
55 full_metadata = {
56 'image/png' : {"png" : "exists"},
56 'image/png' : {"png" : "exists"},
57 'image/jpeg' : {"jpeg" : "exists"},
57 'image/jpeg' : {"jpeg" : "exists"},
58 'image/svg+xml' : {"svg" : "exists"},
58 'image/svg+xml' : {"svg" : "exists"},
59 'text/html' : {"html" : "exists"},
59 'text/html' : {"html" : "exists"},
60 'application/javascript' : {"js" : "exists"},
60 'application/javascript' : {"js" : "exists"},
61 'application/json' : {"json" : "exists"},
61 'application/json' : {"json" : "exists"},
62 }
62 }
63
63
64 hello_stdout = "hello, stdout"
64 hello_stdout = "hello, stdout"
65 hello_stderr = "hello, stderr"
65 hello_stderr = "hello, stderr"
66
66
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68 # Test Functions
68 # Test Functions
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70
70
71 def test_rich_output_empty():
71 def test_rich_output_empty():
72 """RichOutput with no args"""
72 """RichOutput with no args"""
73 rich = capture.RichOutput()
73 rich = capture.RichOutput()
74 for method, mime in _mime_map.items():
74 for method, mime in _mime_map.items():
75 yield nt.assert_equal, getattr(rich, method)(), None
75 yield nt.assert_equal, getattr(rich, method)(), None
76
76
77 def test_rich_output():
77 def test_rich_output():
78 """test RichOutput basics"""
78 """test RichOutput basics"""
79 data = basic_data
79 data = basic_data
80 metadata = basic_metadata
80 metadata = basic_metadata
81 rich = capture.RichOutput(source="test", data=data, metadata=metadata)
81 rich = capture.RichOutput(data=data, metadata=metadata)
82 yield nt.assert_equal, rich.source, "test"
83 yield nt.assert_equal, rich._repr_html_(), data['text/html']
82 yield nt.assert_equal, rich._repr_html_(), data['text/html']
84 yield nt.assert_equal, rich._repr_png_(), (data['image/png'], metadata['image/png'])
83 yield nt.assert_equal, rich._repr_png_(), (data['image/png'], metadata['image/png'])
85 yield nt.assert_equal, rich._repr_latex_(), None
84 yield nt.assert_equal, rich._repr_latex_(), None
86 yield nt.assert_equal, rich._repr_javascript_(), None
85 yield nt.assert_equal, rich._repr_javascript_(), None
87 yield nt.assert_equal, rich._repr_svg_(), None
86 yield nt.assert_equal, rich._repr_svg_(), None
88
87
89 def test_rich_output_no_metadata():
88 def test_rich_output_no_metadata():
90 """test RichOutput with no metadata"""
89 """test RichOutput with no metadata"""
91 data = full_data
90 data = full_data
92 rich = capture.RichOutput(source="test", data=data)
91 rich = capture.RichOutput(data=data)
93 for method, mime in _mime_map.items():
92 for method, mime in _mime_map.items():
94 yield nt.assert_equal, getattr(rich, method)(), data[mime]
93 yield nt.assert_equal, getattr(rich, method)(), data[mime]
95
94
96 def test_rich_output_metadata():
95 def test_rich_output_metadata():
97 """test RichOutput with metadata"""
96 """test RichOutput with metadata"""
98 data = full_data
97 data = full_data
99 metadata = full_metadata
98 metadata = full_metadata
100 rich = capture.RichOutput(source="test", data=data, metadata=metadata)
99 rich = capture.RichOutput(data=data, metadata=metadata)
101 for method, mime in _mime_map.items():
100 for method, mime in _mime_map.items():
102 yield nt.assert_equal, getattr(rich, method)(), (data[mime], metadata[mime])
101 yield nt.assert_equal, getattr(rich, method)(), (data[mime], metadata[mime])
103
102
104 def test_rich_output_display():
103 def test_rich_output_display():
105 """test RichOutput.display
104 """test RichOutput.display
106
105
107 This is a bit circular, because we are actually using the capture code we are testing
106 This is a bit circular, because we are actually using the capture code we are testing
108 to test itself.
107 to test itself.
109 """
108 """
110 data = full_data
109 data = full_data
111 rich = capture.RichOutput(data=data)
110 rich = capture.RichOutput(data=data)
112 with capture.capture_output() as cap:
111 with capture.capture_output() as cap:
113 rich.display()
112 rich.display()
114 yield nt.assert_equal, len(cap.outputs), 1
113 yield nt.assert_equal, len(cap.outputs), 1
115 rich2 = cap.outputs[0]
114 rich2 = cap.outputs[0]
116 yield nt.assert_equal, rich2.data, rich.data
115 yield nt.assert_equal, rich2.data, rich.data
117 yield nt.assert_equal, rich2.metadata, rich.metadata
116 yield nt.assert_equal, rich2.metadata, rich.metadata
118
117
119 def test_capture_output():
118 def test_capture_output():
120 """capture_output works"""
119 """capture_output works"""
121 rich = capture.RichOutput(data=full_data)
120 rich = capture.RichOutput(data=full_data)
122 with capture.capture_output() as cap:
121 with capture.capture_output() as cap:
123 print(hello_stdout, end="")
122 print(hello_stdout, end="")
124 print(hello_stderr, end="", file=sys.stderr)
123 print(hello_stderr, end="", file=sys.stderr)
125 rich.display()
124 rich.display()
126 yield nt.assert_equal, hello_stdout, cap.stdout
125 yield nt.assert_equal, hello_stdout, cap.stdout
127 yield nt.assert_equal, hello_stderr, cap.stderr
126 yield nt.assert_equal, hello_stderr, cap.stderr
128
127
129 def test_capture_output_no_stdout():
128 def test_capture_output_no_stdout():
130 """test capture_output(stdout=False)"""
129 """test capture_output(stdout=False)"""
131 rich = capture.RichOutput(data=full_data)
130 rich = capture.RichOutput(data=full_data)
132 with capture.capture_output(stdout=False) as cap:
131 with capture.capture_output(stdout=False) as cap:
133 print(hello_stdout, end="")
132 print(hello_stdout, end="")
134 print(hello_stderr, end="", file=sys.stderr)
133 print(hello_stderr, end="", file=sys.stderr)
135 rich.display()
134 rich.display()
136 yield nt.assert_equal, "", cap.stdout
135 yield nt.assert_equal, "", cap.stdout
137 yield nt.assert_equal, hello_stderr, cap.stderr
136 yield nt.assert_equal, hello_stderr, cap.stderr
138 yield nt.assert_equal, len(cap.outputs), 1
137 yield nt.assert_equal, len(cap.outputs), 1
139
138
140 def test_capture_output_no_stderr():
139 def test_capture_output_no_stderr():
141 """test capture_output(stderr=False)"""
140 """test capture_output(stderr=False)"""
142 rich = capture.RichOutput(data=full_data)
141 rich = capture.RichOutput(data=full_data)
143 # add nested capture_output so stderr doesn't make it to nose output
142 # add nested capture_output so stderr doesn't make it to nose output
144 with capture.capture_output(), capture.capture_output(stderr=False) as cap:
143 with capture.capture_output(), capture.capture_output(stderr=False) as cap:
145 print(hello_stdout, end="")
144 print(hello_stdout, end="")
146 print(hello_stderr, end="", file=sys.stderr)
145 print(hello_stderr, end="", file=sys.stderr)
147 rich.display()
146 rich.display()
148 yield nt.assert_equal, hello_stdout, cap.stdout
147 yield nt.assert_equal, hello_stdout, cap.stdout
149 yield nt.assert_equal, "", cap.stderr
148 yield nt.assert_equal, "", cap.stderr
150 yield nt.assert_equal, len(cap.outputs), 1
149 yield nt.assert_equal, len(cap.outputs), 1
151
150
152 def test_capture_output_no_display():
151 def test_capture_output_no_display():
153 """test capture_output(display=False)"""
152 """test capture_output(display=False)"""
154 rich = capture.RichOutput(data=full_data)
153 rich = capture.RichOutput(data=full_data)
155 with capture.capture_output(display=False) as cap:
154 with capture.capture_output(display=False) as cap:
156 print(hello_stdout, end="")
155 print(hello_stdout, end="")
157 print(hello_stderr, end="", file=sys.stderr)
156 print(hello_stderr, end="", file=sys.stderr)
158 rich.display()
157 rich.display()
159 yield nt.assert_equal, hello_stdout, cap.stdout
158 yield nt.assert_equal, hello_stdout, cap.stdout
160 yield nt.assert_equal, hello_stderr, cap.stderr
159 yield nt.assert_equal, hello_stderr, cap.stderr
161 yield nt.assert_equal, cap.outputs, [] No newline at end of file
160 yield nt.assert_equal, cap.outputs, []
General Comments 0
You need to be logged in to leave comments. Login now