##// END OF EJS Templates
Fix imports of builtins module
Thomas Kluyver -
Show More
@@ -1,821 +1,821 b''
1 """A simple configuration system.
1 """A simple configuration system.
2
2
3 Inheritance diagram:
3 Inheritance diagram:
4
4
5 .. inheritance-diagram:: IPython.config.loader
5 .. inheritance-diagram:: IPython.config.loader
6 :parts: 3
6 :parts: 3
7
7
8 Authors
8 Authors
9 -------
9 -------
10 * Brian Granger
10 * Brian Granger
11 * Fernando Perez
11 * Fernando Perez
12 * Min RK
12 * Min RK
13 """
13 """
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Copyright (C) 2008-2011 The IPython Development Team
16 # Copyright (C) 2008-2011 The IPython Development Team
17 #
17 #
18 # Distributed under the terms of the BSD License. The full license is in
18 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
19 # the file COPYING, distributed as part of this software.
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 # Imports
23 # Imports
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 import __builtin__ as builtin_mod
27 import argparse
26 import argparse
28 import copy
27 import copy
29 import os
28 import os
30 import re
29 import re
31 import sys
30 import sys
32
31
33 from IPython.utils.path import filefind, get_ipython_dir
32 from IPython.utils.path import filefind, get_ipython_dir
34 from IPython.utils import py3compat, warn
33 from IPython.utils import py3compat, warn
35 from IPython.utils.encoding import DEFAULT_ENCODING
34 from IPython.utils.encoding import DEFAULT_ENCODING
35 from IPython.utils.py3compat import builtin_mod
36 from IPython.utils.traitlets import HasTraits, List, Any, TraitError
36 from IPython.utils.traitlets import HasTraits, List, Any, TraitError
37
37
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39 # Exceptions
39 # Exceptions
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41
41
42
42
43 class ConfigError(Exception):
43 class ConfigError(Exception):
44 pass
44 pass
45
45
46 class ConfigLoaderError(ConfigError):
46 class ConfigLoaderError(ConfigError):
47 pass
47 pass
48
48
49 class ConfigFileNotFound(ConfigError):
49 class ConfigFileNotFound(ConfigError):
50 pass
50 pass
51
51
52 class ArgumentError(ConfigLoaderError):
52 class ArgumentError(ConfigLoaderError):
53 pass
53 pass
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Argparse fix
56 # Argparse fix
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 # Unfortunately argparse by default prints help messages to stderr instead of
59 # Unfortunately argparse by default prints help messages to stderr instead of
60 # stdout. This makes it annoying to capture long help screens at the command
60 # stdout. This makes it annoying to capture long help screens at the command
61 # line, since one must know how to pipe stderr, which many users don't know how
61 # line, since one must know how to pipe stderr, which many users don't know how
62 # to do. So we override the print_help method with one that defaults to
62 # to do. So we override the print_help method with one that defaults to
63 # stdout and use our class instead.
63 # stdout and use our class instead.
64
64
65 class ArgumentParser(argparse.ArgumentParser):
65 class ArgumentParser(argparse.ArgumentParser):
66 """Simple argparse subclass that prints help to stdout by default."""
66 """Simple argparse subclass that prints help to stdout by default."""
67
67
68 def print_help(self, file=None):
68 def print_help(self, file=None):
69 if file is None:
69 if file is None:
70 file = sys.stdout
70 file = sys.stdout
71 return super(ArgumentParser, self).print_help(file)
71 return super(ArgumentParser, self).print_help(file)
72
72
73 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
73 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
74
74
75 #-----------------------------------------------------------------------------
75 #-----------------------------------------------------------------------------
76 # Config class for holding config information
76 # Config class for holding config information
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78
78
79 class LazyConfigValue(HasTraits):
79 class LazyConfigValue(HasTraits):
80 """Proxy object for exposing methods on configurable containers
80 """Proxy object for exposing methods on configurable containers
81
81
82 Exposes:
82 Exposes:
83
83
84 - append, extend, insert on lists
84 - append, extend, insert on lists
85 - update on dicts
85 - update on dicts
86 - update, add on sets
86 - update, add on sets
87 """
87 """
88
88
89 _value = None
89 _value = None
90
90
91 # list methods
91 # list methods
92 _extend = List()
92 _extend = List()
93 _prepend = List()
93 _prepend = List()
94
94
95 def append(self, obj):
95 def append(self, obj):
96 self._extend.append(obj)
96 self._extend.append(obj)
97
97
98 def extend(self, other):
98 def extend(self, other):
99 self._extend.extend(other)
99 self._extend.extend(other)
100
100
101 def prepend(self, other):
101 def prepend(self, other):
102 """like list.extend, but for the front"""
102 """like list.extend, but for the front"""
103 self._prepend[:0] = other
103 self._prepend[:0] = other
104
104
105 _inserts = List()
105 _inserts = List()
106 def insert(self, index, other):
106 def insert(self, index, other):
107 if not isinstance(index, int):
107 if not isinstance(index, int):
108 raise TypeError("An integer is required")
108 raise TypeError("An integer is required")
109 self._inserts.append((index, other))
109 self._inserts.append((index, other))
110
110
111 # dict methods
111 # dict methods
112 # update is used for both dict and set
112 # update is used for both dict and set
113 _update = Any()
113 _update = Any()
114 def update(self, other):
114 def update(self, other):
115 if self._update is None:
115 if self._update is None:
116 if isinstance(other, dict):
116 if isinstance(other, dict):
117 self._update = {}
117 self._update = {}
118 else:
118 else:
119 self._update = set()
119 self._update = set()
120 self._update.update(other)
120 self._update.update(other)
121
121
122 # set methods
122 # set methods
123 def add(self, obj):
123 def add(self, obj):
124 self.update({obj})
124 self.update({obj})
125
125
126 def get_value(self, initial):
126 def get_value(self, initial):
127 """construct the value from the initial one
127 """construct the value from the initial one
128
128
129 after applying any insert / extend / update changes
129 after applying any insert / extend / update changes
130 """
130 """
131 if self._value is not None:
131 if self._value is not None:
132 return self._value
132 return self._value
133 value = copy.deepcopy(initial)
133 value = copy.deepcopy(initial)
134 if isinstance(value, list):
134 if isinstance(value, list):
135 for idx, obj in self._inserts:
135 for idx, obj in self._inserts:
136 value.insert(idx, obj)
136 value.insert(idx, obj)
137 value[:0] = self._prepend
137 value[:0] = self._prepend
138 value.extend(self._extend)
138 value.extend(self._extend)
139
139
140 elif isinstance(value, dict):
140 elif isinstance(value, dict):
141 if self._update:
141 if self._update:
142 value.update(self._update)
142 value.update(self._update)
143 elif isinstance(value, set):
143 elif isinstance(value, set):
144 if self._update:
144 if self._update:
145 value.update(self._update)
145 value.update(self._update)
146 self._value = value
146 self._value = value
147 return value
147 return value
148
148
149 def to_dict(self):
149 def to_dict(self):
150 """return JSONable dict form of my data
150 """return JSONable dict form of my data
151
151
152 Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
152 Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
153 """
153 """
154 d = {}
154 d = {}
155 if self._update:
155 if self._update:
156 d['update'] = self._update
156 d['update'] = self._update
157 if self._extend:
157 if self._extend:
158 d['extend'] = self._extend
158 d['extend'] = self._extend
159 if self._prepend:
159 if self._prepend:
160 d['prepend'] = self._prepend
160 d['prepend'] = self._prepend
161 elif self._inserts:
161 elif self._inserts:
162 d['inserts'] = self._inserts
162 d['inserts'] = self._inserts
163 return d
163 return d
164
164
165
165
166 class Config(dict):
166 class Config(dict):
167 """An attribute based dict that can do smart merges."""
167 """An attribute based dict that can do smart merges."""
168
168
169 def __init__(self, *args, **kwds):
169 def __init__(self, *args, **kwds):
170 dict.__init__(self, *args, **kwds)
170 dict.__init__(self, *args, **kwds)
171 # This sets self.__dict__ = self, but it has to be done this way
171 # This sets self.__dict__ = self, but it has to be done this way
172 # because we are also overriding __setattr__.
172 # because we are also overriding __setattr__.
173 dict.__setattr__(self, '__dict__', self)
173 dict.__setattr__(self, '__dict__', self)
174 self._ensure_subconfig()
174 self._ensure_subconfig()
175
175
176 def _ensure_subconfig(self):
176 def _ensure_subconfig(self):
177 """ensure that sub-dicts that should be Config objects are
177 """ensure that sub-dicts that should be Config objects are
178
178
179 casts dicts that are under section keys to Config objects,
179 casts dicts that are under section keys to Config objects,
180 which is necessary for constructing Config objects from dict literals.
180 which is necessary for constructing Config objects from dict literals.
181 """
181 """
182 for key in self:
182 for key in self:
183 obj = self[key]
183 obj = self[key]
184 if self._is_section_key(key) \
184 if self._is_section_key(key) \
185 and isinstance(obj, dict) \
185 and isinstance(obj, dict) \
186 and not isinstance(obj, Config):
186 and not isinstance(obj, Config):
187 dict.__setattr__(self, key, Config(obj))
187 dict.__setattr__(self, key, Config(obj))
188
188
189 def _merge(self, other):
189 def _merge(self, other):
190 """deprecated alias, use Config.merge()"""
190 """deprecated alias, use Config.merge()"""
191 self.merge(other)
191 self.merge(other)
192
192
193 def merge(self, other):
193 def merge(self, other):
194 """merge another config object into this one"""
194 """merge another config object into this one"""
195 to_update = {}
195 to_update = {}
196 for k, v in other.iteritems():
196 for k, v in other.iteritems():
197 if k not in self:
197 if k not in self:
198 to_update[k] = v
198 to_update[k] = v
199 else: # I have this key
199 else: # I have this key
200 if isinstance(v, Config) and isinstance(self[k], Config):
200 if isinstance(v, Config) and isinstance(self[k], Config):
201 # Recursively merge common sub Configs
201 # Recursively merge common sub Configs
202 self[k].merge(v)
202 self[k].merge(v)
203 else:
203 else:
204 # Plain updates for non-Configs
204 # Plain updates for non-Configs
205 to_update[k] = v
205 to_update[k] = v
206
206
207 self.update(to_update)
207 self.update(to_update)
208
208
209 def _is_section_key(self, key):
209 def _is_section_key(self, key):
210 if key[0].upper()==key[0] and not key.startswith('_'):
210 if key[0].upper()==key[0] and not key.startswith('_'):
211 return True
211 return True
212 else:
212 else:
213 return False
213 return False
214
214
215 def __contains__(self, key):
215 def __contains__(self, key):
216 # allow nested contains of the form `"Section.key" in config`
216 # allow nested contains of the form `"Section.key" in config`
217 if '.' in key:
217 if '.' in key:
218 first, remainder = key.split('.', 1)
218 first, remainder = key.split('.', 1)
219 if first not in self:
219 if first not in self:
220 return False
220 return False
221 return remainder in self[first]
221 return remainder in self[first]
222
222
223 # we always have Sections
223 # we always have Sections
224 if self._is_section_key(key):
224 if self._is_section_key(key):
225 return True
225 return True
226 else:
226 else:
227 return super(Config, self).__contains__(key)
227 return super(Config, self).__contains__(key)
228 # .has_key is deprecated for dictionaries.
228 # .has_key is deprecated for dictionaries.
229 has_key = __contains__
229 has_key = __contains__
230
230
231 def _has_section(self, key):
231 def _has_section(self, key):
232 if self._is_section_key(key):
232 if self._is_section_key(key):
233 if super(Config, self).__contains__(key):
233 if super(Config, self).__contains__(key):
234 return True
234 return True
235 return False
235 return False
236
236
237 def copy(self):
237 def copy(self):
238 return type(self)(dict.copy(self))
238 return type(self)(dict.copy(self))
239
239
240 def __copy__(self):
240 def __copy__(self):
241 return self.copy()
241 return self.copy()
242
242
243 def __deepcopy__(self, memo):
243 def __deepcopy__(self, memo):
244 import copy
244 import copy
245 return type(self)(copy.deepcopy(self.items()))
245 return type(self)(copy.deepcopy(self.items()))
246
246
247 def __getitem__(self, key):
247 def __getitem__(self, key):
248 # We cannot use directly self._is_section_key, because it triggers
248 # We cannot use directly self._is_section_key, because it triggers
249 # infinite recursion on top of PyPy. Instead, we manually fish the
249 # infinite recursion on top of PyPy. Instead, we manually fish the
250 # bound method.
250 # bound method.
251 is_section_key = self.__class__._is_section_key.__get__(self)
251 is_section_key = self.__class__._is_section_key.__get__(self)
252
252
253 # Because we use this for an exec namespace, we need to delegate
253 # Because we use this for an exec namespace, we need to delegate
254 # the lookup of names in __builtin__ to itself. This means
254 # the lookup of names in __builtin__ to itself. This means
255 # that you can't have section or attribute names that are
255 # that you can't have section or attribute names that are
256 # builtins.
256 # builtins.
257 try:
257 try:
258 return getattr(builtin_mod, key)
258 return getattr(builtin_mod, key)
259 except AttributeError:
259 except AttributeError:
260 pass
260 pass
261 if is_section_key(key):
261 if is_section_key(key):
262 try:
262 try:
263 return dict.__getitem__(self, key)
263 return dict.__getitem__(self, key)
264 except KeyError:
264 except KeyError:
265 c = Config()
265 c = Config()
266 dict.__setitem__(self, key, c)
266 dict.__setitem__(self, key, c)
267 return c
267 return c
268 else:
268 else:
269 try:
269 try:
270 return dict.__getitem__(self, key)
270 return dict.__getitem__(self, key)
271 except KeyError:
271 except KeyError:
272 # undefined
272 # undefined
273 v = LazyConfigValue()
273 v = LazyConfigValue()
274 dict.__setitem__(self, key, v)
274 dict.__setitem__(self, key, v)
275 return v
275 return v
276
276
277
277
278 def __setitem__(self, key, value):
278 def __setitem__(self, key, value):
279 if self._is_section_key(key):
279 if self._is_section_key(key):
280 if not isinstance(value, Config):
280 if not isinstance(value, Config):
281 raise ValueError('values whose keys begin with an uppercase '
281 raise ValueError('values whose keys begin with an uppercase '
282 'char must be Config instances: %r, %r' % (key, value))
282 'char must be Config instances: %r, %r' % (key, value))
283 else:
283 else:
284 dict.__setitem__(self, key, value)
284 dict.__setitem__(self, key, value)
285
285
286 def __getattr__(self, key):
286 def __getattr__(self, key):
287 try:
287 try:
288 return self.__getitem__(key)
288 return self.__getitem__(key)
289 except KeyError as e:
289 except KeyError as e:
290 raise AttributeError(e)
290 raise AttributeError(e)
291
291
292 def __setattr__(self, key, value):
292 def __setattr__(self, key, value):
293 try:
293 try:
294 self.__setitem__(key, value)
294 self.__setitem__(key, value)
295 except KeyError as e:
295 except KeyError as e:
296 raise AttributeError(e)
296 raise AttributeError(e)
297
297
298 def __delattr__(self, key):
298 def __delattr__(self, key):
299 try:
299 try:
300 dict.__delitem__(self, key)
300 dict.__delitem__(self, key)
301 except KeyError as e:
301 except KeyError as e:
302 raise AttributeError(e)
302 raise AttributeError(e)
303
303
304
304
305 #-----------------------------------------------------------------------------
305 #-----------------------------------------------------------------------------
306 # Config loading classes
306 # Config loading classes
307 #-----------------------------------------------------------------------------
307 #-----------------------------------------------------------------------------
308
308
309
309
310 class ConfigLoader(object):
310 class ConfigLoader(object):
311 """A object for loading configurations from just about anywhere.
311 """A object for loading configurations from just about anywhere.
312
312
313 The resulting configuration is packaged as a :class:`Struct`.
313 The resulting configuration is packaged as a :class:`Struct`.
314
314
315 Notes
315 Notes
316 -----
316 -----
317 A :class:`ConfigLoader` does one thing: load a config from a source
317 A :class:`ConfigLoader` does one thing: load a config from a source
318 (file, command line arguments) and returns the data as a :class:`Struct`.
318 (file, command line arguments) and returns the data as a :class:`Struct`.
319 There are lots of things that :class:`ConfigLoader` does not do. It does
319 There are lots of things that :class:`ConfigLoader` does not do. It does
320 not implement complex logic for finding config files. It does not handle
320 not implement complex logic for finding config files. It does not handle
321 default values or merge multiple configs. These things need to be
321 default values or merge multiple configs. These things need to be
322 handled elsewhere.
322 handled elsewhere.
323 """
323 """
324
324
325 def __init__(self):
325 def __init__(self):
326 """A base class for config loaders.
326 """A base class for config loaders.
327
327
328 Examples
328 Examples
329 --------
329 --------
330
330
331 >>> cl = ConfigLoader()
331 >>> cl = ConfigLoader()
332 >>> config = cl.load_config()
332 >>> config = cl.load_config()
333 >>> config
333 >>> config
334 {}
334 {}
335 """
335 """
336 self.clear()
336 self.clear()
337
337
338 def clear(self):
338 def clear(self):
339 self.config = Config()
339 self.config = Config()
340
340
341 def load_config(self):
341 def load_config(self):
342 """Load a config from somewhere, return a :class:`Config` instance.
342 """Load a config from somewhere, return a :class:`Config` instance.
343
343
344 Usually, this will cause self.config to be set and then returned.
344 Usually, this will cause self.config to be set and then returned.
345 However, in most cases, :meth:`ConfigLoader.clear` should be called
345 However, in most cases, :meth:`ConfigLoader.clear` should be called
346 to erase any previous state.
346 to erase any previous state.
347 """
347 """
348 self.clear()
348 self.clear()
349 return self.config
349 return self.config
350
350
351
351
352 class FileConfigLoader(ConfigLoader):
352 class FileConfigLoader(ConfigLoader):
353 """A base class for file based configurations.
353 """A base class for file based configurations.
354
354
355 As we add more file based config loaders, the common logic should go
355 As we add more file based config loaders, the common logic should go
356 here.
356 here.
357 """
357 """
358 pass
358 pass
359
359
360
360
361 class PyFileConfigLoader(FileConfigLoader):
361 class PyFileConfigLoader(FileConfigLoader):
362 """A config loader for pure python files.
362 """A config loader for pure python files.
363
363
364 This calls execfile on a plain python file and looks for attributes
364 This calls execfile on a plain python file and looks for attributes
365 that are all caps. These attribute are added to the config Struct.
365 that are all caps. These attribute are added to the config Struct.
366 """
366 """
367
367
368 def __init__(self, filename, path=None):
368 def __init__(self, filename, path=None):
369 """Build a config loader for a filename and path.
369 """Build a config loader for a filename and path.
370
370
371 Parameters
371 Parameters
372 ----------
372 ----------
373 filename : str
373 filename : str
374 The file name of the config file.
374 The file name of the config file.
375 path : str, list, tuple
375 path : str, list, tuple
376 The path to search for the config file on, or a sequence of
376 The path to search for the config file on, or a sequence of
377 paths to try in order.
377 paths to try in order.
378 """
378 """
379 super(PyFileConfigLoader, self).__init__()
379 super(PyFileConfigLoader, self).__init__()
380 self.filename = filename
380 self.filename = filename
381 self.path = path
381 self.path = path
382 self.full_filename = ''
382 self.full_filename = ''
383 self.data = None
383 self.data = None
384
384
385 def load_config(self):
385 def load_config(self):
386 """Load the config from a file and return it as a Struct."""
386 """Load the config from a file and return it as a Struct."""
387 self.clear()
387 self.clear()
388 try:
388 try:
389 self._find_file()
389 self._find_file()
390 except IOError as e:
390 except IOError as e:
391 raise ConfigFileNotFound(str(e))
391 raise ConfigFileNotFound(str(e))
392 self._read_file_as_dict()
392 self._read_file_as_dict()
393 self._convert_to_config()
393 self._convert_to_config()
394 return self.config
394 return self.config
395
395
396 def _find_file(self):
396 def _find_file(self):
397 """Try to find the file by searching the paths."""
397 """Try to find the file by searching the paths."""
398 self.full_filename = filefind(self.filename, self.path)
398 self.full_filename = filefind(self.filename, self.path)
399
399
400 def _read_file_as_dict(self):
400 def _read_file_as_dict(self):
401 """Load the config file into self.config, with recursive loading."""
401 """Load the config file into self.config, with recursive loading."""
402 # This closure is made available in the namespace that is used
402 # This closure is made available in the namespace that is used
403 # to exec the config file. It allows users to call
403 # to exec the config file. It allows users to call
404 # load_subconfig('myconfig.py') to load config files recursively.
404 # load_subconfig('myconfig.py') to load config files recursively.
405 # It needs to be a closure because it has references to self.path
405 # It needs to be a closure because it has references to self.path
406 # and self.config. The sub-config is loaded with the same path
406 # and self.config. The sub-config is loaded with the same path
407 # as the parent, but it uses an empty config which is then merged
407 # as the parent, but it uses an empty config which is then merged
408 # with the parents.
408 # with the parents.
409
409
410 # If a profile is specified, the config file will be loaded
410 # If a profile is specified, the config file will be loaded
411 # from that profile
411 # from that profile
412
412
413 def load_subconfig(fname, profile=None):
413 def load_subconfig(fname, profile=None):
414 # import here to prevent circular imports
414 # import here to prevent circular imports
415 from IPython.core.profiledir import ProfileDir, ProfileDirError
415 from IPython.core.profiledir import ProfileDir, ProfileDirError
416 if profile is not None:
416 if profile is not None:
417 try:
417 try:
418 profile_dir = ProfileDir.find_profile_dir_by_name(
418 profile_dir = ProfileDir.find_profile_dir_by_name(
419 get_ipython_dir(),
419 get_ipython_dir(),
420 profile,
420 profile,
421 )
421 )
422 except ProfileDirError:
422 except ProfileDirError:
423 return
423 return
424 path = profile_dir.location
424 path = profile_dir.location
425 else:
425 else:
426 path = self.path
426 path = self.path
427 loader = PyFileConfigLoader(fname, path)
427 loader = PyFileConfigLoader(fname, path)
428 try:
428 try:
429 sub_config = loader.load_config()
429 sub_config = loader.load_config()
430 except ConfigFileNotFound:
430 except ConfigFileNotFound:
431 # Pass silently if the sub config is not there. This happens
431 # Pass silently if the sub config is not there. This happens
432 # when a user s using a profile, but not the default config.
432 # when a user s using a profile, but not the default config.
433 pass
433 pass
434 else:
434 else:
435 self.config.merge(sub_config)
435 self.config.merge(sub_config)
436
436
437 # Again, this needs to be a closure and should be used in config
437 # Again, this needs to be a closure and should be used in config
438 # files to get the config being loaded.
438 # files to get the config being loaded.
439 def get_config():
439 def get_config():
440 return self.config
440 return self.config
441
441
442 namespace = dict(
442 namespace = dict(
443 load_subconfig=load_subconfig,
443 load_subconfig=load_subconfig,
444 get_config=get_config,
444 get_config=get_config,
445 __file__=self.full_filename,
445 __file__=self.full_filename,
446 )
446 )
447 fs_encoding = sys.getfilesystemencoding() or 'ascii'
447 fs_encoding = sys.getfilesystemencoding() or 'ascii'
448 conf_filename = self.full_filename.encode(fs_encoding)
448 conf_filename = self.full_filename.encode(fs_encoding)
449 py3compat.execfile(conf_filename, namespace)
449 py3compat.execfile(conf_filename, namespace)
450
450
451 def _convert_to_config(self):
451 def _convert_to_config(self):
452 if self.data is None:
452 if self.data is None:
453 ConfigLoaderError('self.data does not exist')
453 ConfigLoaderError('self.data does not exist')
454
454
455
455
456 class CommandLineConfigLoader(ConfigLoader):
456 class CommandLineConfigLoader(ConfigLoader):
457 """A config loader for command line arguments.
457 """A config loader for command line arguments.
458
458
459 As we add more command line based loaders, the common logic should go
459 As we add more command line based loaders, the common logic should go
460 here.
460 here.
461 """
461 """
462
462
463 def _exec_config_str(self, lhs, rhs):
463 def _exec_config_str(self, lhs, rhs):
464 """execute self.config.<lhs> = <rhs>
464 """execute self.config.<lhs> = <rhs>
465
465
466 * expands ~ with expanduser
466 * expands ~ with expanduser
467 * tries to assign with raw eval, otherwise assigns with just the string,
467 * tries to assign with raw eval, otherwise assigns with just the string,
468 allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent. *Not*
468 allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent. *Not*
469 equivalent are `--C.a=4` and `--C.a='4'`.
469 equivalent are `--C.a=4` and `--C.a='4'`.
470 """
470 """
471 rhs = os.path.expanduser(rhs)
471 rhs = os.path.expanduser(rhs)
472 try:
472 try:
473 # Try to see if regular Python syntax will work. This
473 # Try to see if regular Python syntax will work. This
474 # won't handle strings as the quote marks are removed
474 # won't handle strings as the quote marks are removed
475 # by the system shell.
475 # by the system shell.
476 value = eval(rhs)
476 value = eval(rhs)
477 except (NameError, SyntaxError):
477 except (NameError, SyntaxError):
478 # This case happens if the rhs is a string.
478 # This case happens if the rhs is a string.
479 value = rhs
479 value = rhs
480
480
481 exec(u'self.config.%s = value' % lhs)
481 exec(u'self.config.%s = value' % lhs)
482
482
483 def _load_flag(self, cfg):
483 def _load_flag(self, cfg):
484 """update self.config from a flag, which can be a dict or Config"""
484 """update self.config from a flag, which can be a dict or Config"""
485 if isinstance(cfg, (dict, Config)):
485 if isinstance(cfg, (dict, Config)):
486 # don't clobber whole config sections, update
486 # don't clobber whole config sections, update
487 # each section from config:
487 # each section from config:
488 for sec,c in cfg.iteritems():
488 for sec,c in cfg.iteritems():
489 self.config[sec].update(c)
489 self.config[sec].update(c)
490 else:
490 else:
491 raise TypeError("Invalid flag: %r" % cfg)
491 raise TypeError("Invalid flag: %r" % cfg)
492
492
493 # raw --identifier=value pattern
493 # raw --identifier=value pattern
494 # but *also* accept '-' as wordsep, for aliases
494 # but *also* accept '-' as wordsep, for aliases
495 # accepts: --foo=a
495 # accepts: --foo=a
496 # --Class.trait=value
496 # --Class.trait=value
497 # --alias-name=value
497 # --alias-name=value
498 # rejects: -foo=value
498 # rejects: -foo=value
499 # --foo
499 # --foo
500 # --Class.trait
500 # --Class.trait
501 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
501 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
502
502
503 # just flags, no assignments, with two *or one* leading '-'
503 # just flags, no assignments, with two *or one* leading '-'
504 # accepts: --foo
504 # accepts: --foo
505 # -foo-bar-again
505 # -foo-bar-again
506 # rejects: --anything=anything
506 # rejects: --anything=anything
507 # --two.word
507 # --two.word
508
508
509 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
509 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
510
510
511 class KeyValueConfigLoader(CommandLineConfigLoader):
511 class KeyValueConfigLoader(CommandLineConfigLoader):
512 """A config loader that loads key value pairs from the command line.
512 """A config loader that loads key value pairs from the command line.
513
513
514 This allows command line options to be gives in the following form::
514 This allows command line options to be gives in the following form::
515
515
516 ipython --profile="foo" --InteractiveShell.autocall=False
516 ipython --profile="foo" --InteractiveShell.autocall=False
517 """
517 """
518
518
519 def __init__(self, argv=None, aliases=None, flags=None):
519 def __init__(self, argv=None, aliases=None, flags=None):
520 """Create a key value pair config loader.
520 """Create a key value pair config loader.
521
521
522 Parameters
522 Parameters
523 ----------
523 ----------
524 argv : list
524 argv : list
525 A list that has the form of sys.argv[1:] which has unicode
525 A list that has the form of sys.argv[1:] which has unicode
526 elements of the form u"key=value". If this is None (default),
526 elements of the form u"key=value". If this is None (default),
527 then sys.argv[1:] will be used.
527 then sys.argv[1:] will be used.
528 aliases : dict
528 aliases : dict
529 A dict of aliases for configurable traits.
529 A dict of aliases for configurable traits.
530 Keys are the short aliases, Values are the resolved trait.
530 Keys are the short aliases, Values are the resolved trait.
531 Of the form: `{'alias' : 'Configurable.trait'}`
531 Of the form: `{'alias' : 'Configurable.trait'}`
532 flags : dict
532 flags : dict
533 A dict of flags, keyed by str name. Vaues can be Config objects,
533 A dict of flags, keyed by str name. Vaues can be Config objects,
534 dicts, or "key=value" strings. If Config or dict, when the flag
534 dicts, or "key=value" strings. If Config or dict, when the flag
535 is triggered, The flag is loaded as `self.config.update(m)`.
535 is triggered, The flag is loaded as `self.config.update(m)`.
536
536
537 Returns
537 Returns
538 -------
538 -------
539 config : Config
539 config : Config
540 The resulting Config object.
540 The resulting Config object.
541
541
542 Examples
542 Examples
543 --------
543 --------
544
544
545 >>> from IPython.config.loader import KeyValueConfigLoader
545 >>> from IPython.config.loader import KeyValueConfigLoader
546 >>> cl = KeyValueConfigLoader()
546 >>> cl = KeyValueConfigLoader()
547 >>> d = cl.load_config(["--A.name='brian'","--B.number=0"])
547 >>> d = cl.load_config(["--A.name='brian'","--B.number=0"])
548 >>> sorted(d.items())
548 >>> sorted(d.items())
549 [('A', {'name': 'brian'}), ('B', {'number': 0})]
549 [('A', {'name': 'brian'}), ('B', {'number': 0})]
550 """
550 """
551 self.clear()
551 self.clear()
552 if argv is None:
552 if argv is None:
553 argv = sys.argv[1:]
553 argv = sys.argv[1:]
554 self.argv = argv
554 self.argv = argv
555 self.aliases = aliases or {}
555 self.aliases = aliases or {}
556 self.flags = flags or {}
556 self.flags = flags or {}
557
557
558
558
559 def clear(self):
559 def clear(self):
560 super(KeyValueConfigLoader, self).clear()
560 super(KeyValueConfigLoader, self).clear()
561 self.extra_args = []
561 self.extra_args = []
562
562
563
563
564 def _decode_argv(self, argv, enc=None):
564 def _decode_argv(self, argv, enc=None):
565 """decode argv if bytes, using stin.encoding, falling back on default enc"""
565 """decode argv if bytes, using stin.encoding, falling back on default enc"""
566 uargv = []
566 uargv = []
567 if enc is None:
567 if enc is None:
568 enc = DEFAULT_ENCODING
568 enc = DEFAULT_ENCODING
569 for arg in argv:
569 for arg in argv:
570 if not isinstance(arg, unicode):
570 if not isinstance(arg, unicode):
571 # only decode if not already decoded
571 # only decode if not already decoded
572 arg = arg.decode(enc)
572 arg = arg.decode(enc)
573 uargv.append(arg)
573 uargv.append(arg)
574 return uargv
574 return uargv
575
575
576
576
577 def load_config(self, argv=None, aliases=None, flags=None):
577 def load_config(self, argv=None, aliases=None, flags=None):
578 """Parse the configuration and generate the Config object.
578 """Parse the configuration and generate the Config object.
579
579
580 After loading, any arguments that are not key-value or
580 After loading, any arguments that are not key-value or
581 flags will be stored in self.extra_args - a list of
581 flags will be stored in self.extra_args - a list of
582 unparsed command-line arguments. This is used for
582 unparsed command-line arguments. This is used for
583 arguments such as input files or subcommands.
583 arguments such as input files or subcommands.
584
584
585 Parameters
585 Parameters
586 ----------
586 ----------
587 argv : list, optional
587 argv : list, optional
588 A list that has the form of sys.argv[1:] which has unicode
588 A list that has the form of sys.argv[1:] which has unicode
589 elements of the form u"key=value". If this is None (default),
589 elements of the form u"key=value". If this is None (default),
590 then self.argv will be used.
590 then self.argv will be used.
591 aliases : dict
591 aliases : dict
592 A dict of aliases for configurable traits.
592 A dict of aliases for configurable traits.
593 Keys are the short aliases, Values are the resolved trait.
593 Keys are the short aliases, Values are the resolved trait.
594 Of the form: `{'alias' : 'Configurable.trait'}`
594 Of the form: `{'alias' : 'Configurable.trait'}`
595 flags : dict
595 flags : dict
596 A dict of flags, keyed by str name. Values can be Config objects
596 A dict of flags, keyed by str name. Values can be Config objects
597 or dicts. When the flag is triggered, The config is loaded as
597 or dicts. When the flag is triggered, The config is loaded as
598 `self.config.update(cfg)`.
598 `self.config.update(cfg)`.
599 """
599 """
600 self.clear()
600 self.clear()
601 if argv is None:
601 if argv is None:
602 argv = self.argv
602 argv = self.argv
603 if aliases is None:
603 if aliases is None:
604 aliases = self.aliases
604 aliases = self.aliases
605 if flags is None:
605 if flags is None:
606 flags = self.flags
606 flags = self.flags
607
607
608 # ensure argv is a list of unicode strings:
608 # ensure argv is a list of unicode strings:
609 uargv = self._decode_argv(argv)
609 uargv = self._decode_argv(argv)
610 for idx,raw in enumerate(uargv):
610 for idx,raw in enumerate(uargv):
611 # strip leading '-'
611 # strip leading '-'
612 item = raw.lstrip('-')
612 item = raw.lstrip('-')
613
613
614 if raw == '--':
614 if raw == '--':
615 # don't parse arguments after '--'
615 # don't parse arguments after '--'
616 # this is useful for relaying arguments to scripts, e.g.
616 # this is useful for relaying arguments to scripts, e.g.
617 # ipython -i foo.py --matplotlib=qt -- args after '--' go-to-foo.py
617 # ipython -i foo.py --matplotlib=qt -- args after '--' go-to-foo.py
618 self.extra_args.extend(uargv[idx+1:])
618 self.extra_args.extend(uargv[idx+1:])
619 break
619 break
620
620
621 if kv_pattern.match(raw):
621 if kv_pattern.match(raw):
622 lhs,rhs = item.split('=',1)
622 lhs,rhs = item.split('=',1)
623 # Substitute longnames for aliases.
623 # Substitute longnames for aliases.
624 if lhs in aliases:
624 if lhs in aliases:
625 lhs = aliases[lhs]
625 lhs = aliases[lhs]
626 if '.' not in lhs:
626 if '.' not in lhs:
627 # probably a mistyped alias, but not technically illegal
627 # probably a mistyped alias, but not technically illegal
628 warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
628 warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
629 try:
629 try:
630 self._exec_config_str(lhs, rhs)
630 self._exec_config_str(lhs, rhs)
631 except Exception:
631 except Exception:
632 raise ArgumentError("Invalid argument: '%s'" % raw)
632 raise ArgumentError("Invalid argument: '%s'" % raw)
633
633
634 elif flag_pattern.match(raw):
634 elif flag_pattern.match(raw):
635 if item in flags:
635 if item in flags:
636 cfg,help = flags[item]
636 cfg,help = flags[item]
637 self._load_flag(cfg)
637 self._load_flag(cfg)
638 else:
638 else:
639 raise ArgumentError("Unrecognized flag: '%s'"%raw)
639 raise ArgumentError("Unrecognized flag: '%s'"%raw)
640 elif raw.startswith('-'):
640 elif raw.startswith('-'):
641 kv = '--'+item
641 kv = '--'+item
642 if kv_pattern.match(kv):
642 if kv_pattern.match(kv):
643 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
643 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
644 else:
644 else:
645 raise ArgumentError("Invalid argument: '%s'"%raw)
645 raise ArgumentError("Invalid argument: '%s'"%raw)
646 else:
646 else:
647 # keep all args that aren't valid in a list,
647 # keep all args that aren't valid in a list,
648 # in case our parent knows what to do with them.
648 # in case our parent knows what to do with them.
649 self.extra_args.append(item)
649 self.extra_args.append(item)
650 return self.config
650 return self.config
651
651
652 class ArgParseConfigLoader(CommandLineConfigLoader):
652 class ArgParseConfigLoader(CommandLineConfigLoader):
653 """A loader that uses the argparse module to load from the command line."""
653 """A loader that uses the argparse module to load from the command line."""
654
654
655 def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
655 def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
656 """Create a config loader for use with argparse.
656 """Create a config loader for use with argparse.
657
657
658 Parameters
658 Parameters
659 ----------
659 ----------
660
660
661 argv : optional, list
661 argv : optional, list
662 If given, used to read command-line arguments from, otherwise
662 If given, used to read command-line arguments from, otherwise
663 sys.argv[1:] is used.
663 sys.argv[1:] is used.
664
664
665 parser_args : tuple
665 parser_args : tuple
666 A tuple of positional arguments that will be passed to the
666 A tuple of positional arguments that will be passed to the
667 constructor of :class:`argparse.ArgumentParser`.
667 constructor of :class:`argparse.ArgumentParser`.
668
668
669 parser_kw : dict
669 parser_kw : dict
670 A tuple of keyword arguments that will be passed to the
670 A tuple of keyword arguments that will be passed to the
671 constructor of :class:`argparse.ArgumentParser`.
671 constructor of :class:`argparse.ArgumentParser`.
672
672
673 Returns
673 Returns
674 -------
674 -------
675 config : Config
675 config : Config
676 The resulting Config object.
676 The resulting Config object.
677 """
677 """
678 super(CommandLineConfigLoader, self).__init__()
678 super(CommandLineConfigLoader, self).__init__()
679 self.clear()
679 self.clear()
680 if argv is None:
680 if argv is None:
681 argv = sys.argv[1:]
681 argv = sys.argv[1:]
682 self.argv = argv
682 self.argv = argv
683 self.aliases = aliases or {}
683 self.aliases = aliases or {}
684 self.flags = flags or {}
684 self.flags = flags or {}
685
685
686 self.parser_args = parser_args
686 self.parser_args = parser_args
687 self.version = parser_kw.pop("version", None)
687 self.version = parser_kw.pop("version", None)
688 kwargs = dict(argument_default=argparse.SUPPRESS)
688 kwargs = dict(argument_default=argparse.SUPPRESS)
689 kwargs.update(parser_kw)
689 kwargs.update(parser_kw)
690 self.parser_kw = kwargs
690 self.parser_kw = kwargs
691
691
692 def load_config(self, argv=None, aliases=None, flags=None):
692 def load_config(self, argv=None, aliases=None, flags=None):
693 """Parse command line arguments and return as a Config object.
693 """Parse command line arguments and return as a Config object.
694
694
695 Parameters
695 Parameters
696 ----------
696 ----------
697
697
698 args : optional, list
698 args : optional, list
699 If given, a list with the structure of sys.argv[1:] to parse
699 If given, a list with the structure of sys.argv[1:] to parse
700 arguments from. If not given, the instance's self.argv attribute
700 arguments from. If not given, the instance's self.argv attribute
701 (given at construction time) is used."""
701 (given at construction time) is used."""
702 self.clear()
702 self.clear()
703 if argv is None:
703 if argv is None:
704 argv = self.argv
704 argv = self.argv
705 if aliases is None:
705 if aliases is None:
706 aliases = self.aliases
706 aliases = self.aliases
707 if flags is None:
707 if flags is None:
708 flags = self.flags
708 flags = self.flags
709 self._create_parser(aliases, flags)
709 self._create_parser(aliases, flags)
710 self._parse_args(argv)
710 self._parse_args(argv)
711 self._convert_to_config()
711 self._convert_to_config()
712 return self.config
712 return self.config
713
713
714 def get_extra_args(self):
714 def get_extra_args(self):
715 if hasattr(self, 'extra_args'):
715 if hasattr(self, 'extra_args'):
716 return self.extra_args
716 return self.extra_args
717 else:
717 else:
718 return []
718 return []
719
719
720 def _create_parser(self, aliases=None, flags=None):
720 def _create_parser(self, aliases=None, flags=None):
721 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
721 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
722 self._add_arguments(aliases, flags)
722 self._add_arguments(aliases, flags)
723
723
724 def _add_arguments(self, aliases=None, flags=None):
724 def _add_arguments(self, aliases=None, flags=None):
725 raise NotImplementedError("subclasses must implement _add_arguments")
725 raise NotImplementedError("subclasses must implement _add_arguments")
726
726
727 def _parse_args(self, args):
727 def _parse_args(self, args):
728 """self.parser->self.parsed_data"""
728 """self.parser->self.parsed_data"""
729 # decode sys.argv to support unicode command-line options
729 # decode sys.argv to support unicode command-line options
730 enc = DEFAULT_ENCODING
730 enc = DEFAULT_ENCODING
731 uargs = [py3compat.cast_unicode(a, enc) for a in args]
731 uargs = [py3compat.cast_unicode(a, enc) for a in args]
732 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
732 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
733
733
734 def _convert_to_config(self):
734 def _convert_to_config(self):
735 """self.parsed_data->self.config"""
735 """self.parsed_data->self.config"""
736 for k, v in vars(self.parsed_data).iteritems():
736 for k, v in vars(self.parsed_data).iteritems():
737 exec("self.config.%s = v"%k, locals(), globals())
737 exec("self.config.%s = v"%k, locals(), globals())
738
738
739 class KVArgParseConfigLoader(ArgParseConfigLoader):
739 class KVArgParseConfigLoader(ArgParseConfigLoader):
740 """A config loader that loads aliases and flags with argparse,
740 """A config loader that loads aliases and flags with argparse,
741 but will use KVLoader for the rest. This allows better parsing
741 but will use KVLoader for the rest. This allows better parsing
742 of common args, such as `ipython -c 'print 5'`, but still gets
742 of common args, such as `ipython -c 'print 5'`, but still gets
743 arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
743 arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
744
744
745 def _add_arguments(self, aliases=None, flags=None):
745 def _add_arguments(self, aliases=None, flags=None):
746 self.alias_flags = {}
746 self.alias_flags = {}
747 # print aliases, flags
747 # print aliases, flags
748 if aliases is None:
748 if aliases is None:
749 aliases = self.aliases
749 aliases = self.aliases
750 if flags is None:
750 if flags is None:
751 flags = self.flags
751 flags = self.flags
752 paa = self.parser.add_argument
752 paa = self.parser.add_argument
753 for key,value in aliases.iteritems():
753 for key,value in aliases.iteritems():
754 if key in flags:
754 if key in flags:
755 # flags
755 # flags
756 nargs = '?'
756 nargs = '?'
757 else:
757 else:
758 nargs = None
758 nargs = None
759 if len(key) is 1:
759 if len(key) is 1:
760 paa('-'+key, '--'+key, type=unicode, dest=value, nargs=nargs)
760 paa('-'+key, '--'+key, type=unicode, dest=value, nargs=nargs)
761 else:
761 else:
762 paa('--'+key, type=unicode, dest=value, nargs=nargs)
762 paa('--'+key, type=unicode, dest=value, nargs=nargs)
763 for key, (value, help) in flags.iteritems():
763 for key, (value, help) in flags.iteritems():
764 if key in self.aliases:
764 if key in self.aliases:
765 #
765 #
766 self.alias_flags[self.aliases[key]] = value
766 self.alias_flags[self.aliases[key]] = value
767 continue
767 continue
768 if len(key) is 1:
768 if len(key) is 1:
769 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
769 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
770 else:
770 else:
771 paa('--'+key, action='append_const', dest='_flags', const=value)
771 paa('--'+key, action='append_const', dest='_flags', const=value)
772
772
773 def _convert_to_config(self):
773 def _convert_to_config(self):
774 """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
774 """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
775 # remove subconfigs list from namespace before transforming the Namespace
775 # remove subconfigs list from namespace before transforming the Namespace
776 if '_flags' in self.parsed_data:
776 if '_flags' in self.parsed_data:
777 subcs = self.parsed_data._flags
777 subcs = self.parsed_data._flags
778 del self.parsed_data._flags
778 del self.parsed_data._flags
779 else:
779 else:
780 subcs = []
780 subcs = []
781
781
782 for k, v in vars(self.parsed_data).iteritems():
782 for k, v in vars(self.parsed_data).iteritems():
783 if v is None:
783 if v is None:
784 # it was a flag that shares the name of an alias
784 # it was a flag that shares the name of an alias
785 subcs.append(self.alias_flags[k])
785 subcs.append(self.alias_flags[k])
786 else:
786 else:
787 # eval the KV assignment
787 # eval the KV assignment
788 self._exec_config_str(k, v)
788 self._exec_config_str(k, v)
789
789
790 for subc in subcs:
790 for subc in subcs:
791 self._load_flag(subc)
791 self._load_flag(subc)
792
792
793 if self.extra_args:
793 if self.extra_args:
794 sub_parser = KeyValueConfigLoader()
794 sub_parser = KeyValueConfigLoader()
795 sub_parser.load_config(self.extra_args)
795 sub_parser.load_config(self.extra_args)
796 self.config.merge(sub_parser.config)
796 self.config.merge(sub_parser.config)
797 self.extra_args = sub_parser.extra_args
797 self.extra_args = sub_parser.extra_args
798
798
799
799
800 def load_pyconfig_files(config_files, path):
800 def load_pyconfig_files(config_files, path):
801 """Load multiple Python config files, merging each of them in turn.
801 """Load multiple Python config files, merging each of them in turn.
802
802
803 Parameters
803 Parameters
804 ==========
804 ==========
805 config_files : list of str
805 config_files : list of str
806 List of config files names to load and merge into the config.
806 List of config files names to load and merge into the config.
807 path : unicode
807 path : unicode
808 The full path to the location of the config files.
808 The full path to the location of the config files.
809 """
809 """
810 config = Config()
810 config = Config()
811 for cf in config_files:
811 for cf in config_files:
812 loader = PyFileConfigLoader(cf, path=path)
812 loader = PyFileConfigLoader(cf, path=path)
813 try:
813 try:
814 next_config = loader.load_config()
814 next_config = loader.load_config()
815 except ConfigFileNotFound:
815 except ConfigFileNotFound:
816 pass
816 pass
817 except:
817 except:
818 raise
818 raise
819 else:
819 else:
820 config.merge(next_config)
820 config.merge(next_config)
821 return config
821 return config
@@ -1,112 +1,111 b''
1 """
1 """
2 A context manager for managing things injected into :mod:`__builtin__`.
2 A context manager for managing things injected into :mod:`__builtin__`.
3
3
4 Authors:
4 Authors:
5
5
6 * Brian Granger
6 * Brian Granger
7 * Fernando Perez
7 * Fernando Perez
8 """
8 """
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2010-2011 The IPython Development Team.
10 # Copyright (C) 2010-2011 The IPython Development Team.
11 #
11 #
12 # Distributed under the terms of the BSD License.
12 # Distributed under the terms of the BSD License.
13 #
13 #
14 # Complete license in the file COPYING.txt, distributed with this software.
14 # Complete license in the file COPYING.txt, distributed with this software.
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 import __builtin__
22
23 from IPython.config.configurable import Configurable
21 from IPython.config.configurable import Configurable
24
22
23 from IPython.utils.py3compat import builtin_mod
25 from IPython.utils.traitlets import Instance
24 from IPython.utils.traitlets import Instance
26
25
27 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
28 # Classes and functions
27 # Classes and functions
29 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
30
29
31 class __BuiltinUndefined(object): pass
30 class __BuiltinUndefined(object): pass
32 BuiltinUndefined = __BuiltinUndefined()
31 BuiltinUndefined = __BuiltinUndefined()
33
32
34 class __HideBuiltin(object): pass
33 class __HideBuiltin(object): pass
35 HideBuiltin = __HideBuiltin()
34 HideBuiltin = __HideBuiltin()
36
35
37
36
38 class BuiltinTrap(Configurable):
37 class BuiltinTrap(Configurable):
39
38
40 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
39 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
41
40
42 def __init__(self, shell=None):
41 def __init__(self, shell=None):
43 super(BuiltinTrap, self).__init__(shell=shell, config=None)
42 super(BuiltinTrap, self).__init__(shell=shell, config=None)
44 self._orig_builtins = {}
43 self._orig_builtins = {}
45 # We define this to track if a single BuiltinTrap is nested.
44 # We define this to track if a single BuiltinTrap is nested.
46 # Only turn off the trap when the outermost call to __exit__ is made.
45 # Only turn off the trap when the outermost call to __exit__ is made.
47 self._nested_level = 0
46 self._nested_level = 0
48 self.shell = shell
47 self.shell = shell
49 # builtins we always add - if set to HideBuiltin, they will just
48 # builtins we always add - if set to HideBuiltin, they will just
50 # be removed instead of being replaced by something else
49 # be removed instead of being replaced by something else
51 self.auto_builtins = {'exit': HideBuiltin,
50 self.auto_builtins = {'exit': HideBuiltin,
52 'quit': HideBuiltin,
51 'quit': HideBuiltin,
53 'get_ipython': self.shell.get_ipython,
52 'get_ipython': self.shell.get_ipython,
54 }
53 }
55 # Recursive reload function
54 # Recursive reload function
56 try:
55 try:
57 from IPython.lib import deepreload
56 from IPython.lib import deepreload
58 if self.shell.deep_reload:
57 if self.shell.deep_reload:
59 self.auto_builtins['reload'] = deepreload.reload
58 self.auto_builtins['reload'] = deepreload.reload
60 else:
59 else:
61 self.auto_builtins['dreload']= deepreload.reload
60 self.auto_builtins['dreload']= deepreload.reload
62 except ImportError:
61 except ImportError:
63 pass
62 pass
64
63
65 def __enter__(self):
64 def __enter__(self):
66 if self._nested_level == 0:
65 if self._nested_level == 0:
67 self.activate()
66 self.activate()
68 self._nested_level += 1
67 self._nested_level += 1
69 # I return self, so callers can use add_builtin in a with clause.
68 # I return self, so callers can use add_builtin in a with clause.
70 return self
69 return self
71
70
72 def __exit__(self, type, value, traceback):
71 def __exit__(self, type, value, traceback):
73 if self._nested_level == 1:
72 if self._nested_level == 1:
74 self.deactivate()
73 self.deactivate()
75 self._nested_level -= 1
74 self._nested_level -= 1
76 # Returning False will cause exceptions to propagate
75 # Returning False will cause exceptions to propagate
77 return False
76 return False
78
77
79 def add_builtin(self, key, value):
78 def add_builtin(self, key, value):
80 """Add a builtin and save the original."""
79 """Add a builtin and save the original."""
81 bdict = __builtin__.__dict__
80 bdict = builtin_mod.__dict__
82 orig = bdict.get(key, BuiltinUndefined)
81 orig = bdict.get(key, BuiltinUndefined)
83 if value is HideBuiltin:
82 if value is HideBuiltin:
84 if orig is not BuiltinUndefined: #same as 'key in bdict'
83 if orig is not BuiltinUndefined: #same as 'key in bdict'
85 self._orig_builtins[key] = orig
84 self._orig_builtins[key] = orig
86 del bdict[key]
85 del bdict[key]
87 else:
86 else:
88 self._orig_builtins[key] = orig
87 self._orig_builtins[key] = orig
89 bdict[key] = value
88 bdict[key] = value
90
89
91 def remove_builtin(self, key, orig):
90 def remove_builtin(self, key, orig):
92 """Remove an added builtin and re-set the original."""
91 """Remove an added builtin and re-set the original."""
93 if orig is BuiltinUndefined:
92 if orig is BuiltinUndefined:
94 del __builtin__.__dict__[key]
93 del builtin_mod.__dict__[key]
95 else:
94 else:
96 __builtin__.__dict__[key] = orig
95 builtin_mod.__dict__[key] = orig
97
96
98 def activate(self):
97 def activate(self):
99 """Store ipython references in the __builtin__ namespace."""
98 """Store ipython references in the __builtin__ namespace."""
100
99
101 add_builtin = self.add_builtin
100 add_builtin = self.add_builtin
102 for name, func in self.auto_builtins.iteritems():
101 for name, func in self.auto_builtins.iteritems():
103 add_builtin(name, func)
102 add_builtin(name, func)
104
103
105 def deactivate(self):
104 def deactivate(self):
106 """Remove any builtins which might have been added by add_builtins, or
105 """Remove any builtins which might have been added by add_builtins, or
107 restore overwritten ones to their previous values."""
106 restore overwritten ones to their previous values."""
108 remove_builtin = self.remove_builtin
107 remove_builtin = self.remove_builtin
109 for key, val in self._orig_builtins.iteritems():
108 for key, val in self._orig_builtins.iteritems():
110 remove_builtin(key, val)
109 remove_builtin(key, val)
111 self._orig_builtins.clear()
110 self._orig_builtins.clear()
112 self._builtins_added = False
111 self._builtins_added = False
@@ -1,997 +1,997 b''
1 """Word completion for IPython.
1 """Word completion for IPython.
2
2
3 This module is a fork of the rlcompleter module in the Python standard
3 This module is a fork of the rlcompleter module in the Python standard
4 library. The original enhancements made to rlcompleter have been sent
4 library. The original enhancements made to rlcompleter have been sent
5 upstream and were accepted as of Python 2.3, but we need a lot more
5 upstream and were accepted as of Python 2.3, but we need a lot more
6 functionality specific to IPython, so this module will continue to live as an
6 functionality specific to IPython, so this module will continue to live as an
7 IPython-specific utility.
7 IPython-specific utility.
8
8
9 Original rlcompleter documentation:
9 Original rlcompleter documentation:
10
10
11 This requires the latest extension to the readline module (the
11 This requires the latest extension to the readline module (the
12 completes keywords, built-ins and globals in __main__; when completing
12 completes keywords, built-ins and globals in __main__; when completing
13 NAME.NAME..., it evaluates (!) the expression up to the last dot and
13 NAME.NAME..., it evaluates (!) the expression up to the last dot and
14 completes its attributes.
14 completes its attributes.
15
15
16 It's very cool to do "import string" type "string.", hit the
16 It's very cool to do "import string" type "string.", hit the
17 completion key (twice), and see the list of names defined by the
17 completion key (twice), and see the list of names defined by the
18 string module!
18 string module!
19
19
20 Tip: to use the tab key as the completion key, call
20 Tip: to use the tab key as the completion key, call
21
21
22 readline.parse_and_bind("tab: complete")
22 readline.parse_and_bind("tab: complete")
23
23
24 Notes:
24 Notes:
25
25
26 - Exceptions raised by the completer function are *ignored* (and
26 - Exceptions raised by the completer function are *ignored* (and
27 generally cause the completion to fail). This is a feature -- since
27 generally cause the completion to fail). This is a feature -- since
28 readline sets the tty device in raw (or cbreak) mode, printing a
28 readline sets the tty device in raw (or cbreak) mode, printing a
29 traceback wouldn't work well without some complicated hoopla to save,
29 traceback wouldn't work well without some complicated hoopla to save,
30 reset and restore the tty state.
30 reset and restore the tty state.
31
31
32 - The evaluation of the NAME.NAME... form may cause arbitrary
32 - The evaluation of the NAME.NAME... form may cause arbitrary
33 application defined code to be executed if an object with a
33 application defined code to be executed if an object with a
34 ``__getattr__`` hook is found. Since it is the responsibility of the
34 ``__getattr__`` hook is found. Since it is the responsibility of the
35 application (or the user) to enable this feature, I consider this an
35 application (or the user) to enable this feature, I consider this an
36 acceptable risk. More complicated expressions (e.g. function calls or
36 acceptable risk. More complicated expressions (e.g. function calls or
37 indexing operations) are *not* evaluated.
37 indexing operations) are *not* evaluated.
38
38
39 - GNU readline is also used by the built-in functions input() and
39 - GNU readline is also used by the built-in functions input() and
40 raw_input(), and thus these also benefit/suffer from the completer
40 raw_input(), and thus these also benefit/suffer from the completer
41 features. Clearly an interactive application can benefit by
41 features. Clearly an interactive application can benefit by
42 specifying its own completer function and using raw_input() for all
42 specifying its own completer function and using raw_input() for all
43 its input.
43 its input.
44
44
45 - When the original stdin is not a tty device, GNU readline is never
45 - When the original stdin is not a tty device, GNU readline is never
46 used, and this module (and the readline module) are silently inactive.
46 used, and this module (and the readline module) are silently inactive.
47 """
47 """
48
48
49 #*****************************************************************************
49 #*****************************************************************************
50 #
50 #
51 # Since this file is essentially a minimally modified copy of the rlcompleter
51 # Since this file is essentially a minimally modified copy of the rlcompleter
52 # module which is part of the standard Python distribution, I assume that the
52 # module which is part of the standard Python distribution, I assume that the
53 # proper procedure is to maintain its copyright as belonging to the Python
53 # proper procedure is to maintain its copyright as belonging to the Python
54 # Software Foundation (in addition to my own, for all new code).
54 # Software Foundation (in addition to my own, for all new code).
55 #
55 #
56 # Copyright (C) 2008 IPython Development Team
56 # Copyright (C) 2008 IPython Development Team
57 # Copyright (C) 2001 Fernando Perez. <fperez@colorado.edu>
57 # Copyright (C) 2001 Fernando Perez. <fperez@colorado.edu>
58 # Copyright (C) 2001 Python Software Foundation, www.python.org
58 # Copyright (C) 2001 Python Software Foundation, www.python.org
59 #
59 #
60 # Distributed under the terms of the BSD License. The full license is in
60 # Distributed under the terms of the BSD License. The full license is in
61 # the file COPYING, distributed as part of this software.
61 # the file COPYING, distributed as part of this software.
62 #
62 #
63 #*****************************************************************************
63 #*****************************************************************************
64
64
65 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
66 # Imports
66 # Imports
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68
68
69 import __builtin__
70 import __main__
69 import __main__
71 import glob
70 import glob
72 import inspect
71 import inspect
73 import itertools
72 import itertools
74 import keyword
73 import keyword
75 import os
74 import os
76 import re
75 import re
77 import sys
76 import sys
78
77
79 from IPython.config.configurable import Configurable
78 from IPython.config.configurable import Configurable
80 from IPython.core.error import TryNext
79 from IPython.core.error import TryNext
81 from IPython.core.inputsplitter import ESC_MAGIC
80 from IPython.core.inputsplitter import ESC_MAGIC
82 from IPython.utils import generics
81 from IPython.utils import generics
83 from IPython.utils import io
82 from IPython.utils import io
84 from IPython.utils.dir2 import dir2
83 from IPython.utils.dir2 import dir2
85 from IPython.utils.process import arg_split
84 from IPython.utils.process import arg_split
85 from IPython.utils.py3compat import builtin_mod
86 from IPython.utils.traitlets import CBool, Enum
86 from IPython.utils.traitlets import CBool, Enum
87
87
88 #-----------------------------------------------------------------------------
88 #-----------------------------------------------------------------------------
89 # Globals
89 # Globals
90 #-----------------------------------------------------------------------------
90 #-----------------------------------------------------------------------------
91
91
92 # Public API
92 # Public API
93 __all__ = ['Completer','IPCompleter']
93 __all__ = ['Completer','IPCompleter']
94
94
95 if sys.platform == 'win32':
95 if sys.platform == 'win32':
96 PROTECTABLES = ' '
96 PROTECTABLES = ' '
97 else:
97 else:
98 PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
98 PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
99
99
100 #-----------------------------------------------------------------------------
100 #-----------------------------------------------------------------------------
101 # Main functions and classes
101 # Main functions and classes
102 #-----------------------------------------------------------------------------
102 #-----------------------------------------------------------------------------
103
103
104 def has_open_quotes(s):
104 def has_open_quotes(s):
105 """Return whether a string has open quotes.
105 """Return whether a string has open quotes.
106
106
107 This simply counts whether the number of quote characters of either type in
107 This simply counts whether the number of quote characters of either type in
108 the string is odd.
108 the string is odd.
109
109
110 Returns
110 Returns
111 -------
111 -------
112 If there is an open quote, the quote character is returned. Else, return
112 If there is an open quote, the quote character is returned. Else, return
113 False.
113 False.
114 """
114 """
115 # We check " first, then ', so complex cases with nested quotes will get
115 # We check " first, then ', so complex cases with nested quotes will get
116 # the " to take precedence.
116 # the " to take precedence.
117 if s.count('"') % 2:
117 if s.count('"') % 2:
118 return '"'
118 return '"'
119 elif s.count("'") % 2:
119 elif s.count("'") % 2:
120 return "'"
120 return "'"
121 else:
121 else:
122 return False
122 return False
123
123
124
124
125 def protect_filename(s):
125 def protect_filename(s):
126 """Escape a string to protect certain characters."""
126 """Escape a string to protect certain characters."""
127
127
128 return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
128 return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
129 for ch in s])
129 for ch in s])
130
130
131 def expand_user(path):
131 def expand_user(path):
132 """Expand '~'-style usernames in strings.
132 """Expand '~'-style usernames in strings.
133
133
134 This is similar to :func:`os.path.expanduser`, but it computes and returns
134 This is similar to :func:`os.path.expanduser`, but it computes and returns
135 extra information that will be useful if the input was being used in
135 extra information that will be useful if the input was being used in
136 computing completions, and you wish to return the completions with the
136 computing completions, and you wish to return the completions with the
137 original '~' instead of its expanded value.
137 original '~' instead of its expanded value.
138
138
139 Parameters
139 Parameters
140 ----------
140 ----------
141 path : str
141 path : str
142 String to be expanded. If no ~ is present, the output is the same as the
142 String to be expanded. If no ~ is present, the output is the same as the
143 input.
143 input.
144
144
145 Returns
145 Returns
146 -------
146 -------
147 newpath : str
147 newpath : str
148 Result of ~ expansion in the input path.
148 Result of ~ expansion in the input path.
149 tilde_expand : bool
149 tilde_expand : bool
150 Whether any expansion was performed or not.
150 Whether any expansion was performed or not.
151 tilde_val : str
151 tilde_val : str
152 The value that ~ was replaced with.
152 The value that ~ was replaced with.
153 """
153 """
154 # Default values
154 # Default values
155 tilde_expand = False
155 tilde_expand = False
156 tilde_val = ''
156 tilde_val = ''
157 newpath = path
157 newpath = path
158
158
159 if path.startswith('~'):
159 if path.startswith('~'):
160 tilde_expand = True
160 tilde_expand = True
161 rest = len(path)-1
161 rest = len(path)-1
162 newpath = os.path.expanduser(path)
162 newpath = os.path.expanduser(path)
163 if rest:
163 if rest:
164 tilde_val = newpath[:-rest]
164 tilde_val = newpath[:-rest]
165 else:
165 else:
166 tilde_val = newpath
166 tilde_val = newpath
167
167
168 return newpath, tilde_expand, tilde_val
168 return newpath, tilde_expand, tilde_val
169
169
170
170
171 def compress_user(path, tilde_expand, tilde_val):
171 def compress_user(path, tilde_expand, tilde_val):
172 """Does the opposite of expand_user, with its outputs.
172 """Does the opposite of expand_user, with its outputs.
173 """
173 """
174 if tilde_expand:
174 if tilde_expand:
175 return path.replace(tilde_val, '~')
175 return path.replace(tilde_val, '~')
176 else:
176 else:
177 return path
177 return path
178
178
179
179
180
180
181 def penalize_magics_key(word):
181 def penalize_magics_key(word):
182 """key for sorting that penalizes magic commands in the ordering
182 """key for sorting that penalizes magic commands in the ordering
183
183
184 Normal words are left alone.
184 Normal words are left alone.
185
185
186 Magic commands have the initial % moved to the end, e.g.
186 Magic commands have the initial % moved to the end, e.g.
187 %matplotlib is transformed as follows:
187 %matplotlib is transformed as follows:
188
188
189 %matplotlib -> matplotlib%
189 %matplotlib -> matplotlib%
190
190
191 [The choice of the final % is arbitrary.]
191 [The choice of the final % is arbitrary.]
192
192
193 Since "matplotlib" < "matplotlib%" as strings,
193 Since "matplotlib" < "matplotlib%" as strings,
194 "timeit" will appear before the magic "%timeit" in the ordering
194 "timeit" will appear before the magic "%timeit" in the ordering
195
195
196 For consistency, move "%%" to the end, so cell magics appear *after*
196 For consistency, move "%%" to the end, so cell magics appear *after*
197 line magics with the same name.
197 line magics with the same name.
198
198
199 A check is performed that there are no other "%" in the string;
199 A check is performed that there are no other "%" in the string;
200 if there are, then the string is not a magic command and is left unchanged.
200 if there are, then the string is not a magic command and is left unchanged.
201
201
202 """
202 """
203
203
204 # Move any % signs from start to end of the key
204 # Move any % signs from start to end of the key
205 # provided there are no others elsewhere in the string
205 # provided there are no others elsewhere in the string
206
206
207 if word[:2] == "%%":
207 if word[:2] == "%%":
208 if not "%" in word[2:]:
208 if not "%" in word[2:]:
209 return word[2:] + "%%"
209 return word[2:] + "%%"
210
210
211 if word[:1] == "%":
211 if word[:1] == "%":
212 if not "%" in word[1:]:
212 if not "%" in word[1:]:
213 return word[1:] + "%"
213 return word[1:] + "%"
214
214
215 return word
215 return word
216
216
217
217
218
218
219 class Bunch(object): pass
219 class Bunch(object): pass
220
220
221
221
222 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
222 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
223 GREEDY_DELIMS = ' =\r\n'
223 GREEDY_DELIMS = ' =\r\n'
224
224
225
225
226 class CompletionSplitter(object):
226 class CompletionSplitter(object):
227 """An object to split an input line in a manner similar to readline.
227 """An object to split an input line in a manner similar to readline.
228
228
229 By having our own implementation, we can expose readline-like completion in
229 By having our own implementation, we can expose readline-like completion in
230 a uniform manner to all frontends. This object only needs to be given the
230 a uniform manner to all frontends. This object only needs to be given the
231 line of text to be split and the cursor position on said line, and it
231 line of text to be split and the cursor position on said line, and it
232 returns the 'word' to be completed on at the cursor after splitting the
232 returns the 'word' to be completed on at the cursor after splitting the
233 entire line.
233 entire line.
234
234
235 What characters are used as splitting delimiters can be controlled by
235 What characters are used as splitting delimiters can be controlled by
236 setting the `delims` attribute (this is a property that internally
236 setting the `delims` attribute (this is a property that internally
237 automatically builds the necessary regular expression)"""
237 automatically builds the necessary regular expression)"""
238
238
239 # Private interface
239 # Private interface
240
240
241 # A string of delimiter characters. The default value makes sense for
241 # A string of delimiter characters. The default value makes sense for
242 # IPython's most typical usage patterns.
242 # IPython's most typical usage patterns.
243 _delims = DELIMS
243 _delims = DELIMS
244
244
245 # The expression (a normal string) to be compiled into a regular expression
245 # The expression (a normal string) to be compiled into a regular expression
246 # for actual splitting. We store it as an attribute mostly for ease of
246 # for actual splitting. We store it as an attribute mostly for ease of
247 # debugging, since this type of code can be so tricky to debug.
247 # debugging, since this type of code can be so tricky to debug.
248 _delim_expr = None
248 _delim_expr = None
249
249
250 # The regular expression that does the actual splitting
250 # The regular expression that does the actual splitting
251 _delim_re = None
251 _delim_re = None
252
252
253 def __init__(self, delims=None):
253 def __init__(self, delims=None):
254 delims = CompletionSplitter._delims if delims is None else delims
254 delims = CompletionSplitter._delims if delims is None else delims
255 self.delims = delims
255 self.delims = delims
256
256
257 @property
257 @property
258 def delims(self):
258 def delims(self):
259 """Return the string of delimiter characters."""
259 """Return the string of delimiter characters."""
260 return self._delims
260 return self._delims
261
261
262 @delims.setter
262 @delims.setter
263 def delims(self, delims):
263 def delims(self, delims):
264 """Set the delimiters for line splitting."""
264 """Set the delimiters for line splitting."""
265 expr = '[' + ''.join('\\'+ c for c in delims) + ']'
265 expr = '[' + ''.join('\\'+ c for c in delims) + ']'
266 self._delim_re = re.compile(expr)
266 self._delim_re = re.compile(expr)
267 self._delims = delims
267 self._delims = delims
268 self._delim_expr = expr
268 self._delim_expr = expr
269
269
270 def split_line(self, line, cursor_pos=None):
270 def split_line(self, line, cursor_pos=None):
271 """Split a line of text with a cursor at the given position.
271 """Split a line of text with a cursor at the given position.
272 """
272 """
273 l = line if cursor_pos is None else line[:cursor_pos]
273 l = line if cursor_pos is None else line[:cursor_pos]
274 return self._delim_re.split(l)[-1]
274 return self._delim_re.split(l)[-1]
275
275
276
276
277 class Completer(Configurable):
277 class Completer(Configurable):
278
278
279 greedy = CBool(False, config=True,
279 greedy = CBool(False, config=True,
280 help="""Activate greedy completion
280 help="""Activate greedy completion
281
281
282 This will enable completion on elements of lists, results of function calls, etc.,
282 This will enable completion on elements of lists, results of function calls, etc.,
283 but can be unsafe because the code is actually evaluated on TAB.
283 but can be unsafe because the code is actually evaluated on TAB.
284 """
284 """
285 )
285 )
286
286
287
287
288 def __init__(self, namespace=None, global_namespace=None, **kwargs):
288 def __init__(self, namespace=None, global_namespace=None, **kwargs):
289 """Create a new completer for the command line.
289 """Create a new completer for the command line.
290
290
291 Completer(namespace=ns,global_namespace=ns2) -> completer instance.
291 Completer(namespace=ns,global_namespace=ns2) -> completer instance.
292
292
293 If unspecified, the default namespace where completions are performed
293 If unspecified, the default namespace where completions are performed
294 is __main__ (technically, __main__.__dict__). Namespaces should be
294 is __main__ (technically, __main__.__dict__). Namespaces should be
295 given as dictionaries.
295 given as dictionaries.
296
296
297 An optional second namespace can be given. This allows the completer
297 An optional second namespace can be given. This allows the completer
298 to handle cases where both the local and global scopes need to be
298 to handle cases where both the local and global scopes need to be
299 distinguished.
299 distinguished.
300
300
301 Completer instances should be used as the completion mechanism of
301 Completer instances should be used as the completion mechanism of
302 readline via the set_completer() call:
302 readline via the set_completer() call:
303
303
304 readline.set_completer(Completer(my_namespace).complete)
304 readline.set_completer(Completer(my_namespace).complete)
305 """
305 """
306
306
307 # Don't bind to namespace quite yet, but flag whether the user wants a
307 # Don't bind to namespace quite yet, but flag whether the user wants a
308 # specific namespace or to use __main__.__dict__. This will allow us
308 # specific namespace or to use __main__.__dict__. This will allow us
309 # to bind to __main__.__dict__ at completion time, not now.
309 # to bind to __main__.__dict__ at completion time, not now.
310 if namespace is None:
310 if namespace is None:
311 self.use_main_ns = 1
311 self.use_main_ns = 1
312 else:
312 else:
313 self.use_main_ns = 0
313 self.use_main_ns = 0
314 self.namespace = namespace
314 self.namespace = namespace
315
315
316 # The global namespace, if given, can be bound directly
316 # The global namespace, if given, can be bound directly
317 if global_namespace is None:
317 if global_namespace is None:
318 self.global_namespace = {}
318 self.global_namespace = {}
319 else:
319 else:
320 self.global_namespace = global_namespace
320 self.global_namespace = global_namespace
321
321
322 super(Completer, self).__init__(**kwargs)
322 super(Completer, self).__init__(**kwargs)
323
323
324 def complete(self, text, state):
324 def complete(self, text, state):
325 """Return the next possible completion for 'text'.
325 """Return the next possible completion for 'text'.
326
326
327 This is called successively with state == 0, 1, 2, ... until it
327 This is called successively with state == 0, 1, 2, ... until it
328 returns None. The completion should begin with 'text'.
328 returns None. The completion should begin with 'text'.
329
329
330 """
330 """
331 if self.use_main_ns:
331 if self.use_main_ns:
332 self.namespace = __main__.__dict__
332 self.namespace = __main__.__dict__
333
333
334 if state == 0:
334 if state == 0:
335 if "." in text:
335 if "." in text:
336 self.matches = self.attr_matches(text)
336 self.matches = self.attr_matches(text)
337 else:
337 else:
338 self.matches = self.global_matches(text)
338 self.matches = self.global_matches(text)
339 try:
339 try:
340 return self.matches[state]
340 return self.matches[state]
341 except IndexError:
341 except IndexError:
342 return None
342 return None
343
343
344 def global_matches(self, text):
344 def global_matches(self, text):
345 """Compute matches when text is a simple name.
345 """Compute matches when text is a simple name.
346
346
347 Return a list of all keywords, built-in functions and names currently
347 Return a list of all keywords, built-in functions and names currently
348 defined in self.namespace or self.global_namespace that match.
348 defined in self.namespace or self.global_namespace that match.
349
349
350 """
350 """
351 #print 'Completer->global_matches, txt=%r' % text # dbg
351 #print 'Completer->global_matches, txt=%r' % text # dbg
352 matches = []
352 matches = []
353 match_append = matches.append
353 match_append = matches.append
354 n = len(text)
354 n = len(text)
355 for lst in [keyword.kwlist,
355 for lst in [keyword.kwlist,
356 __builtin__.__dict__.keys(),
356 builtin_mod.__dict__.keys(),
357 self.namespace.keys(),
357 self.namespace.keys(),
358 self.global_namespace.keys()]:
358 self.global_namespace.keys()]:
359 for word in lst:
359 for word in lst:
360 if word[:n] == text and word != "__builtins__":
360 if word[:n] == text and word != "__builtins__":
361 match_append(word)
361 match_append(word)
362 return matches
362 return matches
363
363
364 def attr_matches(self, text):
364 def attr_matches(self, text):
365 """Compute matches when text contains a dot.
365 """Compute matches when text contains a dot.
366
366
367 Assuming the text is of the form NAME.NAME....[NAME], and is
367 Assuming the text is of the form NAME.NAME....[NAME], and is
368 evaluatable in self.namespace or self.global_namespace, it will be
368 evaluatable in self.namespace or self.global_namespace, it will be
369 evaluated and its attributes (as revealed by dir()) are used as
369 evaluated and its attributes (as revealed by dir()) are used as
370 possible completions. (For class instances, class members are are
370 possible completions. (For class instances, class members are are
371 also considered.)
371 also considered.)
372
372
373 WARNING: this can still invoke arbitrary C code, if an object
373 WARNING: this can still invoke arbitrary C code, if an object
374 with a __getattr__ hook is evaluated.
374 with a __getattr__ hook is evaluated.
375
375
376 """
376 """
377
377
378 #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg
378 #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg
379 # Another option, seems to work great. Catches things like ''.<tab>
379 # Another option, seems to work great. Catches things like ''.<tab>
380 m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
380 m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
381
381
382 if m:
382 if m:
383 expr, attr = m.group(1, 3)
383 expr, attr = m.group(1, 3)
384 elif self.greedy:
384 elif self.greedy:
385 m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
385 m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
386 if not m2:
386 if not m2:
387 return []
387 return []
388 expr, attr = m2.group(1,2)
388 expr, attr = m2.group(1,2)
389 else:
389 else:
390 return []
390 return []
391
391
392 try:
392 try:
393 obj = eval(expr, self.namespace)
393 obj = eval(expr, self.namespace)
394 except:
394 except:
395 try:
395 try:
396 obj = eval(expr, self.global_namespace)
396 obj = eval(expr, self.global_namespace)
397 except:
397 except:
398 return []
398 return []
399
399
400 if self.limit_to__all__ and hasattr(obj, '__all__'):
400 if self.limit_to__all__ and hasattr(obj, '__all__'):
401 words = get__all__entries(obj)
401 words = get__all__entries(obj)
402 else:
402 else:
403 words = dir2(obj)
403 words = dir2(obj)
404
404
405 try:
405 try:
406 words = generics.complete_object(obj, words)
406 words = generics.complete_object(obj, words)
407 except TryNext:
407 except TryNext:
408 pass
408 pass
409 except Exception:
409 except Exception:
410 # Silence errors from completion function
410 # Silence errors from completion function
411 #raise # dbg
411 #raise # dbg
412 pass
412 pass
413 # Build match list to return
413 # Build match list to return
414 n = len(attr)
414 n = len(attr)
415 res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
415 res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
416 return res
416 return res
417
417
418
418
419 def get__all__entries(obj):
419 def get__all__entries(obj):
420 """returns the strings in the __all__ attribute"""
420 """returns the strings in the __all__ attribute"""
421 try:
421 try:
422 words = getattr(obj, '__all__')
422 words = getattr(obj, '__all__')
423 except:
423 except:
424 return []
424 return []
425
425
426 return [w for w in words if isinstance(w, basestring)]
426 return [w for w in words if isinstance(w, basestring)]
427
427
428
428
429 class IPCompleter(Completer):
429 class IPCompleter(Completer):
430 """Extension of the completer class with IPython-specific features"""
430 """Extension of the completer class with IPython-specific features"""
431
431
432 def _greedy_changed(self, name, old, new):
432 def _greedy_changed(self, name, old, new):
433 """update the splitter and readline delims when greedy is changed"""
433 """update the splitter and readline delims when greedy is changed"""
434 if new:
434 if new:
435 self.splitter.delims = GREEDY_DELIMS
435 self.splitter.delims = GREEDY_DELIMS
436 else:
436 else:
437 self.splitter.delims = DELIMS
437 self.splitter.delims = DELIMS
438
438
439 if self.readline:
439 if self.readline:
440 self.readline.set_completer_delims(self.splitter.delims)
440 self.readline.set_completer_delims(self.splitter.delims)
441
441
442 merge_completions = CBool(True, config=True,
442 merge_completions = CBool(True, config=True,
443 help="""Whether to merge completion results into a single list
443 help="""Whether to merge completion results into a single list
444
444
445 If False, only the completion results from the first non-empty
445 If False, only the completion results from the first non-empty
446 completer will be returned.
446 completer will be returned.
447 """
447 """
448 )
448 )
449 omit__names = Enum((0,1,2), default_value=2, config=True,
449 omit__names = Enum((0,1,2), default_value=2, config=True,
450 help="""Instruct the completer to omit private method names
450 help="""Instruct the completer to omit private method names
451
451
452 Specifically, when completing on ``object.<tab>``.
452 Specifically, when completing on ``object.<tab>``.
453
453
454 When 2 [default]: all names that start with '_' will be excluded.
454 When 2 [default]: all names that start with '_' will be excluded.
455
455
456 When 1: all 'magic' names (``__foo__``) will be excluded.
456 When 1: all 'magic' names (``__foo__``) will be excluded.
457
457
458 When 0: nothing will be excluded.
458 When 0: nothing will be excluded.
459 """
459 """
460 )
460 )
461 limit_to__all__ = CBool(default_value=False, config=True,
461 limit_to__all__ = CBool(default_value=False, config=True,
462 help="""Instruct the completer to use __all__ for the completion
462 help="""Instruct the completer to use __all__ for the completion
463
463
464 Specifically, when completing on ``object.<tab>``.
464 Specifically, when completing on ``object.<tab>``.
465
465
466 When True: only those names in obj.__all__ will be included.
466 When True: only those names in obj.__all__ will be included.
467
467
468 When False [default]: the __all__ attribute is ignored
468 When False [default]: the __all__ attribute is ignored
469 """
469 """
470 )
470 )
471
471
472 def __init__(self, shell=None, namespace=None, global_namespace=None,
472 def __init__(self, shell=None, namespace=None, global_namespace=None,
473 use_readline=True, config=None, **kwargs):
473 use_readline=True, config=None, **kwargs):
474 """IPCompleter() -> completer
474 """IPCompleter() -> completer
475
475
476 Return a completer object suitable for use by the readline library
476 Return a completer object suitable for use by the readline library
477 via readline.set_completer().
477 via readline.set_completer().
478
478
479 Inputs:
479 Inputs:
480
480
481 - shell: a pointer to the ipython shell itself. This is needed
481 - shell: a pointer to the ipython shell itself. This is needed
482 because this completer knows about magic functions, and those can
482 because this completer knows about magic functions, and those can
483 only be accessed via the ipython instance.
483 only be accessed via the ipython instance.
484
484
485 - namespace: an optional dict where completions are performed.
485 - namespace: an optional dict where completions are performed.
486
486
487 - global_namespace: secondary optional dict for completions, to
487 - global_namespace: secondary optional dict for completions, to
488 handle cases (such as IPython embedded inside functions) where
488 handle cases (such as IPython embedded inside functions) where
489 both Python scopes are visible.
489 both Python scopes are visible.
490
490
491 use_readline : bool, optional
491 use_readline : bool, optional
492 If true, use the readline library. This completer can still function
492 If true, use the readline library. This completer can still function
493 without readline, though in that case callers must provide some extra
493 without readline, though in that case callers must provide some extra
494 information on each call about the current line."""
494 information on each call about the current line."""
495
495
496 self.magic_escape = ESC_MAGIC
496 self.magic_escape = ESC_MAGIC
497 self.splitter = CompletionSplitter()
497 self.splitter = CompletionSplitter()
498
498
499 # Readline configuration, only used by the rlcompleter method.
499 # Readline configuration, only used by the rlcompleter method.
500 if use_readline:
500 if use_readline:
501 # We store the right version of readline so that later code
501 # We store the right version of readline so that later code
502 import IPython.utils.rlineimpl as readline
502 import IPython.utils.rlineimpl as readline
503 self.readline = readline
503 self.readline = readline
504 else:
504 else:
505 self.readline = None
505 self.readline = None
506
506
507 # _greedy_changed() depends on splitter and readline being defined:
507 # _greedy_changed() depends on splitter and readline being defined:
508 Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
508 Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
509 config=config, **kwargs)
509 config=config, **kwargs)
510
510
511 # List where completion matches will be stored
511 # List where completion matches will be stored
512 self.matches = []
512 self.matches = []
513 self.shell = shell
513 self.shell = shell
514 # Regexp to split filenames with spaces in them
514 # Regexp to split filenames with spaces in them
515 self.space_name_re = re.compile(r'([^\\] )')
515 self.space_name_re = re.compile(r'([^\\] )')
516 # Hold a local ref. to glob.glob for speed
516 # Hold a local ref. to glob.glob for speed
517 self.glob = glob.glob
517 self.glob = glob.glob
518
518
519 # Determine if we are running on 'dumb' terminals, like (X)Emacs
519 # Determine if we are running on 'dumb' terminals, like (X)Emacs
520 # buffers, to avoid completion problems.
520 # buffers, to avoid completion problems.
521 term = os.environ.get('TERM','xterm')
521 term = os.environ.get('TERM','xterm')
522 self.dumb_terminal = term in ['dumb','emacs']
522 self.dumb_terminal = term in ['dumb','emacs']
523
523
524 # Special handling of backslashes needed in win32 platforms
524 # Special handling of backslashes needed in win32 platforms
525 if sys.platform == "win32":
525 if sys.platform == "win32":
526 self.clean_glob = self._clean_glob_win32
526 self.clean_glob = self._clean_glob_win32
527 else:
527 else:
528 self.clean_glob = self._clean_glob
528 self.clean_glob = self._clean_glob
529
529
530 #regexp to parse docstring for function signature
530 #regexp to parse docstring for function signature
531 self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
531 self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
532 self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
532 self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
533 #use this if positional argument name is also needed
533 #use this if positional argument name is also needed
534 #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
534 #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
535
535
536 # All active matcher routines for completion
536 # All active matcher routines for completion
537 self.matchers = [self.python_matches,
537 self.matchers = [self.python_matches,
538 self.file_matches,
538 self.file_matches,
539 self.magic_matches,
539 self.magic_matches,
540 self.python_func_kw_matches,
540 self.python_func_kw_matches,
541 ]
541 ]
542
542
543 def all_completions(self, text):
543 def all_completions(self, text):
544 """
544 """
545 Wrapper around the complete method for the benefit of emacs
545 Wrapper around the complete method for the benefit of emacs
546 and pydb.
546 and pydb.
547 """
547 """
548 return self.complete(text)[1]
548 return self.complete(text)[1]
549
549
550 def _clean_glob(self,text):
550 def _clean_glob(self,text):
551 return self.glob("%s*" % text)
551 return self.glob("%s*" % text)
552
552
553 def _clean_glob_win32(self,text):
553 def _clean_glob_win32(self,text):
554 return [f.replace("\\","/")
554 return [f.replace("\\","/")
555 for f in self.glob("%s*" % text)]
555 for f in self.glob("%s*" % text)]
556
556
557 def file_matches(self, text):
557 def file_matches(self, text):
558 """Match filenames, expanding ~USER type strings.
558 """Match filenames, expanding ~USER type strings.
559
559
560 Most of the seemingly convoluted logic in this completer is an
560 Most of the seemingly convoluted logic in this completer is an
561 attempt to handle filenames with spaces in them. And yet it's not
561 attempt to handle filenames with spaces in them. And yet it's not
562 quite perfect, because Python's readline doesn't expose all of the
562 quite perfect, because Python's readline doesn't expose all of the
563 GNU readline details needed for this to be done correctly.
563 GNU readline details needed for this to be done correctly.
564
564
565 For a filename with a space in it, the printed completions will be
565 For a filename with a space in it, the printed completions will be
566 only the parts after what's already been typed (instead of the
566 only the parts after what's already been typed (instead of the
567 full completions, as is normally done). I don't think with the
567 full completions, as is normally done). I don't think with the
568 current (as of Python 2.3) Python readline it's possible to do
568 current (as of Python 2.3) Python readline it's possible to do
569 better."""
569 better."""
570
570
571 #io.rprint('Completer->file_matches: <%r>' % text) # dbg
571 #io.rprint('Completer->file_matches: <%r>' % text) # dbg
572
572
573 # chars that require escaping with backslash - i.e. chars
573 # chars that require escaping with backslash - i.e. chars
574 # that readline treats incorrectly as delimiters, but we
574 # that readline treats incorrectly as delimiters, but we
575 # don't want to treat as delimiters in filename matching
575 # don't want to treat as delimiters in filename matching
576 # when escaped with backslash
576 # when escaped with backslash
577 if text.startswith('!'):
577 if text.startswith('!'):
578 text = text[1:]
578 text = text[1:]
579 text_prefix = '!'
579 text_prefix = '!'
580 else:
580 else:
581 text_prefix = ''
581 text_prefix = ''
582
582
583 text_until_cursor = self.text_until_cursor
583 text_until_cursor = self.text_until_cursor
584 # track strings with open quotes
584 # track strings with open quotes
585 open_quotes = has_open_quotes(text_until_cursor)
585 open_quotes = has_open_quotes(text_until_cursor)
586
586
587 if '(' in text_until_cursor or '[' in text_until_cursor:
587 if '(' in text_until_cursor or '[' in text_until_cursor:
588 lsplit = text
588 lsplit = text
589 else:
589 else:
590 try:
590 try:
591 # arg_split ~ shlex.split, but with unicode bugs fixed by us
591 # arg_split ~ shlex.split, but with unicode bugs fixed by us
592 lsplit = arg_split(text_until_cursor)[-1]
592 lsplit = arg_split(text_until_cursor)[-1]
593 except ValueError:
593 except ValueError:
594 # typically an unmatched ", or backslash without escaped char.
594 # typically an unmatched ", or backslash without escaped char.
595 if open_quotes:
595 if open_quotes:
596 lsplit = text_until_cursor.split(open_quotes)[-1]
596 lsplit = text_until_cursor.split(open_quotes)[-1]
597 else:
597 else:
598 return []
598 return []
599 except IndexError:
599 except IndexError:
600 # tab pressed on empty line
600 # tab pressed on empty line
601 lsplit = ""
601 lsplit = ""
602
602
603 if not open_quotes and lsplit != protect_filename(lsplit):
603 if not open_quotes and lsplit != protect_filename(lsplit):
604 # if protectables are found, do matching on the whole escaped name
604 # if protectables are found, do matching on the whole escaped name
605 has_protectables = True
605 has_protectables = True
606 text0,text = text,lsplit
606 text0,text = text,lsplit
607 else:
607 else:
608 has_protectables = False
608 has_protectables = False
609 text = os.path.expanduser(text)
609 text = os.path.expanduser(text)
610
610
611 if text == "":
611 if text == "":
612 return [text_prefix + protect_filename(f) for f in self.glob("*")]
612 return [text_prefix + protect_filename(f) for f in self.glob("*")]
613
613
614 # Compute the matches from the filesystem
614 # Compute the matches from the filesystem
615 m0 = self.clean_glob(text.replace('\\',''))
615 m0 = self.clean_glob(text.replace('\\',''))
616
616
617 if has_protectables:
617 if has_protectables:
618 # If we had protectables, we need to revert our changes to the
618 # If we had protectables, we need to revert our changes to the
619 # beginning of filename so that we don't double-write the part
619 # beginning of filename so that we don't double-write the part
620 # of the filename we have so far
620 # of the filename we have so far
621 len_lsplit = len(lsplit)
621 len_lsplit = len(lsplit)
622 matches = [text_prefix + text0 +
622 matches = [text_prefix + text0 +
623 protect_filename(f[len_lsplit:]) for f in m0]
623 protect_filename(f[len_lsplit:]) for f in m0]
624 else:
624 else:
625 if open_quotes:
625 if open_quotes:
626 # if we have a string with an open quote, we don't need to
626 # if we have a string with an open quote, we don't need to
627 # protect the names at all (and we _shouldn't_, as it
627 # protect the names at all (and we _shouldn't_, as it
628 # would cause bugs when the filesystem call is made).
628 # would cause bugs when the filesystem call is made).
629 matches = m0
629 matches = m0
630 else:
630 else:
631 matches = [text_prefix +
631 matches = [text_prefix +
632 protect_filename(f) for f in m0]
632 protect_filename(f) for f in m0]
633
633
634 #io.rprint('mm', matches) # dbg
634 #io.rprint('mm', matches) # dbg
635
635
636 # Mark directories in input list by appending '/' to their names.
636 # Mark directories in input list by appending '/' to their names.
637 matches = [x+'/' if os.path.isdir(x) else x for x in matches]
637 matches = [x+'/' if os.path.isdir(x) else x for x in matches]
638 return matches
638 return matches
639
639
640 def magic_matches(self, text):
640 def magic_matches(self, text):
641 """Match magics"""
641 """Match magics"""
642 #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
642 #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
643 # Get all shell magics now rather than statically, so magics loaded at
643 # Get all shell magics now rather than statically, so magics loaded at
644 # runtime show up too.
644 # runtime show up too.
645 lsm = self.shell.magics_manager.lsmagic()
645 lsm = self.shell.magics_manager.lsmagic()
646 line_magics = lsm['line']
646 line_magics = lsm['line']
647 cell_magics = lsm['cell']
647 cell_magics = lsm['cell']
648 pre = self.magic_escape
648 pre = self.magic_escape
649 pre2 = pre+pre
649 pre2 = pre+pre
650
650
651 # Completion logic:
651 # Completion logic:
652 # - user gives %%: only do cell magics
652 # - user gives %%: only do cell magics
653 # - user gives %: do both line and cell magics
653 # - user gives %: do both line and cell magics
654 # - no prefix: do both
654 # - no prefix: do both
655 # In other words, line magics are skipped if the user gives %% explicitly
655 # In other words, line magics are skipped if the user gives %% explicitly
656 bare_text = text.lstrip(pre)
656 bare_text = text.lstrip(pre)
657 comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]
657 comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]
658 if not text.startswith(pre2):
658 if not text.startswith(pre2):
659 comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
659 comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
660 return comp
660 return comp
661
661
662 def python_matches(self,text):
662 def python_matches(self,text):
663 """Match attributes or global python names"""
663 """Match attributes or global python names"""
664
664
665 #io.rprint('Completer->python_matches, txt=%r' % text) # dbg
665 #io.rprint('Completer->python_matches, txt=%r' % text) # dbg
666 if "." in text:
666 if "." in text:
667 try:
667 try:
668 matches = self.attr_matches(text)
668 matches = self.attr_matches(text)
669 if text.endswith('.') and self.omit__names:
669 if text.endswith('.') and self.omit__names:
670 if self.omit__names == 1:
670 if self.omit__names == 1:
671 # true if txt is _not_ a __ name, false otherwise:
671 # true if txt is _not_ a __ name, false otherwise:
672 no__name = (lambda txt:
672 no__name = (lambda txt:
673 re.match(r'.*\.__.*?__',txt) is None)
673 re.match(r'.*\.__.*?__',txt) is None)
674 else:
674 else:
675 # true if txt is _not_ a _ name, false otherwise:
675 # true if txt is _not_ a _ name, false otherwise:
676 no__name = (lambda txt:
676 no__name = (lambda txt:
677 re.match(r'.*\._.*?',txt) is None)
677 re.match(r'.*\._.*?',txt) is None)
678 matches = filter(no__name, matches)
678 matches = filter(no__name, matches)
679 except NameError:
679 except NameError:
680 # catches <undefined attributes>.<tab>
680 # catches <undefined attributes>.<tab>
681 matches = []
681 matches = []
682 else:
682 else:
683 matches = self.global_matches(text)
683 matches = self.global_matches(text)
684
684
685 return matches
685 return matches
686
686
687 def _default_arguments_from_docstring(self, doc):
687 def _default_arguments_from_docstring(self, doc):
688 """Parse the first line of docstring for call signature.
688 """Parse the first line of docstring for call signature.
689
689
690 Docstring should be of the form 'min(iterable[, key=func])\n'.
690 Docstring should be of the form 'min(iterable[, key=func])\n'.
691 It can also parse cython docstring of the form
691 It can also parse cython docstring of the form
692 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
692 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
693 """
693 """
694 if doc is None:
694 if doc is None:
695 return []
695 return []
696
696
697 #care only the firstline
697 #care only the firstline
698 line = doc.lstrip().splitlines()[0]
698 line = doc.lstrip().splitlines()[0]
699
699
700 #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
700 #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
701 #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
701 #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
702 sig = self.docstring_sig_re.search(line)
702 sig = self.docstring_sig_re.search(line)
703 if sig is None:
703 if sig is None:
704 return []
704 return []
705 # iterable[, key=func]' -> ['iterable[' ,' key=func]']
705 # iterable[, key=func]' -> ['iterable[' ,' key=func]']
706 sig = sig.groups()[0].split(',')
706 sig = sig.groups()[0].split(',')
707 ret = []
707 ret = []
708 for s in sig:
708 for s in sig:
709 #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
709 #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
710 ret += self.docstring_kwd_re.findall(s)
710 ret += self.docstring_kwd_re.findall(s)
711 return ret
711 return ret
712
712
713 def _default_arguments(self, obj):
713 def _default_arguments(self, obj):
714 """Return the list of default arguments of obj if it is callable,
714 """Return the list of default arguments of obj if it is callable,
715 or empty list otherwise."""
715 or empty list otherwise."""
716 call_obj = obj
716 call_obj = obj
717 ret = []
717 ret = []
718 if inspect.isbuiltin(obj):
718 if inspect.isbuiltin(obj):
719 pass
719 pass
720 elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
720 elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
721 if inspect.isclass(obj):
721 if inspect.isclass(obj):
722 #for cython embededsignature=True the constructor docstring
722 #for cython embededsignature=True the constructor docstring
723 #belongs to the object itself not __init__
723 #belongs to the object itself not __init__
724 ret += self._default_arguments_from_docstring(
724 ret += self._default_arguments_from_docstring(
725 getattr(obj, '__doc__', ''))
725 getattr(obj, '__doc__', ''))
726 # for classes, check for __init__,__new__
726 # for classes, check for __init__,__new__
727 call_obj = (getattr(obj, '__init__', None) or
727 call_obj = (getattr(obj, '__init__', None) or
728 getattr(obj, '__new__', None))
728 getattr(obj, '__new__', None))
729 # for all others, check if they are __call__able
729 # for all others, check if they are __call__able
730 elif hasattr(obj, '__call__'):
730 elif hasattr(obj, '__call__'):
731 call_obj = obj.__call__
731 call_obj = obj.__call__
732
732
733 ret += self._default_arguments_from_docstring(
733 ret += self._default_arguments_from_docstring(
734 getattr(call_obj, '__doc__', ''))
734 getattr(call_obj, '__doc__', ''))
735
735
736 try:
736 try:
737 args,_,_1,defaults = inspect.getargspec(call_obj)
737 args,_,_1,defaults = inspect.getargspec(call_obj)
738 if defaults:
738 if defaults:
739 ret+=args[-len(defaults):]
739 ret+=args[-len(defaults):]
740 except TypeError:
740 except TypeError:
741 pass
741 pass
742
742
743 return list(set(ret))
743 return list(set(ret))
744
744
745 def python_func_kw_matches(self,text):
745 def python_func_kw_matches(self,text):
746 """Match named parameters (kwargs) of the last open function"""
746 """Match named parameters (kwargs) of the last open function"""
747
747
748 if "." in text: # a parameter cannot be dotted
748 if "." in text: # a parameter cannot be dotted
749 return []
749 return []
750 try: regexp = self.__funcParamsRegex
750 try: regexp = self.__funcParamsRegex
751 except AttributeError:
751 except AttributeError:
752 regexp = self.__funcParamsRegex = re.compile(r'''
752 regexp = self.__funcParamsRegex = re.compile(r'''
753 '.*?(?<!\\)' | # single quoted strings or
753 '.*?(?<!\\)' | # single quoted strings or
754 ".*?(?<!\\)" | # double quoted strings or
754 ".*?(?<!\\)" | # double quoted strings or
755 \w+ | # identifier
755 \w+ | # identifier
756 \S # other characters
756 \S # other characters
757 ''', re.VERBOSE | re.DOTALL)
757 ''', re.VERBOSE | re.DOTALL)
758 # 1. find the nearest identifier that comes before an unclosed
758 # 1. find the nearest identifier that comes before an unclosed
759 # parenthesis before the cursor
759 # parenthesis before the cursor
760 # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
760 # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
761 tokens = regexp.findall(self.text_until_cursor)
761 tokens = regexp.findall(self.text_until_cursor)
762 tokens.reverse()
762 tokens.reverse()
763 iterTokens = iter(tokens); openPar = 0
763 iterTokens = iter(tokens); openPar = 0
764
764
765 for token in iterTokens:
765 for token in iterTokens:
766 if token == ')':
766 if token == ')':
767 openPar -= 1
767 openPar -= 1
768 elif token == '(':
768 elif token == '(':
769 openPar += 1
769 openPar += 1
770 if openPar > 0:
770 if openPar > 0:
771 # found the last unclosed parenthesis
771 # found the last unclosed parenthesis
772 break
772 break
773 else:
773 else:
774 return []
774 return []
775 # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
775 # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
776 ids = []
776 ids = []
777 isId = re.compile(r'\w+$').match
777 isId = re.compile(r'\w+$').match
778
778
779 while True:
779 while True:
780 try:
780 try:
781 ids.append(next(iterTokens))
781 ids.append(next(iterTokens))
782 if not isId(ids[-1]):
782 if not isId(ids[-1]):
783 ids.pop(); break
783 ids.pop(); break
784 if not next(iterTokens) == '.':
784 if not next(iterTokens) == '.':
785 break
785 break
786 except StopIteration:
786 except StopIteration:
787 break
787 break
788 # lookup the candidate callable matches either using global_matches
788 # lookup the candidate callable matches either using global_matches
789 # or attr_matches for dotted names
789 # or attr_matches for dotted names
790 if len(ids) == 1:
790 if len(ids) == 1:
791 callableMatches = self.global_matches(ids[0])
791 callableMatches = self.global_matches(ids[0])
792 else:
792 else:
793 callableMatches = self.attr_matches('.'.join(ids[::-1]))
793 callableMatches = self.attr_matches('.'.join(ids[::-1]))
794 argMatches = []
794 argMatches = []
795 for callableMatch in callableMatches:
795 for callableMatch in callableMatches:
796 try:
796 try:
797 namedArgs = self._default_arguments(eval(callableMatch,
797 namedArgs = self._default_arguments(eval(callableMatch,
798 self.namespace))
798 self.namespace))
799 except:
799 except:
800 continue
800 continue
801
801
802 for namedArg in namedArgs:
802 for namedArg in namedArgs:
803 if namedArg.startswith(text):
803 if namedArg.startswith(text):
804 argMatches.append("%s=" %namedArg)
804 argMatches.append("%s=" %namedArg)
805 return argMatches
805 return argMatches
806
806
807 def dispatch_custom_completer(self, text):
807 def dispatch_custom_completer(self, text):
808 #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
808 #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
809 line = self.line_buffer
809 line = self.line_buffer
810 if not line.strip():
810 if not line.strip():
811 return None
811 return None
812
812
813 # Create a little structure to pass all the relevant information about
813 # Create a little structure to pass all the relevant information about
814 # the current completion to any custom completer.
814 # the current completion to any custom completer.
815 event = Bunch()
815 event = Bunch()
816 event.line = line
816 event.line = line
817 event.symbol = text
817 event.symbol = text
818 cmd = line.split(None,1)[0]
818 cmd = line.split(None,1)[0]
819 event.command = cmd
819 event.command = cmd
820 event.text_until_cursor = self.text_until_cursor
820 event.text_until_cursor = self.text_until_cursor
821
821
822 #print "\ncustom:{%s]\n" % event # dbg
822 #print "\ncustom:{%s]\n" % event # dbg
823
823
824 # for foo etc, try also to find completer for %foo
824 # for foo etc, try also to find completer for %foo
825 if not cmd.startswith(self.magic_escape):
825 if not cmd.startswith(self.magic_escape):
826 try_magic = self.custom_completers.s_matches(
826 try_magic = self.custom_completers.s_matches(
827 self.magic_escape + cmd)
827 self.magic_escape + cmd)
828 else:
828 else:
829 try_magic = []
829 try_magic = []
830
830
831 for c in itertools.chain(self.custom_completers.s_matches(cmd),
831 for c in itertools.chain(self.custom_completers.s_matches(cmd),
832 try_magic,
832 try_magic,
833 self.custom_completers.flat_matches(self.text_until_cursor)):
833 self.custom_completers.flat_matches(self.text_until_cursor)):
834 #print "try",c # dbg
834 #print "try",c # dbg
835 try:
835 try:
836 res = c(event)
836 res = c(event)
837 if res:
837 if res:
838 # first, try case sensitive match
838 # first, try case sensitive match
839 withcase = [r for r in res if r.startswith(text)]
839 withcase = [r for r in res if r.startswith(text)]
840 if withcase:
840 if withcase:
841 return withcase
841 return withcase
842 # if none, then case insensitive ones are ok too
842 # if none, then case insensitive ones are ok too
843 text_low = text.lower()
843 text_low = text.lower()
844 return [r for r in res if r.lower().startswith(text_low)]
844 return [r for r in res if r.lower().startswith(text_low)]
845 except TryNext:
845 except TryNext:
846 pass
846 pass
847
847
848 return None
848 return None
849
849
850 def complete(self, text=None, line_buffer=None, cursor_pos=None):
850 def complete(self, text=None, line_buffer=None, cursor_pos=None):
851 """Find completions for the given text and line context.
851 """Find completions for the given text and line context.
852
852
853 This is called successively with state == 0, 1, 2, ... until it
853 This is called successively with state == 0, 1, 2, ... until it
854 returns None. The completion should begin with 'text'.
854 returns None. The completion should begin with 'text'.
855
855
856 Note that both the text and the line_buffer are optional, but at least
856 Note that both the text and the line_buffer are optional, but at least
857 one of them must be given.
857 one of them must be given.
858
858
859 Parameters
859 Parameters
860 ----------
860 ----------
861 text : string, optional
861 text : string, optional
862 Text to perform the completion on. If not given, the line buffer
862 Text to perform the completion on. If not given, the line buffer
863 is split using the instance's CompletionSplitter object.
863 is split using the instance's CompletionSplitter object.
864
864
865 line_buffer : string, optional
865 line_buffer : string, optional
866 If not given, the completer attempts to obtain the current line
866 If not given, the completer attempts to obtain the current line
867 buffer via readline. This keyword allows clients which are
867 buffer via readline. This keyword allows clients which are
868 requesting for text completions in non-readline contexts to inform
868 requesting for text completions in non-readline contexts to inform
869 the completer of the entire text.
869 the completer of the entire text.
870
870
871 cursor_pos : int, optional
871 cursor_pos : int, optional
872 Index of the cursor in the full line buffer. Should be provided by
872 Index of the cursor in the full line buffer. Should be provided by
873 remote frontends where kernel has no access to frontend state.
873 remote frontends where kernel has no access to frontend state.
874
874
875 Returns
875 Returns
876 -------
876 -------
877 text : str
877 text : str
878 Text that was actually used in the completion.
878 Text that was actually used in the completion.
879
879
880 matches : list
880 matches : list
881 A list of completion matches.
881 A list of completion matches.
882 """
882 """
883 #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
883 #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
884
884
885 # if the cursor position isn't given, the only sane assumption we can
885 # if the cursor position isn't given, the only sane assumption we can
886 # make is that it's at the end of the line (the common case)
886 # make is that it's at the end of the line (the common case)
887 if cursor_pos is None:
887 if cursor_pos is None:
888 cursor_pos = len(line_buffer) if text is None else len(text)
888 cursor_pos = len(line_buffer) if text is None else len(text)
889
889
890 # if text is either None or an empty string, rely on the line buffer
890 # if text is either None or an empty string, rely on the line buffer
891 if not text:
891 if not text:
892 text = self.splitter.split_line(line_buffer, cursor_pos)
892 text = self.splitter.split_line(line_buffer, cursor_pos)
893
893
894 # If no line buffer is given, assume the input text is all there was
894 # If no line buffer is given, assume the input text is all there was
895 if line_buffer is None:
895 if line_buffer is None:
896 line_buffer = text
896 line_buffer = text
897
897
898 self.line_buffer = line_buffer
898 self.line_buffer = line_buffer
899 self.text_until_cursor = self.line_buffer[:cursor_pos]
899 self.text_until_cursor = self.line_buffer[:cursor_pos]
900 #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
900 #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
901
901
902 # Start with a clean slate of completions
902 # Start with a clean slate of completions
903 self.matches[:] = []
903 self.matches[:] = []
904 custom_res = self.dispatch_custom_completer(text)
904 custom_res = self.dispatch_custom_completer(text)
905 if custom_res is not None:
905 if custom_res is not None:
906 # did custom completers produce something?
906 # did custom completers produce something?
907 self.matches = custom_res
907 self.matches = custom_res
908 else:
908 else:
909 # Extend the list of completions with the results of each
909 # Extend the list of completions with the results of each
910 # matcher, so we return results to the user from all
910 # matcher, so we return results to the user from all
911 # namespaces.
911 # namespaces.
912 if self.merge_completions:
912 if self.merge_completions:
913 self.matches = []
913 self.matches = []
914 for matcher in self.matchers:
914 for matcher in self.matchers:
915 try:
915 try:
916 self.matches.extend(matcher(text))
916 self.matches.extend(matcher(text))
917 except:
917 except:
918 # Show the ugly traceback if the matcher causes an
918 # Show the ugly traceback if the matcher causes an
919 # exception, but do NOT crash the kernel!
919 # exception, but do NOT crash the kernel!
920 sys.excepthook(*sys.exc_info())
920 sys.excepthook(*sys.exc_info())
921 else:
921 else:
922 for matcher in self.matchers:
922 for matcher in self.matchers:
923 self.matches = matcher(text)
923 self.matches = matcher(text)
924 if self.matches:
924 if self.matches:
925 break
925 break
926 # FIXME: we should extend our api to return a dict with completions for
926 # FIXME: we should extend our api to return a dict with completions for
927 # different types of objects. The rlcomplete() method could then
927 # different types of objects. The rlcomplete() method could then
928 # simply collapse the dict into a list for readline, but we'd have
928 # simply collapse the dict into a list for readline, but we'd have
929 # richer completion semantics in other evironments.
929 # richer completion semantics in other evironments.
930
930
931 # use penalize_magics_key to put magics after variables with same name
931 # use penalize_magics_key to put magics after variables with same name
932 self.matches = sorted(set(self.matches), key=penalize_magics_key)
932 self.matches = sorted(set(self.matches), key=penalize_magics_key)
933
933
934 #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
934 #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
935 return text, self.matches
935 return text, self.matches
936
936
937 def rlcomplete(self, text, state):
937 def rlcomplete(self, text, state):
938 """Return the state-th possible completion for 'text'.
938 """Return the state-th possible completion for 'text'.
939
939
940 This is called successively with state == 0, 1, 2, ... until it
940 This is called successively with state == 0, 1, 2, ... until it
941 returns None. The completion should begin with 'text'.
941 returns None. The completion should begin with 'text'.
942
942
943 Parameters
943 Parameters
944 ----------
944 ----------
945 text : string
945 text : string
946 Text to perform the completion on.
946 Text to perform the completion on.
947
947
948 state : int
948 state : int
949 Counter used by readline.
949 Counter used by readline.
950 """
950 """
951 if state==0:
951 if state==0:
952
952
953 self.line_buffer = line_buffer = self.readline.get_line_buffer()
953 self.line_buffer = line_buffer = self.readline.get_line_buffer()
954 cursor_pos = self.readline.get_endidx()
954 cursor_pos = self.readline.get_endidx()
955
955
956 #io.rprint("\nRLCOMPLETE: %r %r %r" %
956 #io.rprint("\nRLCOMPLETE: %r %r %r" %
957 # (text, line_buffer, cursor_pos) ) # dbg
957 # (text, line_buffer, cursor_pos) ) # dbg
958
958
959 # if there is only a tab on a line with only whitespace, instead of
959 # if there is only a tab on a line with only whitespace, instead of
960 # the mostly useless 'do you want to see all million completions'
960 # the mostly useless 'do you want to see all million completions'
961 # message, just do the right thing and give the user his tab!
961 # message, just do the right thing and give the user his tab!
962 # Incidentally, this enables pasting of tabbed text from an editor
962 # Incidentally, this enables pasting of tabbed text from an editor
963 # (as long as autoindent is off).
963 # (as long as autoindent is off).
964
964
965 # It should be noted that at least pyreadline still shows file
965 # It should be noted that at least pyreadline still shows file
966 # completions - is there a way around it?
966 # completions - is there a way around it?
967
967
968 # don't apply this on 'dumb' terminals, such as emacs buffers, so
968 # don't apply this on 'dumb' terminals, such as emacs buffers, so
969 # we don't interfere with their own tab-completion mechanism.
969 # we don't interfere with their own tab-completion mechanism.
970 if not (self.dumb_terminal or line_buffer.strip()):
970 if not (self.dumb_terminal or line_buffer.strip()):
971 self.readline.insert_text('\t')
971 self.readline.insert_text('\t')
972 sys.stdout.flush()
972 sys.stdout.flush()
973 return None
973 return None
974
974
975 # Note: debugging exceptions that may occur in completion is very
975 # Note: debugging exceptions that may occur in completion is very
976 # tricky, because readline unconditionally silences them. So if
976 # tricky, because readline unconditionally silences them. So if
977 # during development you suspect a bug in the completion code, turn
977 # during development you suspect a bug in the completion code, turn
978 # this flag on temporarily by uncommenting the second form (don't
978 # this flag on temporarily by uncommenting the second form (don't
979 # flip the value in the first line, as the '# dbg' marker can be
979 # flip the value in the first line, as the '# dbg' marker can be
980 # automatically detected and is used elsewhere).
980 # automatically detected and is used elsewhere).
981 DEBUG = False
981 DEBUG = False
982 #DEBUG = True # dbg
982 #DEBUG = True # dbg
983 if DEBUG:
983 if DEBUG:
984 try:
984 try:
985 self.complete(text, line_buffer, cursor_pos)
985 self.complete(text, line_buffer, cursor_pos)
986 except:
986 except:
987 import traceback; traceback.print_exc()
987 import traceback; traceback.print_exc()
988 else:
988 else:
989 # The normal production version is here
989 # The normal production version is here
990
990
991 # This method computes the self.matches array
991 # This method computes the self.matches array
992 self.complete(text, line_buffer, cursor_pos)
992 self.complete(text, line_buffer, cursor_pos)
993
993
994 try:
994 try:
995 return self.matches[state]
995 return self.matches[state]
996 except IndexError:
996 except IndexError:
997 return None
997 return None
@@ -1,280 +1,279 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Displayhook for IPython.
2 """Displayhook for IPython.
3
3
4 This defines a callable class that IPython uses for `sys.displayhook`.
4 This defines a callable class that IPython uses for `sys.displayhook`.
5
5
6 Authors:
6 Authors:
7
7
8 * Fernando Perez
8 * Fernando Perez
9 * Brian Granger
9 * Brian Granger
10 * Robert Kern
10 * Robert Kern
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
15 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
16 #
16 #
17 # Distributed under the terms of the BSD License. The full license is in
17 # Distributed under the terms of the BSD License. The full license is in
18 # the file COPYING, distributed as part of this software.
18 # the file COPYING, distributed as part of this software.
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22 # Imports
22 # Imports
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24 from __future__ import print_function
24 from __future__ import print_function
25
25
26 import __builtin__
27
28 import sys
26 import sys
29
27
30
28
31 from IPython.config.configurable import Configurable
29 from IPython.config.configurable import Configurable
32 from IPython.utils import io
30 from IPython.utils import io
31 from IPython.utils.py3compat import builtin_mod
33 from IPython.utils.traitlets import Instance
32 from IPython.utils.traitlets import Instance
34 from IPython.utils.warn import warn
33 from IPython.utils.warn import warn
35
34
36 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
37 # Main displayhook class
36 # Main displayhook class
38 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
39
38
40 # TODO: Move the various attributes (cache_size, [others now moved]). Some
39 # TODO: Move the various attributes (cache_size, [others now moved]). Some
41 # of these are also attributes of InteractiveShell. They should be on ONE object
40 # of these are also attributes of InteractiveShell. They should be on ONE object
42 # only and the other objects should ask that one object for their values.
41 # only and the other objects should ask that one object for their values.
43
42
44 class DisplayHook(Configurable):
43 class DisplayHook(Configurable):
45 """The custom IPython displayhook to replace sys.displayhook.
44 """The custom IPython displayhook to replace sys.displayhook.
46
45
47 This class does many things, but the basic idea is that it is a callable
46 This class does many things, but the basic idea is that it is a callable
48 that gets called anytime user code returns a value.
47 that gets called anytime user code returns a value.
49 """
48 """
50
49
51 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
50 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
52
51
53 def __init__(self, shell=None, cache_size=1000, **kwargs):
52 def __init__(self, shell=None, cache_size=1000, **kwargs):
54 super(DisplayHook, self).__init__(shell=shell, **kwargs)
53 super(DisplayHook, self).__init__(shell=shell, **kwargs)
55
54
56 cache_size_min = 3
55 cache_size_min = 3
57 if cache_size <= 0:
56 if cache_size <= 0:
58 self.do_full_cache = 0
57 self.do_full_cache = 0
59 cache_size = 0
58 cache_size = 0
60 elif cache_size < cache_size_min:
59 elif cache_size < cache_size_min:
61 self.do_full_cache = 0
60 self.do_full_cache = 0
62 cache_size = 0
61 cache_size = 0
63 warn('caching was disabled (min value for cache size is %s).' %
62 warn('caching was disabled (min value for cache size is %s).' %
64 cache_size_min,level=3)
63 cache_size_min,level=3)
65 else:
64 else:
66 self.do_full_cache = 1
65 self.do_full_cache = 1
67
66
68 self.cache_size = cache_size
67 self.cache_size = cache_size
69
68
70 # we need a reference to the user-level namespace
69 # we need a reference to the user-level namespace
71 self.shell = shell
70 self.shell = shell
72
71
73 self._,self.__,self.___ = '','',''
72 self._,self.__,self.___ = '','',''
74
73
75 # these are deliberately global:
74 # these are deliberately global:
76 to_user_ns = {'_':self._,'__':self.__,'___':self.___}
75 to_user_ns = {'_':self._,'__':self.__,'___':self.___}
77 self.shell.user_ns.update(to_user_ns)
76 self.shell.user_ns.update(to_user_ns)
78
77
79 @property
78 @property
80 def prompt_count(self):
79 def prompt_count(self):
81 return self.shell.execution_count
80 return self.shell.execution_count
82
81
83 #-------------------------------------------------------------------------
82 #-------------------------------------------------------------------------
84 # Methods used in __call__. Override these methods to modify the behavior
83 # Methods used in __call__. Override these methods to modify the behavior
85 # of the displayhook.
84 # of the displayhook.
86 #-------------------------------------------------------------------------
85 #-------------------------------------------------------------------------
87
86
88 def check_for_underscore(self):
87 def check_for_underscore(self):
89 """Check if the user has set the '_' variable by hand."""
88 """Check if the user has set the '_' variable by hand."""
90 # If something injected a '_' variable in __builtin__, delete
89 # If something injected a '_' variable in __builtin__, delete
91 # ipython's automatic one so we don't clobber that. gettext() in
90 # ipython's automatic one so we don't clobber that. gettext() in
92 # particular uses _, so we need to stay away from it.
91 # particular uses _, so we need to stay away from it.
93 if '_' in __builtin__.__dict__:
92 if '_' in builtin_mod.__dict__:
94 try:
93 try:
95 del self.shell.user_ns['_']
94 del self.shell.user_ns['_']
96 except KeyError:
95 except KeyError:
97 pass
96 pass
98
97
99 def quiet(self):
98 def quiet(self):
100 """Should we silence the display hook because of ';'?"""
99 """Should we silence the display hook because of ';'?"""
101 # do not print output if input ends in ';'
100 # do not print output if input ends in ';'
102 try:
101 try:
103 cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
102 cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
104 if cell.rstrip().endswith(';'):
103 if cell.rstrip().endswith(';'):
105 return True
104 return True
106 except IndexError:
105 except IndexError:
107 # some uses of ipshellembed may fail here
106 # some uses of ipshellembed may fail here
108 pass
107 pass
109 return False
108 return False
110
109
111 def start_displayhook(self):
110 def start_displayhook(self):
112 """Start the displayhook, initializing resources."""
111 """Start the displayhook, initializing resources."""
113 pass
112 pass
114
113
115 def write_output_prompt(self):
114 def write_output_prompt(self):
116 """Write the output prompt.
115 """Write the output prompt.
117
116
118 The default implementation simply writes the prompt to
117 The default implementation simply writes the prompt to
119 ``io.stdout``.
118 ``io.stdout``.
120 """
119 """
121 # Use write, not print which adds an extra space.
120 # Use write, not print which adds an extra space.
122 io.stdout.write(self.shell.separate_out)
121 io.stdout.write(self.shell.separate_out)
123 outprompt = self.shell.prompt_manager.render('out')
122 outprompt = self.shell.prompt_manager.render('out')
124 if self.do_full_cache:
123 if self.do_full_cache:
125 io.stdout.write(outprompt)
124 io.stdout.write(outprompt)
126
125
127 def compute_format_data(self, result):
126 def compute_format_data(self, result):
128 """Compute format data of the object to be displayed.
127 """Compute format data of the object to be displayed.
129
128
130 The format data is a generalization of the :func:`repr` of an object.
129 The format data is a generalization of the :func:`repr` of an object.
131 In the default implementation the format data is a :class:`dict` of
130 In the default implementation the format data is a :class:`dict` of
132 key value pair where the keys are valid MIME types and the values
131 key value pair where the keys are valid MIME types and the values
133 are JSON'able data structure containing the raw data for that MIME
132 are JSON'able data structure containing the raw data for that MIME
134 type. It is up to frontends to determine pick a MIME to to use and
133 type. It is up to frontends to determine pick a MIME to to use and
135 display that data in an appropriate manner.
134 display that data in an appropriate manner.
136
135
137 This method only computes the format data for the object and should
136 This method only computes the format data for the object and should
138 NOT actually print or write that to a stream.
137 NOT actually print or write that to a stream.
139
138
140 Parameters
139 Parameters
141 ----------
140 ----------
142 result : object
141 result : object
143 The Python object passed to the display hook, whose format will be
142 The Python object passed to the display hook, whose format will be
144 computed.
143 computed.
145
144
146 Returns
145 Returns
147 -------
146 -------
148 (format_dict, md_dict) : dict
147 (format_dict, md_dict) : dict
149 format_dict is a :class:`dict` whose keys are valid MIME types and values are
148 format_dict is a :class:`dict` whose keys are valid MIME types and values are
150 JSON'able raw data for that MIME type. It is recommended that
149 JSON'able raw data for that MIME type. It is recommended that
151 all return values of this should always include the "text/plain"
150 all return values of this should always include the "text/plain"
152 MIME type representation of the object.
151 MIME type representation of the object.
153 md_dict is a :class:`dict` with the same MIME type keys
152 md_dict is a :class:`dict` with the same MIME type keys
154 of metadata associated with each output.
153 of metadata associated with each output.
155
154
156 """
155 """
157 return self.shell.display_formatter.format(result)
156 return self.shell.display_formatter.format(result)
158
157
159 def write_format_data(self, format_dict, md_dict=None):
158 def write_format_data(self, format_dict, md_dict=None):
160 """Write the format data dict to the frontend.
159 """Write the format data dict to the frontend.
161
160
162 This default version of this method simply writes the plain text
161 This default version of this method simply writes the plain text
163 representation of the object to ``io.stdout``. Subclasses should
162 representation of the object to ``io.stdout``. Subclasses should
164 override this method to send the entire `format_dict` to the
163 override this method to send the entire `format_dict` to the
165 frontends.
164 frontends.
166
165
167 Parameters
166 Parameters
168 ----------
167 ----------
169 format_dict : dict
168 format_dict : dict
170 The format dict for the object passed to `sys.displayhook`.
169 The format dict for the object passed to `sys.displayhook`.
171 md_dict : dict (optional)
170 md_dict : dict (optional)
172 The metadata dict to be associated with the display data.
171 The metadata dict to be associated with the display data.
173 """
172 """
174 # We want to print because we want to always make sure we have a
173 # We want to print because we want to always make sure we have a
175 # newline, even if all the prompt separators are ''. This is the
174 # newline, even if all the prompt separators are ''. This is the
176 # standard IPython behavior.
175 # standard IPython behavior.
177 result_repr = format_dict['text/plain']
176 result_repr = format_dict['text/plain']
178 if '\n' in result_repr:
177 if '\n' in result_repr:
179 # So that multi-line strings line up with the left column of
178 # So that multi-line strings line up with the left column of
180 # the screen, instead of having the output prompt mess up
179 # the screen, instead of having the output prompt mess up
181 # their first line.
180 # their first line.
182 # We use the prompt template instead of the expanded prompt
181 # We use the prompt template instead of the expanded prompt
183 # because the expansion may add ANSI escapes that will interfere
182 # because the expansion may add ANSI escapes that will interfere
184 # with our ability to determine whether or not we should add
183 # with our ability to determine whether or not we should add
185 # a newline.
184 # a newline.
186 prompt_template = self.shell.prompt_manager.out_template
185 prompt_template = self.shell.prompt_manager.out_template
187 if prompt_template and not prompt_template.endswith('\n'):
186 if prompt_template and not prompt_template.endswith('\n'):
188 # But avoid extraneous empty lines.
187 # But avoid extraneous empty lines.
189 result_repr = '\n' + result_repr
188 result_repr = '\n' + result_repr
190
189
191 print(result_repr, file=io.stdout)
190 print(result_repr, file=io.stdout)
192
191
193 def update_user_ns(self, result):
192 def update_user_ns(self, result):
194 """Update user_ns with various things like _, __, _1, etc."""
193 """Update user_ns with various things like _, __, _1, etc."""
195
194
196 # Avoid recursive reference when displaying _oh/Out
195 # Avoid recursive reference when displaying _oh/Out
197 if result is not self.shell.user_ns['_oh']:
196 if result is not self.shell.user_ns['_oh']:
198 if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
197 if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
199 warn('Output cache limit (currently '+
198 warn('Output cache limit (currently '+
200 repr(self.cache_size)+' entries) hit.\n'
199 repr(self.cache_size)+' entries) hit.\n'
201 'Flushing cache and resetting history counter...\n'
200 'Flushing cache and resetting history counter...\n'
202 'The only history variables available will be _,__,___ and _1\n'
201 'The only history variables available will be _,__,___ and _1\n'
203 'with the current result.')
202 'with the current result.')
204
203
205 self.flush()
204 self.flush()
206 # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
205 # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
207 # we cause buggy behavior for things like gettext).
206 # we cause buggy behavior for things like gettext).
208
207
209 if '_' not in __builtin__.__dict__:
208 if '_' not in builtin_mod.__dict__:
210 self.___ = self.__
209 self.___ = self.__
211 self.__ = self._
210 self.__ = self._
212 self._ = result
211 self._ = result
213 self.shell.push({'_':self._,
212 self.shell.push({'_':self._,
214 '__':self.__,
213 '__':self.__,
215 '___':self.___}, interactive=False)
214 '___':self.___}, interactive=False)
216
215
217 # hackish access to top-level namespace to create _1,_2... dynamically
216 # hackish access to top-level namespace to create _1,_2... dynamically
218 to_main = {}
217 to_main = {}
219 if self.do_full_cache:
218 if self.do_full_cache:
220 new_result = '_'+repr(self.prompt_count)
219 new_result = '_'+repr(self.prompt_count)
221 to_main[new_result] = result
220 to_main[new_result] = result
222 self.shell.push(to_main, interactive=False)
221 self.shell.push(to_main, interactive=False)
223 self.shell.user_ns['_oh'][self.prompt_count] = result
222 self.shell.user_ns['_oh'][self.prompt_count] = result
224
223
225 def log_output(self, format_dict):
224 def log_output(self, format_dict):
226 """Log the output."""
225 """Log the output."""
227 if self.shell.logger.log_output:
226 if self.shell.logger.log_output:
228 self.shell.logger.log_write(format_dict['text/plain'], 'output')
227 self.shell.logger.log_write(format_dict['text/plain'], 'output')
229 self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
228 self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
230 format_dict['text/plain']
229 format_dict['text/plain']
231
230
232 def finish_displayhook(self):
231 def finish_displayhook(self):
233 """Finish up all displayhook activities."""
232 """Finish up all displayhook activities."""
234 io.stdout.write(self.shell.separate_out2)
233 io.stdout.write(self.shell.separate_out2)
235 io.stdout.flush()
234 io.stdout.flush()
236
235
237 def __call__(self, result=None):
236 def __call__(self, result=None):
238 """Printing with history cache management.
237 """Printing with history cache management.
239
238
240 This is invoked everytime the interpreter needs to print, and is
239 This is invoked everytime the interpreter needs to print, and is
241 activated by setting the variable sys.displayhook to it.
240 activated by setting the variable sys.displayhook to it.
242 """
241 """
243 self.check_for_underscore()
242 self.check_for_underscore()
244 if result is not None and not self.quiet():
243 if result is not None and not self.quiet():
245 self.start_displayhook()
244 self.start_displayhook()
246 self.write_output_prompt()
245 self.write_output_prompt()
247 format_dict, md_dict = self.compute_format_data(result)
246 format_dict, md_dict = self.compute_format_data(result)
248 self.write_format_data(format_dict, md_dict)
247 self.write_format_data(format_dict, md_dict)
249 self.update_user_ns(result)
248 self.update_user_ns(result)
250 self.log_output(format_dict)
249 self.log_output(format_dict)
251 self.finish_displayhook()
250 self.finish_displayhook()
252
251
253 def flush(self):
252 def flush(self):
254 if not self.do_full_cache:
253 if not self.do_full_cache:
255 raise ValueError("You shouldn't have reached the cache flush "
254 raise ValueError("You shouldn't have reached the cache flush "
256 "if full caching is not enabled!")
255 "if full caching is not enabled!")
257 # delete auto-generated vars from global namespace
256 # delete auto-generated vars from global namespace
258
257
259 for n in range(1,self.prompt_count + 1):
258 for n in range(1,self.prompt_count + 1):
260 key = '_'+repr(n)
259 key = '_'+repr(n)
261 try:
260 try:
262 del self.shell.user_ns[key]
261 del self.shell.user_ns[key]
263 except: pass
262 except: pass
264 # In some embedded circumstances, the user_ns doesn't have the
263 # In some embedded circumstances, the user_ns doesn't have the
265 # '_oh' key set up.
264 # '_oh' key set up.
266 oh = self.shell.user_ns.get('_oh', None)
265 oh = self.shell.user_ns.get('_oh', None)
267 if oh is not None:
266 if oh is not None:
268 oh.clear()
267 oh.clear()
269
268
270 # Release our own references to objects:
269 # Release our own references to objects:
271 self._, self.__, self.___ = '', '', ''
270 self._, self.__, self.___ = '', '', ''
272
271
273 if '_' not in __builtin__.__dict__:
272 if '_' not in builtin_mod.__dict__:
274 self.shell.user_ns.update({'_':None,'__':None, '___':None})
273 self.shell.user_ns.update({'_':None,'__':None, '___':None})
275 import gc
274 import gc
276 # TODO: Is this really needed?
275 # TODO: Is this really needed?
277 # IronPython blocks here forever
276 # IronPython blocks here forever
278 if sys.platform != "cli":
277 if sys.platform != "cli":
279 gc.collect()
278 gc.collect()
280
279
@@ -1,3164 +1,3164 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Main IPython class."""
2 """Main IPython class."""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 # Copyright (C) 2008-2011 The IPython Development Team
7 # Copyright (C) 2008-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 from __future__ import absolute_import
17 from __future__ import absolute_import
18 from __future__ import print_function
18 from __future__ import print_function
19
19
20 import __builtin__ as builtin_mod
21 import __future__
20 import __future__
22 import abc
21 import abc
23 import ast
22 import ast
24 import atexit
23 import atexit
25 import functools
24 import functools
26 import os
25 import os
27 import re
26 import re
28 import runpy
27 import runpy
29 import sys
28 import sys
30 import tempfile
29 import tempfile
31 import types
30 import types
32 import subprocess
31 import subprocess
33 from io import open as io_open
32 from io import open as io_open
34
33
35 from IPython.config.configurable import SingletonConfigurable
34 from IPython.config.configurable import SingletonConfigurable
36 from IPython.core import debugger, oinspect
35 from IPython.core import debugger, oinspect
37 from IPython.core import magic
36 from IPython.core import magic
38 from IPython.core import page
37 from IPython.core import page
39 from IPython.core import prefilter
38 from IPython.core import prefilter
40 from IPython.core import shadowns
39 from IPython.core import shadowns
41 from IPython.core import ultratb
40 from IPython.core import ultratb
42 from IPython.core.alias import AliasManager, AliasError
41 from IPython.core.alias import AliasManager, AliasError
43 from IPython.core.autocall import ExitAutocall
42 from IPython.core.autocall import ExitAutocall
44 from IPython.core.builtin_trap import BuiltinTrap
43 from IPython.core.builtin_trap import BuiltinTrap
45 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
44 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
46 from IPython.core.display_trap import DisplayTrap
45 from IPython.core.display_trap import DisplayTrap
47 from IPython.core.displayhook import DisplayHook
46 from IPython.core.displayhook import DisplayHook
48 from IPython.core.displaypub import DisplayPublisher
47 from IPython.core.displaypub import DisplayPublisher
49 from IPython.core.error import UsageError
48 from IPython.core.error import UsageError
50 from IPython.core.extensions import ExtensionManager
49 from IPython.core.extensions import ExtensionManager
51 from IPython.core.formatters import DisplayFormatter
50 from IPython.core.formatters import DisplayFormatter
52 from IPython.core.history import HistoryManager
51 from IPython.core.history import HistoryManager
53 from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2
52 from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2
54 from IPython.core.logger import Logger
53 from IPython.core.logger import Logger
55 from IPython.core.macro import Macro
54 from IPython.core.macro import Macro
56 from IPython.core.payload import PayloadManager
55 from IPython.core.payload import PayloadManager
57 from IPython.core.prefilter import PrefilterManager
56 from IPython.core.prefilter import PrefilterManager
58 from IPython.core.profiledir import ProfileDir
57 from IPython.core.profiledir import ProfileDir
59 from IPython.core.prompts import PromptManager
58 from IPython.core.prompts import PromptManager
60 from IPython.lib.latextools import LaTeXTool
59 from IPython.lib.latextools import LaTeXTool
61 from IPython.testing.skipdoctest import skip_doctest
60 from IPython.testing.skipdoctest import skip_doctest
62 from IPython.utils import PyColorize
61 from IPython.utils import PyColorize
63 from IPython.utils import io
62 from IPython.utils import io
64 from IPython.utils import py3compat
63 from IPython.utils import py3compat
65 from IPython.utils import openpy
64 from IPython.utils import openpy
66 from IPython.utils.decorators import undoc
65 from IPython.utils.decorators import undoc
67 from IPython.utils.io import ask_yes_no
66 from IPython.utils.io import ask_yes_no
68 from IPython.utils.ipstruct import Struct
67 from IPython.utils.ipstruct import Struct
69 from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
68 from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
70 from IPython.utils.pickleshare import PickleShareDB
69 from IPython.utils.pickleshare import PickleShareDB
71 from IPython.utils.process import system, getoutput
70 from IPython.utils.process import system, getoutput
71 from IPython.utils.py3compat import builtin_mod
72 from IPython.utils.strdispatch import StrDispatch
72 from IPython.utils.strdispatch import StrDispatch
73 from IPython.utils.syspathcontext import prepended_to_syspath
73 from IPython.utils.syspathcontext import prepended_to_syspath
74 from IPython.utils.text import (format_screen, LSString, SList,
74 from IPython.utils.text import (format_screen, LSString, SList,
75 DollarFormatter)
75 DollarFormatter)
76 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
76 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
77 List, Unicode, Instance, Type)
77 List, Unicode, Instance, Type)
78 from IPython.utils.warn import warn, error
78 from IPython.utils.warn import warn, error
79 import IPython.core.hooks
79 import IPython.core.hooks
80
80
81 #-----------------------------------------------------------------------------
81 #-----------------------------------------------------------------------------
82 # Globals
82 # Globals
83 #-----------------------------------------------------------------------------
83 #-----------------------------------------------------------------------------
84
84
85 # compiled regexps for autoindent management
85 # compiled regexps for autoindent management
86 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
86 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
87
87
88 #-----------------------------------------------------------------------------
88 #-----------------------------------------------------------------------------
89 # Utilities
89 # Utilities
90 #-----------------------------------------------------------------------------
90 #-----------------------------------------------------------------------------
91
91
92 @undoc
92 @undoc
93 def softspace(file, newvalue):
93 def softspace(file, newvalue):
94 """Copied from code.py, to remove the dependency"""
94 """Copied from code.py, to remove the dependency"""
95
95
96 oldvalue = 0
96 oldvalue = 0
97 try:
97 try:
98 oldvalue = file.softspace
98 oldvalue = file.softspace
99 except AttributeError:
99 except AttributeError:
100 pass
100 pass
101 try:
101 try:
102 file.softspace = newvalue
102 file.softspace = newvalue
103 except (AttributeError, TypeError):
103 except (AttributeError, TypeError):
104 # "attribute-less object" or "read-only attributes"
104 # "attribute-less object" or "read-only attributes"
105 pass
105 pass
106 return oldvalue
106 return oldvalue
107
107
108 @undoc
108 @undoc
109 def no_op(*a, **kw): pass
109 def no_op(*a, **kw): pass
110
110
111 @undoc
111 @undoc
112 class NoOpContext(object):
112 class NoOpContext(object):
113 def __enter__(self): pass
113 def __enter__(self): pass
114 def __exit__(self, type, value, traceback): pass
114 def __exit__(self, type, value, traceback): pass
115 no_op_context = NoOpContext()
115 no_op_context = NoOpContext()
116
116
117 class SpaceInInput(Exception): pass
117 class SpaceInInput(Exception): pass
118
118
119 @undoc
119 @undoc
120 class Bunch: pass
120 class Bunch: pass
121
121
122
122
123 def get_default_colors():
123 def get_default_colors():
124 if sys.platform=='darwin':
124 if sys.platform=='darwin':
125 return "LightBG"
125 return "LightBG"
126 elif os.name=='nt':
126 elif os.name=='nt':
127 return 'Linux'
127 return 'Linux'
128 else:
128 else:
129 return 'Linux'
129 return 'Linux'
130
130
131
131
132 class SeparateUnicode(Unicode):
132 class SeparateUnicode(Unicode):
133 """A Unicode subclass to validate separate_in, separate_out, etc.
133 """A Unicode subclass to validate separate_in, separate_out, etc.
134
134
135 This is a Unicode based trait that converts '0'->'' and '\\n'->'\n'.
135 This is a Unicode based trait that converts '0'->'' and '\\n'->'\n'.
136 """
136 """
137
137
138 def validate(self, obj, value):
138 def validate(self, obj, value):
139 if value == '0': value = ''
139 if value == '0': value = ''
140 value = value.replace('\\n','\n')
140 value = value.replace('\\n','\n')
141 return super(SeparateUnicode, self).validate(obj, value)
141 return super(SeparateUnicode, self).validate(obj, value)
142
142
143
143
144 class ReadlineNoRecord(object):
144 class ReadlineNoRecord(object):
145 """Context manager to execute some code, then reload readline history
145 """Context manager to execute some code, then reload readline history
146 so that interactive input to the code doesn't appear when pressing up."""
146 so that interactive input to the code doesn't appear when pressing up."""
147 def __init__(self, shell):
147 def __init__(self, shell):
148 self.shell = shell
148 self.shell = shell
149 self._nested_level = 0
149 self._nested_level = 0
150
150
151 def __enter__(self):
151 def __enter__(self):
152 if self._nested_level == 0:
152 if self._nested_level == 0:
153 try:
153 try:
154 self.orig_length = self.current_length()
154 self.orig_length = self.current_length()
155 self.readline_tail = self.get_readline_tail()
155 self.readline_tail = self.get_readline_tail()
156 except (AttributeError, IndexError): # Can fail with pyreadline
156 except (AttributeError, IndexError): # Can fail with pyreadline
157 self.orig_length, self.readline_tail = 999999, []
157 self.orig_length, self.readline_tail = 999999, []
158 self._nested_level += 1
158 self._nested_level += 1
159
159
160 def __exit__(self, type, value, traceback):
160 def __exit__(self, type, value, traceback):
161 self._nested_level -= 1
161 self._nested_level -= 1
162 if self._nested_level == 0:
162 if self._nested_level == 0:
163 # Try clipping the end if it's got longer
163 # Try clipping the end if it's got longer
164 try:
164 try:
165 e = self.current_length() - self.orig_length
165 e = self.current_length() - self.orig_length
166 if e > 0:
166 if e > 0:
167 for _ in range(e):
167 for _ in range(e):
168 self.shell.readline.remove_history_item(self.orig_length)
168 self.shell.readline.remove_history_item(self.orig_length)
169
169
170 # If it still doesn't match, just reload readline history.
170 # If it still doesn't match, just reload readline history.
171 if self.current_length() != self.orig_length \
171 if self.current_length() != self.orig_length \
172 or self.get_readline_tail() != self.readline_tail:
172 or self.get_readline_tail() != self.readline_tail:
173 self.shell.refill_readline_hist()
173 self.shell.refill_readline_hist()
174 except (AttributeError, IndexError):
174 except (AttributeError, IndexError):
175 pass
175 pass
176 # Returning False will cause exceptions to propagate
176 # Returning False will cause exceptions to propagate
177 return False
177 return False
178
178
179 def current_length(self):
179 def current_length(self):
180 return self.shell.readline.get_current_history_length()
180 return self.shell.readline.get_current_history_length()
181
181
182 def get_readline_tail(self, n=10):
182 def get_readline_tail(self, n=10):
183 """Get the last n items in readline history."""
183 """Get the last n items in readline history."""
184 end = self.shell.readline.get_current_history_length() + 1
184 end = self.shell.readline.get_current_history_length() + 1
185 start = max(end-n, 1)
185 start = max(end-n, 1)
186 ghi = self.shell.readline.get_history_item
186 ghi = self.shell.readline.get_history_item
187 return [ghi(x) for x in range(start, end)]
187 return [ghi(x) for x in range(start, end)]
188
188
189
189
190 @undoc
190 @undoc
191 class DummyMod(object):
191 class DummyMod(object):
192 """A dummy module used for IPython's interactive module when
192 """A dummy module used for IPython's interactive module when
193 a namespace must be assigned to the module's __dict__."""
193 a namespace must be assigned to the module's __dict__."""
194 pass
194 pass
195
195
196 #-----------------------------------------------------------------------------
196 #-----------------------------------------------------------------------------
197 # Main IPython class
197 # Main IPython class
198 #-----------------------------------------------------------------------------
198 #-----------------------------------------------------------------------------
199
199
200 class InteractiveShell(SingletonConfigurable):
200 class InteractiveShell(SingletonConfigurable):
201 """An enhanced, interactive shell for Python."""
201 """An enhanced, interactive shell for Python."""
202
202
203 _instance = None
203 _instance = None
204
204
205 ast_transformers = List([], config=True, help=
205 ast_transformers = List([], config=True, help=
206 """
206 """
207 A list of ast.NodeTransformer subclass instances, which will be applied
207 A list of ast.NodeTransformer subclass instances, which will be applied
208 to user input before code is run.
208 to user input before code is run.
209 """
209 """
210 )
210 )
211
211
212 autocall = Enum((0,1,2), default_value=0, config=True, help=
212 autocall = Enum((0,1,2), default_value=0, config=True, help=
213 """
213 """
214 Make IPython automatically call any callable object even if you didn't
214 Make IPython automatically call any callable object even if you didn't
215 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
215 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
216 automatically. The value can be '0' to disable the feature, '1' for
216 automatically. The value can be '0' to disable the feature, '1' for
217 'smart' autocall, where it is not applied if there are no more
217 'smart' autocall, where it is not applied if there are no more
218 arguments on the line, and '2' for 'full' autocall, where all callable
218 arguments on the line, and '2' for 'full' autocall, where all callable
219 objects are automatically called (even if no arguments are present).
219 objects are automatically called (even if no arguments are present).
220 """
220 """
221 )
221 )
222 # TODO: remove all autoindent logic and put into frontends.
222 # TODO: remove all autoindent logic and put into frontends.
223 # We can't do this yet because even runlines uses the autoindent.
223 # We can't do this yet because even runlines uses the autoindent.
224 autoindent = CBool(True, config=True, help=
224 autoindent = CBool(True, config=True, help=
225 """
225 """
226 Autoindent IPython code entered interactively.
226 Autoindent IPython code entered interactively.
227 """
227 """
228 )
228 )
229 automagic = CBool(True, config=True, help=
229 automagic = CBool(True, config=True, help=
230 """
230 """
231 Enable magic commands to be called without the leading %.
231 Enable magic commands to be called without the leading %.
232 """
232 """
233 )
233 )
234 cache_size = Integer(1000, config=True, help=
234 cache_size = Integer(1000, config=True, help=
235 """
235 """
236 Set the size of the output cache. The default is 1000, you can
236 Set the size of the output cache. The default is 1000, you can
237 change it permanently in your config file. Setting it to 0 completely
237 change it permanently in your config file. Setting it to 0 completely
238 disables the caching system, and the minimum value accepted is 20 (if
238 disables the caching system, and the minimum value accepted is 20 (if
239 you provide a value less than 20, it is reset to 0 and a warning is
239 you provide a value less than 20, it is reset to 0 and a warning is
240 issued). This limit is defined because otherwise you'll spend more
240 issued). This limit is defined because otherwise you'll spend more
241 time re-flushing a too small cache than working
241 time re-flushing a too small cache than working
242 """
242 """
243 )
243 )
244 color_info = CBool(True, config=True, help=
244 color_info = CBool(True, config=True, help=
245 """
245 """
246 Use colors for displaying information about objects. Because this
246 Use colors for displaying information about objects. Because this
247 information is passed through a pager (like 'less'), and some pagers
247 information is passed through a pager (like 'less'), and some pagers
248 get confused with color codes, this capability can be turned off.
248 get confused with color codes, this capability can be turned off.
249 """
249 """
250 )
250 )
251 colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
251 colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
252 default_value=get_default_colors(), config=True,
252 default_value=get_default_colors(), config=True,
253 help="Set the color scheme (NoColor, Linux, or LightBG)."
253 help="Set the color scheme (NoColor, Linux, or LightBG)."
254 )
254 )
255 colors_force = CBool(False, help=
255 colors_force = CBool(False, help=
256 """
256 """
257 Force use of ANSI color codes, regardless of OS and readline
257 Force use of ANSI color codes, regardless of OS and readline
258 availability.
258 availability.
259 """
259 """
260 # FIXME: This is essentially a hack to allow ZMQShell to show colors
260 # FIXME: This is essentially a hack to allow ZMQShell to show colors
261 # without readline on Win32. When the ZMQ formatting system is
261 # without readline on Win32. When the ZMQ formatting system is
262 # refactored, this should be removed.
262 # refactored, this should be removed.
263 )
263 )
264 debug = CBool(False, config=True)
264 debug = CBool(False, config=True)
265 deep_reload = CBool(False, config=True, help=
265 deep_reload = CBool(False, config=True, help=
266 """
266 """
267 Enable deep (recursive) reloading by default. IPython can use the
267 Enable deep (recursive) reloading by default. IPython can use the
268 deep_reload module which reloads changes in modules recursively (it
268 deep_reload module which reloads changes in modules recursively (it
269 replaces the reload() function, so you don't need to change anything to
269 replaces the reload() function, so you don't need to change anything to
270 use it). deep_reload() forces a full reload of modules whose code may
270 use it). deep_reload() forces a full reload of modules whose code may
271 have changed, which the default reload() function does not. When
271 have changed, which the default reload() function does not. When
272 deep_reload is off, IPython will use the normal reload(), but
272 deep_reload is off, IPython will use the normal reload(), but
273 deep_reload will still be available as dreload().
273 deep_reload will still be available as dreload().
274 """
274 """
275 )
275 )
276 disable_failing_post_execute = CBool(False, config=True,
276 disable_failing_post_execute = CBool(False, config=True,
277 help="Don't call post-execute functions that have failed in the past."
277 help="Don't call post-execute functions that have failed in the past."
278 )
278 )
279 display_formatter = Instance(DisplayFormatter)
279 display_formatter = Instance(DisplayFormatter)
280 displayhook_class = Type(DisplayHook)
280 displayhook_class = Type(DisplayHook)
281 display_pub_class = Type(DisplayPublisher)
281 display_pub_class = Type(DisplayPublisher)
282 data_pub_class = None
282 data_pub_class = None
283
283
284 exit_now = CBool(False)
284 exit_now = CBool(False)
285 exiter = Instance(ExitAutocall)
285 exiter = Instance(ExitAutocall)
286 def _exiter_default(self):
286 def _exiter_default(self):
287 return ExitAutocall(self)
287 return ExitAutocall(self)
288 # Monotonically increasing execution counter
288 # Monotonically increasing execution counter
289 execution_count = Integer(1)
289 execution_count = Integer(1)
290 filename = Unicode("<ipython console>")
290 filename = Unicode("<ipython console>")
291 ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
291 ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
292
292
293 # Input splitter, to transform input line by line and detect when a block
293 # Input splitter, to transform input line by line and detect when a block
294 # is ready to be executed.
294 # is ready to be executed.
295 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
295 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
296 (), {'line_input_checker': True})
296 (), {'line_input_checker': True})
297
297
298 # This InputSplitter instance is used to transform completed cells before
298 # This InputSplitter instance is used to transform completed cells before
299 # running them. It allows cell magics to contain blank lines.
299 # running them. It allows cell magics to contain blank lines.
300 input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
300 input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
301 (), {'line_input_checker': False})
301 (), {'line_input_checker': False})
302
302
303 logstart = CBool(False, config=True, help=
303 logstart = CBool(False, config=True, help=
304 """
304 """
305 Start logging to the default log file.
305 Start logging to the default log file.
306 """
306 """
307 )
307 )
308 logfile = Unicode('', config=True, help=
308 logfile = Unicode('', config=True, help=
309 """
309 """
310 The name of the logfile to use.
310 The name of the logfile to use.
311 """
311 """
312 )
312 )
313 logappend = Unicode('', config=True, help=
313 logappend = Unicode('', config=True, help=
314 """
314 """
315 Start logging to the given file in append mode.
315 Start logging to the given file in append mode.
316 """
316 """
317 )
317 )
318 object_info_string_level = Enum((0,1,2), default_value=0,
318 object_info_string_level = Enum((0,1,2), default_value=0,
319 config=True)
319 config=True)
320 pdb = CBool(False, config=True, help=
320 pdb = CBool(False, config=True, help=
321 """
321 """
322 Automatically call the pdb debugger after every exception.
322 Automatically call the pdb debugger after every exception.
323 """
323 """
324 )
324 )
325 multiline_history = CBool(sys.platform != 'win32', config=True,
325 multiline_history = CBool(sys.platform != 'win32', config=True,
326 help="Save multi-line entries as one entry in readline history"
326 help="Save multi-line entries as one entry in readline history"
327 )
327 )
328
328
329 # deprecated prompt traits:
329 # deprecated prompt traits:
330
330
331 prompt_in1 = Unicode('In [\\#]: ', config=True,
331 prompt_in1 = Unicode('In [\\#]: ', config=True,
332 help="Deprecated, use PromptManager.in_template")
332 help="Deprecated, use PromptManager.in_template")
333 prompt_in2 = Unicode(' .\\D.: ', config=True,
333 prompt_in2 = Unicode(' .\\D.: ', config=True,
334 help="Deprecated, use PromptManager.in2_template")
334 help="Deprecated, use PromptManager.in2_template")
335 prompt_out = Unicode('Out[\\#]: ', config=True,
335 prompt_out = Unicode('Out[\\#]: ', config=True,
336 help="Deprecated, use PromptManager.out_template")
336 help="Deprecated, use PromptManager.out_template")
337 prompts_pad_left = CBool(True, config=True,
337 prompts_pad_left = CBool(True, config=True,
338 help="Deprecated, use PromptManager.justify")
338 help="Deprecated, use PromptManager.justify")
339
339
340 def _prompt_trait_changed(self, name, old, new):
340 def _prompt_trait_changed(self, name, old, new):
341 table = {
341 table = {
342 'prompt_in1' : 'in_template',
342 'prompt_in1' : 'in_template',
343 'prompt_in2' : 'in2_template',
343 'prompt_in2' : 'in2_template',
344 'prompt_out' : 'out_template',
344 'prompt_out' : 'out_template',
345 'prompts_pad_left' : 'justify',
345 'prompts_pad_left' : 'justify',
346 }
346 }
347 warn("InteractiveShell.{name} is deprecated, use PromptManager.{newname}".format(
347 warn("InteractiveShell.{name} is deprecated, use PromptManager.{newname}".format(
348 name=name, newname=table[name])
348 name=name, newname=table[name])
349 )
349 )
350 # protect against weird cases where self.config may not exist:
350 # protect against weird cases where self.config may not exist:
351 if self.config is not None:
351 if self.config is not None:
352 # propagate to corresponding PromptManager trait
352 # propagate to corresponding PromptManager trait
353 setattr(self.config.PromptManager, table[name], new)
353 setattr(self.config.PromptManager, table[name], new)
354
354
355 _prompt_in1_changed = _prompt_trait_changed
355 _prompt_in1_changed = _prompt_trait_changed
356 _prompt_in2_changed = _prompt_trait_changed
356 _prompt_in2_changed = _prompt_trait_changed
357 _prompt_out_changed = _prompt_trait_changed
357 _prompt_out_changed = _prompt_trait_changed
358 _prompt_pad_left_changed = _prompt_trait_changed
358 _prompt_pad_left_changed = _prompt_trait_changed
359
359
360 show_rewritten_input = CBool(True, config=True,
360 show_rewritten_input = CBool(True, config=True,
361 help="Show rewritten input, e.g. for autocall."
361 help="Show rewritten input, e.g. for autocall."
362 )
362 )
363
363
364 quiet = CBool(False, config=True)
364 quiet = CBool(False, config=True)
365
365
366 history_length = Integer(10000, config=True)
366 history_length = Integer(10000, config=True)
367
367
368 # The readline stuff will eventually be moved to the terminal subclass
368 # The readline stuff will eventually be moved to the terminal subclass
369 # but for now, we can't do that as readline is welded in everywhere.
369 # but for now, we can't do that as readline is welded in everywhere.
370 readline_use = CBool(True, config=True)
370 readline_use = CBool(True, config=True)
371 readline_remove_delims = Unicode('-/~', config=True)
371 readline_remove_delims = Unicode('-/~', config=True)
372 readline_delims = Unicode() # set by init_readline()
372 readline_delims = Unicode() # set by init_readline()
373 # don't use \M- bindings by default, because they
373 # don't use \M- bindings by default, because they
374 # conflict with 8-bit encodings. See gh-58,gh-88
374 # conflict with 8-bit encodings. See gh-58,gh-88
375 readline_parse_and_bind = List([
375 readline_parse_and_bind = List([
376 'tab: complete',
376 'tab: complete',
377 '"\C-l": clear-screen',
377 '"\C-l": clear-screen',
378 'set show-all-if-ambiguous on',
378 'set show-all-if-ambiguous on',
379 '"\C-o": tab-insert',
379 '"\C-o": tab-insert',
380 '"\C-r": reverse-search-history',
380 '"\C-r": reverse-search-history',
381 '"\C-s": forward-search-history',
381 '"\C-s": forward-search-history',
382 '"\C-p": history-search-backward',
382 '"\C-p": history-search-backward',
383 '"\C-n": history-search-forward',
383 '"\C-n": history-search-forward',
384 '"\e[A": history-search-backward',
384 '"\e[A": history-search-backward',
385 '"\e[B": history-search-forward',
385 '"\e[B": history-search-forward',
386 '"\C-k": kill-line',
386 '"\C-k": kill-line',
387 '"\C-u": unix-line-discard',
387 '"\C-u": unix-line-discard',
388 ], allow_none=False, config=True)
388 ], allow_none=False, config=True)
389
389
390 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
390 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
391 default_value='last_expr', config=True,
391 default_value='last_expr', config=True,
392 help="""
392 help="""
393 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
393 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
394 run interactively (displaying output from expressions).""")
394 run interactively (displaying output from expressions).""")
395
395
396 # TODO: this part of prompt management should be moved to the frontends.
396 # TODO: this part of prompt management should be moved to the frontends.
397 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
397 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
398 separate_in = SeparateUnicode('\n', config=True)
398 separate_in = SeparateUnicode('\n', config=True)
399 separate_out = SeparateUnicode('', config=True)
399 separate_out = SeparateUnicode('', config=True)
400 separate_out2 = SeparateUnicode('', config=True)
400 separate_out2 = SeparateUnicode('', config=True)
401 wildcards_case_sensitive = CBool(True, config=True)
401 wildcards_case_sensitive = CBool(True, config=True)
402 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
402 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
403 default_value='Context', config=True)
403 default_value='Context', config=True)
404
404
405 # Subcomponents of InteractiveShell
405 # Subcomponents of InteractiveShell
406 alias_manager = Instance('IPython.core.alias.AliasManager')
406 alias_manager = Instance('IPython.core.alias.AliasManager')
407 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
407 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
408 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
408 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
409 display_trap = Instance('IPython.core.display_trap.DisplayTrap')
409 display_trap = Instance('IPython.core.display_trap.DisplayTrap')
410 extension_manager = Instance('IPython.core.extensions.ExtensionManager')
410 extension_manager = Instance('IPython.core.extensions.ExtensionManager')
411 payload_manager = Instance('IPython.core.payload.PayloadManager')
411 payload_manager = Instance('IPython.core.payload.PayloadManager')
412 history_manager = Instance('IPython.core.history.HistoryManager')
412 history_manager = Instance('IPython.core.history.HistoryManager')
413 magics_manager = Instance('IPython.core.magic.MagicsManager')
413 magics_manager = Instance('IPython.core.magic.MagicsManager')
414
414
415 profile_dir = Instance('IPython.core.application.ProfileDir')
415 profile_dir = Instance('IPython.core.application.ProfileDir')
416 @property
416 @property
417 def profile(self):
417 def profile(self):
418 if self.profile_dir is not None:
418 if self.profile_dir is not None:
419 name = os.path.basename(self.profile_dir.location)
419 name = os.path.basename(self.profile_dir.location)
420 return name.replace('profile_','')
420 return name.replace('profile_','')
421
421
422
422
423 # Private interface
423 # Private interface
424 _post_execute = Instance(dict)
424 _post_execute = Instance(dict)
425
425
426 # Tracks any GUI loop loaded for pylab
426 # Tracks any GUI loop loaded for pylab
427 pylab_gui_select = None
427 pylab_gui_select = None
428
428
429 def __init__(self, ipython_dir=None, profile_dir=None,
429 def __init__(self, ipython_dir=None, profile_dir=None,
430 user_module=None, user_ns=None,
430 user_module=None, user_ns=None,
431 custom_exceptions=((), None), **kwargs):
431 custom_exceptions=((), None), **kwargs):
432
432
433 # This is where traits with a config_key argument are updated
433 # This is where traits with a config_key argument are updated
434 # from the values on config.
434 # from the values on config.
435 super(InteractiveShell, self).__init__(**kwargs)
435 super(InteractiveShell, self).__init__(**kwargs)
436 self.configurables = [self]
436 self.configurables = [self]
437
437
438 # These are relatively independent and stateless
438 # These are relatively independent and stateless
439 self.init_ipython_dir(ipython_dir)
439 self.init_ipython_dir(ipython_dir)
440 self.init_profile_dir(profile_dir)
440 self.init_profile_dir(profile_dir)
441 self.init_instance_attrs()
441 self.init_instance_attrs()
442 self.init_environment()
442 self.init_environment()
443
443
444 # Check if we're in a virtualenv, and set up sys.path.
444 # Check if we're in a virtualenv, and set up sys.path.
445 self.init_virtualenv()
445 self.init_virtualenv()
446
446
447 # Create namespaces (user_ns, user_global_ns, etc.)
447 # Create namespaces (user_ns, user_global_ns, etc.)
448 self.init_create_namespaces(user_module, user_ns)
448 self.init_create_namespaces(user_module, user_ns)
449 # This has to be done after init_create_namespaces because it uses
449 # This has to be done after init_create_namespaces because it uses
450 # something in self.user_ns, but before init_sys_modules, which
450 # something in self.user_ns, but before init_sys_modules, which
451 # is the first thing to modify sys.
451 # is the first thing to modify sys.
452 # TODO: When we override sys.stdout and sys.stderr before this class
452 # TODO: When we override sys.stdout and sys.stderr before this class
453 # is created, we are saving the overridden ones here. Not sure if this
453 # is created, we are saving the overridden ones here. Not sure if this
454 # is what we want to do.
454 # is what we want to do.
455 self.save_sys_module_state()
455 self.save_sys_module_state()
456 self.init_sys_modules()
456 self.init_sys_modules()
457
457
458 # While we're trying to have each part of the code directly access what
458 # While we're trying to have each part of the code directly access what
459 # it needs without keeping redundant references to objects, we have too
459 # it needs without keeping redundant references to objects, we have too
460 # much legacy code that expects ip.db to exist.
460 # much legacy code that expects ip.db to exist.
461 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
461 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
462
462
463 self.init_history()
463 self.init_history()
464 self.init_encoding()
464 self.init_encoding()
465 self.init_prefilter()
465 self.init_prefilter()
466
466
467 self.init_syntax_highlighting()
467 self.init_syntax_highlighting()
468 self.init_hooks()
468 self.init_hooks()
469 self.init_pushd_popd_magic()
469 self.init_pushd_popd_magic()
470 # self.init_traceback_handlers use to be here, but we moved it below
470 # self.init_traceback_handlers use to be here, but we moved it below
471 # because it and init_io have to come after init_readline.
471 # because it and init_io have to come after init_readline.
472 self.init_user_ns()
472 self.init_user_ns()
473 self.init_logger()
473 self.init_logger()
474 self.init_builtins()
474 self.init_builtins()
475
475
476 # The following was in post_config_initialization
476 # The following was in post_config_initialization
477 self.init_inspector()
477 self.init_inspector()
478 # init_readline() must come before init_io(), because init_io uses
478 # init_readline() must come before init_io(), because init_io uses
479 # readline related things.
479 # readline related things.
480 self.init_readline()
480 self.init_readline()
481 # We save this here in case user code replaces raw_input, but it needs
481 # We save this here in case user code replaces raw_input, but it needs
482 # to be after init_readline(), because PyPy's readline works by replacing
482 # to be after init_readline(), because PyPy's readline works by replacing
483 # raw_input.
483 # raw_input.
484 if py3compat.PY3:
484 if py3compat.PY3:
485 self.raw_input_original = input
485 self.raw_input_original = input
486 else:
486 else:
487 self.raw_input_original = raw_input
487 self.raw_input_original = raw_input
488 # init_completer must come after init_readline, because it needs to
488 # init_completer must come after init_readline, because it needs to
489 # know whether readline is present or not system-wide to configure the
489 # know whether readline is present or not system-wide to configure the
490 # completers, since the completion machinery can now operate
490 # completers, since the completion machinery can now operate
491 # independently of readline (e.g. over the network)
491 # independently of readline (e.g. over the network)
492 self.init_completer()
492 self.init_completer()
493 # TODO: init_io() needs to happen before init_traceback handlers
493 # TODO: init_io() needs to happen before init_traceback handlers
494 # because the traceback handlers hardcode the stdout/stderr streams.
494 # because the traceback handlers hardcode the stdout/stderr streams.
495 # This logic in in debugger.Pdb and should eventually be changed.
495 # This logic in in debugger.Pdb and should eventually be changed.
496 self.init_io()
496 self.init_io()
497 self.init_traceback_handlers(custom_exceptions)
497 self.init_traceback_handlers(custom_exceptions)
498 self.init_prompts()
498 self.init_prompts()
499 self.init_display_formatter()
499 self.init_display_formatter()
500 self.init_display_pub()
500 self.init_display_pub()
501 self.init_data_pub()
501 self.init_data_pub()
502 self.init_displayhook()
502 self.init_displayhook()
503 self.init_latextool()
503 self.init_latextool()
504 self.init_magics()
504 self.init_magics()
505 self.init_alias()
505 self.init_alias()
506 self.init_logstart()
506 self.init_logstart()
507 self.init_pdb()
507 self.init_pdb()
508 self.init_extension_manager()
508 self.init_extension_manager()
509 self.init_payload()
509 self.init_payload()
510 self.init_comms()
510 self.init_comms()
511 self.hooks.late_startup_hook()
511 self.hooks.late_startup_hook()
512 atexit.register(self.atexit_operations)
512 atexit.register(self.atexit_operations)
513
513
514 def get_ipython(self):
514 def get_ipython(self):
515 """Return the currently running IPython instance."""
515 """Return the currently running IPython instance."""
516 return self
516 return self
517
517
518 #-------------------------------------------------------------------------
518 #-------------------------------------------------------------------------
519 # Trait changed handlers
519 # Trait changed handlers
520 #-------------------------------------------------------------------------
520 #-------------------------------------------------------------------------
521
521
522 def _ipython_dir_changed(self, name, new):
522 def _ipython_dir_changed(self, name, new):
523 if not os.path.isdir(new):
523 if not os.path.isdir(new):
524 os.makedirs(new, mode = 0o777)
524 os.makedirs(new, mode = 0o777)
525
525
526 def set_autoindent(self,value=None):
526 def set_autoindent(self,value=None):
527 """Set the autoindent flag, checking for readline support.
527 """Set the autoindent flag, checking for readline support.
528
528
529 If called with no arguments, it acts as a toggle."""
529 If called with no arguments, it acts as a toggle."""
530
530
531 if value != 0 and not self.has_readline:
531 if value != 0 and not self.has_readline:
532 if os.name == 'posix':
532 if os.name == 'posix':
533 warn("The auto-indent feature requires the readline library")
533 warn("The auto-indent feature requires the readline library")
534 self.autoindent = 0
534 self.autoindent = 0
535 return
535 return
536 if value is None:
536 if value is None:
537 self.autoindent = not self.autoindent
537 self.autoindent = not self.autoindent
538 else:
538 else:
539 self.autoindent = value
539 self.autoindent = value
540
540
541 #-------------------------------------------------------------------------
541 #-------------------------------------------------------------------------
542 # init_* methods called by __init__
542 # init_* methods called by __init__
543 #-------------------------------------------------------------------------
543 #-------------------------------------------------------------------------
544
544
545 def init_ipython_dir(self, ipython_dir):
545 def init_ipython_dir(self, ipython_dir):
546 if ipython_dir is not None:
546 if ipython_dir is not None:
547 self.ipython_dir = ipython_dir
547 self.ipython_dir = ipython_dir
548 return
548 return
549
549
550 self.ipython_dir = get_ipython_dir()
550 self.ipython_dir = get_ipython_dir()
551
551
552 def init_profile_dir(self, profile_dir):
552 def init_profile_dir(self, profile_dir):
553 if profile_dir is not None:
553 if profile_dir is not None:
554 self.profile_dir = profile_dir
554 self.profile_dir = profile_dir
555 return
555 return
556 self.profile_dir =\
556 self.profile_dir =\
557 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
557 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
558
558
559 def init_instance_attrs(self):
559 def init_instance_attrs(self):
560 self.more = False
560 self.more = False
561
561
562 # command compiler
562 # command compiler
563 self.compile = CachingCompiler()
563 self.compile = CachingCompiler()
564
564
565 # Make an empty namespace, which extension writers can rely on both
565 # Make an empty namespace, which extension writers can rely on both
566 # existing and NEVER being used by ipython itself. This gives them a
566 # existing and NEVER being used by ipython itself. This gives them a
567 # convenient location for storing additional information and state
567 # convenient location for storing additional information and state
568 # their extensions may require, without fear of collisions with other
568 # their extensions may require, without fear of collisions with other
569 # ipython names that may develop later.
569 # ipython names that may develop later.
570 self.meta = Struct()
570 self.meta = Struct()
571
571
572 # Temporary files used for various purposes. Deleted at exit.
572 # Temporary files used for various purposes. Deleted at exit.
573 self.tempfiles = []
573 self.tempfiles = []
574
574
575 # Keep track of readline usage (later set by init_readline)
575 # Keep track of readline usage (later set by init_readline)
576 self.has_readline = False
576 self.has_readline = False
577
577
578 # keep track of where we started running (mainly for crash post-mortem)
578 # keep track of where we started running (mainly for crash post-mortem)
579 # This is not being used anywhere currently.
579 # This is not being used anywhere currently.
580 self.starting_dir = os.getcwdu()
580 self.starting_dir = os.getcwdu()
581
581
582 # Indentation management
582 # Indentation management
583 self.indent_current_nsp = 0
583 self.indent_current_nsp = 0
584
584
585 # Dict to track post-execution functions that have been registered
585 # Dict to track post-execution functions that have been registered
586 self._post_execute = {}
586 self._post_execute = {}
587
587
588 def init_environment(self):
588 def init_environment(self):
589 """Any changes we need to make to the user's environment."""
589 """Any changes we need to make to the user's environment."""
590 pass
590 pass
591
591
592 def init_encoding(self):
592 def init_encoding(self):
593 # Get system encoding at startup time. Certain terminals (like Emacs
593 # Get system encoding at startup time. Certain terminals (like Emacs
594 # under Win32 have it set to None, and we need to have a known valid
594 # under Win32 have it set to None, and we need to have a known valid
595 # encoding to use in the raw_input() method
595 # encoding to use in the raw_input() method
596 try:
596 try:
597 self.stdin_encoding = sys.stdin.encoding or 'ascii'
597 self.stdin_encoding = sys.stdin.encoding or 'ascii'
598 except AttributeError:
598 except AttributeError:
599 self.stdin_encoding = 'ascii'
599 self.stdin_encoding = 'ascii'
600
600
601 def init_syntax_highlighting(self):
601 def init_syntax_highlighting(self):
602 # Python source parser/formatter for syntax highlighting
602 # Python source parser/formatter for syntax highlighting
603 pyformat = PyColorize.Parser().format
603 pyformat = PyColorize.Parser().format
604 self.pycolorize = lambda src: pyformat(src,'str',self.colors)
604 self.pycolorize = lambda src: pyformat(src,'str',self.colors)
605
605
606 def init_pushd_popd_magic(self):
606 def init_pushd_popd_magic(self):
607 # for pushd/popd management
607 # for pushd/popd management
608 self.home_dir = get_home_dir()
608 self.home_dir = get_home_dir()
609
609
610 self.dir_stack = []
610 self.dir_stack = []
611
611
612 def init_logger(self):
612 def init_logger(self):
613 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
613 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
614 logmode='rotate')
614 logmode='rotate')
615
615
616 def init_logstart(self):
616 def init_logstart(self):
617 """Initialize logging in case it was requested at the command line.
617 """Initialize logging in case it was requested at the command line.
618 """
618 """
619 if self.logappend:
619 if self.logappend:
620 self.magic('logstart %s append' % self.logappend)
620 self.magic('logstart %s append' % self.logappend)
621 elif self.logfile:
621 elif self.logfile:
622 self.magic('logstart %s' % self.logfile)
622 self.magic('logstart %s' % self.logfile)
623 elif self.logstart:
623 elif self.logstart:
624 self.magic('logstart')
624 self.magic('logstart')
625
625
626 def init_builtins(self):
626 def init_builtins(self):
627 # A single, static flag that we set to True. Its presence indicates
627 # A single, static flag that we set to True. Its presence indicates
628 # that an IPython shell has been created, and we make no attempts at
628 # that an IPython shell has been created, and we make no attempts at
629 # removing on exit or representing the existence of more than one
629 # removing on exit or representing the existence of more than one
630 # IPython at a time.
630 # IPython at a time.
631 builtin_mod.__dict__['__IPYTHON__'] = True
631 builtin_mod.__dict__['__IPYTHON__'] = True
632
632
633 # In 0.11 we introduced '__IPYTHON__active' as an integer we'd try to
633 # In 0.11 we introduced '__IPYTHON__active' as an integer we'd try to
634 # manage on enter/exit, but with all our shells it's virtually
634 # manage on enter/exit, but with all our shells it's virtually
635 # impossible to get all the cases right. We're leaving the name in for
635 # impossible to get all the cases right. We're leaving the name in for
636 # those who adapted their codes to check for this flag, but will
636 # those who adapted their codes to check for this flag, but will
637 # eventually remove it after a few more releases.
637 # eventually remove it after a few more releases.
638 builtin_mod.__dict__['__IPYTHON__active'] = \
638 builtin_mod.__dict__['__IPYTHON__active'] = \
639 'Deprecated, check for __IPYTHON__'
639 'Deprecated, check for __IPYTHON__'
640
640
641 self.builtin_trap = BuiltinTrap(shell=self)
641 self.builtin_trap = BuiltinTrap(shell=self)
642
642
643 def init_inspector(self):
643 def init_inspector(self):
644 # Object inspector
644 # Object inspector
645 self.inspector = oinspect.Inspector(oinspect.InspectColors,
645 self.inspector = oinspect.Inspector(oinspect.InspectColors,
646 PyColorize.ANSICodeColors,
646 PyColorize.ANSICodeColors,
647 'NoColor',
647 'NoColor',
648 self.object_info_string_level)
648 self.object_info_string_level)
649
649
650 def init_io(self):
650 def init_io(self):
651 # This will just use sys.stdout and sys.stderr. If you want to
651 # This will just use sys.stdout and sys.stderr. If you want to
652 # override sys.stdout and sys.stderr themselves, you need to do that
652 # override sys.stdout and sys.stderr themselves, you need to do that
653 # *before* instantiating this class, because io holds onto
653 # *before* instantiating this class, because io holds onto
654 # references to the underlying streams.
654 # references to the underlying streams.
655 if (sys.platform == 'win32' or sys.platform == 'cli') and self.has_readline:
655 if (sys.platform == 'win32' or sys.platform == 'cli') and self.has_readline:
656 io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
656 io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
657 else:
657 else:
658 io.stdout = io.IOStream(sys.stdout)
658 io.stdout = io.IOStream(sys.stdout)
659 io.stderr = io.IOStream(sys.stderr)
659 io.stderr = io.IOStream(sys.stderr)
660
660
661 def init_prompts(self):
661 def init_prompts(self):
662 self.prompt_manager = PromptManager(shell=self, parent=self)
662 self.prompt_manager = PromptManager(shell=self, parent=self)
663 self.configurables.append(self.prompt_manager)
663 self.configurables.append(self.prompt_manager)
664 # Set system prompts, so that scripts can decide if they are running
664 # Set system prompts, so that scripts can decide if they are running
665 # interactively.
665 # interactively.
666 sys.ps1 = 'In : '
666 sys.ps1 = 'In : '
667 sys.ps2 = '...: '
667 sys.ps2 = '...: '
668 sys.ps3 = 'Out: '
668 sys.ps3 = 'Out: '
669
669
670 def init_display_formatter(self):
670 def init_display_formatter(self):
671 self.display_formatter = DisplayFormatter(parent=self)
671 self.display_formatter = DisplayFormatter(parent=self)
672 self.configurables.append(self.display_formatter)
672 self.configurables.append(self.display_formatter)
673
673
674 def init_display_pub(self):
674 def init_display_pub(self):
675 self.display_pub = self.display_pub_class(parent=self)
675 self.display_pub = self.display_pub_class(parent=self)
676 self.configurables.append(self.display_pub)
676 self.configurables.append(self.display_pub)
677
677
678 def init_data_pub(self):
678 def init_data_pub(self):
679 if not self.data_pub_class:
679 if not self.data_pub_class:
680 self.data_pub = None
680 self.data_pub = None
681 return
681 return
682 self.data_pub = self.data_pub_class(parent=self)
682 self.data_pub = self.data_pub_class(parent=self)
683 self.configurables.append(self.data_pub)
683 self.configurables.append(self.data_pub)
684
684
685 def init_displayhook(self):
685 def init_displayhook(self):
686 # Initialize displayhook, set in/out prompts and printing system
686 # Initialize displayhook, set in/out prompts and printing system
687 self.displayhook = self.displayhook_class(
687 self.displayhook = self.displayhook_class(
688 parent=self,
688 parent=self,
689 shell=self,
689 shell=self,
690 cache_size=self.cache_size,
690 cache_size=self.cache_size,
691 )
691 )
692 self.configurables.append(self.displayhook)
692 self.configurables.append(self.displayhook)
693 # This is a context manager that installs/revmoes the displayhook at
693 # This is a context manager that installs/revmoes the displayhook at
694 # the appropriate time.
694 # the appropriate time.
695 self.display_trap = DisplayTrap(hook=self.displayhook)
695 self.display_trap = DisplayTrap(hook=self.displayhook)
696
696
697 def init_latextool(self):
697 def init_latextool(self):
698 """Configure LaTeXTool."""
698 """Configure LaTeXTool."""
699 cfg = LaTeXTool.instance(parent=self)
699 cfg = LaTeXTool.instance(parent=self)
700 if cfg not in self.configurables:
700 if cfg not in self.configurables:
701 self.configurables.append(cfg)
701 self.configurables.append(cfg)
702
702
703 def init_virtualenv(self):
703 def init_virtualenv(self):
704 """Add a virtualenv to sys.path so the user can import modules from it.
704 """Add a virtualenv to sys.path so the user can import modules from it.
705 This isn't perfect: it doesn't use the Python interpreter with which the
705 This isn't perfect: it doesn't use the Python interpreter with which the
706 virtualenv was built, and it ignores the --no-site-packages option. A
706 virtualenv was built, and it ignores the --no-site-packages option. A
707 warning will appear suggesting the user installs IPython in the
707 warning will appear suggesting the user installs IPython in the
708 virtualenv, but for many cases, it probably works well enough.
708 virtualenv, but for many cases, it probably works well enough.
709
709
710 Adapted from code snippets online.
710 Adapted from code snippets online.
711
711
712 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
712 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
713 """
713 """
714 if 'VIRTUAL_ENV' not in os.environ:
714 if 'VIRTUAL_ENV' not in os.environ:
715 # Not in a virtualenv
715 # Not in a virtualenv
716 return
716 return
717
717
718 if sys.executable.startswith(os.environ['VIRTUAL_ENV']):
718 if sys.executable.startswith(os.environ['VIRTUAL_ENV']):
719 # Running properly in the virtualenv, don't need to do anything
719 # Running properly in the virtualenv, don't need to do anything
720 return
720 return
721
721
722 warn("Attempting to work in a virtualenv. If you encounter problems, please "
722 warn("Attempting to work in a virtualenv. If you encounter problems, please "
723 "install IPython inside the virtualenv.")
723 "install IPython inside the virtualenv.")
724 if sys.platform == "win32":
724 if sys.platform == "win32":
725 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
725 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
726 else:
726 else:
727 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
727 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
728 'python%d.%d' % sys.version_info[:2], 'site-packages')
728 'python%d.%d' % sys.version_info[:2], 'site-packages')
729
729
730 import site
730 import site
731 sys.path.insert(0, virtual_env)
731 sys.path.insert(0, virtual_env)
732 site.addsitedir(virtual_env)
732 site.addsitedir(virtual_env)
733
733
734 #-------------------------------------------------------------------------
734 #-------------------------------------------------------------------------
735 # Things related to injections into the sys module
735 # Things related to injections into the sys module
736 #-------------------------------------------------------------------------
736 #-------------------------------------------------------------------------
737
737
738 def save_sys_module_state(self):
738 def save_sys_module_state(self):
739 """Save the state of hooks in the sys module.
739 """Save the state of hooks in the sys module.
740
740
741 This has to be called after self.user_module is created.
741 This has to be called after self.user_module is created.
742 """
742 """
743 self._orig_sys_module_state = {}
743 self._orig_sys_module_state = {}
744 self._orig_sys_module_state['stdin'] = sys.stdin
744 self._orig_sys_module_state['stdin'] = sys.stdin
745 self._orig_sys_module_state['stdout'] = sys.stdout
745 self._orig_sys_module_state['stdout'] = sys.stdout
746 self._orig_sys_module_state['stderr'] = sys.stderr
746 self._orig_sys_module_state['stderr'] = sys.stderr
747 self._orig_sys_module_state['excepthook'] = sys.excepthook
747 self._orig_sys_module_state['excepthook'] = sys.excepthook
748 self._orig_sys_modules_main_name = self.user_module.__name__
748 self._orig_sys_modules_main_name = self.user_module.__name__
749 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
749 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
750
750
751 def restore_sys_module_state(self):
751 def restore_sys_module_state(self):
752 """Restore the state of the sys module."""
752 """Restore the state of the sys module."""
753 try:
753 try:
754 for k, v in self._orig_sys_module_state.iteritems():
754 for k, v in self._orig_sys_module_state.iteritems():
755 setattr(sys, k, v)
755 setattr(sys, k, v)
756 except AttributeError:
756 except AttributeError:
757 pass
757 pass
758 # Reset what what done in self.init_sys_modules
758 # Reset what what done in self.init_sys_modules
759 if self._orig_sys_modules_main_mod is not None:
759 if self._orig_sys_modules_main_mod is not None:
760 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
760 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
761
761
762 #-------------------------------------------------------------------------
762 #-------------------------------------------------------------------------
763 # Things related to hooks
763 # Things related to hooks
764 #-------------------------------------------------------------------------
764 #-------------------------------------------------------------------------
765
765
766 def init_hooks(self):
766 def init_hooks(self):
767 # hooks holds pointers used for user-side customizations
767 # hooks holds pointers used for user-side customizations
768 self.hooks = Struct()
768 self.hooks = Struct()
769
769
770 self.strdispatchers = {}
770 self.strdispatchers = {}
771
771
772 # Set all default hooks, defined in the IPython.hooks module.
772 # Set all default hooks, defined in the IPython.hooks module.
773 hooks = IPython.core.hooks
773 hooks = IPython.core.hooks
774 for hook_name in hooks.__all__:
774 for hook_name in hooks.__all__:
775 # default hooks have priority 100, i.e. low; user hooks should have
775 # default hooks have priority 100, i.e. low; user hooks should have
776 # 0-100 priority
776 # 0-100 priority
777 self.set_hook(hook_name,getattr(hooks,hook_name), 100)
777 self.set_hook(hook_name,getattr(hooks,hook_name), 100)
778
778
779 def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
779 def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
780 """set_hook(name,hook) -> sets an internal IPython hook.
780 """set_hook(name,hook) -> sets an internal IPython hook.
781
781
782 IPython exposes some of its internal API as user-modifiable hooks. By
782 IPython exposes some of its internal API as user-modifiable hooks. By
783 adding your function to one of these hooks, you can modify IPython's
783 adding your function to one of these hooks, you can modify IPython's
784 behavior to call at runtime your own routines."""
784 behavior to call at runtime your own routines."""
785
785
786 # At some point in the future, this should validate the hook before it
786 # At some point in the future, this should validate the hook before it
787 # accepts it. Probably at least check that the hook takes the number
787 # accepts it. Probably at least check that the hook takes the number
788 # of args it's supposed to.
788 # of args it's supposed to.
789
789
790 f = types.MethodType(hook,self)
790 f = types.MethodType(hook,self)
791
791
792 # check if the hook is for strdispatcher first
792 # check if the hook is for strdispatcher first
793 if str_key is not None:
793 if str_key is not None:
794 sdp = self.strdispatchers.get(name, StrDispatch())
794 sdp = self.strdispatchers.get(name, StrDispatch())
795 sdp.add_s(str_key, f, priority )
795 sdp.add_s(str_key, f, priority )
796 self.strdispatchers[name] = sdp
796 self.strdispatchers[name] = sdp
797 return
797 return
798 if re_key is not None:
798 if re_key is not None:
799 sdp = self.strdispatchers.get(name, StrDispatch())
799 sdp = self.strdispatchers.get(name, StrDispatch())
800 sdp.add_re(re.compile(re_key), f, priority )
800 sdp.add_re(re.compile(re_key), f, priority )
801 self.strdispatchers[name] = sdp
801 self.strdispatchers[name] = sdp
802 return
802 return
803
803
804 dp = getattr(self.hooks, name, None)
804 dp = getattr(self.hooks, name, None)
805 if name not in IPython.core.hooks.__all__:
805 if name not in IPython.core.hooks.__all__:
806 print("Warning! Hook '%s' is not one of %s" % \
806 print("Warning! Hook '%s' is not one of %s" % \
807 (name, IPython.core.hooks.__all__ ))
807 (name, IPython.core.hooks.__all__ ))
808 if not dp:
808 if not dp:
809 dp = IPython.core.hooks.CommandChainDispatcher()
809 dp = IPython.core.hooks.CommandChainDispatcher()
810
810
811 try:
811 try:
812 dp.add(f,priority)
812 dp.add(f,priority)
813 except AttributeError:
813 except AttributeError:
814 # it was not commandchain, plain old func - replace
814 # it was not commandchain, plain old func - replace
815 dp = f
815 dp = f
816
816
817 setattr(self.hooks,name, dp)
817 setattr(self.hooks,name, dp)
818
818
819 def register_post_execute(self, func):
819 def register_post_execute(self, func):
820 """Register a function for calling after code execution.
820 """Register a function for calling after code execution.
821 """
821 """
822 if not callable(func):
822 if not callable(func):
823 raise ValueError('argument %s must be callable' % func)
823 raise ValueError('argument %s must be callable' % func)
824 self._post_execute[func] = True
824 self._post_execute[func] = True
825
825
826 #-------------------------------------------------------------------------
826 #-------------------------------------------------------------------------
827 # Things related to the "main" module
827 # Things related to the "main" module
828 #-------------------------------------------------------------------------
828 #-------------------------------------------------------------------------
829
829
830 def new_main_mod(self, filename, modname):
830 def new_main_mod(self, filename, modname):
831 """Return a new 'main' module object for user code execution.
831 """Return a new 'main' module object for user code execution.
832
832
833 ``filename`` should be the path of the script which will be run in the
833 ``filename`` should be the path of the script which will be run in the
834 module. Requests with the same filename will get the same module, with
834 module. Requests with the same filename will get the same module, with
835 its namespace cleared.
835 its namespace cleared.
836
836
837 ``modname`` should be the module name - normally either '__main__' or
837 ``modname`` should be the module name - normally either '__main__' or
838 the basename of the file without the extension.
838 the basename of the file without the extension.
839
839
840 When scripts are executed via %run, we must keep a reference to their
840 When scripts are executed via %run, we must keep a reference to their
841 __main__ module around so that Python doesn't
841 __main__ module around so that Python doesn't
842 clear it, rendering references to module globals useless.
842 clear it, rendering references to module globals useless.
843
843
844 This method keeps said reference in a private dict, keyed by the
844 This method keeps said reference in a private dict, keyed by the
845 absolute path of the script. This way, for multiple executions of the
845 absolute path of the script. This way, for multiple executions of the
846 same script we only keep one copy of the namespace (the last one),
846 same script we only keep one copy of the namespace (the last one),
847 thus preventing memory leaks from old references while allowing the
847 thus preventing memory leaks from old references while allowing the
848 objects from the last execution to be accessible.
848 objects from the last execution to be accessible.
849 """
849 """
850 filename = os.path.abspath(filename)
850 filename = os.path.abspath(filename)
851 try:
851 try:
852 main_mod = self._main_mod_cache[filename]
852 main_mod = self._main_mod_cache[filename]
853 except KeyError:
853 except KeyError:
854 main_mod = self._main_mod_cache[filename] = types.ModuleType(modname,
854 main_mod = self._main_mod_cache[filename] = types.ModuleType(modname,
855 doc="Module created for script run in IPython")
855 doc="Module created for script run in IPython")
856 else:
856 else:
857 main_mod.__dict__.clear()
857 main_mod.__dict__.clear()
858 main_mod.__name__ = modname
858 main_mod.__name__ = modname
859
859
860 main_mod.__file__ = filename
860 main_mod.__file__ = filename
861 # It seems pydoc (and perhaps others) needs any module instance to
861 # It seems pydoc (and perhaps others) needs any module instance to
862 # implement a __nonzero__ method
862 # implement a __nonzero__ method
863 main_mod.__nonzero__ = lambda : True
863 main_mod.__nonzero__ = lambda : True
864
864
865 return main_mod
865 return main_mod
866
866
867 def clear_main_mod_cache(self):
867 def clear_main_mod_cache(self):
868 """Clear the cache of main modules.
868 """Clear the cache of main modules.
869
869
870 Mainly for use by utilities like %reset.
870 Mainly for use by utilities like %reset.
871
871
872 Examples
872 Examples
873 --------
873 --------
874
874
875 In [15]: import IPython
875 In [15]: import IPython
876
876
877 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
877 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
878
878
879 In [17]: len(_ip._main_mod_cache) > 0
879 In [17]: len(_ip._main_mod_cache) > 0
880 Out[17]: True
880 Out[17]: True
881
881
882 In [18]: _ip.clear_main_mod_cache()
882 In [18]: _ip.clear_main_mod_cache()
883
883
884 In [19]: len(_ip._main_mod_cache) == 0
884 In [19]: len(_ip._main_mod_cache) == 0
885 Out[19]: True
885 Out[19]: True
886 """
886 """
887 self._main_mod_cache.clear()
887 self._main_mod_cache.clear()
888
888
889 #-------------------------------------------------------------------------
889 #-------------------------------------------------------------------------
890 # Things related to debugging
890 # Things related to debugging
891 #-------------------------------------------------------------------------
891 #-------------------------------------------------------------------------
892
892
893 def init_pdb(self):
893 def init_pdb(self):
894 # Set calling of pdb on exceptions
894 # Set calling of pdb on exceptions
895 # self.call_pdb is a property
895 # self.call_pdb is a property
896 self.call_pdb = self.pdb
896 self.call_pdb = self.pdb
897
897
898 def _get_call_pdb(self):
898 def _get_call_pdb(self):
899 return self._call_pdb
899 return self._call_pdb
900
900
901 def _set_call_pdb(self,val):
901 def _set_call_pdb(self,val):
902
902
903 if val not in (0,1,False,True):
903 if val not in (0,1,False,True):
904 raise ValueError('new call_pdb value must be boolean')
904 raise ValueError('new call_pdb value must be boolean')
905
905
906 # store value in instance
906 # store value in instance
907 self._call_pdb = val
907 self._call_pdb = val
908
908
909 # notify the actual exception handlers
909 # notify the actual exception handlers
910 self.InteractiveTB.call_pdb = val
910 self.InteractiveTB.call_pdb = val
911
911
912 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
912 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
913 'Control auto-activation of pdb at exceptions')
913 'Control auto-activation of pdb at exceptions')
914
914
915 def debugger(self,force=False):
915 def debugger(self,force=False):
916 """Call the pydb/pdb debugger.
916 """Call the pydb/pdb debugger.
917
917
918 Keywords:
918 Keywords:
919
919
920 - force(False): by default, this routine checks the instance call_pdb
920 - force(False): by default, this routine checks the instance call_pdb
921 flag and does not actually invoke the debugger if the flag is false.
921 flag and does not actually invoke the debugger if the flag is false.
922 The 'force' option forces the debugger to activate even if the flag
922 The 'force' option forces the debugger to activate even if the flag
923 is false.
923 is false.
924 """
924 """
925
925
926 if not (force or self.call_pdb):
926 if not (force or self.call_pdb):
927 return
927 return
928
928
929 if not hasattr(sys,'last_traceback'):
929 if not hasattr(sys,'last_traceback'):
930 error('No traceback has been produced, nothing to debug.')
930 error('No traceback has been produced, nothing to debug.')
931 return
931 return
932
932
933 # use pydb if available
933 # use pydb if available
934 if debugger.has_pydb:
934 if debugger.has_pydb:
935 from pydb import pm
935 from pydb import pm
936 else:
936 else:
937 # fallback to our internal debugger
937 # fallback to our internal debugger
938 pm = lambda : self.InteractiveTB.debugger(force=True)
938 pm = lambda : self.InteractiveTB.debugger(force=True)
939
939
940 with self.readline_no_record:
940 with self.readline_no_record:
941 pm()
941 pm()
942
942
943 #-------------------------------------------------------------------------
943 #-------------------------------------------------------------------------
944 # Things related to IPython's various namespaces
944 # Things related to IPython's various namespaces
945 #-------------------------------------------------------------------------
945 #-------------------------------------------------------------------------
946 default_user_namespaces = True
946 default_user_namespaces = True
947
947
948 def init_create_namespaces(self, user_module=None, user_ns=None):
948 def init_create_namespaces(self, user_module=None, user_ns=None):
949 # Create the namespace where the user will operate. user_ns is
949 # Create the namespace where the user will operate. user_ns is
950 # normally the only one used, and it is passed to the exec calls as
950 # normally the only one used, and it is passed to the exec calls as
951 # the locals argument. But we do carry a user_global_ns namespace
951 # the locals argument. But we do carry a user_global_ns namespace
952 # given as the exec 'globals' argument, This is useful in embedding
952 # given as the exec 'globals' argument, This is useful in embedding
953 # situations where the ipython shell opens in a context where the
953 # situations where the ipython shell opens in a context where the
954 # distinction between locals and globals is meaningful. For
954 # distinction between locals and globals is meaningful. For
955 # non-embedded contexts, it is just the same object as the user_ns dict.
955 # non-embedded contexts, it is just the same object as the user_ns dict.
956
956
957 # FIXME. For some strange reason, __builtins__ is showing up at user
957 # FIXME. For some strange reason, __builtins__ is showing up at user
958 # level as a dict instead of a module. This is a manual fix, but I
958 # level as a dict instead of a module. This is a manual fix, but I
959 # should really track down where the problem is coming from. Alex
959 # should really track down where the problem is coming from. Alex
960 # Schmolck reported this problem first.
960 # Schmolck reported this problem first.
961
961
962 # A useful post by Alex Martelli on this topic:
962 # A useful post by Alex Martelli on this topic:
963 # Re: inconsistent value from __builtins__
963 # Re: inconsistent value from __builtins__
964 # Von: Alex Martelli <aleaxit@yahoo.com>
964 # Von: Alex Martelli <aleaxit@yahoo.com>
965 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
965 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
966 # Gruppen: comp.lang.python
966 # Gruppen: comp.lang.python
967
967
968 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
968 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
969 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
969 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
970 # > <type 'dict'>
970 # > <type 'dict'>
971 # > >>> print type(__builtins__)
971 # > >>> print type(__builtins__)
972 # > <type 'module'>
972 # > <type 'module'>
973 # > Is this difference in return value intentional?
973 # > Is this difference in return value intentional?
974
974
975 # Well, it's documented that '__builtins__' can be either a dictionary
975 # Well, it's documented that '__builtins__' can be either a dictionary
976 # or a module, and it's been that way for a long time. Whether it's
976 # or a module, and it's been that way for a long time. Whether it's
977 # intentional (or sensible), I don't know. In any case, the idea is
977 # intentional (or sensible), I don't know. In any case, the idea is
978 # that if you need to access the built-in namespace directly, you
978 # that if you need to access the built-in namespace directly, you
979 # should start with "import __builtin__" (note, no 's') which will
979 # should start with "import __builtin__" (note, no 's') which will
980 # definitely give you a module. Yeah, it's somewhat confusing:-(.
980 # definitely give you a module. Yeah, it's somewhat confusing:-(.
981
981
982 # These routines return a properly built module and dict as needed by
982 # These routines return a properly built module and dict as needed by
983 # the rest of the code, and can also be used by extension writers to
983 # the rest of the code, and can also be used by extension writers to
984 # generate properly initialized namespaces.
984 # generate properly initialized namespaces.
985 if (user_ns is not None) or (user_module is not None):
985 if (user_ns is not None) or (user_module is not None):
986 self.default_user_namespaces = False
986 self.default_user_namespaces = False
987 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
987 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
988
988
989 # A record of hidden variables we have added to the user namespace, so
989 # A record of hidden variables we have added to the user namespace, so
990 # we can list later only variables defined in actual interactive use.
990 # we can list later only variables defined in actual interactive use.
991 self.user_ns_hidden = {}
991 self.user_ns_hidden = {}
992
992
993 # Now that FakeModule produces a real module, we've run into a nasty
993 # Now that FakeModule produces a real module, we've run into a nasty
994 # problem: after script execution (via %run), the module where the user
994 # problem: after script execution (via %run), the module where the user
995 # code ran is deleted. Now that this object is a true module (needed
995 # code ran is deleted. Now that this object is a true module (needed
996 # so docetst and other tools work correctly), the Python module
996 # so docetst and other tools work correctly), the Python module
997 # teardown mechanism runs over it, and sets to None every variable
997 # teardown mechanism runs over it, and sets to None every variable
998 # present in that module. Top-level references to objects from the
998 # present in that module. Top-level references to objects from the
999 # script survive, because the user_ns is updated with them. However,
999 # script survive, because the user_ns is updated with them. However,
1000 # calling functions defined in the script that use other things from
1000 # calling functions defined in the script that use other things from
1001 # the script will fail, because the function's closure had references
1001 # the script will fail, because the function's closure had references
1002 # to the original objects, which are now all None. So we must protect
1002 # to the original objects, which are now all None. So we must protect
1003 # these modules from deletion by keeping a cache.
1003 # these modules from deletion by keeping a cache.
1004 #
1004 #
1005 # To avoid keeping stale modules around (we only need the one from the
1005 # To avoid keeping stale modules around (we only need the one from the
1006 # last run), we use a dict keyed with the full path to the script, so
1006 # last run), we use a dict keyed with the full path to the script, so
1007 # only the last version of the module is held in the cache. Note,
1007 # only the last version of the module is held in the cache. Note,
1008 # however, that we must cache the module *namespace contents* (their
1008 # however, that we must cache the module *namespace contents* (their
1009 # __dict__). Because if we try to cache the actual modules, old ones
1009 # __dict__). Because if we try to cache the actual modules, old ones
1010 # (uncached) could be destroyed while still holding references (such as
1010 # (uncached) could be destroyed while still holding references (such as
1011 # those held by GUI objects that tend to be long-lived)>
1011 # those held by GUI objects that tend to be long-lived)>
1012 #
1012 #
1013 # The %reset command will flush this cache. See the cache_main_mod()
1013 # The %reset command will flush this cache. See the cache_main_mod()
1014 # and clear_main_mod_cache() methods for details on use.
1014 # and clear_main_mod_cache() methods for details on use.
1015
1015
1016 # This is the cache used for 'main' namespaces
1016 # This is the cache used for 'main' namespaces
1017 self._main_mod_cache = {}
1017 self._main_mod_cache = {}
1018
1018
1019 # A table holding all the namespaces IPython deals with, so that
1019 # A table holding all the namespaces IPython deals with, so that
1020 # introspection facilities can search easily.
1020 # introspection facilities can search easily.
1021 self.ns_table = {'user_global':self.user_module.__dict__,
1021 self.ns_table = {'user_global':self.user_module.__dict__,
1022 'user_local':self.user_ns,
1022 'user_local':self.user_ns,
1023 'builtin':builtin_mod.__dict__
1023 'builtin':builtin_mod.__dict__
1024 }
1024 }
1025
1025
1026 @property
1026 @property
1027 def user_global_ns(self):
1027 def user_global_ns(self):
1028 return self.user_module.__dict__
1028 return self.user_module.__dict__
1029
1029
1030 def prepare_user_module(self, user_module=None, user_ns=None):
1030 def prepare_user_module(self, user_module=None, user_ns=None):
1031 """Prepare the module and namespace in which user code will be run.
1031 """Prepare the module and namespace in which user code will be run.
1032
1032
1033 When IPython is started normally, both parameters are None: a new module
1033 When IPython is started normally, both parameters are None: a new module
1034 is created automatically, and its __dict__ used as the namespace.
1034 is created automatically, and its __dict__ used as the namespace.
1035
1035
1036 If only user_module is provided, its __dict__ is used as the namespace.
1036 If only user_module is provided, its __dict__ is used as the namespace.
1037 If only user_ns is provided, a dummy module is created, and user_ns
1037 If only user_ns is provided, a dummy module is created, and user_ns
1038 becomes the global namespace. If both are provided (as they may be
1038 becomes the global namespace. If both are provided (as they may be
1039 when embedding), user_ns is the local namespace, and user_module
1039 when embedding), user_ns is the local namespace, and user_module
1040 provides the global namespace.
1040 provides the global namespace.
1041
1041
1042 Parameters
1042 Parameters
1043 ----------
1043 ----------
1044 user_module : module, optional
1044 user_module : module, optional
1045 The current user module in which IPython is being run. If None,
1045 The current user module in which IPython is being run. If None,
1046 a clean module will be created.
1046 a clean module will be created.
1047 user_ns : dict, optional
1047 user_ns : dict, optional
1048 A namespace in which to run interactive commands.
1048 A namespace in which to run interactive commands.
1049
1049
1050 Returns
1050 Returns
1051 -------
1051 -------
1052 A tuple of user_module and user_ns, each properly initialised.
1052 A tuple of user_module and user_ns, each properly initialised.
1053 """
1053 """
1054 if user_module is None and user_ns is not None:
1054 if user_module is None and user_ns is not None:
1055 user_ns.setdefault("__name__", "__main__")
1055 user_ns.setdefault("__name__", "__main__")
1056 user_module = DummyMod()
1056 user_module = DummyMod()
1057 user_module.__dict__ = user_ns
1057 user_module.__dict__ = user_ns
1058
1058
1059 if user_module is None:
1059 if user_module is None:
1060 user_module = types.ModuleType("__main__",
1060 user_module = types.ModuleType("__main__",
1061 doc="Automatically created module for IPython interactive environment")
1061 doc="Automatically created module for IPython interactive environment")
1062
1062
1063 # We must ensure that __builtin__ (without the final 's') is always
1063 # We must ensure that __builtin__ (without the final 's') is always
1064 # available and pointing to the __builtin__ *module*. For more details:
1064 # available and pointing to the __builtin__ *module*. For more details:
1065 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1065 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1066 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1066 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1067 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1067 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1068
1068
1069 if user_ns is None:
1069 if user_ns is None:
1070 user_ns = user_module.__dict__
1070 user_ns = user_module.__dict__
1071
1071
1072 return user_module, user_ns
1072 return user_module, user_ns
1073
1073
1074 def init_sys_modules(self):
1074 def init_sys_modules(self):
1075 # We need to insert into sys.modules something that looks like a
1075 # We need to insert into sys.modules something that looks like a
1076 # module but which accesses the IPython namespace, for shelve and
1076 # module but which accesses the IPython namespace, for shelve and
1077 # pickle to work interactively. Normally they rely on getting
1077 # pickle to work interactively. Normally they rely on getting
1078 # everything out of __main__, but for embedding purposes each IPython
1078 # everything out of __main__, but for embedding purposes each IPython
1079 # instance has its own private namespace, so we can't go shoving
1079 # instance has its own private namespace, so we can't go shoving
1080 # everything into __main__.
1080 # everything into __main__.
1081
1081
1082 # note, however, that we should only do this for non-embedded
1082 # note, however, that we should only do this for non-embedded
1083 # ipythons, which really mimic the __main__.__dict__ with their own
1083 # ipythons, which really mimic the __main__.__dict__ with their own
1084 # namespace. Embedded instances, on the other hand, should not do
1084 # namespace. Embedded instances, on the other hand, should not do
1085 # this because they need to manage the user local/global namespaces
1085 # this because they need to manage the user local/global namespaces
1086 # only, but they live within a 'normal' __main__ (meaning, they
1086 # only, but they live within a 'normal' __main__ (meaning, they
1087 # shouldn't overtake the execution environment of the script they're
1087 # shouldn't overtake the execution environment of the script they're
1088 # embedded in).
1088 # embedded in).
1089
1089
1090 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1090 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1091 main_name = self.user_module.__name__
1091 main_name = self.user_module.__name__
1092 sys.modules[main_name] = self.user_module
1092 sys.modules[main_name] = self.user_module
1093
1093
1094 def init_user_ns(self):
1094 def init_user_ns(self):
1095 """Initialize all user-visible namespaces to their minimum defaults.
1095 """Initialize all user-visible namespaces to their minimum defaults.
1096
1096
1097 Certain history lists are also initialized here, as they effectively
1097 Certain history lists are also initialized here, as they effectively
1098 act as user namespaces.
1098 act as user namespaces.
1099
1099
1100 Notes
1100 Notes
1101 -----
1101 -----
1102 All data structures here are only filled in, they are NOT reset by this
1102 All data structures here are only filled in, they are NOT reset by this
1103 method. If they were not empty before, data will simply be added to
1103 method. If they were not empty before, data will simply be added to
1104 therm.
1104 therm.
1105 """
1105 """
1106 # This function works in two parts: first we put a few things in
1106 # This function works in two parts: first we put a few things in
1107 # user_ns, and we sync that contents into user_ns_hidden so that these
1107 # user_ns, and we sync that contents into user_ns_hidden so that these
1108 # initial variables aren't shown by %who. After the sync, we add the
1108 # initial variables aren't shown by %who. After the sync, we add the
1109 # rest of what we *do* want the user to see with %who even on a new
1109 # rest of what we *do* want the user to see with %who even on a new
1110 # session (probably nothing, so theye really only see their own stuff)
1110 # session (probably nothing, so theye really only see their own stuff)
1111
1111
1112 # The user dict must *always* have a __builtin__ reference to the
1112 # The user dict must *always* have a __builtin__ reference to the
1113 # Python standard __builtin__ namespace, which must be imported.
1113 # Python standard __builtin__ namespace, which must be imported.
1114 # This is so that certain operations in prompt evaluation can be
1114 # This is so that certain operations in prompt evaluation can be
1115 # reliably executed with builtins. Note that we can NOT use
1115 # reliably executed with builtins. Note that we can NOT use
1116 # __builtins__ (note the 's'), because that can either be a dict or a
1116 # __builtins__ (note the 's'), because that can either be a dict or a
1117 # module, and can even mutate at runtime, depending on the context
1117 # module, and can even mutate at runtime, depending on the context
1118 # (Python makes no guarantees on it). In contrast, __builtin__ is
1118 # (Python makes no guarantees on it). In contrast, __builtin__ is
1119 # always a module object, though it must be explicitly imported.
1119 # always a module object, though it must be explicitly imported.
1120
1120
1121 # For more details:
1121 # For more details:
1122 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1122 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1123 ns = dict()
1123 ns = dict()
1124
1124
1125 # Put 'help' in the user namespace
1125 # Put 'help' in the user namespace
1126 try:
1126 try:
1127 from site import _Helper
1127 from site import _Helper
1128 ns['help'] = _Helper()
1128 ns['help'] = _Helper()
1129 except ImportError:
1129 except ImportError:
1130 warn('help() not available - check site.py')
1130 warn('help() not available - check site.py')
1131
1131
1132 # make global variables for user access to the histories
1132 # make global variables for user access to the histories
1133 ns['_ih'] = self.history_manager.input_hist_parsed
1133 ns['_ih'] = self.history_manager.input_hist_parsed
1134 ns['_oh'] = self.history_manager.output_hist
1134 ns['_oh'] = self.history_manager.output_hist
1135 ns['_dh'] = self.history_manager.dir_hist
1135 ns['_dh'] = self.history_manager.dir_hist
1136
1136
1137 ns['_sh'] = shadowns
1137 ns['_sh'] = shadowns
1138
1138
1139 # user aliases to input and output histories. These shouldn't show up
1139 # user aliases to input and output histories. These shouldn't show up
1140 # in %who, as they can have very large reprs.
1140 # in %who, as they can have very large reprs.
1141 ns['In'] = self.history_manager.input_hist_parsed
1141 ns['In'] = self.history_manager.input_hist_parsed
1142 ns['Out'] = self.history_manager.output_hist
1142 ns['Out'] = self.history_manager.output_hist
1143
1143
1144 # Store myself as the public api!!!
1144 # Store myself as the public api!!!
1145 ns['get_ipython'] = self.get_ipython
1145 ns['get_ipython'] = self.get_ipython
1146
1146
1147 ns['exit'] = self.exiter
1147 ns['exit'] = self.exiter
1148 ns['quit'] = self.exiter
1148 ns['quit'] = self.exiter
1149
1149
1150 # Sync what we've added so far to user_ns_hidden so these aren't seen
1150 # Sync what we've added so far to user_ns_hidden so these aren't seen
1151 # by %who
1151 # by %who
1152 self.user_ns_hidden.update(ns)
1152 self.user_ns_hidden.update(ns)
1153
1153
1154 # Anything put into ns now would show up in %who. Think twice before
1154 # Anything put into ns now would show up in %who. Think twice before
1155 # putting anything here, as we really want %who to show the user their
1155 # putting anything here, as we really want %who to show the user their
1156 # stuff, not our variables.
1156 # stuff, not our variables.
1157
1157
1158 # Finally, update the real user's namespace
1158 # Finally, update the real user's namespace
1159 self.user_ns.update(ns)
1159 self.user_ns.update(ns)
1160
1160
1161 @property
1161 @property
1162 def all_ns_refs(self):
1162 def all_ns_refs(self):
1163 """Get a list of references to all the namespace dictionaries in which
1163 """Get a list of references to all the namespace dictionaries in which
1164 IPython might store a user-created object.
1164 IPython might store a user-created object.
1165
1165
1166 Note that this does not include the displayhook, which also caches
1166 Note that this does not include the displayhook, which also caches
1167 objects from the output."""
1167 objects from the output."""
1168 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1168 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1169 [m.__dict__ for m in self._main_mod_cache.values()]
1169 [m.__dict__ for m in self._main_mod_cache.values()]
1170
1170
1171 def reset(self, new_session=True):
1171 def reset(self, new_session=True):
1172 """Clear all internal namespaces, and attempt to release references to
1172 """Clear all internal namespaces, and attempt to release references to
1173 user objects.
1173 user objects.
1174
1174
1175 If new_session is True, a new history session will be opened.
1175 If new_session is True, a new history session will be opened.
1176 """
1176 """
1177 # Clear histories
1177 # Clear histories
1178 self.history_manager.reset(new_session)
1178 self.history_manager.reset(new_session)
1179 # Reset counter used to index all histories
1179 # Reset counter used to index all histories
1180 if new_session:
1180 if new_session:
1181 self.execution_count = 1
1181 self.execution_count = 1
1182
1182
1183 # Flush cached output items
1183 # Flush cached output items
1184 if self.displayhook.do_full_cache:
1184 if self.displayhook.do_full_cache:
1185 self.displayhook.flush()
1185 self.displayhook.flush()
1186
1186
1187 # The main execution namespaces must be cleared very carefully,
1187 # The main execution namespaces must be cleared very carefully,
1188 # skipping the deletion of the builtin-related keys, because doing so
1188 # skipping the deletion of the builtin-related keys, because doing so
1189 # would cause errors in many object's __del__ methods.
1189 # would cause errors in many object's __del__ methods.
1190 if self.user_ns is not self.user_global_ns:
1190 if self.user_ns is not self.user_global_ns:
1191 self.user_ns.clear()
1191 self.user_ns.clear()
1192 ns = self.user_global_ns
1192 ns = self.user_global_ns
1193 drop_keys = set(ns.keys())
1193 drop_keys = set(ns.keys())
1194 drop_keys.discard('__builtin__')
1194 drop_keys.discard('__builtin__')
1195 drop_keys.discard('__builtins__')
1195 drop_keys.discard('__builtins__')
1196 drop_keys.discard('__name__')
1196 drop_keys.discard('__name__')
1197 for k in drop_keys:
1197 for k in drop_keys:
1198 del ns[k]
1198 del ns[k]
1199
1199
1200 self.user_ns_hidden.clear()
1200 self.user_ns_hidden.clear()
1201
1201
1202 # Restore the user namespaces to minimal usability
1202 # Restore the user namespaces to minimal usability
1203 self.init_user_ns()
1203 self.init_user_ns()
1204
1204
1205 # Restore the default and user aliases
1205 # Restore the default and user aliases
1206 self.alias_manager.clear_aliases()
1206 self.alias_manager.clear_aliases()
1207 self.alias_manager.init_aliases()
1207 self.alias_manager.init_aliases()
1208
1208
1209 # Flush the private list of module references kept for script
1209 # Flush the private list of module references kept for script
1210 # execution protection
1210 # execution protection
1211 self.clear_main_mod_cache()
1211 self.clear_main_mod_cache()
1212
1212
1213 def del_var(self, varname, by_name=False):
1213 def del_var(self, varname, by_name=False):
1214 """Delete a variable from the various namespaces, so that, as
1214 """Delete a variable from the various namespaces, so that, as
1215 far as possible, we're not keeping any hidden references to it.
1215 far as possible, we're not keeping any hidden references to it.
1216
1216
1217 Parameters
1217 Parameters
1218 ----------
1218 ----------
1219 varname : str
1219 varname : str
1220 The name of the variable to delete.
1220 The name of the variable to delete.
1221 by_name : bool
1221 by_name : bool
1222 If True, delete variables with the given name in each
1222 If True, delete variables with the given name in each
1223 namespace. If False (default), find the variable in the user
1223 namespace. If False (default), find the variable in the user
1224 namespace, and delete references to it.
1224 namespace, and delete references to it.
1225 """
1225 """
1226 if varname in ('__builtin__', '__builtins__'):
1226 if varname in ('__builtin__', '__builtins__'):
1227 raise ValueError("Refusing to delete %s" % varname)
1227 raise ValueError("Refusing to delete %s" % varname)
1228
1228
1229 ns_refs = self.all_ns_refs
1229 ns_refs = self.all_ns_refs
1230
1230
1231 if by_name: # Delete by name
1231 if by_name: # Delete by name
1232 for ns in ns_refs:
1232 for ns in ns_refs:
1233 try:
1233 try:
1234 del ns[varname]
1234 del ns[varname]
1235 except KeyError:
1235 except KeyError:
1236 pass
1236 pass
1237 else: # Delete by object
1237 else: # Delete by object
1238 try:
1238 try:
1239 obj = self.user_ns[varname]
1239 obj = self.user_ns[varname]
1240 except KeyError:
1240 except KeyError:
1241 raise NameError("name '%s' is not defined" % varname)
1241 raise NameError("name '%s' is not defined" % varname)
1242 # Also check in output history
1242 # Also check in output history
1243 ns_refs.append(self.history_manager.output_hist)
1243 ns_refs.append(self.history_manager.output_hist)
1244 for ns in ns_refs:
1244 for ns in ns_refs:
1245 to_delete = [n for n, o in ns.iteritems() if o is obj]
1245 to_delete = [n for n, o in ns.iteritems() if o is obj]
1246 for name in to_delete:
1246 for name in to_delete:
1247 del ns[name]
1247 del ns[name]
1248
1248
1249 # displayhook keeps extra references, but not in a dictionary
1249 # displayhook keeps extra references, but not in a dictionary
1250 for name in ('_', '__', '___'):
1250 for name in ('_', '__', '___'):
1251 if getattr(self.displayhook, name) is obj:
1251 if getattr(self.displayhook, name) is obj:
1252 setattr(self.displayhook, name, None)
1252 setattr(self.displayhook, name, None)
1253
1253
1254 def reset_selective(self, regex=None):
1254 def reset_selective(self, regex=None):
1255 """Clear selective variables from internal namespaces based on a
1255 """Clear selective variables from internal namespaces based on a
1256 specified regular expression.
1256 specified regular expression.
1257
1257
1258 Parameters
1258 Parameters
1259 ----------
1259 ----------
1260 regex : string or compiled pattern, optional
1260 regex : string or compiled pattern, optional
1261 A regular expression pattern that will be used in searching
1261 A regular expression pattern that will be used in searching
1262 variable names in the users namespaces.
1262 variable names in the users namespaces.
1263 """
1263 """
1264 if regex is not None:
1264 if regex is not None:
1265 try:
1265 try:
1266 m = re.compile(regex)
1266 m = re.compile(regex)
1267 except TypeError:
1267 except TypeError:
1268 raise TypeError('regex must be a string or compiled pattern')
1268 raise TypeError('regex must be a string or compiled pattern')
1269 # Search for keys in each namespace that match the given regex
1269 # Search for keys in each namespace that match the given regex
1270 # If a match is found, delete the key/value pair.
1270 # If a match is found, delete the key/value pair.
1271 for ns in self.all_ns_refs:
1271 for ns in self.all_ns_refs:
1272 for var in ns:
1272 for var in ns:
1273 if m.search(var):
1273 if m.search(var):
1274 del ns[var]
1274 del ns[var]
1275
1275
1276 def push(self, variables, interactive=True):
1276 def push(self, variables, interactive=True):
1277 """Inject a group of variables into the IPython user namespace.
1277 """Inject a group of variables into the IPython user namespace.
1278
1278
1279 Parameters
1279 Parameters
1280 ----------
1280 ----------
1281 variables : dict, str or list/tuple of str
1281 variables : dict, str or list/tuple of str
1282 The variables to inject into the user's namespace. If a dict, a
1282 The variables to inject into the user's namespace. If a dict, a
1283 simple update is done. If a str, the string is assumed to have
1283 simple update is done. If a str, the string is assumed to have
1284 variable names separated by spaces. A list/tuple of str can also
1284 variable names separated by spaces. A list/tuple of str can also
1285 be used to give the variable names. If just the variable names are
1285 be used to give the variable names. If just the variable names are
1286 give (list/tuple/str) then the variable values looked up in the
1286 give (list/tuple/str) then the variable values looked up in the
1287 callers frame.
1287 callers frame.
1288 interactive : bool
1288 interactive : bool
1289 If True (default), the variables will be listed with the ``who``
1289 If True (default), the variables will be listed with the ``who``
1290 magic.
1290 magic.
1291 """
1291 """
1292 vdict = None
1292 vdict = None
1293
1293
1294 # We need a dict of name/value pairs to do namespace updates.
1294 # We need a dict of name/value pairs to do namespace updates.
1295 if isinstance(variables, dict):
1295 if isinstance(variables, dict):
1296 vdict = variables
1296 vdict = variables
1297 elif isinstance(variables, (basestring, list, tuple)):
1297 elif isinstance(variables, (basestring, list, tuple)):
1298 if isinstance(variables, basestring):
1298 if isinstance(variables, basestring):
1299 vlist = variables.split()
1299 vlist = variables.split()
1300 else:
1300 else:
1301 vlist = variables
1301 vlist = variables
1302 vdict = {}
1302 vdict = {}
1303 cf = sys._getframe(1)
1303 cf = sys._getframe(1)
1304 for name in vlist:
1304 for name in vlist:
1305 try:
1305 try:
1306 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1306 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1307 except:
1307 except:
1308 print('Could not get variable %s from %s' %
1308 print('Could not get variable %s from %s' %
1309 (name,cf.f_code.co_name))
1309 (name,cf.f_code.co_name))
1310 else:
1310 else:
1311 raise ValueError('variables must be a dict/str/list/tuple')
1311 raise ValueError('variables must be a dict/str/list/tuple')
1312
1312
1313 # Propagate variables to user namespace
1313 # Propagate variables to user namespace
1314 self.user_ns.update(vdict)
1314 self.user_ns.update(vdict)
1315
1315
1316 # And configure interactive visibility
1316 # And configure interactive visibility
1317 user_ns_hidden = self.user_ns_hidden
1317 user_ns_hidden = self.user_ns_hidden
1318 if interactive:
1318 if interactive:
1319 for name in vdict:
1319 for name in vdict:
1320 user_ns_hidden.pop(name, None)
1320 user_ns_hidden.pop(name, None)
1321 else:
1321 else:
1322 user_ns_hidden.update(vdict)
1322 user_ns_hidden.update(vdict)
1323
1323
1324 def drop_by_id(self, variables):
1324 def drop_by_id(self, variables):
1325 """Remove a dict of variables from the user namespace, if they are the
1325 """Remove a dict of variables from the user namespace, if they are the
1326 same as the values in the dictionary.
1326 same as the values in the dictionary.
1327
1327
1328 This is intended for use by extensions: variables that they've added can
1328 This is intended for use by extensions: variables that they've added can
1329 be taken back out if they are unloaded, without removing any that the
1329 be taken back out if they are unloaded, without removing any that the
1330 user has overwritten.
1330 user has overwritten.
1331
1331
1332 Parameters
1332 Parameters
1333 ----------
1333 ----------
1334 variables : dict
1334 variables : dict
1335 A dictionary mapping object names (as strings) to the objects.
1335 A dictionary mapping object names (as strings) to the objects.
1336 """
1336 """
1337 for name, obj in variables.iteritems():
1337 for name, obj in variables.iteritems():
1338 if name in self.user_ns and self.user_ns[name] is obj:
1338 if name in self.user_ns and self.user_ns[name] is obj:
1339 del self.user_ns[name]
1339 del self.user_ns[name]
1340 self.user_ns_hidden.pop(name, None)
1340 self.user_ns_hidden.pop(name, None)
1341
1341
1342 #-------------------------------------------------------------------------
1342 #-------------------------------------------------------------------------
1343 # Things related to object introspection
1343 # Things related to object introspection
1344 #-------------------------------------------------------------------------
1344 #-------------------------------------------------------------------------
1345
1345
1346 def _ofind(self, oname, namespaces=None):
1346 def _ofind(self, oname, namespaces=None):
1347 """Find an object in the available namespaces.
1347 """Find an object in the available namespaces.
1348
1348
1349 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1349 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1350
1350
1351 Has special code to detect magic functions.
1351 Has special code to detect magic functions.
1352 """
1352 """
1353 oname = oname.strip()
1353 oname = oname.strip()
1354 #print '1- oname: <%r>' % oname # dbg
1354 #print '1- oname: <%r>' % oname # dbg
1355 if not oname.startswith(ESC_MAGIC) and \
1355 if not oname.startswith(ESC_MAGIC) and \
1356 not oname.startswith(ESC_MAGIC2) and \
1356 not oname.startswith(ESC_MAGIC2) and \
1357 not py3compat.isidentifier(oname, dotted=True):
1357 not py3compat.isidentifier(oname, dotted=True):
1358 return dict(found=False)
1358 return dict(found=False)
1359
1359
1360 alias_ns = None
1360 alias_ns = None
1361 if namespaces is None:
1361 if namespaces is None:
1362 # Namespaces to search in:
1362 # Namespaces to search in:
1363 # Put them in a list. The order is important so that we
1363 # Put them in a list. The order is important so that we
1364 # find things in the same order that Python finds them.
1364 # find things in the same order that Python finds them.
1365 namespaces = [ ('Interactive', self.user_ns),
1365 namespaces = [ ('Interactive', self.user_ns),
1366 ('Interactive (global)', self.user_global_ns),
1366 ('Interactive (global)', self.user_global_ns),
1367 ('Python builtin', builtin_mod.__dict__),
1367 ('Python builtin', builtin_mod.__dict__),
1368 ]
1368 ]
1369
1369
1370 # initialize results to 'null'
1370 # initialize results to 'null'
1371 found = False; obj = None; ospace = None; ds = None;
1371 found = False; obj = None; ospace = None; ds = None;
1372 ismagic = False; isalias = False; parent = None
1372 ismagic = False; isalias = False; parent = None
1373
1373
1374 # We need to special-case 'print', which as of python2.6 registers as a
1374 # We need to special-case 'print', which as of python2.6 registers as a
1375 # function but should only be treated as one if print_function was
1375 # function but should only be treated as one if print_function was
1376 # loaded with a future import. In this case, just bail.
1376 # loaded with a future import. In this case, just bail.
1377 if (oname == 'print' and not py3compat.PY3 and not \
1377 if (oname == 'print' and not py3compat.PY3 and not \
1378 (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
1378 (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
1379 return {'found':found, 'obj':obj, 'namespace':ospace,
1379 return {'found':found, 'obj':obj, 'namespace':ospace,
1380 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1380 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1381
1381
1382 # Look for the given name by splitting it in parts. If the head is
1382 # Look for the given name by splitting it in parts. If the head is
1383 # found, then we look for all the remaining parts as members, and only
1383 # found, then we look for all the remaining parts as members, and only
1384 # declare success if we can find them all.
1384 # declare success if we can find them all.
1385 oname_parts = oname.split('.')
1385 oname_parts = oname.split('.')
1386 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1386 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1387 for nsname,ns in namespaces:
1387 for nsname,ns in namespaces:
1388 try:
1388 try:
1389 obj = ns[oname_head]
1389 obj = ns[oname_head]
1390 except KeyError:
1390 except KeyError:
1391 continue
1391 continue
1392 else:
1392 else:
1393 #print 'oname_rest:', oname_rest # dbg
1393 #print 'oname_rest:', oname_rest # dbg
1394 for part in oname_rest:
1394 for part in oname_rest:
1395 try:
1395 try:
1396 parent = obj
1396 parent = obj
1397 obj = getattr(obj,part)
1397 obj = getattr(obj,part)
1398 except:
1398 except:
1399 # Blanket except b/c some badly implemented objects
1399 # Blanket except b/c some badly implemented objects
1400 # allow __getattr__ to raise exceptions other than
1400 # allow __getattr__ to raise exceptions other than
1401 # AttributeError, which then crashes IPython.
1401 # AttributeError, which then crashes IPython.
1402 break
1402 break
1403 else:
1403 else:
1404 # If we finish the for loop (no break), we got all members
1404 # If we finish the for loop (no break), we got all members
1405 found = True
1405 found = True
1406 ospace = nsname
1406 ospace = nsname
1407 break # namespace loop
1407 break # namespace loop
1408
1408
1409 # Try to see if it's magic
1409 # Try to see if it's magic
1410 if not found:
1410 if not found:
1411 obj = None
1411 obj = None
1412 if oname.startswith(ESC_MAGIC2):
1412 if oname.startswith(ESC_MAGIC2):
1413 oname = oname.lstrip(ESC_MAGIC2)
1413 oname = oname.lstrip(ESC_MAGIC2)
1414 obj = self.find_cell_magic(oname)
1414 obj = self.find_cell_magic(oname)
1415 elif oname.startswith(ESC_MAGIC):
1415 elif oname.startswith(ESC_MAGIC):
1416 oname = oname.lstrip(ESC_MAGIC)
1416 oname = oname.lstrip(ESC_MAGIC)
1417 obj = self.find_line_magic(oname)
1417 obj = self.find_line_magic(oname)
1418 else:
1418 else:
1419 # search without prefix, so run? will find %run?
1419 # search without prefix, so run? will find %run?
1420 obj = self.find_line_magic(oname)
1420 obj = self.find_line_magic(oname)
1421 if obj is None:
1421 if obj is None:
1422 obj = self.find_cell_magic(oname)
1422 obj = self.find_cell_magic(oname)
1423 if obj is not None:
1423 if obj is not None:
1424 found = True
1424 found = True
1425 ospace = 'IPython internal'
1425 ospace = 'IPython internal'
1426 ismagic = True
1426 ismagic = True
1427
1427
1428 # Last try: special-case some literals like '', [], {}, etc:
1428 # Last try: special-case some literals like '', [], {}, etc:
1429 if not found and oname_head in ["''",'""','[]','{}','()']:
1429 if not found and oname_head in ["''",'""','[]','{}','()']:
1430 obj = eval(oname_head)
1430 obj = eval(oname_head)
1431 found = True
1431 found = True
1432 ospace = 'Interactive'
1432 ospace = 'Interactive'
1433
1433
1434 return {'found':found, 'obj':obj, 'namespace':ospace,
1434 return {'found':found, 'obj':obj, 'namespace':ospace,
1435 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1435 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1436
1436
1437 def _ofind_property(self, oname, info):
1437 def _ofind_property(self, oname, info):
1438 """Second part of object finding, to look for property details."""
1438 """Second part of object finding, to look for property details."""
1439 if info.found:
1439 if info.found:
1440 # Get the docstring of the class property if it exists.
1440 # Get the docstring of the class property if it exists.
1441 path = oname.split('.')
1441 path = oname.split('.')
1442 root = '.'.join(path[:-1])
1442 root = '.'.join(path[:-1])
1443 if info.parent is not None:
1443 if info.parent is not None:
1444 try:
1444 try:
1445 target = getattr(info.parent, '__class__')
1445 target = getattr(info.parent, '__class__')
1446 # The object belongs to a class instance.
1446 # The object belongs to a class instance.
1447 try:
1447 try:
1448 target = getattr(target, path[-1])
1448 target = getattr(target, path[-1])
1449 # The class defines the object.
1449 # The class defines the object.
1450 if isinstance(target, property):
1450 if isinstance(target, property):
1451 oname = root + '.__class__.' + path[-1]
1451 oname = root + '.__class__.' + path[-1]
1452 info = Struct(self._ofind(oname))
1452 info = Struct(self._ofind(oname))
1453 except AttributeError: pass
1453 except AttributeError: pass
1454 except AttributeError: pass
1454 except AttributeError: pass
1455
1455
1456 # We return either the new info or the unmodified input if the object
1456 # We return either the new info or the unmodified input if the object
1457 # hadn't been found
1457 # hadn't been found
1458 return info
1458 return info
1459
1459
1460 def _object_find(self, oname, namespaces=None):
1460 def _object_find(self, oname, namespaces=None):
1461 """Find an object and return a struct with info about it."""
1461 """Find an object and return a struct with info about it."""
1462 inf = Struct(self._ofind(oname, namespaces))
1462 inf = Struct(self._ofind(oname, namespaces))
1463 return Struct(self._ofind_property(oname, inf))
1463 return Struct(self._ofind_property(oname, inf))
1464
1464
1465 def _inspect(self, meth, oname, namespaces=None, **kw):
1465 def _inspect(self, meth, oname, namespaces=None, **kw):
1466 """Generic interface to the inspector system.
1466 """Generic interface to the inspector system.
1467
1467
1468 This function is meant to be called by pdef, pdoc & friends."""
1468 This function is meant to be called by pdef, pdoc & friends."""
1469 info = self._object_find(oname, namespaces)
1469 info = self._object_find(oname, namespaces)
1470 if info.found:
1470 if info.found:
1471 pmethod = getattr(self.inspector, meth)
1471 pmethod = getattr(self.inspector, meth)
1472 formatter = format_screen if info.ismagic else None
1472 formatter = format_screen if info.ismagic else None
1473 if meth == 'pdoc':
1473 if meth == 'pdoc':
1474 pmethod(info.obj, oname, formatter)
1474 pmethod(info.obj, oname, formatter)
1475 elif meth == 'pinfo':
1475 elif meth == 'pinfo':
1476 pmethod(info.obj, oname, formatter, info, **kw)
1476 pmethod(info.obj, oname, formatter, info, **kw)
1477 else:
1477 else:
1478 pmethod(info.obj, oname)
1478 pmethod(info.obj, oname)
1479 else:
1479 else:
1480 print('Object `%s` not found.' % oname)
1480 print('Object `%s` not found.' % oname)
1481 return 'not found' # so callers can take other action
1481 return 'not found' # so callers can take other action
1482
1482
1483 def object_inspect(self, oname, detail_level=0):
1483 def object_inspect(self, oname, detail_level=0):
1484 with self.builtin_trap:
1484 with self.builtin_trap:
1485 info = self._object_find(oname)
1485 info = self._object_find(oname)
1486 if info.found:
1486 if info.found:
1487 return self.inspector.info(info.obj, oname, info=info,
1487 return self.inspector.info(info.obj, oname, info=info,
1488 detail_level=detail_level
1488 detail_level=detail_level
1489 )
1489 )
1490 else:
1490 else:
1491 return oinspect.object_info(name=oname, found=False)
1491 return oinspect.object_info(name=oname, found=False)
1492
1492
1493 #-------------------------------------------------------------------------
1493 #-------------------------------------------------------------------------
1494 # Things related to history management
1494 # Things related to history management
1495 #-------------------------------------------------------------------------
1495 #-------------------------------------------------------------------------
1496
1496
1497 def init_history(self):
1497 def init_history(self):
1498 """Sets up the command history, and starts regular autosaves."""
1498 """Sets up the command history, and starts regular autosaves."""
1499 self.history_manager = HistoryManager(shell=self, parent=self)
1499 self.history_manager = HistoryManager(shell=self, parent=self)
1500 self.configurables.append(self.history_manager)
1500 self.configurables.append(self.history_manager)
1501
1501
1502 #-------------------------------------------------------------------------
1502 #-------------------------------------------------------------------------
1503 # Things related to exception handling and tracebacks (not debugging)
1503 # Things related to exception handling and tracebacks (not debugging)
1504 #-------------------------------------------------------------------------
1504 #-------------------------------------------------------------------------
1505
1505
1506 def init_traceback_handlers(self, custom_exceptions):
1506 def init_traceback_handlers(self, custom_exceptions):
1507 # Syntax error handler.
1507 # Syntax error handler.
1508 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
1508 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
1509
1509
1510 # The interactive one is initialized with an offset, meaning we always
1510 # The interactive one is initialized with an offset, meaning we always
1511 # want to remove the topmost item in the traceback, which is our own
1511 # want to remove the topmost item in the traceback, which is our own
1512 # internal code. Valid modes: ['Plain','Context','Verbose']
1512 # internal code. Valid modes: ['Plain','Context','Verbose']
1513 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1513 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1514 color_scheme='NoColor',
1514 color_scheme='NoColor',
1515 tb_offset = 1,
1515 tb_offset = 1,
1516 check_cache=check_linecache_ipython)
1516 check_cache=check_linecache_ipython)
1517
1517
1518 # The instance will store a pointer to the system-wide exception hook,
1518 # The instance will store a pointer to the system-wide exception hook,
1519 # so that runtime code (such as magics) can access it. This is because
1519 # so that runtime code (such as magics) can access it. This is because
1520 # during the read-eval loop, it may get temporarily overwritten.
1520 # during the read-eval loop, it may get temporarily overwritten.
1521 self.sys_excepthook = sys.excepthook
1521 self.sys_excepthook = sys.excepthook
1522
1522
1523 # and add any custom exception handlers the user may have specified
1523 # and add any custom exception handlers the user may have specified
1524 self.set_custom_exc(*custom_exceptions)
1524 self.set_custom_exc(*custom_exceptions)
1525
1525
1526 # Set the exception mode
1526 # Set the exception mode
1527 self.InteractiveTB.set_mode(mode=self.xmode)
1527 self.InteractiveTB.set_mode(mode=self.xmode)
1528
1528
1529 def set_custom_exc(self, exc_tuple, handler):
1529 def set_custom_exc(self, exc_tuple, handler):
1530 """set_custom_exc(exc_tuple,handler)
1530 """set_custom_exc(exc_tuple,handler)
1531
1531
1532 Set a custom exception handler, which will be called if any of the
1532 Set a custom exception handler, which will be called if any of the
1533 exceptions in exc_tuple occur in the mainloop (specifically, in the
1533 exceptions in exc_tuple occur in the mainloop (specifically, in the
1534 run_code() method).
1534 run_code() method).
1535
1535
1536 Parameters
1536 Parameters
1537 ----------
1537 ----------
1538
1538
1539 exc_tuple : tuple of exception classes
1539 exc_tuple : tuple of exception classes
1540 A *tuple* of exception classes, for which to call the defined
1540 A *tuple* of exception classes, for which to call the defined
1541 handler. It is very important that you use a tuple, and NOT A
1541 handler. It is very important that you use a tuple, and NOT A
1542 LIST here, because of the way Python's except statement works. If
1542 LIST here, because of the way Python's except statement works. If
1543 you only want to trap a single exception, use a singleton tuple::
1543 you only want to trap a single exception, use a singleton tuple::
1544
1544
1545 exc_tuple == (MyCustomException,)
1545 exc_tuple == (MyCustomException,)
1546
1546
1547 handler : callable
1547 handler : callable
1548 handler must have the following signature::
1548 handler must have the following signature::
1549
1549
1550 def my_handler(self, etype, value, tb, tb_offset=None):
1550 def my_handler(self, etype, value, tb, tb_offset=None):
1551 ...
1551 ...
1552 return structured_traceback
1552 return structured_traceback
1553
1553
1554 Your handler must return a structured traceback (a list of strings),
1554 Your handler must return a structured traceback (a list of strings),
1555 or None.
1555 or None.
1556
1556
1557 This will be made into an instance method (via types.MethodType)
1557 This will be made into an instance method (via types.MethodType)
1558 of IPython itself, and it will be called if any of the exceptions
1558 of IPython itself, and it will be called if any of the exceptions
1559 listed in the exc_tuple are caught. If the handler is None, an
1559 listed in the exc_tuple are caught. If the handler is None, an
1560 internal basic one is used, which just prints basic info.
1560 internal basic one is used, which just prints basic info.
1561
1561
1562 To protect IPython from crashes, if your handler ever raises an
1562 To protect IPython from crashes, if your handler ever raises an
1563 exception or returns an invalid result, it will be immediately
1563 exception or returns an invalid result, it will be immediately
1564 disabled.
1564 disabled.
1565
1565
1566 WARNING: by putting in your own exception handler into IPython's main
1566 WARNING: by putting in your own exception handler into IPython's main
1567 execution loop, you run a very good chance of nasty crashes. This
1567 execution loop, you run a very good chance of nasty crashes. This
1568 facility should only be used if you really know what you are doing."""
1568 facility should only be used if you really know what you are doing."""
1569
1569
1570 assert type(exc_tuple)==type(()) , \
1570 assert type(exc_tuple)==type(()) , \
1571 "The custom exceptions must be given AS A TUPLE."
1571 "The custom exceptions must be given AS A TUPLE."
1572
1572
1573 def dummy_handler(self,etype,value,tb,tb_offset=None):
1573 def dummy_handler(self,etype,value,tb,tb_offset=None):
1574 print('*** Simple custom exception handler ***')
1574 print('*** Simple custom exception handler ***')
1575 print('Exception type :',etype)
1575 print('Exception type :',etype)
1576 print('Exception value:',value)
1576 print('Exception value:',value)
1577 print('Traceback :',tb)
1577 print('Traceback :',tb)
1578 #print 'Source code :','\n'.join(self.buffer)
1578 #print 'Source code :','\n'.join(self.buffer)
1579
1579
1580 def validate_stb(stb):
1580 def validate_stb(stb):
1581 """validate structured traceback return type
1581 """validate structured traceback return type
1582
1582
1583 return type of CustomTB *should* be a list of strings, but allow
1583 return type of CustomTB *should* be a list of strings, but allow
1584 single strings or None, which are harmless.
1584 single strings or None, which are harmless.
1585
1585
1586 This function will *always* return a list of strings,
1586 This function will *always* return a list of strings,
1587 and will raise a TypeError if stb is inappropriate.
1587 and will raise a TypeError if stb is inappropriate.
1588 """
1588 """
1589 msg = "CustomTB must return list of strings, not %r" % stb
1589 msg = "CustomTB must return list of strings, not %r" % stb
1590 if stb is None:
1590 if stb is None:
1591 return []
1591 return []
1592 elif isinstance(stb, basestring):
1592 elif isinstance(stb, basestring):
1593 return [stb]
1593 return [stb]
1594 elif not isinstance(stb, list):
1594 elif not isinstance(stb, list):
1595 raise TypeError(msg)
1595 raise TypeError(msg)
1596 # it's a list
1596 # it's a list
1597 for line in stb:
1597 for line in stb:
1598 # check every element
1598 # check every element
1599 if not isinstance(line, basestring):
1599 if not isinstance(line, basestring):
1600 raise TypeError(msg)
1600 raise TypeError(msg)
1601 return stb
1601 return stb
1602
1602
1603 if handler is None:
1603 if handler is None:
1604 wrapped = dummy_handler
1604 wrapped = dummy_handler
1605 else:
1605 else:
1606 def wrapped(self,etype,value,tb,tb_offset=None):
1606 def wrapped(self,etype,value,tb,tb_offset=None):
1607 """wrap CustomTB handler, to protect IPython from user code
1607 """wrap CustomTB handler, to protect IPython from user code
1608
1608
1609 This makes it harder (but not impossible) for custom exception
1609 This makes it harder (but not impossible) for custom exception
1610 handlers to crash IPython.
1610 handlers to crash IPython.
1611 """
1611 """
1612 try:
1612 try:
1613 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1613 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1614 return validate_stb(stb)
1614 return validate_stb(stb)
1615 except:
1615 except:
1616 # clear custom handler immediately
1616 # clear custom handler immediately
1617 self.set_custom_exc((), None)
1617 self.set_custom_exc((), None)
1618 print("Custom TB Handler failed, unregistering", file=io.stderr)
1618 print("Custom TB Handler failed, unregistering", file=io.stderr)
1619 # show the exception in handler first
1619 # show the exception in handler first
1620 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1620 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1621 print(self.InteractiveTB.stb2text(stb), file=io.stdout)
1621 print(self.InteractiveTB.stb2text(stb), file=io.stdout)
1622 print("The original exception:", file=io.stdout)
1622 print("The original exception:", file=io.stdout)
1623 stb = self.InteractiveTB.structured_traceback(
1623 stb = self.InteractiveTB.structured_traceback(
1624 (etype,value,tb), tb_offset=tb_offset
1624 (etype,value,tb), tb_offset=tb_offset
1625 )
1625 )
1626 return stb
1626 return stb
1627
1627
1628 self.CustomTB = types.MethodType(wrapped,self)
1628 self.CustomTB = types.MethodType(wrapped,self)
1629 self.custom_exceptions = exc_tuple
1629 self.custom_exceptions = exc_tuple
1630
1630
1631 def excepthook(self, etype, value, tb):
1631 def excepthook(self, etype, value, tb):
1632 """One more defense for GUI apps that call sys.excepthook.
1632 """One more defense for GUI apps that call sys.excepthook.
1633
1633
1634 GUI frameworks like wxPython trap exceptions and call
1634 GUI frameworks like wxPython trap exceptions and call
1635 sys.excepthook themselves. I guess this is a feature that
1635 sys.excepthook themselves. I guess this is a feature that
1636 enables them to keep running after exceptions that would
1636 enables them to keep running after exceptions that would
1637 otherwise kill their mainloop. This is a bother for IPython
1637 otherwise kill their mainloop. This is a bother for IPython
1638 which excepts to catch all of the program exceptions with a try:
1638 which excepts to catch all of the program exceptions with a try:
1639 except: statement.
1639 except: statement.
1640
1640
1641 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1641 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1642 any app directly invokes sys.excepthook, it will look to the user like
1642 any app directly invokes sys.excepthook, it will look to the user like
1643 IPython crashed. In order to work around this, we can disable the
1643 IPython crashed. In order to work around this, we can disable the
1644 CrashHandler and replace it with this excepthook instead, which prints a
1644 CrashHandler and replace it with this excepthook instead, which prints a
1645 regular traceback using our InteractiveTB. In this fashion, apps which
1645 regular traceback using our InteractiveTB. In this fashion, apps which
1646 call sys.excepthook will generate a regular-looking exception from
1646 call sys.excepthook will generate a regular-looking exception from
1647 IPython, and the CrashHandler will only be triggered by real IPython
1647 IPython, and the CrashHandler will only be triggered by real IPython
1648 crashes.
1648 crashes.
1649
1649
1650 This hook should be used sparingly, only in places which are not likely
1650 This hook should be used sparingly, only in places which are not likely
1651 to be true IPython errors.
1651 to be true IPython errors.
1652 """
1652 """
1653 self.showtraceback((etype,value,tb),tb_offset=0)
1653 self.showtraceback((etype,value,tb),tb_offset=0)
1654
1654
1655 def _get_exc_info(self, exc_tuple=None):
1655 def _get_exc_info(self, exc_tuple=None):
1656 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1656 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1657
1657
1658 Ensures sys.last_type,value,traceback hold the exc_info we found,
1658 Ensures sys.last_type,value,traceback hold the exc_info we found,
1659 from whichever source.
1659 from whichever source.
1660
1660
1661 raises ValueError if none of these contain any information
1661 raises ValueError if none of these contain any information
1662 """
1662 """
1663 if exc_tuple is None:
1663 if exc_tuple is None:
1664 etype, value, tb = sys.exc_info()
1664 etype, value, tb = sys.exc_info()
1665 else:
1665 else:
1666 etype, value, tb = exc_tuple
1666 etype, value, tb = exc_tuple
1667
1667
1668 if etype is None:
1668 if etype is None:
1669 if hasattr(sys, 'last_type'):
1669 if hasattr(sys, 'last_type'):
1670 etype, value, tb = sys.last_type, sys.last_value, \
1670 etype, value, tb = sys.last_type, sys.last_value, \
1671 sys.last_traceback
1671 sys.last_traceback
1672
1672
1673 if etype is None:
1673 if etype is None:
1674 raise ValueError("No exception to find")
1674 raise ValueError("No exception to find")
1675
1675
1676 # Now store the exception info in sys.last_type etc.
1676 # Now store the exception info in sys.last_type etc.
1677 # WARNING: these variables are somewhat deprecated and not
1677 # WARNING: these variables are somewhat deprecated and not
1678 # necessarily safe to use in a threaded environment, but tools
1678 # necessarily safe to use in a threaded environment, but tools
1679 # like pdb depend on their existence, so let's set them. If we
1679 # like pdb depend on their existence, so let's set them. If we
1680 # find problems in the field, we'll need to revisit their use.
1680 # find problems in the field, we'll need to revisit their use.
1681 sys.last_type = etype
1681 sys.last_type = etype
1682 sys.last_value = value
1682 sys.last_value = value
1683 sys.last_traceback = tb
1683 sys.last_traceback = tb
1684
1684
1685 return etype, value, tb
1685 return etype, value, tb
1686
1686
1687 def show_usage_error(self, exc):
1687 def show_usage_error(self, exc):
1688 """Show a short message for UsageErrors
1688 """Show a short message for UsageErrors
1689
1689
1690 These are special exceptions that shouldn't show a traceback.
1690 These are special exceptions that shouldn't show a traceback.
1691 """
1691 """
1692 self.write_err("UsageError: %s" % exc)
1692 self.write_err("UsageError: %s" % exc)
1693
1693
1694 def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
1694 def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
1695 exception_only=False):
1695 exception_only=False):
1696 """Display the exception that just occurred.
1696 """Display the exception that just occurred.
1697
1697
1698 If nothing is known about the exception, this is the method which
1698 If nothing is known about the exception, this is the method which
1699 should be used throughout the code for presenting user tracebacks,
1699 should be used throughout the code for presenting user tracebacks,
1700 rather than directly invoking the InteractiveTB object.
1700 rather than directly invoking the InteractiveTB object.
1701
1701
1702 A specific showsyntaxerror() also exists, but this method can take
1702 A specific showsyntaxerror() also exists, but this method can take
1703 care of calling it if needed, so unless you are explicitly catching a
1703 care of calling it if needed, so unless you are explicitly catching a
1704 SyntaxError exception, don't try to analyze the stack manually and
1704 SyntaxError exception, don't try to analyze the stack manually and
1705 simply call this method."""
1705 simply call this method."""
1706
1706
1707 try:
1707 try:
1708 try:
1708 try:
1709 etype, value, tb = self._get_exc_info(exc_tuple)
1709 etype, value, tb = self._get_exc_info(exc_tuple)
1710 except ValueError:
1710 except ValueError:
1711 self.write_err('No traceback available to show.\n')
1711 self.write_err('No traceback available to show.\n')
1712 return
1712 return
1713
1713
1714 if issubclass(etype, SyntaxError):
1714 if issubclass(etype, SyntaxError):
1715 # Though this won't be called by syntax errors in the input
1715 # Though this won't be called by syntax errors in the input
1716 # line, there may be SyntaxError cases with imported code.
1716 # line, there may be SyntaxError cases with imported code.
1717 self.showsyntaxerror(filename)
1717 self.showsyntaxerror(filename)
1718 elif etype is UsageError:
1718 elif etype is UsageError:
1719 self.show_usage_error(value)
1719 self.show_usage_error(value)
1720 else:
1720 else:
1721 if exception_only:
1721 if exception_only:
1722 stb = ['An exception has occurred, use %tb to see '
1722 stb = ['An exception has occurred, use %tb to see '
1723 'the full traceback.\n']
1723 'the full traceback.\n']
1724 stb.extend(self.InteractiveTB.get_exception_only(etype,
1724 stb.extend(self.InteractiveTB.get_exception_only(etype,
1725 value))
1725 value))
1726 else:
1726 else:
1727 try:
1727 try:
1728 # Exception classes can customise their traceback - we
1728 # Exception classes can customise their traceback - we
1729 # use this in IPython.parallel for exceptions occurring
1729 # use this in IPython.parallel for exceptions occurring
1730 # in the engines. This should return a list of strings.
1730 # in the engines. This should return a list of strings.
1731 stb = value._render_traceback_()
1731 stb = value._render_traceback_()
1732 except Exception:
1732 except Exception:
1733 stb = self.InteractiveTB.structured_traceback(etype,
1733 stb = self.InteractiveTB.structured_traceback(etype,
1734 value, tb, tb_offset=tb_offset)
1734 value, tb, tb_offset=tb_offset)
1735
1735
1736 self._showtraceback(etype, value, stb)
1736 self._showtraceback(etype, value, stb)
1737 if self.call_pdb:
1737 if self.call_pdb:
1738 # drop into debugger
1738 # drop into debugger
1739 self.debugger(force=True)
1739 self.debugger(force=True)
1740 return
1740 return
1741
1741
1742 # Actually show the traceback
1742 # Actually show the traceback
1743 self._showtraceback(etype, value, stb)
1743 self._showtraceback(etype, value, stb)
1744
1744
1745 except KeyboardInterrupt:
1745 except KeyboardInterrupt:
1746 self.write_err("\nKeyboardInterrupt\n")
1746 self.write_err("\nKeyboardInterrupt\n")
1747
1747
1748 def _showtraceback(self, etype, evalue, stb):
1748 def _showtraceback(self, etype, evalue, stb):
1749 """Actually show a traceback.
1749 """Actually show a traceback.
1750
1750
1751 Subclasses may override this method to put the traceback on a different
1751 Subclasses may override this method to put the traceback on a different
1752 place, like a side channel.
1752 place, like a side channel.
1753 """
1753 """
1754 print(self.InteractiveTB.stb2text(stb), file=io.stdout)
1754 print(self.InteractiveTB.stb2text(stb), file=io.stdout)
1755
1755
1756 def showsyntaxerror(self, filename=None):
1756 def showsyntaxerror(self, filename=None):
1757 """Display the syntax error that just occurred.
1757 """Display the syntax error that just occurred.
1758
1758
1759 This doesn't display a stack trace because there isn't one.
1759 This doesn't display a stack trace because there isn't one.
1760
1760
1761 If a filename is given, it is stuffed in the exception instead
1761 If a filename is given, it is stuffed in the exception instead
1762 of what was there before (because Python's parser always uses
1762 of what was there before (because Python's parser always uses
1763 "<string>" when reading from a string).
1763 "<string>" when reading from a string).
1764 """
1764 """
1765 etype, value, last_traceback = self._get_exc_info()
1765 etype, value, last_traceback = self._get_exc_info()
1766
1766
1767 if filename and issubclass(etype, SyntaxError):
1767 if filename and issubclass(etype, SyntaxError):
1768 try:
1768 try:
1769 value.filename = filename
1769 value.filename = filename
1770 except:
1770 except:
1771 # Not the format we expect; leave it alone
1771 # Not the format we expect; leave it alone
1772 pass
1772 pass
1773
1773
1774 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1774 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1775 self._showtraceback(etype, value, stb)
1775 self._showtraceback(etype, value, stb)
1776
1776
1777 # This is overridden in TerminalInteractiveShell to show a message about
1777 # This is overridden in TerminalInteractiveShell to show a message about
1778 # the %paste magic.
1778 # the %paste magic.
1779 def showindentationerror(self):
1779 def showindentationerror(self):
1780 """Called by run_cell when there's an IndentationError in code entered
1780 """Called by run_cell when there's an IndentationError in code entered
1781 at the prompt.
1781 at the prompt.
1782
1782
1783 This is overridden in TerminalInteractiveShell to show a message about
1783 This is overridden in TerminalInteractiveShell to show a message about
1784 the %paste magic."""
1784 the %paste magic."""
1785 self.showsyntaxerror()
1785 self.showsyntaxerror()
1786
1786
1787 #-------------------------------------------------------------------------
1787 #-------------------------------------------------------------------------
1788 # Things related to readline
1788 # Things related to readline
1789 #-------------------------------------------------------------------------
1789 #-------------------------------------------------------------------------
1790
1790
1791 def init_readline(self):
1791 def init_readline(self):
1792 """Command history completion/saving/reloading."""
1792 """Command history completion/saving/reloading."""
1793
1793
1794 if self.readline_use:
1794 if self.readline_use:
1795 import IPython.utils.rlineimpl as readline
1795 import IPython.utils.rlineimpl as readline
1796
1796
1797 self.rl_next_input = None
1797 self.rl_next_input = None
1798 self.rl_do_indent = False
1798 self.rl_do_indent = False
1799
1799
1800 if not self.readline_use or not readline.have_readline:
1800 if not self.readline_use or not readline.have_readline:
1801 self.has_readline = False
1801 self.has_readline = False
1802 self.readline = None
1802 self.readline = None
1803 # Set a number of methods that depend on readline to be no-op
1803 # Set a number of methods that depend on readline to be no-op
1804 self.readline_no_record = no_op_context
1804 self.readline_no_record = no_op_context
1805 self.set_readline_completer = no_op
1805 self.set_readline_completer = no_op
1806 self.set_custom_completer = no_op
1806 self.set_custom_completer = no_op
1807 if self.readline_use:
1807 if self.readline_use:
1808 warn('Readline services not available or not loaded.')
1808 warn('Readline services not available or not loaded.')
1809 else:
1809 else:
1810 self.has_readline = True
1810 self.has_readline = True
1811 self.readline = readline
1811 self.readline = readline
1812 sys.modules['readline'] = readline
1812 sys.modules['readline'] = readline
1813
1813
1814 # Platform-specific configuration
1814 # Platform-specific configuration
1815 if os.name == 'nt':
1815 if os.name == 'nt':
1816 # FIXME - check with Frederick to see if we can harmonize
1816 # FIXME - check with Frederick to see if we can harmonize
1817 # naming conventions with pyreadline to avoid this
1817 # naming conventions with pyreadline to avoid this
1818 # platform-dependent check
1818 # platform-dependent check
1819 self.readline_startup_hook = readline.set_pre_input_hook
1819 self.readline_startup_hook = readline.set_pre_input_hook
1820 else:
1820 else:
1821 self.readline_startup_hook = readline.set_startup_hook
1821 self.readline_startup_hook = readline.set_startup_hook
1822
1822
1823 # Load user's initrc file (readline config)
1823 # Load user's initrc file (readline config)
1824 # Or if libedit is used, load editrc.
1824 # Or if libedit is used, load editrc.
1825 inputrc_name = os.environ.get('INPUTRC')
1825 inputrc_name = os.environ.get('INPUTRC')
1826 if inputrc_name is None:
1826 if inputrc_name is None:
1827 inputrc_name = '.inputrc'
1827 inputrc_name = '.inputrc'
1828 if readline.uses_libedit:
1828 if readline.uses_libedit:
1829 inputrc_name = '.editrc'
1829 inputrc_name = '.editrc'
1830 inputrc_name = os.path.join(self.home_dir, inputrc_name)
1830 inputrc_name = os.path.join(self.home_dir, inputrc_name)
1831 if os.path.isfile(inputrc_name):
1831 if os.path.isfile(inputrc_name):
1832 try:
1832 try:
1833 readline.read_init_file(inputrc_name)
1833 readline.read_init_file(inputrc_name)
1834 except:
1834 except:
1835 warn('Problems reading readline initialization file <%s>'
1835 warn('Problems reading readline initialization file <%s>'
1836 % inputrc_name)
1836 % inputrc_name)
1837
1837
1838 # Configure readline according to user's prefs
1838 # Configure readline according to user's prefs
1839 # This is only done if GNU readline is being used. If libedit
1839 # This is only done if GNU readline is being used. If libedit
1840 # is being used (as on Leopard) the readline config is
1840 # is being used (as on Leopard) the readline config is
1841 # not run as the syntax for libedit is different.
1841 # not run as the syntax for libedit is different.
1842 if not readline.uses_libedit:
1842 if not readline.uses_libedit:
1843 for rlcommand in self.readline_parse_and_bind:
1843 for rlcommand in self.readline_parse_and_bind:
1844 #print "loading rl:",rlcommand # dbg
1844 #print "loading rl:",rlcommand # dbg
1845 readline.parse_and_bind(rlcommand)
1845 readline.parse_and_bind(rlcommand)
1846
1846
1847 # Remove some chars from the delimiters list. If we encounter
1847 # Remove some chars from the delimiters list. If we encounter
1848 # unicode chars, discard them.
1848 # unicode chars, discard them.
1849 delims = readline.get_completer_delims()
1849 delims = readline.get_completer_delims()
1850 if not py3compat.PY3:
1850 if not py3compat.PY3:
1851 delims = delims.encode("ascii", "ignore")
1851 delims = delims.encode("ascii", "ignore")
1852 for d in self.readline_remove_delims:
1852 for d in self.readline_remove_delims:
1853 delims = delims.replace(d, "")
1853 delims = delims.replace(d, "")
1854 delims = delims.replace(ESC_MAGIC, '')
1854 delims = delims.replace(ESC_MAGIC, '')
1855 readline.set_completer_delims(delims)
1855 readline.set_completer_delims(delims)
1856 # Store these so we can restore them if something like rpy2 modifies
1856 # Store these so we can restore them if something like rpy2 modifies
1857 # them.
1857 # them.
1858 self.readline_delims = delims
1858 self.readline_delims = delims
1859 # otherwise we end up with a monster history after a while:
1859 # otherwise we end up with a monster history after a while:
1860 readline.set_history_length(self.history_length)
1860 readline.set_history_length(self.history_length)
1861
1861
1862 self.refill_readline_hist()
1862 self.refill_readline_hist()
1863 self.readline_no_record = ReadlineNoRecord(self)
1863 self.readline_no_record = ReadlineNoRecord(self)
1864
1864
1865 # Configure auto-indent for all platforms
1865 # Configure auto-indent for all platforms
1866 self.set_autoindent(self.autoindent)
1866 self.set_autoindent(self.autoindent)
1867
1867
1868 def refill_readline_hist(self):
1868 def refill_readline_hist(self):
1869 # Load the last 1000 lines from history
1869 # Load the last 1000 lines from history
1870 self.readline.clear_history()
1870 self.readline.clear_history()
1871 stdin_encoding = sys.stdin.encoding or "utf-8"
1871 stdin_encoding = sys.stdin.encoding or "utf-8"
1872 last_cell = u""
1872 last_cell = u""
1873 for _, _, cell in self.history_manager.get_tail(1000,
1873 for _, _, cell in self.history_manager.get_tail(1000,
1874 include_latest=True):
1874 include_latest=True):
1875 # Ignore blank lines and consecutive duplicates
1875 # Ignore blank lines and consecutive duplicates
1876 cell = cell.rstrip()
1876 cell = cell.rstrip()
1877 if cell and (cell != last_cell):
1877 if cell and (cell != last_cell):
1878 try:
1878 try:
1879 if self.multiline_history:
1879 if self.multiline_history:
1880 self.readline.add_history(py3compat.unicode_to_str(cell,
1880 self.readline.add_history(py3compat.unicode_to_str(cell,
1881 stdin_encoding))
1881 stdin_encoding))
1882 else:
1882 else:
1883 for line in cell.splitlines():
1883 for line in cell.splitlines():
1884 self.readline.add_history(py3compat.unicode_to_str(line,
1884 self.readline.add_history(py3compat.unicode_to_str(line,
1885 stdin_encoding))
1885 stdin_encoding))
1886 last_cell = cell
1886 last_cell = cell
1887
1887
1888 except TypeError:
1888 except TypeError:
1889 # The history DB can get corrupted so it returns strings
1889 # The history DB can get corrupted so it returns strings
1890 # containing null bytes, which readline objects to.
1890 # containing null bytes, which readline objects to.
1891 continue
1891 continue
1892
1892
1893 @skip_doctest
1893 @skip_doctest
1894 def set_next_input(self, s):
1894 def set_next_input(self, s):
1895 """ Sets the 'default' input string for the next command line.
1895 """ Sets the 'default' input string for the next command line.
1896
1896
1897 Requires readline.
1897 Requires readline.
1898
1898
1899 Example::
1899 Example::
1900
1900
1901 In [1]: _ip.set_next_input("Hello Word")
1901 In [1]: _ip.set_next_input("Hello Word")
1902 In [2]: Hello Word_ # cursor is here
1902 In [2]: Hello Word_ # cursor is here
1903 """
1903 """
1904 self.rl_next_input = py3compat.cast_bytes_py2(s)
1904 self.rl_next_input = py3compat.cast_bytes_py2(s)
1905
1905
1906 # Maybe move this to the terminal subclass?
1906 # Maybe move this to the terminal subclass?
1907 def pre_readline(self):
1907 def pre_readline(self):
1908 """readline hook to be used at the start of each line.
1908 """readline hook to be used at the start of each line.
1909
1909
1910 Currently it handles auto-indent only."""
1910 Currently it handles auto-indent only."""
1911
1911
1912 if self.rl_do_indent:
1912 if self.rl_do_indent:
1913 self.readline.insert_text(self._indent_current_str())
1913 self.readline.insert_text(self._indent_current_str())
1914 if self.rl_next_input is not None:
1914 if self.rl_next_input is not None:
1915 self.readline.insert_text(self.rl_next_input)
1915 self.readline.insert_text(self.rl_next_input)
1916 self.rl_next_input = None
1916 self.rl_next_input = None
1917
1917
1918 def _indent_current_str(self):
1918 def _indent_current_str(self):
1919 """return the current level of indentation as a string"""
1919 """return the current level of indentation as a string"""
1920 return self.input_splitter.indent_spaces * ' '
1920 return self.input_splitter.indent_spaces * ' '
1921
1921
1922 #-------------------------------------------------------------------------
1922 #-------------------------------------------------------------------------
1923 # Things related to text completion
1923 # Things related to text completion
1924 #-------------------------------------------------------------------------
1924 #-------------------------------------------------------------------------
1925
1925
1926 def init_completer(self):
1926 def init_completer(self):
1927 """Initialize the completion machinery.
1927 """Initialize the completion machinery.
1928
1928
1929 This creates completion machinery that can be used by client code,
1929 This creates completion machinery that can be used by client code,
1930 either interactively in-process (typically triggered by the readline
1930 either interactively in-process (typically triggered by the readline
1931 library), programatically (such as in test suites) or out-of-prcess
1931 library), programatically (such as in test suites) or out-of-prcess
1932 (typically over the network by remote frontends).
1932 (typically over the network by remote frontends).
1933 """
1933 """
1934 from IPython.core.completer import IPCompleter
1934 from IPython.core.completer import IPCompleter
1935 from IPython.core.completerlib import (module_completer,
1935 from IPython.core.completerlib import (module_completer,
1936 magic_run_completer, cd_completer, reset_completer)
1936 magic_run_completer, cd_completer, reset_completer)
1937
1937
1938 self.Completer = IPCompleter(shell=self,
1938 self.Completer = IPCompleter(shell=self,
1939 namespace=self.user_ns,
1939 namespace=self.user_ns,
1940 global_namespace=self.user_global_ns,
1940 global_namespace=self.user_global_ns,
1941 use_readline=self.has_readline,
1941 use_readline=self.has_readline,
1942 parent=self,
1942 parent=self,
1943 )
1943 )
1944 self.configurables.append(self.Completer)
1944 self.configurables.append(self.Completer)
1945
1945
1946 # Add custom completers to the basic ones built into IPCompleter
1946 # Add custom completers to the basic ones built into IPCompleter
1947 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1947 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1948 self.strdispatchers['complete_command'] = sdisp
1948 self.strdispatchers['complete_command'] = sdisp
1949 self.Completer.custom_completers = sdisp
1949 self.Completer.custom_completers = sdisp
1950
1950
1951 self.set_hook('complete_command', module_completer, str_key = 'import')
1951 self.set_hook('complete_command', module_completer, str_key = 'import')
1952 self.set_hook('complete_command', module_completer, str_key = 'from')
1952 self.set_hook('complete_command', module_completer, str_key = 'from')
1953 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1953 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1954 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1954 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1955 self.set_hook('complete_command', reset_completer, str_key = '%reset')
1955 self.set_hook('complete_command', reset_completer, str_key = '%reset')
1956
1956
1957 # Only configure readline if we truly are using readline. IPython can
1957 # Only configure readline if we truly are using readline. IPython can
1958 # do tab-completion over the network, in GUIs, etc, where readline
1958 # do tab-completion over the network, in GUIs, etc, where readline
1959 # itself may be absent
1959 # itself may be absent
1960 if self.has_readline:
1960 if self.has_readline:
1961 self.set_readline_completer()
1961 self.set_readline_completer()
1962
1962
1963 def complete(self, text, line=None, cursor_pos=None):
1963 def complete(self, text, line=None, cursor_pos=None):
1964 """Return the completed text and a list of completions.
1964 """Return the completed text and a list of completions.
1965
1965
1966 Parameters
1966 Parameters
1967 ----------
1967 ----------
1968
1968
1969 text : string
1969 text : string
1970 A string of text to be completed on. It can be given as empty and
1970 A string of text to be completed on. It can be given as empty and
1971 instead a line/position pair are given. In this case, the
1971 instead a line/position pair are given. In this case, the
1972 completer itself will split the line like readline does.
1972 completer itself will split the line like readline does.
1973
1973
1974 line : string, optional
1974 line : string, optional
1975 The complete line that text is part of.
1975 The complete line that text is part of.
1976
1976
1977 cursor_pos : int, optional
1977 cursor_pos : int, optional
1978 The position of the cursor on the input line.
1978 The position of the cursor on the input line.
1979
1979
1980 Returns
1980 Returns
1981 -------
1981 -------
1982 text : string
1982 text : string
1983 The actual text that was completed.
1983 The actual text that was completed.
1984
1984
1985 matches : list
1985 matches : list
1986 A sorted list with all possible completions.
1986 A sorted list with all possible completions.
1987
1987
1988 The optional arguments allow the completion to take more context into
1988 The optional arguments allow the completion to take more context into
1989 account, and are part of the low-level completion API.
1989 account, and are part of the low-level completion API.
1990
1990
1991 This is a wrapper around the completion mechanism, similar to what
1991 This is a wrapper around the completion mechanism, similar to what
1992 readline does at the command line when the TAB key is hit. By
1992 readline does at the command line when the TAB key is hit. By
1993 exposing it as a method, it can be used by other non-readline
1993 exposing it as a method, it can be used by other non-readline
1994 environments (such as GUIs) for text completion.
1994 environments (such as GUIs) for text completion.
1995
1995
1996 Simple usage example:
1996 Simple usage example:
1997
1997
1998 In [1]: x = 'hello'
1998 In [1]: x = 'hello'
1999
1999
2000 In [2]: _ip.complete('x.l')
2000 In [2]: _ip.complete('x.l')
2001 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
2001 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
2002 """
2002 """
2003
2003
2004 # Inject names into __builtin__ so we can complete on the added names.
2004 # Inject names into __builtin__ so we can complete on the added names.
2005 with self.builtin_trap:
2005 with self.builtin_trap:
2006 return self.Completer.complete(text, line, cursor_pos)
2006 return self.Completer.complete(text, line, cursor_pos)
2007
2007
2008 def set_custom_completer(self, completer, pos=0):
2008 def set_custom_completer(self, completer, pos=0):
2009 """Adds a new custom completer function.
2009 """Adds a new custom completer function.
2010
2010
2011 The position argument (defaults to 0) is the index in the completers
2011 The position argument (defaults to 0) is the index in the completers
2012 list where you want the completer to be inserted."""
2012 list where you want the completer to be inserted."""
2013
2013
2014 newcomp = types.MethodType(completer,self.Completer)
2014 newcomp = types.MethodType(completer,self.Completer)
2015 self.Completer.matchers.insert(pos,newcomp)
2015 self.Completer.matchers.insert(pos,newcomp)
2016
2016
2017 def set_readline_completer(self):
2017 def set_readline_completer(self):
2018 """Reset readline's completer to be our own."""
2018 """Reset readline's completer to be our own."""
2019 self.readline.set_completer(self.Completer.rlcomplete)
2019 self.readline.set_completer(self.Completer.rlcomplete)
2020
2020
2021 def set_completer_frame(self, frame=None):
2021 def set_completer_frame(self, frame=None):
2022 """Set the frame of the completer."""
2022 """Set the frame of the completer."""
2023 if frame:
2023 if frame:
2024 self.Completer.namespace = frame.f_locals
2024 self.Completer.namespace = frame.f_locals
2025 self.Completer.global_namespace = frame.f_globals
2025 self.Completer.global_namespace = frame.f_globals
2026 else:
2026 else:
2027 self.Completer.namespace = self.user_ns
2027 self.Completer.namespace = self.user_ns
2028 self.Completer.global_namespace = self.user_global_ns
2028 self.Completer.global_namespace = self.user_global_ns
2029
2029
2030 #-------------------------------------------------------------------------
2030 #-------------------------------------------------------------------------
2031 # Things related to magics
2031 # Things related to magics
2032 #-------------------------------------------------------------------------
2032 #-------------------------------------------------------------------------
2033
2033
2034 def init_magics(self):
2034 def init_magics(self):
2035 from IPython.core import magics as m
2035 from IPython.core import magics as m
2036 self.magics_manager = magic.MagicsManager(shell=self,
2036 self.magics_manager = magic.MagicsManager(shell=self,
2037 parent=self,
2037 parent=self,
2038 user_magics=m.UserMagics(self))
2038 user_magics=m.UserMagics(self))
2039 self.configurables.append(self.magics_manager)
2039 self.configurables.append(self.magics_manager)
2040
2040
2041 # Expose as public API from the magics manager
2041 # Expose as public API from the magics manager
2042 self.register_magics = self.magics_manager.register
2042 self.register_magics = self.magics_manager.register
2043 self.define_magic = self.magics_manager.define_magic
2043 self.define_magic = self.magics_manager.define_magic
2044
2044
2045 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2045 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2046 m.ConfigMagics, m.DeprecatedMagics, m.DisplayMagics, m.ExecutionMagics,
2046 m.ConfigMagics, m.DeprecatedMagics, m.DisplayMagics, m.ExecutionMagics,
2047 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2047 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2048 m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
2048 m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
2049 )
2049 )
2050
2050
2051 # Register Magic Aliases
2051 # Register Magic Aliases
2052 mman = self.magics_manager
2052 mman = self.magics_manager
2053 # FIXME: magic aliases should be defined by the Magics classes
2053 # FIXME: magic aliases should be defined by the Magics classes
2054 # or in MagicsManager, not here
2054 # or in MagicsManager, not here
2055 mman.register_alias('ed', 'edit')
2055 mman.register_alias('ed', 'edit')
2056 mman.register_alias('hist', 'history')
2056 mman.register_alias('hist', 'history')
2057 mman.register_alias('rep', 'recall')
2057 mman.register_alias('rep', 'recall')
2058 mman.register_alias('SVG', 'svg', 'cell')
2058 mman.register_alias('SVG', 'svg', 'cell')
2059 mman.register_alias('HTML', 'html', 'cell')
2059 mman.register_alias('HTML', 'html', 'cell')
2060 mman.register_alias('file', 'writefile', 'cell')
2060 mman.register_alias('file', 'writefile', 'cell')
2061
2061
2062 # FIXME: Move the color initialization to the DisplayHook, which
2062 # FIXME: Move the color initialization to the DisplayHook, which
2063 # should be split into a prompt manager and displayhook. We probably
2063 # should be split into a prompt manager and displayhook. We probably
2064 # even need a centralize colors management object.
2064 # even need a centralize colors management object.
2065 self.magic('colors %s' % self.colors)
2065 self.magic('colors %s' % self.colors)
2066
2066
2067 # Defined here so that it's included in the documentation
2067 # Defined here so that it's included in the documentation
2068 @functools.wraps(magic.MagicsManager.register_function)
2068 @functools.wraps(magic.MagicsManager.register_function)
2069 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2069 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2070 self.magics_manager.register_function(func,
2070 self.magics_manager.register_function(func,
2071 magic_kind=magic_kind, magic_name=magic_name)
2071 magic_kind=magic_kind, magic_name=magic_name)
2072
2072
2073 def run_line_magic(self, magic_name, line):
2073 def run_line_magic(self, magic_name, line):
2074 """Execute the given line magic.
2074 """Execute the given line magic.
2075
2075
2076 Parameters
2076 Parameters
2077 ----------
2077 ----------
2078 magic_name : str
2078 magic_name : str
2079 Name of the desired magic function, without '%' prefix.
2079 Name of the desired magic function, without '%' prefix.
2080
2080
2081 line : str
2081 line : str
2082 The rest of the input line as a single string.
2082 The rest of the input line as a single string.
2083 """
2083 """
2084 fn = self.find_line_magic(magic_name)
2084 fn = self.find_line_magic(magic_name)
2085 if fn is None:
2085 if fn is None:
2086 cm = self.find_cell_magic(magic_name)
2086 cm = self.find_cell_magic(magic_name)
2087 etpl = "Line magic function `%%%s` not found%s."
2087 etpl = "Line magic function `%%%s` not found%s."
2088 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2088 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2089 'did you mean that instead?)' % magic_name )
2089 'did you mean that instead?)' % magic_name )
2090 error(etpl % (magic_name, extra))
2090 error(etpl % (magic_name, extra))
2091 else:
2091 else:
2092 # Note: this is the distance in the stack to the user's frame.
2092 # Note: this is the distance in the stack to the user's frame.
2093 # This will need to be updated if the internal calling logic gets
2093 # This will need to be updated if the internal calling logic gets
2094 # refactored, or else we'll be expanding the wrong variables.
2094 # refactored, or else we'll be expanding the wrong variables.
2095 stack_depth = 2
2095 stack_depth = 2
2096 magic_arg_s = self.var_expand(line, stack_depth)
2096 magic_arg_s = self.var_expand(line, stack_depth)
2097 # Put magic args in a list so we can call with f(*a) syntax
2097 # Put magic args in a list so we can call with f(*a) syntax
2098 args = [magic_arg_s]
2098 args = [magic_arg_s]
2099 kwargs = {}
2099 kwargs = {}
2100 # Grab local namespace if we need it:
2100 # Grab local namespace if we need it:
2101 if getattr(fn, "needs_local_scope", False):
2101 if getattr(fn, "needs_local_scope", False):
2102 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2102 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2103 with self.builtin_trap:
2103 with self.builtin_trap:
2104 result = fn(*args,**kwargs)
2104 result = fn(*args,**kwargs)
2105 return result
2105 return result
2106
2106
2107 def run_cell_magic(self, magic_name, line, cell):
2107 def run_cell_magic(self, magic_name, line, cell):
2108 """Execute the given cell magic.
2108 """Execute the given cell magic.
2109
2109
2110 Parameters
2110 Parameters
2111 ----------
2111 ----------
2112 magic_name : str
2112 magic_name : str
2113 Name of the desired magic function, without '%' prefix.
2113 Name of the desired magic function, without '%' prefix.
2114
2114
2115 line : str
2115 line : str
2116 The rest of the first input line as a single string.
2116 The rest of the first input line as a single string.
2117
2117
2118 cell : str
2118 cell : str
2119 The body of the cell as a (possibly multiline) string.
2119 The body of the cell as a (possibly multiline) string.
2120 """
2120 """
2121 fn = self.find_cell_magic(magic_name)
2121 fn = self.find_cell_magic(magic_name)
2122 if fn is None:
2122 if fn is None:
2123 lm = self.find_line_magic(magic_name)
2123 lm = self.find_line_magic(magic_name)
2124 etpl = "Cell magic `%%{0}` not found{1}."
2124 etpl = "Cell magic `%%{0}` not found{1}."
2125 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2125 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2126 'did you mean that instead?)'.format(magic_name))
2126 'did you mean that instead?)'.format(magic_name))
2127 error(etpl.format(magic_name, extra))
2127 error(etpl.format(magic_name, extra))
2128 elif cell == '':
2128 elif cell == '':
2129 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2129 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2130 if self.find_line_magic(magic_name) is not None:
2130 if self.find_line_magic(magic_name) is not None:
2131 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2131 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2132 raise UsageError(message)
2132 raise UsageError(message)
2133 else:
2133 else:
2134 # Note: this is the distance in the stack to the user's frame.
2134 # Note: this is the distance in the stack to the user's frame.
2135 # This will need to be updated if the internal calling logic gets
2135 # This will need to be updated if the internal calling logic gets
2136 # refactored, or else we'll be expanding the wrong variables.
2136 # refactored, or else we'll be expanding the wrong variables.
2137 stack_depth = 2
2137 stack_depth = 2
2138 magic_arg_s = self.var_expand(line, stack_depth)
2138 magic_arg_s = self.var_expand(line, stack_depth)
2139 with self.builtin_trap:
2139 with self.builtin_trap:
2140 result = fn(magic_arg_s, cell)
2140 result = fn(magic_arg_s, cell)
2141 return result
2141 return result
2142
2142
2143 def find_line_magic(self, magic_name):
2143 def find_line_magic(self, magic_name):
2144 """Find and return a line magic by name.
2144 """Find and return a line magic by name.
2145
2145
2146 Returns None if the magic isn't found."""
2146 Returns None if the magic isn't found."""
2147 return self.magics_manager.magics['line'].get(magic_name)
2147 return self.magics_manager.magics['line'].get(magic_name)
2148
2148
2149 def find_cell_magic(self, magic_name):
2149 def find_cell_magic(self, magic_name):
2150 """Find and return a cell magic by name.
2150 """Find and return a cell magic by name.
2151
2151
2152 Returns None if the magic isn't found."""
2152 Returns None if the magic isn't found."""
2153 return self.magics_manager.magics['cell'].get(magic_name)
2153 return self.magics_manager.magics['cell'].get(magic_name)
2154
2154
2155 def find_magic(self, magic_name, magic_kind='line'):
2155 def find_magic(self, magic_name, magic_kind='line'):
2156 """Find and return a magic of the given type by name.
2156 """Find and return a magic of the given type by name.
2157
2157
2158 Returns None if the magic isn't found."""
2158 Returns None if the magic isn't found."""
2159 return self.magics_manager.magics[magic_kind].get(magic_name)
2159 return self.magics_manager.magics[magic_kind].get(magic_name)
2160
2160
2161 def magic(self, arg_s):
2161 def magic(self, arg_s):
2162 """DEPRECATED. Use run_line_magic() instead.
2162 """DEPRECATED. Use run_line_magic() instead.
2163
2163
2164 Call a magic function by name.
2164 Call a magic function by name.
2165
2165
2166 Input: a string containing the name of the magic function to call and
2166 Input: a string containing the name of the magic function to call and
2167 any additional arguments to be passed to the magic.
2167 any additional arguments to be passed to the magic.
2168
2168
2169 magic('name -opt foo bar') is equivalent to typing at the ipython
2169 magic('name -opt foo bar') is equivalent to typing at the ipython
2170 prompt:
2170 prompt:
2171
2171
2172 In[1]: %name -opt foo bar
2172 In[1]: %name -opt foo bar
2173
2173
2174 To call a magic without arguments, simply use magic('name').
2174 To call a magic without arguments, simply use magic('name').
2175
2175
2176 This provides a proper Python function to call IPython's magics in any
2176 This provides a proper Python function to call IPython's magics in any
2177 valid Python code you can type at the interpreter, including loops and
2177 valid Python code you can type at the interpreter, including loops and
2178 compound statements.
2178 compound statements.
2179 """
2179 """
2180 # TODO: should we issue a loud deprecation warning here?
2180 # TODO: should we issue a loud deprecation warning here?
2181 magic_name, _, magic_arg_s = arg_s.partition(' ')
2181 magic_name, _, magic_arg_s = arg_s.partition(' ')
2182 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2182 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2183 return self.run_line_magic(magic_name, magic_arg_s)
2183 return self.run_line_magic(magic_name, magic_arg_s)
2184
2184
2185 #-------------------------------------------------------------------------
2185 #-------------------------------------------------------------------------
2186 # Things related to macros
2186 # Things related to macros
2187 #-------------------------------------------------------------------------
2187 #-------------------------------------------------------------------------
2188
2188
2189 def define_macro(self, name, themacro):
2189 def define_macro(self, name, themacro):
2190 """Define a new macro
2190 """Define a new macro
2191
2191
2192 Parameters
2192 Parameters
2193 ----------
2193 ----------
2194 name : str
2194 name : str
2195 The name of the macro.
2195 The name of the macro.
2196 themacro : str or Macro
2196 themacro : str or Macro
2197 The action to do upon invoking the macro. If a string, a new
2197 The action to do upon invoking the macro. If a string, a new
2198 Macro object is created by passing the string to it.
2198 Macro object is created by passing the string to it.
2199 """
2199 """
2200
2200
2201 from IPython.core import macro
2201 from IPython.core import macro
2202
2202
2203 if isinstance(themacro, basestring):
2203 if isinstance(themacro, basestring):
2204 themacro = macro.Macro(themacro)
2204 themacro = macro.Macro(themacro)
2205 if not isinstance(themacro, macro.Macro):
2205 if not isinstance(themacro, macro.Macro):
2206 raise ValueError('A macro must be a string or a Macro instance.')
2206 raise ValueError('A macro must be a string or a Macro instance.')
2207 self.user_ns[name] = themacro
2207 self.user_ns[name] = themacro
2208
2208
2209 #-------------------------------------------------------------------------
2209 #-------------------------------------------------------------------------
2210 # Things related to the running of system commands
2210 # Things related to the running of system commands
2211 #-------------------------------------------------------------------------
2211 #-------------------------------------------------------------------------
2212
2212
2213 def system_piped(self, cmd):
2213 def system_piped(self, cmd):
2214 """Call the given cmd in a subprocess, piping stdout/err
2214 """Call the given cmd in a subprocess, piping stdout/err
2215
2215
2216 Parameters
2216 Parameters
2217 ----------
2217 ----------
2218 cmd : str
2218 cmd : str
2219 Command to execute (can not end in '&', as background processes are
2219 Command to execute (can not end in '&', as background processes are
2220 not supported. Should not be a command that expects input
2220 not supported. Should not be a command that expects input
2221 other than simple text.
2221 other than simple text.
2222 """
2222 """
2223 if cmd.rstrip().endswith('&'):
2223 if cmd.rstrip().endswith('&'):
2224 # this is *far* from a rigorous test
2224 # this is *far* from a rigorous test
2225 # We do not support backgrounding processes because we either use
2225 # We do not support backgrounding processes because we either use
2226 # pexpect or pipes to read from. Users can always just call
2226 # pexpect or pipes to read from. Users can always just call
2227 # os.system() or use ip.system=ip.system_raw
2227 # os.system() or use ip.system=ip.system_raw
2228 # if they really want a background process.
2228 # if they really want a background process.
2229 raise OSError("Background processes not supported.")
2229 raise OSError("Background processes not supported.")
2230
2230
2231 # we explicitly do NOT return the subprocess status code, because
2231 # we explicitly do NOT return the subprocess status code, because
2232 # a non-None value would trigger :func:`sys.displayhook` calls.
2232 # a non-None value would trigger :func:`sys.displayhook` calls.
2233 # Instead, we store the exit_code in user_ns.
2233 # Instead, we store the exit_code in user_ns.
2234 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2234 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2235
2235
2236 def system_raw(self, cmd):
2236 def system_raw(self, cmd):
2237 """Call the given cmd in a subprocess using os.system on Windows or
2237 """Call the given cmd in a subprocess using os.system on Windows or
2238 subprocess.call using the system shell on other platforms.
2238 subprocess.call using the system shell on other platforms.
2239
2239
2240 Parameters
2240 Parameters
2241 ----------
2241 ----------
2242 cmd : str
2242 cmd : str
2243 Command to execute.
2243 Command to execute.
2244 """
2244 """
2245 cmd = self.var_expand(cmd, depth=1)
2245 cmd = self.var_expand(cmd, depth=1)
2246 # protect os.system from UNC paths on Windows, which it can't handle:
2246 # protect os.system from UNC paths on Windows, which it can't handle:
2247 if sys.platform == 'win32':
2247 if sys.platform == 'win32':
2248 from IPython.utils._process_win32 import AvoidUNCPath
2248 from IPython.utils._process_win32 import AvoidUNCPath
2249 with AvoidUNCPath() as path:
2249 with AvoidUNCPath() as path:
2250 if path is not None:
2250 if path is not None:
2251 cmd = '"pushd %s &&"%s' % (path, cmd)
2251 cmd = '"pushd %s &&"%s' % (path, cmd)
2252 cmd = py3compat.unicode_to_str(cmd)
2252 cmd = py3compat.unicode_to_str(cmd)
2253 ec = os.system(cmd)
2253 ec = os.system(cmd)
2254 else:
2254 else:
2255 cmd = py3compat.unicode_to_str(cmd)
2255 cmd = py3compat.unicode_to_str(cmd)
2256 # Call the cmd using the OS shell, instead of the default /bin/sh, if set.
2256 # Call the cmd using the OS shell, instead of the default /bin/sh, if set.
2257 ec = subprocess.call(cmd, shell=True, executable=os.environ.get('SHELL', None))
2257 ec = subprocess.call(cmd, shell=True, executable=os.environ.get('SHELL', None))
2258 # exit code is positive for program failure, or negative for
2258 # exit code is positive for program failure, or negative for
2259 # terminating signal number.
2259 # terminating signal number.
2260
2260
2261 # We explicitly do NOT return the subprocess status code, because
2261 # We explicitly do NOT return the subprocess status code, because
2262 # a non-None value would trigger :func:`sys.displayhook` calls.
2262 # a non-None value would trigger :func:`sys.displayhook` calls.
2263 # Instead, we store the exit_code in user_ns.
2263 # Instead, we store the exit_code in user_ns.
2264 self.user_ns['_exit_code'] = ec
2264 self.user_ns['_exit_code'] = ec
2265
2265
2266 # use piped system by default, because it is better behaved
2266 # use piped system by default, because it is better behaved
2267 system = system_piped
2267 system = system_piped
2268
2268
2269 def getoutput(self, cmd, split=True, depth=0):
2269 def getoutput(self, cmd, split=True, depth=0):
2270 """Get output (possibly including stderr) from a subprocess.
2270 """Get output (possibly including stderr) from a subprocess.
2271
2271
2272 Parameters
2272 Parameters
2273 ----------
2273 ----------
2274 cmd : str
2274 cmd : str
2275 Command to execute (can not end in '&', as background processes are
2275 Command to execute (can not end in '&', as background processes are
2276 not supported.
2276 not supported.
2277 split : bool, optional
2277 split : bool, optional
2278 If True, split the output into an IPython SList. Otherwise, an
2278 If True, split the output into an IPython SList. Otherwise, an
2279 IPython LSString is returned. These are objects similar to normal
2279 IPython LSString is returned. These are objects similar to normal
2280 lists and strings, with a few convenience attributes for easier
2280 lists and strings, with a few convenience attributes for easier
2281 manipulation of line-based output. You can use '?' on them for
2281 manipulation of line-based output. You can use '?' on them for
2282 details.
2282 details.
2283 depth : int, optional
2283 depth : int, optional
2284 How many frames above the caller are the local variables which should
2284 How many frames above the caller are the local variables which should
2285 be expanded in the command string? The default (0) assumes that the
2285 be expanded in the command string? The default (0) assumes that the
2286 expansion variables are in the stack frame calling this function.
2286 expansion variables are in the stack frame calling this function.
2287 """
2287 """
2288 if cmd.rstrip().endswith('&'):
2288 if cmd.rstrip().endswith('&'):
2289 # this is *far* from a rigorous test
2289 # this is *far* from a rigorous test
2290 raise OSError("Background processes not supported.")
2290 raise OSError("Background processes not supported.")
2291 out = getoutput(self.var_expand(cmd, depth=depth+1))
2291 out = getoutput(self.var_expand(cmd, depth=depth+1))
2292 if split:
2292 if split:
2293 out = SList(out.splitlines())
2293 out = SList(out.splitlines())
2294 else:
2294 else:
2295 out = LSString(out)
2295 out = LSString(out)
2296 return out
2296 return out
2297
2297
2298 #-------------------------------------------------------------------------
2298 #-------------------------------------------------------------------------
2299 # Things related to aliases
2299 # Things related to aliases
2300 #-------------------------------------------------------------------------
2300 #-------------------------------------------------------------------------
2301
2301
2302 def init_alias(self):
2302 def init_alias(self):
2303 self.alias_manager = AliasManager(shell=self, parent=self)
2303 self.alias_manager = AliasManager(shell=self, parent=self)
2304 self.configurables.append(self.alias_manager)
2304 self.configurables.append(self.alias_manager)
2305
2305
2306 #-------------------------------------------------------------------------
2306 #-------------------------------------------------------------------------
2307 # Things related to extensions
2307 # Things related to extensions
2308 #-------------------------------------------------------------------------
2308 #-------------------------------------------------------------------------
2309
2309
2310 def init_extension_manager(self):
2310 def init_extension_manager(self):
2311 self.extension_manager = ExtensionManager(shell=self, parent=self)
2311 self.extension_manager = ExtensionManager(shell=self, parent=self)
2312 self.configurables.append(self.extension_manager)
2312 self.configurables.append(self.extension_manager)
2313
2313
2314 #-------------------------------------------------------------------------
2314 #-------------------------------------------------------------------------
2315 # Things related to payloads
2315 # Things related to payloads
2316 #-------------------------------------------------------------------------
2316 #-------------------------------------------------------------------------
2317
2317
2318 def init_payload(self):
2318 def init_payload(self):
2319 self.payload_manager = PayloadManager(parent=self)
2319 self.payload_manager = PayloadManager(parent=self)
2320 self.configurables.append(self.payload_manager)
2320 self.configurables.append(self.payload_manager)
2321
2321
2322 #-------------------------------------------------------------------------
2322 #-------------------------------------------------------------------------
2323 # Things related to widgets
2323 # Things related to widgets
2324 #-------------------------------------------------------------------------
2324 #-------------------------------------------------------------------------
2325
2325
2326 def init_comms(self):
2326 def init_comms(self):
2327 # not implemented in the base class
2327 # not implemented in the base class
2328 pass
2328 pass
2329
2329
2330 #-------------------------------------------------------------------------
2330 #-------------------------------------------------------------------------
2331 # Things related to the prefilter
2331 # Things related to the prefilter
2332 #-------------------------------------------------------------------------
2332 #-------------------------------------------------------------------------
2333
2333
2334 def init_prefilter(self):
2334 def init_prefilter(self):
2335 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2335 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2336 self.configurables.append(self.prefilter_manager)
2336 self.configurables.append(self.prefilter_manager)
2337 # Ultimately this will be refactored in the new interpreter code, but
2337 # Ultimately this will be refactored in the new interpreter code, but
2338 # for now, we should expose the main prefilter method (there's legacy
2338 # for now, we should expose the main prefilter method (there's legacy
2339 # code out there that may rely on this).
2339 # code out there that may rely on this).
2340 self.prefilter = self.prefilter_manager.prefilter_lines
2340 self.prefilter = self.prefilter_manager.prefilter_lines
2341
2341
2342 def auto_rewrite_input(self, cmd):
2342 def auto_rewrite_input(self, cmd):
2343 """Print to the screen the rewritten form of the user's command.
2343 """Print to the screen the rewritten form of the user's command.
2344
2344
2345 This shows visual feedback by rewriting input lines that cause
2345 This shows visual feedback by rewriting input lines that cause
2346 automatic calling to kick in, like::
2346 automatic calling to kick in, like::
2347
2347
2348 /f x
2348 /f x
2349
2349
2350 into::
2350 into::
2351
2351
2352 ------> f(x)
2352 ------> f(x)
2353
2353
2354 after the user's input prompt. This helps the user understand that the
2354 after the user's input prompt. This helps the user understand that the
2355 input line was transformed automatically by IPython.
2355 input line was transformed automatically by IPython.
2356 """
2356 """
2357 if not self.show_rewritten_input:
2357 if not self.show_rewritten_input:
2358 return
2358 return
2359
2359
2360 rw = self.prompt_manager.render('rewrite') + cmd
2360 rw = self.prompt_manager.render('rewrite') + cmd
2361
2361
2362 try:
2362 try:
2363 # plain ascii works better w/ pyreadline, on some machines, so
2363 # plain ascii works better w/ pyreadline, on some machines, so
2364 # we use it and only print uncolored rewrite if we have unicode
2364 # we use it and only print uncolored rewrite if we have unicode
2365 rw = str(rw)
2365 rw = str(rw)
2366 print(rw, file=io.stdout)
2366 print(rw, file=io.stdout)
2367 except UnicodeEncodeError:
2367 except UnicodeEncodeError:
2368 print("------> " + cmd)
2368 print("------> " + cmd)
2369
2369
2370 #-------------------------------------------------------------------------
2370 #-------------------------------------------------------------------------
2371 # Things related to extracting values/expressions from kernel and user_ns
2371 # Things related to extracting values/expressions from kernel and user_ns
2372 #-------------------------------------------------------------------------
2372 #-------------------------------------------------------------------------
2373
2373
2374 def _user_obj_error(self):
2374 def _user_obj_error(self):
2375 """return simple exception dict
2375 """return simple exception dict
2376
2376
2377 for use in user_variables / expressions
2377 for use in user_variables / expressions
2378 """
2378 """
2379
2379
2380 etype, evalue, tb = self._get_exc_info()
2380 etype, evalue, tb = self._get_exc_info()
2381 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2381 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2382
2382
2383 exc_info = {
2383 exc_info = {
2384 u'status' : 'error',
2384 u'status' : 'error',
2385 u'traceback' : stb,
2385 u'traceback' : stb,
2386 u'ename' : unicode(etype.__name__),
2386 u'ename' : unicode(etype.__name__),
2387 u'evalue' : py3compat.safe_unicode(evalue),
2387 u'evalue' : py3compat.safe_unicode(evalue),
2388 }
2388 }
2389
2389
2390 return exc_info
2390 return exc_info
2391
2391
2392 def _format_user_obj(self, obj):
2392 def _format_user_obj(self, obj):
2393 """format a user object to display dict
2393 """format a user object to display dict
2394
2394
2395 for use in user_expressions / variables
2395 for use in user_expressions / variables
2396 """
2396 """
2397
2397
2398 data, md = self.display_formatter.format(obj)
2398 data, md = self.display_formatter.format(obj)
2399 value = {
2399 value = {
2400 'status' : 'ok',
2400 'status' : 'ok',
2401 'data' : data,
2401 'data' : data,
2402 'metadata' : md,
2402 'metadata' : md,
2403 }
2403 }
2404 return value
2404 return value
2405
2405
2406 def user_variables(self, names):
2406 def user_variables(self, names):
2407 """Get a list of variable names from the user's namespace.
2407 """Get a list of variable names from the user's namespace.
2408
2408
2409 Parameters
2409 Parameters
2410 ----------
2410 ----------
2411 names : list of strings
2411 names : list of strings
2412 A list of names of variables to be read from the user namespace.
2412 A list of names of variables to be read from the user namespace.
2413
2413
2414 Returns
2414 Returns
2415 -------
2415 -------
2416 A dict, keyed by the input names and with the rich mime-type repr(s) of each value.
2416 A dict, keyed by the input names and with the rich mime-type repr(s) of each value.
2417 Each element will be a sub-dict of the same form as a display_data message.
2417 Each element will be a sub-dict of the same form as a display_data message.
2418 """
2418 """
2419 out = {}
2419 out = {}
2420 user_ns = self.user_ns
2420 user_ns = self.user_ns
2421
2421
2422 for varname in names:
2422 for varname in names:
2423 try:
2423 try:
2424 value = self._format_user_obj(user_ns[varname])
2424 value = self._format_user_obj(user_ns[varname])
2425 except:
2425 except:
2426 value = self._user_obj_error()
2426 value = self._user_obj_error()
2427 out[varname] = value
2427 out[varname] = value
2428 return out
2428 return out
2429
2429
2430 def user_expressions(self, expressions):
2430 def user_expressions(self, expressions):
2431 """Evaluate a dict of expressions in the user's namespace.
2431 """Evaluate a dict of expressions in the user's namespace.
2432
2432
2433 Parameters
2433 Parameters
2434 ----------
2434 ----------
2435 expressions : dict
2435 expressions : dict
2436 A dict with string keys and string values. The expression values
2436 A dict with string keys and string values. The expression values
2437 should be valid Python expressions, each of which will be evaluated
2437 should be valid Python expressions, each of which will be evaluated
2438 in the user namespace.
2438 in the user namespace.
2439
2439
2440 Returns
2440 Returns
2441 -------
2441 -------
2442 A dict, keyed like the input expressions dict, with the rich mime-typed
2442 A dict, keyed like the input expressions dict, with the rich mime-typed
2443 display_data of each value.
2443 display_data of each value.
2444 """
2444 """
2445 out = {}
2445 out = {}
2446 user_ns = self.user_ns
2446 user_ns = self.user_ns
2447 global_ns = self.user_global_ns
2447 global_ns = self.user_global_ns
2448
2448
2449 for key, expr in expressions.iteritems():
2449 for key, expr in expressions.iteritems():
2450 try:
2450 try:
2451 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2451 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2452 except:
2452 except:
2453 value = self._user_obj_error()
2453 value = self._user_obj_error()
2454 out[key] = value
2454 out[key] = value
2455 return out
2455 return out
2456
2456
2457 #-------------------------------------------------------------------------
2457 #-------------------------------------------------------------------------
2458 # Things related to the running of code
2458 # Things related to the running of code
2459 #-------------------------------------------------------------------------
2459 #-------------------------------------------------------------------------
2460
2460
2461 def ex(self, cmd):
2461 def ex(self, cmd):
2462 """Execute a normal python statement in user namespace."""
2462 """Execute a normal python statement in user namespace."""
2463 with self.builtin_trap:
2463 with self.builtin_trap:
2464 exec(cmd, self.user_global_ns, self.user_ns)
2464 exec(cmd, self.user_global_ns, self.user_ns)
2465
2465
2466 def ev(self, expr):
2466 def ev(self, expr):
2467 """Evaluate python expression expr in user namespace.
2467 """Evaluate python expression expr in user namespace.
2468
2468
2469 Returns the result of evaluation
2469 Returns the result of evaluation
2470 """
2470 """
2471 with self.builtin_trap:
2471 with self.builtin_trap:
2472 return eval(expr, self.user_global_ns, self.user_ns)
2472 return eval(expr, self.user_global_ns, self.user_ns)
2473
2473
2474 def safe_execfile(self, fname, *where, **kw):
2474 def safe_execfile(self, fname, *where, **kw):
2475 """A safe version of the builtin execfile().
2475 """A safe version of the builtin execfile().
2476
2476
2477 This version will never throw an exception, but instead print
2477 This version will never throw an exception, but instead print
2478 helpful error messages to the screen. This only works on pure
2478 helpful error messages to the screen. This only works on pure
2479 Python files with the .py extension.
2479 Python files with the .py extension.
2480
2480
2481 Parameters
2481 Parameters
2482 ----------
2482 ----------
2483 fname : string
2483 fname : string
2484 The name of the file to be executed.
2484 The name of the file to be executed.
2485 where : tuple
2485 where : tuple
2486 One or two namespaces, passed to execfile() as (globals,locals).
2486 One or two namespaces, passed to execfile() as (globals,locals).
2487 If only one is given, it is passed as both.
2487 If only one is given, it is passed as both.
2488 exit_ignore : bool (False)
2488 exit_ignore : bool (False)
2489 If True, then silence SystemExit for non-zero status (it is always
2489 If True, then silence SystemExit for non-zero status (it is always
2490 silenced for zero status, as it is so common).
2490 silenced for zero status, as it is so common).
2491 raise_exceptions : bool (False)
2491 raise_exceptions : bool (False)
2492 If True raise exceptions everywhere. Meant for testing.
2492 If True raise exceptions everywhere. Meant for testing.
2493
2493
2494 """
2494 """
2495 kw.setdefault('exit_ignore', False)
2495 kw.setdefault('exit_ignore', False)
2496 kw.setdefault('raise_exceptions', False)
2496 kw.setdefault('raise_exceptions', False)
2497
2497
2498 fname = os.path.abspath(os.path.expanduser(fname))
2498 fname = os.path.abspath(os.path.expanduser(fname))
2499
2499
2500 # Make sure we can open the file
2500 # Make sure we can open the file
2501 try:
2501 try:
2502 with open(fname) as thefile:
2502 with open(fname) as thefile:
2503 pass
2503 pass
2504 except:
2504 except:
2505 warn('Could not open file <%s> for safe execution.' % fname)
2505 warn('Could not open file <%s> for safe execution.' % fname)
2506 return
2506 return
2507
2507
2508 # Find things also in current directory. This is needed to mimic the
2508 # Find things also in current directory. This is needed to mimic the
2509 # behavior of running a script from the system command line, where
2509 # behavior of running a script from the system command line, where
2510 # Python inserts the script's directory into sys.path
2510 # Python inserts the script's directory into sys.path
2511 dname = os.path.dirname(fname)
2511 dname = os.path.dirname(fname)
2512
2512
2513 with prepended_to_syspath(dname):
2513 with prepended_to_syspath(dname):
2514 try:
2514 try:
2515 py3compat.execfile(fname,*where)
2515 py3compat.execfile(fname,*where)
2516 except SystemExit as status:
2516 except SystemExit as status:
2517 # If the call was made with 0 or None exit status (sys.exit(0)
2517 # If the call was made with 0 or None exit status (sys.exit(0)
2518 # or sys.exit() ), don't bother showing a traceback, as both of
2518 # or sys.exit() ), don't bother showing a traceback, as both of
2519 # these are considered normal by the OS:
2519 # these are considered normal by the OS:
2520 # > python -c'import sys;sys.exit(0)'; echo $?
2520 # > python -c'import sys;sys.exit(0)'; echo $?
2521 # 0
2521 # 0
2522 # > python -c'import sys;sys.exit()'; echo $?
2522 # > python -c'import sys;sys.exit()'; echo $?
2523 # 0
2523 # 0
2524 # For other exit status, we show the exception unless
2524 # For other exit status, we show the exception unless
2525 # explicitly silenced, but only in short form.
2525 # explicitly silenced, but only in short form.
2526 if kw['raise_exceptions']:
2526 if kw['raise_exceptions']:
2527 raise
2527 raise
2528 if status.code and not kw['exit_ignore']:
2528 if status.code and not kw['exit_ignore']:
2529 self.showtraceback(exception_only=True)
2529 self.showtraceback(exception_only=True)
2530 except:
2530 except:
2531 if kw['raise_exceptions']:
2531 if kw['raise_exceptions']:
2532 raise
2532 raise
2533 self.showtraceback()
2533 self.showtraceback()
2534
2534
2535 def safe_execfile_ipy(self, fname):
2535 def safe_execfile_ipy(self, fname):
2536 """Like safe_execfile, but for .ipy files with IPython syntax.
2536 """Like safe_execfile, but for .ipy files with IPython syntax.
2537
2537
2538 Parameters
2538 Parameters
2539 ----------
2539 ----------
2540 fname : str
2540 fname : str
2541 The name of the file to execute. The filename must have a
2541 The name of the file to execute. The filename must have a
2542 .ipy extension.
2542 .ipy extension.
2543 """
2543 """
2544 fname = os.path.abspath(os.path.expanduser(fname))
2544 fname = os.path.abspath(os.path.expanduser(fname))
2545
2545
2546 # Make sure we can open the file
2546 # Make sure we can open the file
2547 try:
2547 try:
2548 with open(fname) as thefile:
2548 with open(fname) as thefile:
2549 pass
2549 pass
2550 except:
2550 except:
2551 warn('Could not open file <%s> for safe execution.' % fname)
2551 warn('Could not open file <%s> for safe execution.' % fname)
2552 return
2552 return
2553
2553
2554 # Find things also in current directory. This is needed to mimic the
2554 # Find things also in current directory. This is needed to mimic the
2555 # behavior of running a script from the system command line, where
2555 # behavior of running a script from the system command line, where
2556 # Python inserts the script's directory into sys.path
2556 # Python inserts the script's directory into sys.path
2557 dname = os.path.dirname(fname)
2557 dname = os.path.dirname(fname)
2558
2558
2559 with prepended_to_syspath(dname):
2559 with prepended_to_syspath(dname):
2560 try:
2560 try:
2561 with open(fname) as thefile:
2561 with open(fname) as thefile:
2562 # self.run_cell currently captures all exceptions
2562 # self.run_cell currently captures all exceptions
2563 # raised in user code. It would be nice if there were
2563 # raised in user code. It would be nice if there were
2564 # versions of runlines, execfile that did raise, so
2564 # versions of runlines, execfile that did raise, so
2565 # we could catch the errors.
2565 # we could catch the errors.
2566 self.run_cell(thefile.read(), store_history=False, shell_futures=False)
2566 self.run_cell(thefile.read(), store_history=False, shell_futures=False)
2567 except:
2567 except:
2568 self.showtraceback()
2568 self.showtraceback()
2569 warn('Unknown failure executing file: <%s>' % fname)
2569 warn('Unknown failure executing file: <%s>' % fname)
2570
2570
2571 def safe_run_module(self, mod_name, where):
2571 def safe_run_module(self, mod_name, where):
2572 """A safe version of runpy.run_module().
2572 """A safe version of runpy.run_module().
2573
2573
2574 This version will never throw an exception, but instead print
2574 This version will never throw an exception, but instead print
2575 helpful error messages to the screen.
2575 helpful error messages to the screen.
2576
2576
2577 `SystemExit` exceptions with status code 0 or None are ignored.
2577 `SystemExit` exceptions with status code 0 or None are ignored.
2578
2578
2579 Parameters
2579 Parameters
2580 ----------
2580 ----------
2581 mod_name : string
2581 mod_name : string
2582 The name of the module to be executed.
2582 The name of the module to be executed.
2583 where : dict
2583 where : dict
2584 The globals namespace.
2584 The globals namespace.
2585 """
2585 """
2586 try:
2586 try:
2587 try:
2587 try:
2588 where.update(
2588 where.update(
2589 runpy.run_module(str(mod_name), run_name="__main__",
2589 runpy.run_module(str(mod_name), run_name="__main__",
2590 alter_sys=True)
2590 alter_sys=True)
2591 )
2591 )
2592 except SystemExit as status:
2592 except SystemExit as status:
2593 if status.code:
2593 if status.code:
2594 raise
2594 raise
2595 except:
2595 except:
2596 self.showtraceback()
2596 self.showtraceback()
2597 warn('Unknown failure executing module: <%s>' % mod_name)
2597 warn('Unknown failure executing module: <%s>' % mod_name)
2598
2598
2599 def _run_cached_cell_magic(self, magic_name, line):
2599 def _run_cached_cell_magic(self, magic_name, line):
2600 """Special method to call a cell magic with the data stored in self.
2600 """Special method to call a cell magic with the data stored in self.
2601 """
2601 """
2602 cell = self._current_cell_magic_body
2602 cell = self._current_cell_magic_body
2603 self._current_cell_magic_body = None
2603 self._current_cell_magic_body = None
2604 return self.run_cell_magic(magic_name, line, cell)
2604 return self.run_cell_magic(magic_name, line, cell)
2605
2605
2606 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2606 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2607 """Run a complete IPython cell.
2607 """Run a complete IPython cell.
2608
2608
2609 Parameters
2609 Parameters
2610 ----------
2610 ----------
2611 raw_cell : str
2611 raw_cell : str
2612 The code (including IPython code such as %magic functions) to run.
2612 The code (including IPython code such as %magic functions) to run.
2613 store_history : bool
2613 store_history : bool
2614 If True, the raw and translated cell will be stored in IPython's
2614 If True, the raw and translated cell will be stored in IPython's
2615 history. For user code calling back into IPython's machinery, this
2615 history. For user code calling back into IPython's machinery, this
2616 should be set to False.
2616 should be set to False.
2617 silent : bool
2617 silent : bool
2618 If True, avoid side-effects, such as implicit displayhooks and
2618 If True, avoid side-effects, such as implicit displayhooks and
2619 and logging. silent=True forces store_history=False.
2619 and logging. silent=True forces store_history=False.
2620 shell_futures : bool
2620 shell_futures : bool
2621 If True, the code will share future statements with the interactive
2621 If True, the code will share future statements with the interactive
2622 shell. It will both be affected by previous __future__ imports, and
2622 shell. It will both be affected by previous __future__ imports, and
2623 any __future__ imports in the code will affect the shell. If False,
2623 any __future__ imports in the code will affect the shell. If False,
2624 __future__ imports are not shared in either direction.
2624 __future__ imports are not shared in either direction.
2625 """
2625 """
2626 if (not raw_cell) or raw_cell.isspace():
2626 if (not raw_cell) or raw_cell.isspace():
2627 return
2627 return
2628
2628
2629 if silent:
2629 if silent:
2630 store_history = False
2630 store_history = False
2631
2631
2632 self.input_transformer_manager.push(raw_cell)
2632 self.input_transformer_manager.push(raw_cell)
2633 cell = self.input_transformer_manager.source_reset()
2633 cell = self.input_transformer_manager.source_reset()
2634
2634
2635 # Our own compiler remembers the __future__ environment. If we want to
2635 # Our own compiler remembers the __future__ environment. If we want to
2636 # run code with a separate __future__ environment, use the default
2636 # run code with a separate __future__ environment, use the default
2637 # compiler
2637 # compiler
2638 compiler = self.compile if shell_futures else CachingCompiler()
2638 compiler = self.compile if shell_futures else CachingCompiler()
2639
2639
2640 with self.builtin_trap:
2640 with self.builtin_trap:
2641 prefilter_failed = False
2641 prefilter_failed = False
2642 if len(cell.splitlines()) == 1:
2642 if len(cell.splitlines()) == 1:
2643 try:
2643 try:
2644 # use prefilter_lines to handle trailing newlines
2644 # use prefilter_lines to handle trailing newlines
2645 # restore trailing newline for ast.parse
2645 # restore trailing newline for ast.parse
2646 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2646 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2647 except AliasError as e:
2647 except AliasError as e:
2648 error(e)
2648 error(e)
2649 prefilter_failed = True
2649 prefilter_failed = True
2650 except Exception:
2650 except Exception:
2651 # don't allow prefilter errors to crash IPython
2651 # don't allow prefilter errors to crash IPython
2652 self.showtraceback()
2652 self.showtraceback()
2653 prefilter_failed = True
2653 prefilter_failed = True
2654
2654
2655 # Store raw and processed history
2655 # Store raw and processed history
2656 if store_history:
2656 if store_history:
2657 self.history_manager.store_inputs(self.execution_count,
2657 self.history_manager.store_inputs(self.execution_count,
2658 cell, raw_cell)
2658 cell, raw_cell)
2659 if not silent:
2659 if not silent:
2660 self.logger.log(cell, raw_cell)
2660 self.logger.log(cell, raw_cell)
2661
2661
2662 if not prefilter_failed:
2662 if not prefilter_failed:
2663 # don't run if prefilter failed
2663 # don't run if prefilter failed
2664 cell_name = self.compile.cache(cell, self.execution_count)
2664 cell_name = self.compile.cache(cell, self.execution_count)
2665
2665
2666 with self.display_trap:
2666 with self.display_trap:
2667 try:
2667 try:
2668 code_ast = compiler.ast_parse(cell, filename=cell_name)
2668 code_ast = compiler.ast_parse(cell, filename=cell_name)
2669 except IndentationError:
2669 except IndentationError:
2670 self.showindentationerror()
2670 self.showindentationerror()
2671 if store_history:
2671 if store_history:
2672 self.execution_count += 1
2672 self.execution_count += 1
2673 return None
2673 return None
2674 except (OverflowError, SyntaxError, ValueError, TypeError,
2674 except (OverflowError, SyntaxError, ValueError, TypeError,
2675 MemoryError):
2675 MemoryError):
2676 self.showsyntaxerror()
2676 self.showsyntaxerror()
2677 if store_history:
2677 if store_history:
2678 self.execution_count += 1
2678 self.execution_count += 1
2679 return None
2679 return None
2680
2680
2681 code_ast = self.transform_ast(code_ast)
2681 code_ast = self.transform_ast(code_ast)
2682
2682
2683 interactivity = "none" if silent else self.ast_node_interactivity
2683 interactivity = "none" if silent else self.ast_node_interactivity
2684 self.run_ast_nodes(code_ast.body, cell_name,
2684 self.run_ast_nodes(code_ast.body, cell_name,
2685 interactivity=interactivity, compiler=compiler)
2685 interactivity=interactivity, compiler=compiler)
2686
2686
2687 # Execute any registered post-execution functions.
2687 # Execute any registered post-execution functions.
2688 # unless we are silent
2688 # unless we are silent
2689 post_exec = [] if silent else self._post_execute.iteritems()
2689 post_exec = [] if silent else self._post_execute.iteritems()
2690
2690
2691 for func, status in post_exec:
2691 for func, status in post_exec:
2692 if self.disable_failing_post_execute and not status:
2692 if self.disable_failing_post_execute and not status:
2693 continue
2693 continue
2694 try:
2694 try:
2695 func()
2695 func()
2696 except KeyboardInterrupt:
2696 except KeyboardInterrupt:
2697 print("\nKeyboardInterrupt", file=io.stderr)
2697 print("\nKeyboardInterrupt", file=io.stderr)
2698 except Exception:
2698 except Exception:
2699 # register as failing:
2699 # register as failing:
2700 self._post_execute[func] = False
2700 self._post_execute[func] = False
2701 self.showtraceback()
2701 self.showtraceback()
2702 print('\n'.join([
2702 print('\n'.join([
2703 "post-execution function %r produced an error." % func,
2703 "post-execution function %r produced an error." % func,
2704 "If this problem persists, you can disable failing post-exec functions with:",
2704 "If this problem persists, you can disable failing post-exec functions with:",
2705 "",
2705 "",
2706 " get_ipython().disable_failing_post_execute = True"
2706 " get_ipython().disable_failing_post_execute = True"
2707 ]), file=io.stderr)
2707 ]), file=io.stderr)
2708
2708
2709 if store_history:
2709 if store_history:
2710 # Write output to the database. Does nothing unless
2710 # Write output to the database. Does nothing unless
2711 # history output logging is enabled.
2711 # history output logging is enabled.
2712 self.history_manager.store_output(self.execution_count)
2712 self.history_manager.store_output(self.execution_count)
2713 # Each cell is a *single* input, regardless of how many lines it has
2713 # Each cell is a *single* input, regardless of how many lines it has
2714 self.execution_count += 1
2714 self.execution_count += 1
2715
2715
2716 def transform_ast(self, node):
2716 def transform_ast(self, node):
2717 """Apply the AST transformations from self.ast_transformers
2717 """Apply the AST transformations from self.ast_transformers
2718
2718
2719 Parameters
2719 Parameters
2720 ----------
2720 ----------
2721 node : ast.Node
2721 node : ast.Node
2722 The root node to be transformed. Typically called with the ast.Module
2722 The root node to be transformed. Typically called with the ast.Module
2723 produced by parsing user input.
2723 produced by parsing user input.
2724
2724
2725 Returns
2725 Returns
2726 -------
2726 -------
2727 An ast.Node corresponding to the node it was called with. Note that it
2727 An ast.Node corresponding to the node it was called with. Note that it
2728 may also modify the passed object, so don't rely on references to the
2728 may also modify the passed object, so don't rely on references to the
2729 original AST.
2729 original AST.
2730 """
2730 """
2731 for transformer in self.ast_transformers:
2731 for transformer in self.ast_transformers:
2732 try:
2732 try:
2733 node = transformer.visit(node)
2733 node = transformer.visit(node)
2734 except Exception:
2734 except Exception:
2735 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
2735 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
2736 self.ast_transformers.remove(transformer)
2736 self.ast_transformers.remove(transformer)
2737
2737
2738 if self.ast_transformers:
2738 if self.ast_transformers:
2739 ast.fix_missing_locations(node)
2739 ast.fix_missing_locations(node)
2740 return node
2740 return node
2741
2741
2742
2742
2743 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
2743 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
2744 compiler=compile):
2744 compiler=compile):
2745 """Run a sequence of AST nodes. The execution mode depends on the
2745 """Run a sequence of AST nodes. The execution mode depends on the
2746 interactivity parameter.
2746 interactivity parameter.
2747
2747
2748 Parameters
2748 Parameters
2749 ----------
2749 ----------
2750 nodelist : list
2750 nodelist : list
2751 A sequence of AST nodes to run.
2751 A sequence of AST nodes to run.
2752 cell_name : str
2752 cell_name : str
2753 Will be passed to the compiler as the filename of the cell. Typically
2753 Will be passed to the compiler as the filename of the cell. Typically
2754 the value returned by ip.compile.cache(cell).
2754 the value returned by ip.compile.cache(cell).
2755 interactivity : str
2755 interactivity : str
2756 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2756 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2757 run interactively (displaying output from expressions). 'last_expr'
2757 run interactively (displaying output from expressions). 'last_expr'
2758 will run the last node interactively only if it is an expression (i.e.
2758 will run the last node interactively only if it is an expression (i.e.
2759 expressions in loops or other blocks are not displayed. Other values
2759 expressions in loops or other blocks are not displayed. Other values
2760 for this parameter will raise a ValueError.
2760 for this parameter will raise a ValueError.
2761 compiler : callable
2761 compiler : callable
2762 A function with the same interface as the built-in compile(), to turn
2762 A function with the same interface as the built-in compile(), to turn
2763 the AST nodes into code objects. Default is the built-in compile().
2763 the AST nodes into code objects. Default is the built-in compile().
2764 """
2764 """
2765 if not nodelist:
2765 if not nodelist:
2766 return
2766 return
2767
2767
2768 if interactivity == 'last_expr':
2768 if interactivity == 'last_expr':
2769 if isinstance(nodelist[-1], ast.Expr):
2769 if isinstance(nodelist[-1], ast.Expr):
2770 interactivity = "last"
2770 interactivity = "last"
2771 else:
2771 else:
2772 interactivity = "none"
2772 interactivity = "none"
2773
2773
2774 if interactivity == 'none':
2774 if interactivity == 'none':
2775 to_run_exec, to_run_interactive = nodelist, []
2775 to_run_exec, to_run_interactive = nodelist, []
2776 elif interactivity == 'last':
2776 elif interactivity == 'last':
2777 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2777 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2778 elif interactivity == 'all':
2778 elif interactivity == 'all':
2779 to_run_exec, to_run_interactive = [], nodelist
2779 to_run_exec, to_run_interactive = [], nodelist
2780 else:
2780 else:
2781 raise ValueError("Interactivity was %r" % interactivity)
2781 raise ValueError("Interactivity was %r" % interactivity)
2782
2782
2783 exec_count = self.execution_count
2783 exec_count = self.execution_count
2784
2784
2785 try:
2785 try:
2786 for i, node in enumerate(to_run_exec):
2786 for i, node in enumerate(to_run_exec):
2787 mod = ast.Module([node])
2787 mod = ast.Module([node])
2788 code = compiler(mod, cell_name, "exec")
2788 code = compiler(mod, cell_name, "exec")
2789 if self.run_code(code):
2789 if self.run_code(code):
2790 return True
2790 return True
2791
2791
2792 for i, node in enumerate(to_run_interactive):
2792 for i, node in enumerate(to_run_interactive):
2793 mod = ast.Interactive([node])
2793 mod = ast.Interactive([node])
2794 code = compiler(mod, cell_name, "single")
2794 code = compiler(mod, cell_name, "single")
2795 if self.run_code(code):
2795 if self.run_code(code):
2796 return True
2796 return True
2797
2797
2798 # Flush softspace
2798 # Flush softspace
2799 if softspace(sys.stdout, 0):
2799 if softspace(sys.stdout, 0):
2800 print()
2800 print()
2801
2801
2802 except:
2802 except:
2803 # It's possible to have exceptions raised here, typically by
2803 # It's possible to have exceptions raised here, typically by
2804 # compilation of odd code (such as a naked 'return' outside a
2804 # compilation of odd code (such as a naked 'return' outside a
2805 # function) that did parse but isn't valid. Typically the exception
2805 # function) that did parse but isn't valid. Typically the exception
2806 # is a SyntaxError, but it's safest just to catch anything and show
2806 # is a SyntaxError, but it's safest just to catch anything and show
2807 # the user a traceback.
2807 # the user a traceback.
2808
2808
2809 # We do only one try/except outside the loop to minimize the impact
2809 # We do only one try/except outside the loop to minimize the impact
2810 # on runtime, and also because if any node in the node list is
2810 # on runtime, and also because if any node in the node list is
2811 # broken, we should stop execution completely.
2811 # broken, we should stop execution completely.
2812 self.showtraceback()
2812 self.showtraceback()
2813
2813
2814 return False
2814 return False
2815
2815
2816 def run_code(self, code_obj):
2816 def run_code(self, code_obj):
2817 """Execute a code object.
2817 """Execute a code object.
2818
2818
2819 When an exception occurs, self.showtraceback() is called to display a
2819 When an exception occurs, self.showtraceback() is called to display a
2820 traceback.
2820 traceback.
2821
2821
2822 Parameters
2822 Parameters
2823 ----------
2823 ----------
2824 code_obj : code object
2824 code_obj : code object
2825 A compiled code object, to be executed
2825 A compiled code object, to be executed
2826
2826
2827 Returns
2827 Returns
2828 -------
2828 -------
2829 False : successful execution.
2829 False : successful execution.
2830 True : an error occurred.
2830 True : an error occurred.
2831 """
2831 """
2832
2832
2833 # Set our own excepthook in case the user code tries to call it
2833 # Set our own excepthook in case the user code tries to call it
2834 # directly, so that the IPython crash handler doesn't get triggered
2834 # directly, so that the IPython crash handler doesn't get triggered
2835 old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
2835 old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
2836
2836
2837 # we save the original sys.excepthook in the instance, in case config
2837 # we save the original sys.excepthook in the instance, in case config
2838 # code (such as magics) needs access to it.
2838 # code (such as magics) needs access to it.
2839 self.sys_excepthook = old_excepthook
2839 self.sys_excepthook = old_excepthook
2840 outflag = 1 # happens in more places, so it's easier as default
2840 outflag = 1 # happens in more places, so it's easier as default
2841 try:
2841 try:
2842 try:
2842 try:
2843 self.hooks.pre_run_code_hook()
2843 self.hooks.pre_run_code_hook()
2844 #rprint('Running code', repr(code_obj)) # dbg
2844 #rprint('Running code', repr(code_obj)) # dbg
2845 exec(code_obj, self.user_global_ns, self.user_ns)
2845 exec(code_obj, self.user_global_ns, self.user_ns)
2846 finally:
2846 finally:
2847 # Reset our crash handler in place
2847 # Reset our crash handler in place
2848 sys.excepthook = old_excepthook
2848 sys.excepthook = old_excepthook
2849 except SystemExit:
2849 except SystemExit:
2850 self.showtraceback(exception_only=True)
2850 self.showtraceback(exception_only=True)
2851 warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
2851 warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
2852 except self.custom_exceptions:
2852 except self.custom_exceptions:
2853 etype,value,tb = sys.exc_info()
2853 etype,value,tb = sys.exc_info()
2854 self.CustomTB(etype,value,tb)
2854 self.CustomTB(etype,value,tb)
2855 except:
2855 except:
2856 self.showtraceback()
2856 self.showtraceback()
2857 else:
2857 else:
2858 outflag = 0
2858 outflag = 0
2859 return outflag
2859 return outflag
2860
2860
2861 # For backwards compatibility
2861 # For backwards compatibility
2862 runcode = run_code
2862 runcode = run_code
2863
2863
2864 #-------------------------------------------------------------------------
2864 #-------------------------------------------------------------------------
2865 # Things related to GUI support and pylab
2865 # Things related to GUI support and pylab
2866 #-------------------------------------------------------------------------
2866 #-------------------------------------------------------------------------
2867
2867
2868 def enable_gui(self, gui=None):
2868 def enable_gui(self, gui=None):
2869 raise NotImplementedError('Implement enable_gui in a subclass')
2869 raise NotImplementedError('Implement enable_gui in a subclass')
2870
2870
2871 def enable_matplotlib(self, gui=None):
2871 def enable_matplotlib(self, gui=None):
2872 """Enable interactive matplotlib and inline figure support.
2872 """Enable interactive matplotlib and inline figure support.
2873
2873
2874 This takes the following steps:
2874 This takes the following steps:
2875
2875
2876 1. select the appropriate eventloop and matplotlib backend
2876 1. select the appropriate eventloop and matplotlib backend
2877 2. set up matplotlib for interactive use with that backend
2877 2. set up matplotlib for interactive use with that backend
2878 3. configure formatters for inline figure display
2878 3. configure formatters for inline figure display
2879 4. enable the selected gui eventloop
2879 4. enable the selected gui eventloop
2880
2880
2881 Parameters
2881 Parameters
2882 ----------
2882 ----------
2883 gui : optional, string
2883 gui : optional, string
2884 If given, dictates the choice of matplotlib GUI backend to use
2884 If given, dictates the choice of matplotlib GUI backend to use
2885 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2885 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2886 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2886 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2887 matplotlib (as dictated by the matplotlib build-time options plus the
2887 matplotlib (as dictated by the matplotlib build-time options plus the
2888 user's matplotlibrc configuration file). Note that not all backends
2888 user's matplotlibrc configuration file). Note that not all backends
2889 make sense in all contexts, for example a terminal ipython can't
2889 make sense in all contexts, for example a terminal ipython can't
2890 display figures inline.
2890 display figures inline.
2891 """
2891 """
2892 from IPython.core import pylabtools as pt
2892 from IPython.core import pylabtools as pt
2893 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
2893 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
2894
2894
2895 if gui != 'inline':
2895 if gui != 'inline':
2896 # If we have our first gui selection, store it
2896 # If we have our first gui selection, store it
2897 if self.pylab_gui_select is None:
2897 if self.pylab_gui_select is None:
2898 self.pylab_gui_select = gui
2898 self.pylab_gui_select = gui
2899 # Otherwise if they are different
2899 # Otherwise if they are different
2900 elif gui != self.pylab_gui_select:
2900 elif gui != self.pylab_gui_select:
2901 print ('Warning: Cannot change to a different GUI toolkit: %s.'
2901 print ('Warning: Cannot change to a different GUI toolkit: %s.'
2902 ' Using %s instead.' % (gui, self.pylab_gui_select))
2902 ' Using %s instead.' % (gui, self.pylab_gui_select))
2903 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
2903 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
2904
2904
2905 pt.activate_matplotlib(backend)
2905 pt.activate_matplotlib(backend)
2906 pt.configure_inline_support(self, backend)
2906 pt.configure_inline_support(self, backend)
2907
2907
2908 # Now we must activate the gui pylab wants to use, and fix %run to take
2908 # Now we must activate the gui pylab wants to use, and fix %run to take
2909 # plot updates into account
2909 # plot updates into account
2910 self.enable_gui(gui)
2910 self.enable_gui(gui)
2911 self.magics_manager.registry['ExecutionMagics'].default_runner = \
2911 self.magics_manager.registry['ExecutionMagics'].default_runner = \
2912 pt.mpl_runner(self.safe_execfile)
2912 pt.mpl_runner(self.safe_execfile)
2913
2913
2914 return gui, backend
2914 return gui, backend
2915
2915
2916 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
2916 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
2917 """Activate pylab support at runtime.
2917 """Activate pylab support at runtime.
2918
2918
2919 This turns on support for matplotlib, preloads into the interactive
2919 This turns on support for matplotlib, preloads into the interactive
2920 namespace all of numpy and pylab, and configures IPython to correctly
2920 namespace all of numpy and pylab, and configures IPython to correctly
2921 interact with the GUI event loop. The GUI backend to be used can be
2921 interact with the GUI event loop. The GUI backend to be used can be
2922 optionally selected with the optional ``gui`` argument.
2922 optionally selected with the optional ``gui`` argument.
2923
2923
2924 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
2924 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
2925
2925
2926 Parameters
2926 Parameters
2927 ----------
2927 ----------
2928 gui : optional, string
2928 gui : optional, string
2929 If given, dictates the choice of matplotlib GUI backend to use
2929 If given, dictates the choice of matplotlib GUI backend to use
2930 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2930 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2931 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2931 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2932 matplotlib (as dictated by the matplotlib build-time options plus the
2932 matplotlib (as dictated by the matplotlib build-time options plus the
2933 user's matplotlibrc configuration file). Note that not all backends
2933 user's matplotlibrc configuration file). Note that not all backends
2934 make sense in all contexts, for example a terminal ipython can't
2934 make sense in all contexts, for example a terminal ipython can't
2935 display figures inline.
2935 display figures inline.
2936 import_all : optional, bool, default: True
2936 import_all : optional, bool, default: True
2937 Whether to do `from numpy import *` and `from pylab import *`
2937 Whether to do `from numpy import *` and `from pylab import *`
2938 in addition to module imports.
2938 in addition to module imports.
2939 welcome_message : deprecated
2939 welcome_message : deprecated
2940 This argument is ignored, no welcome message will be displayed.
2940 This argument is ignored, no welcome message will be displayed.
2941 """
2941 """
2942 from IPython.core.pylabtools import import_pylab
2942 from IPython.core.pylabtools import import_pylab
2943
2943
2944 gui, backend = self.enable_matplotlib(gui)
2944 gui, backend = self.enable_matplotlib(gui)
2945
2945
2946 # We want to prevent the loading of pylab to pollute the user's
2946 # We want to prevent the loading of pylab to pollute the user's
2947 # namespace as shown by the %who* magics, so we execute the activation
2947 # namespace as shown by the %who* magics, so we execute the activation
2948 # code in an empty namespace, and we update *both* user_ns and
2948 # code in an empty namespace, and we update *both* user_ns and
2949 # user_ns_hidden with this information.
2949 # user_ns_hidden with this information.
2950 ns = {}
2950 ns = {}
2951 import_pylab(ns, import_all)
2951 import_pylab(ns, import_all)
2952 # warn about clobbered names
2952 # warn about clobbered names
2953 ignored = set(["__builtins__"])
2953 ignored = set(["__builtins__"])
2954 both = set(ns).intersection(self.user_ns).difference(ignored)
2954 both = set(ns).intersection(self.user_ns).difference(ignored)
2955 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
2955 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
2956 self.user_ns.update(ns)
2956 self.user_ns.update(ns)
2957 self.user_ns_hidden.update(ns)
2957 self.user_ns_hidden.update(ns)
2958 return gui, backend, clobbered
2958 return gui, backend, clobbered
2959
2959
2960 #-------------------------------------------------------------------------
2960 #-------------------------------------------------------------------------
2961 # Utilities
2961 # Utilities
2962 #-------------------------------------------------------------------------
2962 #-------------------------------------------------------------------------
2963
2963
2964 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
2964 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
2965 """Expand python variables in a string.
2965 """Expand python variables in a string.
2966
2966
2967 The depth argument indicates how many frames above the caller should
2967 The depth argument indicates how many frames above the caller should
2968 be walked to look for the local namespace where to expand variables.
2968 be walked to look for the local namespace where to expand variables.
2969
2969
2970 The global namespace for expansion is always the user's interactive
2970 The global namespace for expansion is always the user's interactive
2971 namespace.
2971 namespace.
2972 """
2972 """
2973 ns = self.user_ns.copy()
2973 ns = self.user_ns.copy()
2974 ns.update(sys._getframe(depth+1).f_locals)
2974 ns.update(sys._getframe(depth+1).f_locals)
2975 try:
2975 try:
2976 # We have to use .vformat() here, because 'self' is a valid and common
2976 # We have to use .vformat() here, because 'self' is a valid and common
2977 # name, and expanding **ns for .format() would make it collide with
2977 # name, and expanding **ns for .format() would make it collide with
2978 # the 'self' argument of the method.
2978 # the 'self' argument of the method.
2979 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
2979 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
2980 except Exception:
2980 except Exception:
2981 # if formatter couldn't format, just let it go untransformed
2981 # if formatter couldn't format, just let it go untransformed
2982 pass
2982 pass
2983 return cmd
2983 return cmd
2984
2984
2985 def mktempfile(self, data=None, prefix='ipython_edit_'):
2985 def mktempfile(self, data=None, prefix='ipython_edit_'):
2986 """Make a new tempfile and return its filename.
2986 """Make a new tempfile and return its filename.
2987
2987
2988 This makes a call to tempfile.mktemp, but it registers the created
2988 This makes a call to tempfile.mktemp, but it registers the created
2989 filename internally so ipython cleans it up at exit time.
2989 filename internally so ipython cleans it up at exit time.
2990
2990
2991 Optional inputs:
2991 Optional inputs:
2992
2992
2993 - data(None): if data is given, it gets written out to the temp file
2993 - data(None): if data is given, it gets written out to the temp file
2994 immediately, and the file is closed again."""
2994 immediately, and the file is closed again."""
2995
2995
2996 filename = tempfile.mktemp('.py', prefix)
2996 filename = tempfile.mktemp('.py', prefix)
2997 self.tempfiles.append(filename)
2997 self.tempfiles.append(filename)
2998
2998
2999 if data:
2999 if data:
3000 tmp_file = open(filename,'w')
3000 tmp_file = open(filename,'w')
3001 tmp_file.write(data)
3001 tmp_file.write(data)
3002 tmp_file.close()
3002 tmp_file.close()
3003 return filename
3003 return filename
3004
3004
3005 # TODO: This should be removed when Term is refactored.
3005 # TODO: This should be removed when Term is refactored.
3006 def write(self,data):
3006 def write(self,data):
3007 """Write a string to the default output"""
3007 """Write a string to the default output"""
3008 io.stdout.write(data)
3008 io.stdout.write(data)
3009
3009
3010 # TODO: This should be removed when Term is refactored.
3010 # TODO: This should be removed when Term is refactored.
3011 def write_err(self,data):
3011 def write_err(self,data):
3012 """Write a string to the default error output"""
3012 """Write a string to the default error output"""
3013 io.stderr.write(data)
3013 io.stderr.write(data)
3014
3014
3015 def ask_yes_no(self, prompt, default=None):
3015 def ask_yes_no(self, prompt, default=None):
3016 if self.quiet:
3016 if self.quiet:
3017 return True
3017 return True
3018 return ask_yes_no(prompt,default)
3018 return ask_yes_no(prompt,default)
3019
3019
3020 def show_usage(self):
3020 def show_usage(self):
3021 """Show a usage message"""
3021 """Show a usage message"""
3022 page.page(IPython.core.usage.interactive_usage)
3022 page.page(IPython.core.usage.interactive_usage)
3023
3023
3024 def extract_input_lines(self, range_str, raw=False):
3024 def extract_input_lines(self, range_str, raw=False):
3025 """Return as a string a set of input history slices.
3025 """Return as a string a set of input history slices.
3026
3026
3027 Parameters
3027 Parameters
3028 ----------
3028 ----------
3029 range_str : string
3029 range_str : string
3030 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3030 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3031 since this function is for use by magic functions which get their
3031 since this function is for use by magic functions which get their
3032 arguments as strings. The number before the / is the session
3032 arguments as strings. The number before the / is the session
3033 number: ~n goes n back from the current session.
3033 number: ~n goes n back from the current session.
3034
3034
3035 Optional Parameters:
3035 Optional Parameters:
3036 - raw(False): by default, the processed input is used. If this is
3036 - raw(False): by default, the processed input is used. If this is
3037 true, the raw input history is used instead.
3037 true, the raw input history is used instead.
3038
3038
3039 Note that slices can be called with two notations:
3039 Note that slices can be called with two notations:
3040
3040
3041 N:M -> standard python form, means including items N...(M-1).
3041 N:M -> standard python form, means including items N...(M-1).
3042
3042
3043 N-M -> include items N..M (closed endpoint).
3043 N-M -> include items N..M (closed endpoint).
3044 """
3044 """
3045 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3045 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3046 return "\n".join(x for _, _, x in lines)
3046 return "\n".join(x for _, _, x in lines)
3047
3047
3048 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True):
3048 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True):
3049 """Get a code string from history, file, url, or a string or macro.
3049 """Get a code string from history, file, url, or a string or macro.
3050
3050
3051 This is mainly used by magic functions.
3051 This is mainly used by magic functions.
3052
3052
3053 Parameters
3053 Parameters
3054 ----------
3054 ----------
3055
3055
3056 target : str
3056 target : str
3057
3057
3058 A string specifying code to retrieve. This will be tried respectively
3058 A string specifying code to retrieve. This will be tried respectively
3059 as: ranges of input history (see %history for syntax), url,
3059 as: ranges of input history (see %history for syntax), url,
3060 correspnding .py file, filename, or an expression evaluating to a
3060 correspnding .py file, filename, or an expression evaluating to a
3061 string or Macro in the user namespace.
3061 string or Macro in the user namespace.
3062
3062
3063 raw : bool
3063 raw : bool
3064 If true (default), retrieve raw history. Has no effect on the other
3064 If true (default), retrieve raw history. Has no effect on the other
3065 retrieval mechanisms.
3065 retrieval mechanisms.
3066
3066
3067 py_only : bool (default False)
3067 py_only : bool (default False)
3068 Only try to fetch python code, do not try alternative methods to decode file
3068 Only try to fetch python code, do not try alternative methods to decode file
3069 if unicode fails.
3069 if unicode fails.
3070
3070
3071 Returns
3071 Returns
3072 -------
3072 -------
3073 A string of code.
3073 A string of code.
3074
3074
3075 ValueError is raised if nothing is found, and TypeError if it evaluates
3075 ValueError is raised if nothing is found, and TypeError if it evaluates
3076 to an object of another type. In each case, .args[0] is a printable
3076 to an object of another type. In each case, .args[0] is a printable
3077 message.
3077 message.
3078 """
3078 """
3079 code = self.extract_input_lines(target, raw=raw) # Grab history
3079 code = self.extract_input_lines(target, raw=raw) # Grab history
3080 if code:
3080 if code:
3081 return code
3081 return code
3082 utarget = unquote_filename(target)
3082 utarget = unquote_filename(target)
3083 try:
3083 try:
3084 if utarget.startswith(('http://', 'https://')):
3084 if utarget.startswith(('http://', 'https://')):
3085 return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
3085 return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
3086 except UnicodeDecodeError:
3086 except UnicodeDecodeError:
3087 if not py_only :
3087 if not py_only :
3088 from urllib import urlopen # Deferred import
3088 from urllib import urlopen # Deferred import
3089 response = urlopen(target)
3089 response = urlopen(target)
3090 return response.read().decode('latin1')
3090 return response.read().decode('latin1')
3091 raise ValueError(("'%s' seem to be unreadable.") % utarget)
3091 raise ValueError(("'%s' seem to be unreadable.") % utarget)
3092
3092
3093 potential_target = [target]
3093 potential_target = [target]
3094 try :
3094 try :
3095 potential_target.insert(0,get_py_filename(target))
3095 potential_target.insert(0,get_py_filename(target))
3096 except IOError:
3096 except IOError:
3097 pass
3097 pass
3098
3098
3099 for tgt in potential_target :
3099 for tgt in potential_target :
3100 if os.path.isfile(tgt): # Read file
3100 if os.path.isfile(tgt): # Read file
3101 try :
3101 try :
3102 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3102 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3103 except UnicodeDecodeError :
3103 except UnicodeDecodeError :
3104 if not py_only :
3104 if not py_only :
3105 with io_open(tgt,'r', encoding='latin1') as f :
3105 with io_open(tgt,'r', encoding='latin1') as f :
3106 return f.read()
3106 return f.read()
3107 raise ValueError(("'%s' seem to be unreadable.") % target)
3107 raise ValueError(("'%s' seem to be unreadable.") % target)
3108 elif os.path.isdir(os.path.expanduser(tgt)):
3108 elif os.path.isdir(os.path.expanduser(tgt)):
3109 raise ValueError("'%s' is a directory, not a regular file." % target)
3109 raise ValueError("'%s' is a directory, not a regular file." % target)
3110
3110
3111 try: # User namespace
3111 try: # User namespace
3112 codeobj = eval(target, self.user_ns)
3112 codeobj = eval(target, self.user_ns)
3113 except Exception:
3113 except Exception:
3114 raise ValueError(("'%s' was not found in history, as a file, url, "
3114 raise ValueError(("'%s' was not found in history, as a file, url, "
3115 "nor in the user namespace.") % target)
3115 "nor in the user namespace.") % target)
3116 if isinstance(codeobj, basestring):
3116 if isinstance(codeobj, basestring):
3117 return codeobj
3117 return codeobj
3118 elif isinstance(codeobj, Macro):
3118 elif isinstance(codeobj, Macro):
3119 return codeobj.value
3119 return codeobj.value
3120
3120
3121 raise TypeError("%s is neither a string nor a macro." % target,
3121 raise TypeError("%s is neither a string nor a macro." % target,
3122 codeobj)
3122 codeobj)
3123
3123
3124 #-------------------------------------------------------------------------
3124 #-------------------------------------------------------------------------
3125 # Things related to IPython exiting
3125 # Things related to IPython exiting
3126 #-------------------------------------------------------------------------
3126 #-------------------------------------------------------------------------
3127 def atexit_operations(self):
3127 def atexit_operations(self):
3128 """This will be executed at the time of exit.
3128 """This will be executed at the time of exit.
3129
3129
3130 Cleanup operations and saving of persistent data that is done
3130 Cleanup operations and saving of persistent data that is done
3131 unconditionally by IPython should be performed here.
3131 unconditionally by IPython should be performed here.
3132
3132
3133 For things that may depend on startup flags or platform specifics (such
3133 For things that may depend on startup flags or platform specifics (such
3134 as having readline or not), register a separate atexit function in the
3134 as having readline or not), register a separate atexit function in the
3135 code that has the appropriate information, rather than trying to
3135 code that has the appropriate information, rather than trying to
3136 clutter
3136 clutter
3137 """
3137 """
3138 # Close the history session (this stores the end time and line count)
3138 # Close the history session (this stores the end time and line count)
3139 # this must be *before* the tempfile cleanup, in case of temporary
3139 # this must be *before* the tempfile cleanup, in case of temporary
3140 # history db
3140 # history db
3141 self.history_manager.end_session()
3141 self.history_manager.end_session()
3142
3142
3143 # Cleanup all tempfiles left around
3143 # Cleanup all tempfiles left around
3144 for tfile in self.tempfiles:
3144 for tfile in self.tempfiles:
3145 try:
3145 try:
3146 os.unlink(tfile)
3146 os.unlink(tfile)
3147 except OSError:
3147 except OSError:
3148 pass
3148 pass
3149
3149
3150 # Clear all user namespaces to release all references cleanly.
3150 # Clear all user namespaces to release all references cleanly.
3151 self.reset(new_session=False)
3151 self.reset(new_session=False)
3152
3152
3153 # Run user hooks
3153 # Run user hooks
3154 self.hooks.shutdown_hook()
3154 self.hooks.shutdown_hook()
3155
3155
3156 def cleanup(self):
3156 def cleanup(self):
3157 self.restore_sys_module_state()
3157 self.restore_sys_module_state()
3158
3158
3159
3159
3160 class InteractiveShellABC(object):
3160 class InteractiveShellABC(object):
3161 """An abstract base class for InteractiveShell."""
3161 """An abstract base class for InteractiveShell."""
3162 __metaclass__ = abc.ABCMeta
3162 __metaclass__ = abc.ABCMeta
3163
3163
3164 InteractiveShellABC.register(InteractiveShell)
3164 InteractiveShellABC.register(InteractiveShell)
@@ -1,1288 +1,1288 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Implementation of execution-related magic functions.
2 """Implementation of execution-related magic functions.
3 """
3 """
4 from __future__ import print_function
4 from __future__ import print_function
5 #-----------------------------------------------------------------------------
5 #-----------------------------------------------------------------------------
6 # Copyright (c) 2012 The IPython Development Team.
6 # Copyright (c) 2012 The IPython Development Team.
7 #
7 #
8 # Distributed under the terms of the Modified BSD License.
8 # Distributed under the terms of the Modified BSD License.
9 #
9 #
10 # The full license is in the file COPYING.txt, distributed with this software.
10 # The full license is in the file COPYING.txt, distributed with this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 # Stdlib
17 # Stdlib
18 import __builtin__ as builtin_mod
19 import ast
18 import ast
20 import bdb
19 import bdb
21 import os
20 import os
22 import sys
21 import sys
23 import time
22 import time
24 from StringIO import StringIO
23 from StringIO import StringIO
25
24
26 # cProfile was added in Python2.5
25 # cProfile was added in Python2.5
27 try:
26 try:
28 import cProfile as profile
27 import cProfile as profile
29 import pstats
28 import pstats
30 except ImportError:
29 except ImportError:
31 # profile isn't bundled by default in Debian for license reasons
30 # profile isn't bundled by default in Debian for license reasons
32 try:
31 try:
33 import profile, pstats
32 import profile, pstats
34 except ImportError:
33 except ImportError:
35 profile = pstats = None
34 profile = pstats = None
36
35
37 # Our own packages
36 # Our own packages
38 from IPython.core import debugger, oinspect
37 from IPython.core import debugger, oinspect
39 from IPython.core import magic_arguments
38 from IPython.core import magic_arguments
40 from IPython.core import page
39 from IPython.core import page
41 from IPython.core.error import UsageError
40 from IPython.core.error import UsageError
42 from IPython.core.macro import Macro
41 from IPython.core.macro import Macro
43 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
42 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
44 line_cell_magic, on_off, needs_local_scope)
43 line_cell_magic, on_off, needs_local_scope)
45 from IPython.testing.skipdoctest import skip_doctest
44 from IPython.testing.skipdoctest import skip_doctest
46 from IPython.utils import py3compat
45 from IPython.utils import py3compat
46 from IPython.utils.py3compat import builtin_mod
47 from IPython.utils.contexts import preserve_keys
47 from IPython.utils.contexts import preserve_keys
48 from IPython.utils.io import capture_output
48 from IPython.utils.io import capture_output
49 from IPython.utils.ipstruct import Struct
49 from IPython.utils.ipstruct import Struct
50 from IPython.utils.module_paths import find_mod
50 from IPython.utils.module_paths import find_mod
51 from IPython.utils.path import get_py_filename, unquote_filename, shellglob
51 from IPython.utils.path import get_py_filename, unquote_filename, shellglob
52 from IPython.utils.timing import clock, clock2
52 from IPython.utils.timing import clock, clock2
53 from IPython.utils.warn import warn, error
53 from IPython.utils.warn import warn, error
54
54
55
55
56 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
57 # Magic implementation classes
57 # Magic implementation classes
58 #-----------------------------------------------------------------------------
58 #-----------------------------------------------------------------------------
59
59
60
60
61 class TimeitResult(object):
61 class TimeitResult(object):
62 """
62 """
63 Object returned by the timeit magic with info about the run.
63 Object returned by the timeit magic with info about the run.
64
64
65 Contain the following attributes :
65 Contain the following attributes :
66
66
67 loops: (int) number of loop done per measurement
67 loops: (int) number of loop done per measurement
68 repeat: (int) number of time the mesurement has been repeated
68 repeat: (int) number of time the mesurement has been repeated
69 best: (float) best execusion time / number
69 best: (float) best execusion time / number
70 all_runs: (list of float) execusion time of each run (in s)
70 all_runs: (list of float) execusion time of each run (in s)
71 compile_time: (float) time of statement compilation (s)
71 compile_time: (float) time of statement compilation (s)
72
72
73 """
73 """
74
74
75 def __init__(self, loops, repeat, best, all_runs, compile_time, precision):
75 def __init__(self, loops, repeat, best, all_runs, compile_time, precision):
76 self.loops = loops
76 self.loops = loops
77 self.repeat = repeat
77 self.repeat = repeat
78 self.best = best
78 self.best = best
79 self.all_runs = all_runs
79 self.all_runs = all_runs
80 self.compile_time = compile_time
80 self.compile_time = compile_time
81 self._precision = precision
81 self._precision = precision
82
82
83 def _repr_pretty_(self, p , cycle):
83 def _repr_pretty_(self, p , cycle):
84 unic = u"%d loops, best of %d: %s per loop" % (self.loops, self.repeat,
84 unic = u"%d loops, best of %d: %s per loop" % (self.loops, self.repeat,
85 _format_time(self.best, self._precision))
85 _format_time(self.best, self._precision))
86 p.text(u'<TimeitResult : '+unic+u'>')
86 p.text(u'<TimeitResult : '+unic+u'>')
87
87
88
88
89
89
90
90
91 @magics_class
91 @magics_class
92 class ExecutionMagics(Magics):
92 class ExecutionMagics(Magics):
93 """Magics related to code execution, debugging, profiling, etc.
93 """Magics related to code execution, debugging, profiling, etc.
94
94
95 """
95 """
96
96
97 def __init__(self, shell):
97 def __init__(self, shell):
98 super(ExecutionMagics, self).__init__(shell)
98 super(ExecutionMagics, self).__init__(shell)
99 if profile is None:
99 if profile is None:
100 self.prun = self.profile_missing_notice
100 self.prun = self.profile_missing_notice
101 # Default execution function used to actually run user code.
101 # Default execution function used to actually run user code.
102 self.default_runner = None
102 self.default_runner = None
103
103
104 def profile_missing_notice(self, *args, **kwargs):
104 def profile_missing_notice(self, *args, **kwargs):
105 error("""\
105 error("""\
106 The profile module could not be found. It has been removed from the standard
106 The profile module could not be found. It has been removed from the standard
107 python packages because of its non-free license. To use profiling, install the
107 python packages because of its non-free license. To use profiling, install the
108 python-profiler package from non-free.""")
108 python-profiler package from non-free.""")
109
109
110 @skip_doctest
110 @skip_doctest
111 @line_cell_magic
111 @line_cell_magic
112 def prun(self, parameter_s='', cell=None):
112 def prun(self, parameter_s='', cell=None):
113
113
114 """Run a statement through the python code profiler.
114 """Run a statement through the python code profiler.
115
115
116 Usage, in line mode:
116 Usage, in line mode:
117 %prun [options] statement
117 %prun [options] statement
118
118
119 Usage, in cell mode:
119 Usage, in cell mode:
120 %%prun [options] [statement]
120 %%prun [options] [statement]
121 code...
121 code...
122 code...
122 code...
123
123
124 In cell mode, the additional code lines are appended to the (possibly
124 In cell mode, the additional code lines are appended to the (possibly
125 empty) statement in the first line. Cell mode allows you to easily
125 empty) statement in the first line. Cell mode allows you to easily
126 profile multiline blocks without having to put them in a separate
126 profile multiline blocks without having to put them in a separate
127 function.
127 function.
128
128
129 The given statement (which doesn't require quote marks) is run via the
129 The given statement (which doesn't require quote marks) is run via the
130 python profiler in a manner similar to the profile.run() function.
130 python profiler in a manner similar to the profile.run() function.
131 Namespaces are internally managed to work correctly; profile.run
131 Namespaces are internally managed to work correctly; profile.run
132 cannot be used in IPython because it makes certain assumptions about
132 cannot be used in IPython because it makes certain assumptions about
133 namespaces which do not hold under IPython.
133 namespaces which do not hold under IPython.
134
134
135 Options:
135 Options:
136
136
137 -l <limit>
137 -l <limit>
138 you can place restrictions on what or how much of the
138 you can place restrictions on what or how much of the
139 profile gets printed. The limit value can be:
139 profile gets printed. The limit value can be:
140
140
141 * A string: only information for function names containing this string
141 * A string: only information for function names containing this string
142 is printed.
142 is printed.
143
143
144 * An integer: only these many lines are printed.
144 * An integer: only these many lines are printed.
145
145
146 * A float (between 0 and 1): this fraction of the report is printed
146 * A float (between 0 and 1): this fraction of the report is printed
147 (for example, use a limit of 0.4 to see the topmost 40% only).
147 (for example, use a limit of 0.4 to see the topmost 40% only).
148
148
149 You can combine several limits with repeated use of the option. For
149 You can combine several limits with repeated use of the option. For
150 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
150 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
151 information about class constructors.
151 information about class constructors.
152
152
153 -r
153 -r
154 return the pstats.Stats object generated by the profiling. This
154 return the pstats.Stats object generated by the profiling. This
155 object has all the information about the profile in it, and you can
155 object has all the information about the profile in it, and you can
156 later use it for further analysis or in other functions.
156 later use it for further analysis or in other functions.
157
157
158 -s <key>
158 -s <key>
159 sort profile by given key. You can provide more than one key
159 sort profile by given key. You can provide more than one key
160 by using the option several times: '-s key1 -s key2 -s key3...'. The
160 by using the option several times: '-s key1 -s key2 -s key3...'. The
161 default sorting key is 'time'.
161 default sorting key is 'time'.
162
162
163 The following is copied verbatim from the profile documentation
163 The following is copied verbatim from the profile documentation
164 referenced below:
164 referenced below:
165
165
166 When more than one key is provided, additional keys are used as
166 When more than one key is provided, additional keys are used as
167 secondary criteria when the there is equality in all keys selected
167 secondary criteria when the there is equality in all keys selected
168 before them.
168 before them.
169
169
170 Abbreviations can be used for any key names, as long as the
170 Abbreviations can be used for any key names, as long as the
171 abbreviation is unambiguous. The following are the keys currently
171 abbreviation is unambiguous. The following are the keys currently
172 defined:
172 defined:
173
173
174 ============ =====================
174 ============ =====================
175 Valid Arg Meaning
175 Valid Arg Meaning
176 ============ =====================
176 ============ =====================
177 "calls" call count
177 "calls" call count
178 "cumulative" cumulative time
178 "cumulative" cumulative time
179 "file" file name
179 "file" file name
180 "module" file name
180 "module" file name
181 "pcalls" primitive call count
181 "pcalls" primitive call count
182 "line" line number
182 "line" line number
183 "name" function name
183 "name" function name
184 "nfl" name/file/line
184 "nfl" name/file/line
185 "stdname" standard name
185 "stdname" standard name
186 "time" internal time
186 "time" internal time
187 ============ =====================
187 ============ =====================
188
188
189 Note that all sorts on statistics are in descending order (placing
189 Note that all sorts on statistics are in descending order (placing
190 most time consuming items first), where as name, file, and line number
190 most time consuming items first), where as name, file, and line number
191 searches are in ascending order (i.e., alphabetical). The subtle
191 searches are in ascending order (i.e., alphabetical). The subtle
192 distinction between "nfl" and "stdname" is that the standard name is a
192 distinction between "nfl" and "stdname" is that the standard name is a
193 sort of the name as printed, which means that the embedded line
193 sort of the name as printed, which means that the embedded line
194 numbers get compared in an odd way. For example, lines 3, 20, and 40
194 numbers get compared in an odd way. For example, lines 3, 20, and 40
195 would (if the file names were the same) appear in the string order
195 would (if the file names were the same) appear in the string order
196 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
196 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
197 line numbers. In fact, sort_stats("nfl") is the same as
197 line numbers. In fact, sort_stats("nfl") is the same as
198 sort_stats("name", "file", "line").
198 sort_stats("name", "file", "line").
199
199
200 -T <filename>
200 -T <filename>
201 save profile results as shown on screen to a text
201 save profile results as shown on screen to a text
202 file. The profile is still shown on screen.
202 file. The profile is still shown on screen.
203
203
204 -D <filename>
204 -D <filename>
205 save (via dump_stats) profile statistics to given
205 save (via dump_stats) profile statistics to given
206 filename. This data is in a format understood by the pstats module, and
206 filename. This data is in a format understood by the pstats module, and
207 is generated by a call to the dump_stats() method of profile
207 is generated by a call to the dump_stats() method of profile
208 objects. The profile is still shown on screen.
208 objects. The profile is still shown on screen.
209
209
210 -q
210 -q
211 suppress output to the pager. Best used with -T and/or -D above.
211 suppress output to the pager. Best used with -T and/or -D above.
212
212
213 If you want to run complete programs under the profiler's control, use
213 If you want to run complete programs under the profiler's control, use
214 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
214 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
215 contains profiler specific options as described here.
215 contains profiler specific options as described here.
216
216
217 You can read the complete documentation for the profile module with::
217 You can read the complete documentation for the profile module with::
218
218
219 In [1]: import profile; profile.help()
219 In [1]: import profile; profile.help()
220 """
220 """
221 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
221 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
222 list_all=True, posix=False)
222 list_all=True, posix=False)
223 if cell is not None:
223 if cell is not None:
224 arg_str += '\n' + cell
224 arg_str += '\n' + cell
225 arg_str = self.shell.input_splitter.transform_cell(arg_str)
225 arg_str = self.shell.input_splitter.transform_cell(arg_str)
226 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
226 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
227
227
228 def _run_with_profiler(self, code, opts, namespace):
228 def _run_with_profiler(self, code, opts, namespace):
229 """
229 """
230 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
230 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
231
231
232 Parameters
232 Parameters
233 ----------
233 ----------
234 code : str
234 code : str
235 Code to be executed.
235 Code to be executed.
236 opts : Struct
236 opts : Struct
237 Options parsed by `self.parse_options`.
237 Options parsed by `self.parse_options`.
238 namespace : dict
238 namespace : dict
239 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
239 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
240
240
241 """
241 """
242
242
243 # Fill default values for unspecified options:
243 # Fill default values for unspecified options:
244 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
244 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
245
245
246 prof = profile.Profile()
246 prof = profile.Profile()
247 try:
247 try:
248 prof = prof.runctx(code, namespace, namespace)
248 prof = prof.runctx(code, namespace, namespace)
249 sys_exit = ''
249 sys_exit = ''
250 except SystemExit:
250 except SystemExit:
251 sys_exit = """*** SystemExit exception caught in code being profiled."""
251 sys_exit = """*** SystemExit exception caught in code being profiled."""
252
252
253 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
253 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
254
254
255 lims = opts.l
255 lims = opts.l
256 if lims:
256 if lims:
257 lims = [] # rebuild lims with ints/floats/strings
257 lims = [] # rebuild lims with ints/floats/strings
258 for lim in opts.l:
258 for lim in opts.l:
259 try:
259 try:
260 lims.append(int(lim))
260 lims.append(int(lim))
261 except ValueError:
261 except ValueError:
262 try:
262 try:
263 lims.append(float(lim))
263 lims.append(float(lim))
264 except ValueError:
264 except ValueError:
265 lims.append(lim)
265 lims.append(lim)
266
266
267 # Trap output.
267 # Trap output.
268 stdout_trap = StringIO()
268 stdout_trap = StringIO()
269 stats_stream = stats.stream
269 stats_stream = stats.stream
270 try:
270 try:
271 stats.stream = stdout_trap
271 stats.stream = stdout_trap
272 stats.print_stats(*lims)
272 stats.print_stats(*lims)
273 finally:
273 finally:
274 stats.stream = stats_stream
274 stats.stream = stats_stream
275
275
276 output = stdout_trap.getvalue()
276 output = stdout_trap.getvalue()
277 output = output.rstrip()
277 output = output.rstrip()
278
278
279 if 'q' not in opts:
279 if 'q' not in opts:
280 page.page(output)
280 page.page(output)
281 print(sys_exit, end=' ')
281 print(sys_exit, end=' ')
282
282
283 dump_file = opts.D[0]
283 dump_file = opts.D[0]
284 text_file = opts.T[0]
284 text_file = opts.T[0]
285 if dump_file:
285 if dump_file:
286 dump_file = unquote_filename(dump_file)
286 dump_file = unquote_filename(dump_file)
287 prof.dump_stats(dump_file)
287 prof.dump_stats(dump_file)
288 print('\n*** Profile stats marshalled to file',\
288 print('\n*** Profile stats marshalled to file',\
289 repr(dump_file)+'.',sys_exit)
289 repr(dump_file)+'.',sys_exit)
290 if text_file:
290 if text_file:
291 text_file = unquote_filename(text_file)
291 text_file = unquote_filename(text_file)
292 pfile = open(text_file,'w')
292 pfile = open(text_file,'w')
293 pfile.write(output)
293 pfile.write(output)
294 pfile.close()
294 pfile.close()
295 print('\n*** Profile printout saved to text file',\
295 print('\n*** Profile printout saved to text file',\
296 repr(text_file)+'.',sys_exit)
296 repr(text_file)+'.',sys_exit)
297
297
298 if 'r' in opts:
298 if 'r' in opts:
299 return stats
299 return stats
300 else:
300 else:
301 return None
301 return None
302
302
303 @line_magic
303 @line_magic
304 def pdb(self, parameter_s=''):
304 def pdb(self, parameter_s=''):
305 """Control the automatic calling of the pdb interactive debugger.
305 """Control the automatic calling of the pdb interactive debugger.
306
306
307 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
307 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
308 argument it works as a toggle.
308 argument it works as a toggle.
309
309
310 When an exception is triggered, IPython can optionally call the
310 When an exception is triggered, IPython can optionally call the
311 interactive pdb debugger after the traceback printout. %pdb toggles
311 interactive pdb debugger after the traceback printout. %pdb toggles
312 this feature on and off.
312 this feature on and off.
313
313
314 The initial state of this feature is set in your configuration
314 The initial state of this feature is set in your configuration
315 file (the option is ``InteractiveShell.pdb``).
315 file (the option is ``InteractiveShell.pdb``).
316
316
317 If you want to just activate the debugger AFTER an exception has fired,
317 If you want to just activate the debugger AFTER an exception has fired,
318 without having to type '%pdb on' and rerunning your code, you can use
318 without having to type '%pdb on' and rerunning your code, you can use
319 the %debug magic."""
319 the %debug magic."""
320
320
321 par = parameter_s.strip().lower()
321 par = parameter_s.strip().lower()
322
322
323 if par:
323 if par:
324 try:
324 try:
325 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
325 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
326 except KeyError:
326 except KeyError:
327 print ('Incorrect argument. Use on/1, off/0, '
327 print ('Incorrect argument. Use on/1, off/0, '
328 'or nothing for a toggle.')
328 'or nothing for a toggle.')
329 return
329 return
330 else:
330 else:
331 # toggle
331 # toggle
332 new_pdb = not self.shell.call_pdb
332 new_pdb = not self.shell.call_pdb
333
333
334 # set on the shell
334 # set on the shell
335 self.shell.call_pdb = new_pdb
335 self.shell.call_pdb = new_pdb
336 print('Automatic pdb calling has been turned',on_off(new_pdb))
336 print('Automatic pdb calling has been turned',on_off(new_pdb))
337
337
338 @skip_doctest
338 @skip_doctest
339 @magic_arguments.magic_arguments()
339 @magic_arguments.magic_arguments()
340 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
340 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
341 help="""
341 help="""
342 Set break point at LINE in FILE.
342 Set break point at LINE in FILE.
343 """
343 """
344 )
344 )
345 @magic_arguments.argument('statement', nargs='*',
345 @magic_arguments.argument('statement', nargs='*',
346 help="""
346 help="""
347 Code to run in debugger.
347 Code to run in debugger.
348 You can omit this in cell magic mode.
348 You can omit this in cell magic mode.
349 """
349 """
350 )
350 )
351 @line_cell_magic
351 @line_cell_magic
352 def debug(self, line='', cell=None):
352 def debug(self, line='', cell=None):
353 """Activate the interactive debugger.
353 """Activate the interactive debugger.
354
354
355 This magic command support two ways of activating debugger.
355 This magic command support two ways of activating debugger.
356 One is to activate debugger before executing code. This way, you
356 One is to activate debugger before executing code. This way, you
357 can set a break point, to step through the code from the point.
357 can set a break point, to step through the code from the point.
358 You can use this mode by giving statements to execute and optionally
358 You can use this mode by giving statements to execute and optionally
359 a breakpoint.
359 a breakpoint.
360
360
361 The other one is to activate debugger in post-mortem mode. You can
361 The other one is to activate debugger in post-mortem mode. You can
362 activate this mode simply running %debug without any argument.
362 activate this mode simply running %debug without any argument.
363 If an exception has just occurred, this lets you inspect its stack
363 If an exception has just occurred, this lets you inspect its stack
364 frames interactively. Note that this will always work only on the last
364 frames interactively. Note that this will always work only on the last
365 traceback that occurred, so you must call this quickly after an
365 traceback that occurred, so you must call this quickly after an
366 exception that you wish to inspect has fired, because if another one
366 exception that you wish to inspect has fired, because if another one
367 occurs, it clobbers the previous one.
367 occurs, it clobbers the previous one.
368
368
369 If you want IPython to automatically do this on every exception, see
369 If you want IPython to automatically do this on every exception, see
370 the %pdb magic for more details.
370 the %pdb magic for more details.
371 """
371 """
372 args = magic_arguments.parse_argstring(self.debug, line)
372 args = magic_arguments.parse_argstring(self.debug, line)
373
373
374 if not (args.breakpoint or args.statement or cell):
374 if not (args.breakpoint or args.statement or cell):
375 self._debug_post_mortem()
375 self._debug_post_mortem()
376 else:
376 else:
377 code = "\n".join(args.statement)
377 code = "\n".join(args.statement)
378 if cell:
378 if cell:
379 code += "\n" + cell
379 code += "\n" + cell
380 self._debug_exec(code, args.breakpoint)
380 self._debug_exec(code, args.breakpoint)
381
381
382 def _debug_post_mortem(self):
382 def _debug_post_mortem(self):
383 self.shell.debugger(force=True)
383 self.shell.debugger(force=True)
384
384
385 def _debug_exec(self, code, breakpoint):
385 def _debug_exec(self, code, breakpoint):
386 if breakpoint:
386 if breakpoint:
387 (filename, bp_line) = breakpoint.split(':', 1)
387 (filename, bp_line) = breakpoint.split(':', 1)
388 bp_line = int(bp_line)
388 bp_line = int(bp_line)
389 else:
389 else:
390 (filename, bp_line) = (None, None)
390 (filename, bp_line) = (None, None)
391 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
391 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
392
392
393 @line_magic
393 @line_magic
394 def tb(self, s):
394 def tb(self, s):
395 """Print the last traceback with the currently active exception mode.
395 """Print the last traceback with the currently active exception mode.
396
396
397 See %xmode for changing exception reporting modes."""
397 See %xmode for changing exception reporting modes."""
398 self.shell.showtraceback()
398 self.shell.showtraceback()
399
399
400 @skip_doctest
400 @skip_doctest
401 @line_magic
401 @line_magic
402 def run(self, parameter_s='', runner=None,
402 def run(self, parameter_s='', runner=None,
403 file_finder=get_py_filename):
403 file_finder=get_py_filename):
404 """Run the named file inside IPython as a program.
404 """Run the named file inside IPython as a program.
405
405
406 Usage::
406 Usage::
407
407
408 %run [-n -i -e -G]
408 %run [-n -i -e -G]
409 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
409 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
410 ( -m mod | file ) [args]
410 ( -m mod | file ) [args]
411
411
412 Parameters after the filename are passed as command-line arguments to
412 Parameters after the filename are passed as command-line arguments to
413 the program (put in sys.argv). Then, control returns to IPython's
413 the program (put in sys.argv). Then, control returns to IPython's
414 prompt.
414 prompt.
415
415
416 This is similar to running at a system prompt ``python file args``,
416 This is similar to running at a system prompt ``python file args``,
417 but with the advantage of giving you IPython's tracebacks, and of
417 but with the advantage of giving you IPython's tracebacks, and of
418 loading all variables into your interactive namespace for further use
418 loading all variables into your interactive namespace for further use
419 (unless -p is used, see below).
419 (unless -p is used, see below).
420
420
421 The file is executed in a namespace initially consisting only of
421 The file is executed in a namespace initially consisting only of
422 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
422 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
423 sees its environment as if it were being run as a stand-alone program
423 sees its environment as if it were being run as a stand-alone program
424 (except for sharing global objects such as previously imported
424 (except for sharing global objects such as previously imported
425 modules). But after execution, the IPython interactive namespace gets
425 modules). But after execution, the IPython interactive namespace gets
426 updated with all variables defined in the program (except for __name__
426 updated with all variables defined in the program (except for __name__
427 and sys.argv). This allows for very convenient loading of code for
427 and sys.argv). This allows for very convenient loading of code for
428 interactive work, while giving each program a 'clean sheet' to run in.
428 interactive work, while giving each program a 'clean sheet' to run in.
429
429
430 Arguments are expanded using shell-like glob match. Patterns
430 Arguments are expanded using shell-like glob match. Patterns
431 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
431 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
432 tilde '~' will be expanded into user's home directory. Unlike
432 tilde '~' will be expanded into user's home directory. Unlike
433 real shells, quotation does not suppress expansions. Use
433 real shells, quotation does not suppress expansions. Use
434 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
434 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
435 To completely disable these expansions, you can use -G flag.
435 To completely disable these expansions, you can use -G flag.
436
436
437 Options:
437 Options:
438
438
439 -n
439 -n
440 __name__ is NOT set to '__main__', but to the running file's name
440 __name__ is NOT set to '__main__', but to the running file's name
441 without extension (as python does under import). This allows running
441 without extension (as python does under import). This allows running
442 scripts and reloading the definitions in them without calling code
442 scripts and reloading the definitions in them without calling code
443 protected by an ``if __name__ == "__main__"`` clause.
443 protected by an ``if __name__ == "__main__"`` clause.
444
444
445 -i
445 -i
446 run the file in IPython's namespace instead of an empty one. This
446 run the file in IPython's namespace instead of an empty one. This
447 is useful if you are experimenting with code written in a text editor
447 is useful if you are experimenting with code written in a text editor
448 which depends on variables defined interactively.
448 which depends on variables defined interactively.
449
449
450 -e
450 -e
451 ignore sys.exit() calls or SystemExit exceptions in the script
451 ignore sys.exit() calls or SystemExit exceptions in the script
452 being run. This is particularly useful if IPython is being used to
452 being run. This is particularly useful if IPython is being used to
453 run unittests, which always exit with a sys.exit() call. In such
453 run unittests, which always exit with a sys.exit() call. In such
454 cases you are interested in the output of the test results, not in
454 cases you are interested in the output of the test results, not in
455 seeing a traceback of the unittest module.
455 seeing a traceback of the unittest module.
456
456
457 -t
457 -t
458 print timing information at the end of the run. IPython will give
458 print timing information at the end of the run. IPython will give
459 you an estimated CPU time consumption for your script, which under
459 you an estimated CPU time consumption for your script, which under
460 Unix uses the resource module to avoid the wraparound problems of
460 Unix uses the resource module to avoid the wraparound problems of
461 time.clock(). Under Unix, an estimate of time spent on system tasks
461 time.clock(). Under Unix, an estimate of time spent on system tasks
462 is also given (for Windows platforms this is reported as 0.0).
462 is also given (for Windows platforms this is reported as 0.0).
463
463
464 If -t is given, an additional ``-N<N>`` option can be given, where <N>
464 If -t is given, an additional ``-N<N>`` option can be given, where <N>
465 must be an integer indicating how many times you want the script to
465 must be an integer indicating how many times you want the script to
466 run. The final timing report will include total and per run results.
466 run. The final timing report will include total and per run results.
467
467
468 For example (testing the script uniq_stable.py)::
468 For example (testing the script uniq_stable.py)::
469
469
470 In [1]: run -t uniq_stable
470 In [1]: run -t uniq_stable
471
471
472 IPython CPU timings (estimated):
472 IPython CPU timings (estimated):
473 User : 0.19597 s.
473 User : 0.19597 s.
474 System: 0.0 s.
474 System: 0.0 s.
475
475
476 In [2]: run -t -N5 uniq_stable
476 In [2]: run -t -N5 uniq_stable
477
477
478 IPython CPU timings (estimated):
478 IPython CPU timings (estimated):
479 Total runs performed: 5
479 Total runs performed: 5
480 Times : Total Per run
480 Times : Total Per run
481 User : 0.910862 s, 0.1821724 s.
481 User : 0.910862 s, 0.1821724 s.
482 System: 0.0 s, 0.0 s.
482 System: 0.0 s, 0.0 s.
483
483
484 -d
484 -d
485 run your program under the control of pdb, the Python debugger.
485 run your program under the control of pdb, the Python debugger.
486 This allows you to execute your program step by step, watch variables,
486 This allows you to execute your program step by step, watch variables,
487 etc. Internally, what IPython does is similar to calling::
487 etc. Internally, what IPython does is similar to calling::
488
488
489 pdb.run('execfile("YOURFILENAME")')
489 pdb.run('execfile("YOURFILENAME")')
490
490
491 with a breakpoint set on line 1 of your file. You can change the line
491 with a breakpoint set on line 1 of your file. You can change the line
492 number for this automatic breakpoint to be <N> by using the -bN option
492 number for this automatic breakpoint to be <N> by using the -bN option
493 (where N must be an integer). For example::
493 (where N must be an integer). For example::
494
494
495 %run -d -b40 myscript
495 %run -d -b40 myscript
496
496
497 will set the first breakpoint at line 40 in myscript.py. Note that
497 will set the first breakpoint at line 40 in myscript.py. Note that
498 the first breakpoint must be set on a line which actually does
498 the first breakpoint must be set on a line which actually does
499 something (not a comment or docstring) for it to stop execution.
499 something (not a comment or docstring) for it to stop execution.
500
500
501 Or you can specify a breakpoint in a different file::
501 Or you can specify a breakpoint in a different file::
502
502
503 %run -d -b myotherfile.py:20 myscript
503 %run -d -b myotherfile.py:20 myscript
504
504
505 When the pdb debugger starts, you will see a (Pdb) prompt. You must
505 When the pdb debugger starts, you will see a (Pdb) prompt. You must
506 first enter 'c' (without quotes) to start execution up to the first
506 first enter 'c' (without quotes) to start execution up to the first
507 breakpoint.
507 breakpoint.
508
508
509 Entering 'help' gives information about the use of the debugger. You
509 Entering 'help' gives information about the use of the debugger. You
510 can easily see pdb's full documentation with "import pdb;pdb.help()"
510 can easily see pdb's full documentation with "import pdb;pdb.help()"
511 at a prompt.
511 at a prompt.
512
512
513 -p
513 -p
514 run program under the control of the Python profiler module (which
514 run program under the control of the Python profiler module (which
515 prints a detailed report of execution times, function calls, etc).
515 prints a detailed report of execution times, function calls, etc).
516
516
517 You can pass other options after -p which affect the behavior of the
517 You can pass other options after -p which affect the behavior of the
518 profiler itself. See the docs for %prun for details.
518 profiler itself. See the docs for %prun for details.
519
519
520 In this mode, the program's variables do NOT propagate back to the
520 In this mode, the program's variables do NOT propagate back to the
521 IPython interactive namespace (because they remain in the namespace
521 IPython interactive namespace (because they remain in the namespace
522 where the profiler executes them).
522 where the profiler executes them).
523
523
524 Internally this triggers a call to %prun, see its documentation for
524 Internally this triggers a call to %prun, see its documentation for
525 details on the options available specifically for profiling.
525 details on the options available specifically for profiling.
526
526
527 There is one special usage for which the text above doesn't apply:
527 There is one special usage for which the text above doesn't apply:
528 if the filename ends with .ipy, the file is run as ipython script,
528 if the filename ends with .ipy, the file is run as ipython script,
529 just as if the commands were written on IPython prompt.
529 just as if the commands were written on IPython prompt.
530
530
531 -m
531 -m
532 specify module name to load instead of script path. Similar to
532 specify module name to load instead of script path. Similar to
533 the -m option for the python interpreter. Use this option last if you
533 the -m option for the python interpreter. Use this option last if you
534 want to combine with other %run options. Unlike the python interpreter
534 want to combine with other %run options. Unlike the python interpreter
535 only source modules are allowed no .pyc or .pyo files.
535 only source modules are allowed no .pyc or .pyo files.
536 For example::
536 For example::
537
537
538 %run -m example
538 %run -m example
539
539
540 will run the example module.
540 will run the example module.
541
541
542 -G
542 -G
543 disable shell-like glob expansion of arguments.
543 disable shell-like glob expansion of arguments.
544
544
545 """
545 """
546
546
547 # get arguments and set sys.argv for program to be run.
547 # get arguments and set sys.argv for program to be run.
548 opts, arg_lst = self.parse_options(parameter_s,
548 opts, arg_lst = self.parse_options(parameter_s,
549 'nidtN:b:pD:l:rs:T:em:G',
549 'nidtN:b:pD:l:rs:T:em:G',
550 mode='list', list_all=1)
550 mode='list', list_all=1)
551 if "m" in opts:
551 if "m" in opts:
552 modulename = opts["m"][0]
552 modulename = opts["m"][0]
553 modpath = find_mod(modulename)
553 modpath = find_mod(modulename)
554 if modpath is None:
554 if modpath is None:
555 warn('%r is not a valid modulename on sys.path'%modulename)
555 warn('%r is not a valid modulename on sys.path'%modulename)
556 return
556 return
557 arg_lst = [modpath] + arg_lst
557 arg_lst = [modpath] + arg_lst
558 try:
558 try:
559 filename = file_finder(arg_lst[0])
559 filename = file_finder(arg_lst[0])
560 except IndexError:
560 except IndexError:
561 warn('you must provide at least a filename.')
561 warn('you must provide at least a filename.')
562 print('\n%run:\n', oinspect.getdoc(self.run))
562 print('\n%run:\n', oinspect.getdoc(self.run))
563 return
563 return
564 except IOError as e:
564 except IOError as e:
565 try:
565 try:
566 msg = str(e)
566 msg = str(e)
567 except UnicodeError:
567 except UnicodeError:
568 msg = e.message
568 msg = e.message
569 error(msg)
569 error(msg)
570 return
570 return
571
571
572 if filename.lower().endswith('.ipy'):
572 if filename.lower().endswith('.ipy'):
573 with preserve_keys(self.shell.user_ns, '__file__'):
573 with preserve_keys(self.shell.user_ns, '__file__'):
574 self.shell.user_ns['__file__'] = filename
574 self.shell.user_ns['__file__'] = filename
575 self.shell.safe_execfile_ipy(filename)
575 self.shell.safe_execfile_ipy(filename)
576 return
576 return
577
577
578 # Control the response to exit() calls made by the script being run
578 # Control the response to exit() calls made by the script being run
579 exit_ignore = 'e' in opts
579 exit_ignore = 'e' in opts
580
580
581 # Make sure that the running script gets a proper sys.argv as if it
581 # Make sure that the running script gets a proper sys.argv as if it
582 # were run from a system shell.
582 # were run from a system shell.
583 save_argv = sys.argv # save it for later restoring
583 save_argv = sys.argv # save it for later restoring
584
584
585 if 'G' in opts:
585 if 'G' in opts:
586 args = arg_lst[1:]
586 args = arg_lst[1:]
587 else:
587 else:
588 # tilde and glob expansion
588 # tilde and glob expansion
589 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
589 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
590
590
591 sys.argv = [filename] + args # put in the proper filename
591 sys.argv = [filename] + args # put in the proper filename
592 # protect sys.argv from potential unicode strings on Python 2:
592 # protect sys.argv from potential unicode strings on Python 2:
593 if not py3compat.PY3:
593 if not py3compat.PY3:
594 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
594 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
595
595
596 if 'i' in opts:
596 if 'i' in opts:
597 # Run in user's interactive namespace
597 # Run in user's interactive namespace
598 prog_ns = self.shell.user_ns
598 prog_ns = self.shell.user_ns
599 __name__save = self.shell.user_ns['__name__']
599 __name__save = self.shell.user_ns['__name__']
600 prog_ns['__name__'] = '__main__'
600 prog_ns['__name__'] = '__main__'
601 main_mod = self.shell.user_module
601 main_mod = self.shell.user_module
602
602
603 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
603 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
604 # set the __file__ global in the script's namespace
604 # set the __file__ global in the script's namespace
605 # TK: Is this necessary in interactive mode?
605 # TK: Is this necessary in interactive mode?
606 prog_ns['__file__'] = filename
606 prog_ns['__file__'] = filename
607 else:
607 else:
608 # Run in a fresh, empty namespace
608 # Run in a fresh, empty namespace
609 if 'n' in opts:
609 if 'n' in opts:
610 name = os.path.splitext(os.path.basename(filename))[0]
610 name = os.path.splitext(os.path.basename(filename))[0]
611 else:
611 else:
612 name = '__main__'
612 name = '__main__'
613
613
614 # The shell MUST hold a reference to prog_ns so after %run
614 # The shell MUST hold a reference to prog_ns so after %run
615 # exits, the python deletion mechanism doesn't zero it out
615 # exits, the python deletion mechanism doesn't zero it out
616 # (leaving dangling references). See interactiveshell for details
616 # (leaving dangling references). See interactiveshell for details
617 main_mod = self.shell.new_main_mod(filename, name)
617 main_mod = self.shell.new_main_mod(filename, name)
618 prog_ns = main_mod.__dict__
618 prog_ns = main_mod.__dict__
619
619
620 # pickle fix. See interactiveshell for an explanation. But we need to
620 # pickle fix. See interactiveshell for an explanation. But we need to
621 # make sure that, if we overwrite __main__, we replace it at the end
621 # make sure that, if we overwrite __main__, we replace it at the end
622 main_mod_name = prog_ns['__name__']
622 main_mod_name = prog_ns['__name__']
623
623
624 if main_mod_name == '__main__':
624 if main_mod_name == '__main__':
625 restore_main = sys.modules['__main__']
625 restore_main = sys.modules['__main__']
626 else:
626 else:
627 restore_main = False
627 restore_main = False
628
628
629 # This needs to be undone at the end to prevent holding references to
629 # This needs to be undone at the end to prevent holding references to
630 # every single object ever created.
630 # every single object ever created.
631 sys.modules[main_mod_name] = main_mod
631 sys.modules[main_mod_name] = main_mod
632
632
633 if 'p' in opts or 'd' in opts:
633 if 'p' in opts or 'd' in opts:
634 if 'm' in opts:
634 if 'm' in opts:
635 code = 'run_module(modulename, prog_ns)'
635 code = 'run_module(modulename, prog_ns)'
636 code_ns = {
636 code_ns = {
637 'run_module': self.shell.safe_run_module,
637 'run_module': self.shell.safe_run_module,
638 'prog_ns': prog_ns,
638 'prog_ns': prog_ns,
639 'modulename': modulename,
639 'modulename': modulename,
640 }
640 }
641 else:
641 else:
642 code = 'execfile(filename, prog_ns)'
642 code = 'execfile(filename, prog_ns)'
643 code_ns = {
643 code_ns = {
644 'execfile': self.shell.safe_execfile,
644 'execfile': self.shell.safe_execfile,
645 'prog_ns': prog_ns,
645 'prog_ns': prog_ns,
646 'filename': get_py_filename(filename),
646 'filename': get_py_filename(filename),
647 }
647 }
648
648
649 try:
649 try:
650 stats = None
650 stats = None
651 with self.shell.readline_no_record:
651 with self.shell.readline_no_record:
652 if 'p' in opts:
652 if 'p' in opts:
653 stats = self._run_with_profiler(code, opts, code_ns)
653 stats = self._run_with_profiler(code, opts, code_ns)
654 else:
654 else:
655 if 'd' in opts:
655 if 'd' in opts:
656 bp_file, bp_line = parse_breakpoint(
656 bp_file, bp_line = parse_breakpoint(
657 opts.get('b', ['1'])[0], filename)
657 opts.get('b', ['1'])[0], filename)
658 self._run_with_debugger(
658 self._run_with_debugger(
659 code, code_ns, filename, bp_line, bp_file)
659 code, code_ns, filename, bp_line, bp_file)
660 else:
660 else:
661 if 'm' in opts:
661 if 'm' in opts:
662 def run():
662 def run():
663 self.shell.safe_run_module(modulename, prog_ns)
663 self.shell.safe_run_module(modulename, prog_ns)
664 else:
664 else:
665 if runner is None:
665 if runner is None:
666 runner = self.default_runner
666 runner = self.default_runner
667 if runner is None:
667 if runner is None:
668 runner = self.shell.safe_execfile
668 runner = self.shell.safe_execfile
669
669
670 def run():
670 def run():
671 runner(filename, prog_ns, prog_ns,
671 runner(filename, prog_ns, prog_ns,
672 exit_ignore=exit_ignore)
672 exit_ignore=exit_ignore)
673
673
674 if 't' in opts:
674 if 't' in opts:
675 # timed execution
675 # timed execution
676 try:
676 try:
677 nruns = int(opts['N'][0])
677 nruns = int(opts['N'][0])
678 if nruns < 1:
678 if nruns < 1:
679 error('Number of runs must be >=1')
679 error('Number of runs must be >=1')
680 return
680 return
681 except (KeyError):
681 except (KeyError):
682 nruns = 1
682 nruns = 1
683 self._run_with_timing(run, nruns)
683 self._run_with_timing(run, nruns)
684 else:
684 else:
685 # regular execution
685 # regular execution
686 run()
686 run()
687
687
688 if 'i' in opts:
688 if 'i' in opts:
689 self.shell.user_ns['__name__'] = __name__save
689 self.shell.user_ns['__name__'] = __name__save
690 else:
690 else:
691 # update IPython interactive namespace
691 # update IPython interactive namespace
692
692
693 # Some forms of read errors on the file may mean the
693 # Some forms of read errors on the file may mean the
694 # __name__ key was never set; using pop we don't have to
694 # __name__ key was never set; using pop we don't have to
695 # worry about a possible KeyError.
695 # worry about a possible KeyError.
696 prog_ns.pop('__name__', None)
696 prog_ns.pop('__name__', None)
697
697
698 with preserve_keys(self.shell.user_ns, '__file__'):
698 with preserve_keys(self.shell.user_ns, '__file__'):
699 self.shell.user_ns.update(prog_ns)
699 self.shell.user_ns.update(prog_ns)
700 finally:
700 finally:
701 # It's a bit of a mystery why, but __builtins__ can change from
701 # It's a bit of a mystery why, but __builtins__ can change from
702 # being a module to becoming a dict missing some key data after
702 # being a module to becoming a dict missing some key data after
703 # %run. As best I can see, this is NOT something IPython is doing
703 # %run. As best I can see, this is NOT something IPython is doing
704 # at all, and similar problems have been reported before:
704 # at all, and similar problems have been reported before:
705 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
705 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
706 # Since this seems to be done by the interpreter itself, the best
706 # Since this seems to be done by the interpreter itself, the best
707 # we can do is to at least restore __builtins__ for the user on
707 # we can do is to at least restore __builtins__ for the user on
708 # exit.
708 # exit.
709 self.shell.user_ns['__builtins__'] = builtin_mod
709 self.shell.user_ns['__builtins__'] = builtin_mod
710
710
711 # Ensure key global structures are restored
711 # Ensure key global structures are restored
712 sys.argv = save_argv
712 sys.argv = save_argv
713 if restore_main:
713 if restore_main:
714 sys.modules['__main__'] = restore_main
714 sys.modules['__main__'] = restore_main
715 else:
715 else:
716 # Remove from sys.modules the reference to main_mod we'd
716 # Remove from sys.modules the reference to main_mod we'd
717 # added. Otherwise it will trap references to objects
717 # added. Otherwise it will trap references to objects
718 # contained therein.
718 # contained therein.
719 del sys.modules[main_mod_name]
719 del sys.modules[main_mod_name]
720
720
721 return stats
721 return stats
722
722
723 def _run_with_debugger(self, code, code_ns, filename=None,
723 def _run_with_debugger(self, code, code_ns, filename=None,
724 bp_line=None, bp_file=None):
724 bp_line=None, bp_file=None):
725 """
725 """
726 Run `code` in debugger with a break point.
726 Run `code` in debugger with a break point.
727
727
728 Parameters
728 Parameters
729 ----------
729 ----------
730 code : str
730 code : str
731 Code to execute.
731 Code to execute.
732 code_ns : dict
732 code_ns : dict
733 A namespace in which `code` is executed.
733 A namespace in which `code` is executed.
734 filename : str
734 filename : str
735 `code` is ran as if it is in `filename`.
735 `code` is ran as if it is in `filename`.
736 bp_line : int, optional
736 bp_line : int, optional
737 Line number of the break point.
737 Line number of the break point.
738 bp_file : str, optional
738 bp_file : str, optional
739 Path to the file in which break point is specified.
739 Path to the file in which break point is specified.
740 `filename` is used if not given.
740 `filename` is used if not given.
741
741
742 Raises
742 Raises
743 ------
743 ------
744 UsageError
744 UsageError
745 If the break point given by `bp_line` is not valid.
745 If the break point given by `bp_line` is not valid.
746
746
747 """
747 """
748 deb = debugger.Pdb(self.shell.colors)
748 deb = debugger.Pdb(self.shell.colors)
749 # reset Breakpoint state, which is moronically kept
749 # reset Breakpoint state, which is moronically kept
750 # in a class
750 # in a class
751 bdb.Breakpoint.next = 1
751 bdb.Breakpoint.next = 1
752 bdb.Breakpoint.bplist = {}
752 bdb.Breakpoint.bplist = {}
753 bdb.Breakpoint.bpbynumber = [None]
753 bdb.Breakpoint.bpbynumber = [None]
754 if bp_line is not None:
754 if bp_line is not None:
755 # Set an initial breakpoint to stop execution
755 # Set an initial breakpoint to stop execution
756 maxtries = 10
756 maxtries = 10
757 bp_file = bp_file or filename
757 bp_file = bp_file or filename
758 checkline = deb.checkline(bp_file, bp_line)
758 checkline = deb.checkline(bp_file, bp_line)
759 if not checkline:
759 if not checkline:
760 for bp in range(bp_line + 1, bp_line + maxtries + 1):
760 for bp in range(bp_line + 1, bp_line + maxtries + 1):
761 if deb.checkline(bp_file, bp):
761 if deb.checkline(bp_file, bp):
762 break
762 break
763 else:
763 else:
764 msg = ("\nI failed to find a valid line to set "
764 msg = ("\nI failed to find a valid line to set "
765 "a breakpoint\n"
765 "a breakpoint\n"
766 "after trying up to line: %s.\n"
766 "after trying up to line: %s.\n"
767 "Please set a valid breakpoint manually "
767 "Please set a valid breakpoint manually "
768 "with the -b option." % bp)
768 "with the -b option." % bp)
769 raise UsageError(msg)
769 raise UsageError(msg)
770 # if we find a good linenumber, set the breakpoint
770 # if we find a good linenumber, set the breakpoint
771 deb.do_break('%s:%s' % (bp_file, bp_line))
771 deb.do_break('%s:%s' % (bp_file, bp_line))
772
772
773 if filename:
773 if filename:
774 # Mimic Pdb._runscript(...)
774 # Mimic Pdb._runscript(...)
775 deb._wait_for_mainpyfile = True
775 deb._wait_for_mainpyfile = True
776 deb.mainpyfile = deb.canonic(filename)
776 deb.mainpyfile = deb.canonic(filename)
777
777
778 # Start file run
778 # Start file run
779 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
779 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
780 try:
780 try:
781 if filename:
781 if filename:
782 # save filename so it can be used by methods on the deb object
782 # save filename so it can be used by methods on the deb object
783 deb._exec_filename = filename
783 deb._exec_filename = filename
784 deb.run(code, code_ns)
784 deb.run(code, code_ns)
785
785
786 except:
786 except:
787 etype, value, tb = sys.exc_info()
787 etype, value, tb = sys.exc_info()
788 # Skip three frames in the traceback: the %run one,
788 # Skip three frames in the traceback: the %run one,
789 # one inside bdb.py, and the command-line typed by the
789 # one inside bdb.py, and the command-line typed by the
790 # user (run by exec in pdb itself).
790 # user (run by exec in pdb itself).
791 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
791 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
792
792
793 @staticmethod
793 @staticmethod
794 def _run_with_timing(run, nruns):
794 def _run_with_timing(run, nruns):
795 """
795 """
796 Run function `run` and print timing information.
796 Run function `run` and print timing information.
797
797
798 Parameters
798 Parameters
799 ----------
799 ----------
800 run : callable
800 run : callable
801 Any callable object which takes no argument.
801 Any callable object which takes no argument.
802 nruns : int
802 nruns : int
803 Number of times to execute `run`.
803 Number of times to execute `run`.
804
804
805 """
805 """
806 twall0 = time.time()
806 twall0 = time.time()
807 if nruns == 1:
807 if nruns == 1:
808 t0 = clock2()
808 t0 = clock2()
809 run()
809 run()
810 t1 = clock2()
810 t1 = clock2()
811 t_usr = t1[0] - t0[0]
811 t_usr = t1[0] - t0[0]
812 t_sys = t1[1] - t0[1]
812 t_sys = t1[1] - t0[1]
813 print("\nIPython CPU timings (estimated):")
813 print("\nIPython CPU timings (estimated):")
814 print(" User : %10.2f s." % t_usr)
814 print(" User : %10.2f s." % t_usr)
815 print(" System : %10.2f s." % t_sys)
815 print(" System : %10.2f s." % t_sys)
816 else:
816 else:
817 runs = range(nruns)
817 runs = range(nruns)
818 t0 = clock2()
818 t0 = clock2()
819 for nr in runs:
819 for nr in runs:
820 run()
820 run()
821 t1 = clock2()
821 t1 = clock2()
822 t_usr = t1[0] - t0[0]
822 t_usr = t1[0] - t0[0]
823 t_sys = t1[1] - t0[1]
823 t_sys = t1[1] - t0[1]
824 print("\nIPython CPU timings (estimated):")
824 print("\nIPython CPU timings (estimated):")
825 print("Total runs performed:", nruns)
825 print("Total runs performed:", nruns)
826 print(" Times : %10s %10s" % ('Total', 'Per run'))
826 print(" Times : %10s %10s" % ('Total', 'Per run'))
827 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
827 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
828 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
828 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
829 twall1 = time.time()
829 twall1 = time.time()
830 print("Wall time: %10.2f s." % (twall1 - twall0))
830 print("Wall time: %10.2f s." % (twall1 - twall0))
831
831
832 @skip_doctest
832 @skip_doctest
833 @line_cell_magic
833 @line_cell_magic
834 def timeit(self, line='', cell=None):
834 def timeit(self, line='', cell=None):
835 """Time execution of a Python statement or expression
835 """Time execution of a Python statement or expression
836
836
837 Usage, in line mode:
837 Usage, in line mode:
838 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
838 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
839 or in cell mode:
839 or in cell mode:
840 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
840 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
841 code
841 code
842 code...
842 code...
843
843
844 Time execution of a Python statement or expression using the timeit
844 Time execution of a Python statement or expression using the timeit
845 module. This function can be used both as a line and cell magic:
845 module. This function can be used both as a line and cell magic:
846
846
847 - In line mode you can time a single-line statement (though multiple
847 - In line mode you can time a single-line statement (though multiple
848 ones can be chained with using semicolons).
848 ones can be chained with using semicolons).
849
849
850 - In cell mode, the statement in the first line is used as setup code
850 - In cell mode, the statement in the first line is used as setup code
851 (executed but not timed) and the body of the cell is timed. The cell
851 (executed but not timed) and the body of the cell is timed. The cell
852 body has access to any variables created in the setup code.
852 body has access to any variables created in the setup code.
853
853
854 Options:
854 Options:
855 -n<N>: execute the given statement <N> times in a loop. If this value
855 -n<N>: execute the given statement <N> times in a loop. If this value
856 is not given, a fitting value is chosen.
856 is not given, a fitting value is chosen.
857
857
858 -r<R>: repeat the loop iteration <R> times and take the best result.
858 -r<R>: repeat the loop iteration <R> times and take the best result.
859 Default: 3
859 Default: 3
860
860
861 -t: use time.time to measure the time, which is the default on Unix.
861 -t: use time.time to measure the time, which is the default on Unix.
862 This function measures wall time.
862 This function measures wall time.
863
863
864 -c: use time.clock to measure the time, which is the default on
864 -c: use time.clock to measure the time, which is the default on
865 Windows and measures wall time. On Unix, resource.getrusage is used
865 Windows and measures wall time. On Unix, resource.getrusage is used
866 instead and returns the CPU user time.
866 instead and returns the CPU user time.
867
867
868 -p<P>: use a precision of <P> digits to display the timing result.
868 -p<P>: use a precision of <P> digits to display the timing result.
869 Default: 3
869 Default: 3
870
870
871 -q: Quiet, do not print result.
871 -q: Quiet, do not print result.
872
872
873 -o: return a TimeitResult that can be stored in a variable to inspect
873 -o: return a TimeitResult that can be stored in a variable to inspect
874 the result in more details.
874 the result in more details.
875
875
876
876
877 Examples
877 Examples
878 --------
878 --------
879 ::
879 ::
880
880
881 In [1]: %timeit pass
881 In [1]: %timeit pass
882 10000000 loops, best of 3: 53.3 ns per loop
882 10000000 loops, best of 3: 53.3 ns per loop
883
883
884 In [2]: u = None
884 In [2]: u = None
885
885
886 In [3]: %timeit u is None
886 In [3]: %timeit u is None
887 10000000 loops, best of 3: 184 ns per loop
887 10000000 loops, best of 3: 184 ns per loop
888
888
889 In [4]: %timeit -r 4 u == None
889 In [4]: %timeit -r 4 u == None
890 1000000 loops, best of 4: 242 ns per loop
890 1000000 loops, best of 4: 242 ns per loop
891
891
892 In [5]: import time
892 In [5]: import time
893
893
894 In [6]: %timeit -n1 time.sleep(2)
894 In [6]: %timeit -n1 time.sleep(2)
895 1 loops, best of 3: 2 s per loop
895 1 loops, best of 3: 2 s per loop
896
896
897
897
898 The times reported by %timeit will be slightly higher than those
898 The times reported by %timeit will be slightly higher than those
899 reported by the timeit.py script when variables are accessed. This is
899 reported by the timeit.py script when variables are accessed. This is
900 due to the fact that %timeit executes the statement in the namespace
900 due to the fact that %timeit executes the statement in the namespace
901 of the shell, compared with timeit.py, which uses a single setup
901 of the shell, compared with timeit.py, which uses a single setup
902 statement to import function or create variables. Generally, the bias
902 statement to import function or create variables. Generally, the bias
903 does not matter as long as results from timeit.py are not mixed with
903 does not matter as long as results from timeit.py are not mixed with
904 those from %timeit."""
904 those from %timeit."""
905
905
906 import timeit
906 import timeit
907
907
908 opts, stmt = self.parse_options(line,'n:r:tcp:qo',
908 opts, stmt = self.parse_options(line,'n:r:tcp:qo',
909 posix=False, strict=False)
909 posix=False, strict=False)
910 if stmt == "" and cell is None:
910 if stmt == "" and cell is None:
911 return
911 return
912
912
913 timefunc = timeit.default_timer
913 timefunc = timeit.default_timer
914 number = int(getattr(opts, "n", 0))
914 number = int(getattr(opts, "n", 0))
915 repeat = int(getattr(opts, "r", timeit.default_repeat))
915 repeat = int(getattr(opts, "r", timeit.default_repeat))
916 precision = int(getattr(opts, "p", 3))
916 precision = int(getattr(opts, "p", 3))
917 quiet = 'q' in opts
917 quiet = 'q' in opts
918 return_result = 'o' in opts
918 return_result = 'o' in opts
919 if hasattr(opts, "t"):
919 if hasattr(opts, "t"):
920 timefunc = time.time
920 timefunc = time.time
921 if hasattr(opts, "c"):
921 if hasattr(opts, "c"):
922 timefunc = clock
922 timefunc = clock
923
923
924 timer = timeit.Timer(timer=timefunc)
924 timer = timeit.Timer(timer=timefunc)
925 # this code has tight coupling to the inner workings of timeit.Timer,
925 # this code has tight coupling to the inner workings of timeit.Timer,
926 # but is there a better way to achieve that the code stmt has access
926 # but is there a better way to achieve that the code stmt has access
927 # to the shell namespace?
927 # to the shell namespace?
928 transform = self.shell.input_splitter.transform_cell
928 transform = self.shell.input_splitter.transform_cell
929
929
930 if cell is None:
930 if cell is None:
931 # called as line magic
931 # called as line magic
932 ast_setup = ast.parse("pass")
932 ast_setup = ast.parse("pass")
933 ast_stmt = ast.parse(transform(stmt))
933 ast_stmt = ast.parse(transform(stmt))
934 else:
934 else:
935 ast_setup = ast.parse(transform(stmt))
935 ast_setup = ast.parse(transform(stmt))
936 ast_stmt = ast.parse(transform(cell))
936 ast_stmt = ast.parse(transform(cell))
937
937
938 ast_setup = self.shell.transform_ast(ast_setup)
938 ast_setup = self.shell.transform_ast(ast_setup)
939 ast_stmt = self.shell.transform_ast(ast_stmt)
939 ast_stmt = self.shell.transform_ast(ast_stmt)
940
940
941 # This codestring is taken from timeit.template - we fill it in as an
941 # This codestring is taken from timeit.template - we fill it in as an
942 # AST, so that we can apply our AST transformations to the user code
942 # AST, so that we can apply our AST transformations to the user code
943 # without affecting the timing code.
943 # without affecting the timing code.
944 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
944 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
945 ' setup\n'
945 ' setup\n'
946 ' _t0 = _timer()\n'
946 ' _t0 = _timer()\n'
947 ' for _i in _it:\n'
947 ' for _i in _it:\n'
948 ' stmt\n'
948 ' stmt\n'
949 ' _t1 = _timer()\n'
949 ' _t1 = _timer()\n'
950 ' return _t1 - _t0\n')
950 ' return _t1 - _t0\n')
951
951
952 class TimeitTemplateFiller(ast.NodeTransformer):
952 class TimeitTemplateFiller(ast.NodeTransformer):
953 "This is quite tightly tied to the template definition above."
953 "This is quite tightly tied to the template definition above."
954 def visit_FunctionDef(self, node):
954 def visit_FunctionDef(self, node):
955 "Fill in the setup statement"
955 "Fill in the setup statement"
956 self.generic_visit(node)
956 self.generic_visit(node)
957 if node.name == "inner":
957 if node.name == "inner":
958 node.body[:1] = ast_setup.body
958 node.body[:1] = ast_setup.body
959
959
960 return node
960 return node
961
961
962 def visit_For(self, node):
962 def visit_For(self, node):
963 "Fill in the statement to be timed"
963 "Fill in the statement to be timed"
964 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
964 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
965 node.body = ast_stmt.body
965 node.body = ast_stmt.body
966 return node
966 return node
967
967
968 timeit_ast = TimeitTemplateFiller().visit(timeit_ast_template)
968 timeit_ast = TimeitTemplateFiller().visit(timeit_ast_template)
969 timeit_ast = ast.fix_missing_locations(timeit_ast)
969 timeit_ast = ast.fix_missing_locations(timeit_ast)
970
970
971 # Track compilation time so it can be reported if too long
971 # Track compilation time so it can be reported if too long
972 # Minimum time above which compilation time will be reported
972 # Minimum time above which compilation time will be reported
973 tc_min = 0.1
973 tc_min = 0.1
974
974
975 t0 = clock()
975 t0 = clock()
976 code = compile(timeit_ast, "<magic-timeit>", "exec")
976 code = compile(timeit_ast, "<magic-timeit>", "exec")
977 tc = clock()-t0
977 tc = clock()-t0
978
978
979 ns = {}
979 ns = {}
980 exec(code, self.shell.user_ns, ns)
980 exec(code, self.shell.user_ns, ns)
981 timer.inner = ns["inner"]
981 timer.inner = ns["inner"]
982
982
983 if number == 0:
983 if number == 0:
984 # determine number so that 0.2 <= total time < 2.0
984 # determine number so that 0.2 <= total time < 2.0
985 number = 1
985 number = 1
986 for _ in range(1, 10):
986 for _ in range(1, 10):
987 if timer.timeit(number) >= 0.2:
987 if timer.timeit(number) >= 0.2:
988 break
988 break
989 number *= 10
989 number *= 10
990 all_runs = timer.repeat(repeat, number)
990 all_runs = timer.repeat(repeat, number)
991 best = min(all_runs) / number
991 best = min(all_runs) / number
992 if not quiet :
992 if not quiet :
993 print(u"%d loops, best of %d: %s per loop" % (number, repeat,
993 print(u"%d loops, best of %d: %s per loop" % (number, repeat,
994 _format_time(best, precision)))
994 _format_time(best, precision)))
995 if tc > tc_min:
995 if tc > tc_min:
996 print("Compiler time: %.2f s" % tc)
996 print("Compiler time: %.2f s" % tc)
997 if return_result:
997 if return_result:
998 return TimeitResult(number, repeat, best, all_runs, tc, precision)
998 return TimeitResult(number, repeat, best, all_runs, tc, precision)
999
999
1000 @skip_doctest
1000 @skip_doctest
1001 @needs_local_scope
1001 @needs_local_scope
1002 @line_cell_magic
1002 @line_cell_magic
1003 def time(self,line='', cell=None, local_ns=None):
1003 def time(self,line='', cell=None, local_ns=None):
1004 """Time execution of a Python statement or expression.
1004 """Time execution of a Python statement or expression.
1005
1005
1006 The CPU and wall clock times are printed, and the value of the
1006 The CPU and wall clock times are printed, and the value of the
1007 expression (if any) is returned. Note that under Win32, system time
1007 expression (if any) is returned. Note that under Win32, system time
1008 is always reported as 0, since it can not be measured.
1008 is always reported as 0, since it can not be measured.
1009
1009
1010 This function can be used both as a line and cell magic:
1010 This function can be used both as a line and cell magic:
1011
1011
1012 - In line mode you can time a single-line statement (though multiple
1012 - In line mode you can time a single-line statement (though multiple
1013 ones can be chained with using semicolons).
1013 ones can be chained with using semicolons).
1014
1014
1015 - In cell mode, you can time the cell body (a directly
1015 - In cell mode, you can time the cell body (a directly
1016 following statement raises an error).
1016 following statement raises an error).
1017
1017
1018 This function provides very basic timing functionality. Use the timeit
1018 This function provides very basic timing functionality. Use the timeit
1019 magic for more controll over the measurement.
1019 magic for more controll over the measurement.
1020
1020
1021 Examples
1021 Examples
1022 --------
1022 --------
1023 ::
1023 ::
1024
1024
1025 In [1]: %time 2**128
1025 In [1]: %time 2**128
1026 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1026 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1027 Wall time: 0.00
1027 Wall time: 0.00
1028 Out[1]: 340282366920938463463374607431768211456L
1028 Out[1]: 340282366920938463463374607431768211456L
1029
1029
1030 In [2]: n = 1000000
1030 In [2]: n = 1000000
1031
1031
1032 In [3]: %time sum(range(n))
1032 In [3]: %time sum(range(n))
1033 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1033 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1034 Wall time: 1.37
1034 Wall time: 1.37
1035 Out[3]: 499999500000L
1035 Out[3]: 499999500000L
1036
1036
1037 In [4]: %time print 'hello world'
1037 In [4]: %time print 'hello world'
1038 hello world
1038 hello world
1039 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1039 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1040 Wall time: 0.00
1040 Wall time: 0.00
1041
1041
1042 Note that the time needed by Python to compile the given expression
1042 Note that the time needed by Python to compile the given expression
1043 will be reported if it is more than 0.1s. In this example, the
1043 will be reported if it is more than 0.1s. In this example, the
1044 actual exponentiation is done by Python at compilation time, so while
1044 actual exponentiation is done by Python at compilation time, so while
1045 the expression can take a noticeable amount of time to compute, that
1045 the expression can take a noticeable amount of time to compute, that
1046 time is purely due to the compilation:
1046 time is purely due to the compilation:
1047
1047
1048 In [5]: %time 3**9999;
1048 In [5]: %time 3**9999;
1049 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1049 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1050 Wall time: 0.00 s
1050 Wall time: 0.00 s
1051
1051
1052 In [6]: %time 3**999999;
1052 In [6]: %time 3**999999;
1053 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1053 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1054 Wall time: 0.00 s
1054 Wall time: 0.00 s
1055 Compiler : 0.78 s
1055 Compiler : 0.78 s
1056 """
1056 """
1057
1057
1058 # fail immediately if the given expression can't be compiled
1058 # fail immediately if the given expression can't be compiled
1059
1059
1060 if line and cell:
1060 if line and cell:
1061 raise UsageError("Can't use statement directly after '%%time'!")
1061 raise UsageError("Can't use statement directly after '%%time'!")
1062
1062
1063 if cell:
1063 if cell:
1064 expr = self.shell.input_transformer_manager.transform_cell(cell)
1064 expr = self.shell.input_transformer_manager.transform_cell(cell)
1065 else:
1065 else:
1066 expr = self.shell.input_transformer_manager.transform_cell(line)
1066 expr = self.shell.input_transformer_manager.transform_cell(line)
1067
1067
1068 # Minimum time above which parse time will be reported
1068 # Minimum time above which parse time will be reported
1069 tp_min = 0.1
1069 tp_min = 0.1
1070
1070
1071 t0 = clock()
1071 t0 = clock()
1072 expr_ast = ast.parse(expr)
1072 expr_ast = ast.parse(expr)
1073 tp = clock()-t0
1073 tp = clock()-t0
1074
1074
1075 # Apply AST transformations
1075 # Apply AST transformations
1076 expr_ast = self.shell.transform_ast(expr_ast)
1076 expr_ast = self.shell.transform_ast(expr_ast)
1077
1077
1078 # Minimum time above which compilation time will be reported
1078 # Minimum time above which compilation time will be reported
1079 tc_min = 0.1
1079 tc_min = 0.1
1080
1080
1081 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1081 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1082 mode = 'eval'
1082 mode = 'eval'
1083 source = '<timed eval>'
1083 source = '<timed eval>'
1084 expr_ast = ast.Expression(expr_ast.body[0].value)
1084 expr_ast = ast.Expression(expr_ast.body[0].value)
1085 else:
1085 else:
1086 mode = 'exec'
1086 mode = 'exec'
1087 source = '<timed exec>'
1087 source = '<timed exec>'
1088 t0 = clock()
1088 t0 = clock()
1089 code = compile(expr_ast, source, mode)
1089 code = compile(expr_ast, source, mode)
1090 tc = clock()-t0
1090 tc = clock()-t0
1091
1091
1092 # skew measurement as little as possible
1092 # skew measurement as little as possible
1093 glob = self.shell.user_ns
1093 glob = self.shell.user_ns
1094 wtime = time.time
1094 wtime = time.time
1095 # time execution
1095 # time execution
1096 wall_st = wtime()
1096 wall_st = wtime()
1097 if mode=='eval':
1097 if mode=='eval':
1098 st = clock2()
1098 st = clock2()
1099 out = eval(code, glob, local_ns)
1099 out = eval(code, glob, local_ns)
1100 end = clock2()
1100 end = clock2()
1101 else:
1101 else:
1102 st = clock2()
1102 st = clock2()
1103 exec(code, glob, local_ns)
1103 exec(code, glob, local_ns)
1104 end = clock2()
1104 end = clock2()
1105 out = None
1105 out = None
1106 wall_end = wtime()
1106 wall_end = wtime()
1107 # Compute actual times and report
1107 # Compute actual times and report
1108 wall_time = wall_end-wall_st
1108 wall_time = wall_end-wall_st
1109 cpu_user = end[0]-st[0]
1109 cpu_user = end[0]-st[0]
1110 cpu_sys = end[1]-st[1]
1110 cpu_sys = end[1]-st[1]
1111 cpu_tot = cpu_user+cpu_sys
1111 cpu_tot = cpu_user+cpu_sys
1112 # On windows cpu_sys is always zero, so no new information to the next print
1112 # On windows cpu_sys is always zero, so no new information to the next print
1113 if sys.platform != 'win32':
1113 if sys.platform != 'win32':
1114 print("CPU times: user %s, sys: %s, total: %s" % \
1114 print("CPU times: user %s, sys: %s, total: %s" % \
1115 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
1115 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
1116 print("Wall time: %s" % _format_time(wall_time))
1116 print("Wall time: %s" % _format_time(wall_time))
1117 if tc > tc_min:
1117 if tc > tc_min:
1118 print("Compiler : %s" % _format_time(tc))
1118 print("Compiler : %s" % _format_time(tc))
1119 if tp > tp_min:
1119 if tp > tp_min:
1120 print("Parser : %s" % _format_time(tp))
1120 print("Parser : %s" % _format_time(tp))
1121 return out
1121 return out
1122
1122
1123 @skip_doctest
1123 @skip_doctest
1124 @line_magic
1124 @line_magic
1125 def macro(self, parameter_s=''):
1125 def macro(self, parameter_s=''):
1126 """Define a macro for future re-execution. It accepts ranges of history,
1126 """Define a macro for future re-execution. It accepts ranges of history,
1127 filenames or string objects.
1127 filenames or string objects.
1128
1128
1129 Usage:\\
1129 Usage:\\
1130 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1130 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1131
1131
1132 Options:
1132 Options:
1133
1133
1134 -r: use 'raw' input. By default, the 'processed' history is used,
1134 -r: use 'raw' input. By default, the 'processed' history is used,
1135 so that magics are loaded in their transformed version to valid
1135 so that magics are loaded in their transformed version to valid
1136 Python. If this option is given, the raw input as typed at the
1136 Python. If this option is given, the raw input as typed at the
1137 command line is used instead.
1137 command line is used instead.
1138
1138
1139 -q: quiet macro definition. By default, a tag line is printed
1139 -q: quiet macro definition. By default, a tag line is printed
1140 to indicate the macro has been created, and then the contents of
1140 to indicate the macro has been created, and then the contents of
1141 the macro are printed. If this option is given, then no printout
1141 the macro are printed. If this option is given, then no printout
1142 is produced once the macro is created.
1142 is produced once the macro is created.
1143
1143
1144 This will define a global variable called `name` which is a string
1144 This will define a global variable called `name` which is a string
1145 made of joining the slices and lines you specify (n1,n2,... numbers
1145 made of joining the slices and lines you specify (n1,n2,... numbers
1146 above) from your input history into a single string. This variable
1146 above) from your input history into a single string. This variable
1147 acts like an automatic function which re-executes those lines as if
1147 acts like an automatic function which re-executes those lines as if
1148 you had typed them. You just type 'name' at the prompt and the code
1148 you had typed them. You just type 'name' at the prompt and the code
1149 executes.
1149 executes.
1150
1150
1151 The syntax for indicating input ranges is described in %history.
1151 The syntax for indicating input ranges is described in %history.
1152
1152
1153 Note: as a 'hidden' feature, you can also use traditional python slice
1153 Note: as a 'hidden' feature, you can also use traditional python slice
1154 notation, where N:M means numbers N through M-1.
1154 notation, where N:M means numbers N through M-1.
1155
1155
1156 For example, if your history contains (print using %hist -n )::
1156 For example, if your history contains (print using %hist -n )::
1157
1157
1158 44: x=1
1158 44: x=1
1159 45: y=3
1159 45: y=3
1160 46: z=x+y
1160 46: z=x+y
1161 47: print x
1161 47: print x
1162 48: a=5
1162 48: a=5
1163 49: print 'x',x,'y',y
1163 49: print 'x',x,'y',y
1164
1164
1165 you can create a macro with lines 44 through 47 (included) and line 49
1165 you can create a macro with lines 44 through 47 (included) and line 49
1166 called my_macro with::
1166 called my_macro with::
1167
1167
1168 In [55]: %macro my_macro 44-47 49
1168 In [55]: %macro my_macro 44-47 49
1169
1169
1170 Now, typing `my_macro` (without quotes) will re-execute all this code
1170 Now, typing `my_macro` (without quotes) will re-execute all this code
1171 in one pass.
1171 in one pass.
1172
1172
1173 You don't need to give the line-numbers in order, and any given line
1173 You don't need to give the line-numbers in order, and any given line
1174 number can appear multiple times. You can assemble macros with any
1174 number can appear multiple times. You can assemble macros with any
1175 lines from your input history in any order.
1175 lines from your input history in any order.
1176
1176
1177 The macro is a simple object which holds its value in an attribute,
1177 The macro is a simple object which holds its value in an attribute,
1178 but IPython's display system checks for macros and executes them as
1178 but IPython's display system checks for macros and executes them as
1179 code instead of printing them when you type their name.
1179 code instead of printing them when you type their name.
1180
1180
1181 You can view a macro's contents by explicitly printing it with::
1181 You can view a macro's contents by explicitly printing it with::
1182
1182
1183 print macro_name
1183 print macro_name
1184
1184
1185 """
1185 """
1186 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1186 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1187 if not args: # List existing macros
1187 if not args: # List existing macros
1188 return sorted(k for k,v in self.shell.user_ns.iteritems() if\
1188 return sorted(k for k,v in self.shell.user_ns.iteritems() if\
1189 isinstance(v, Macro))
1189 isinstance(v, Macro))
1190 if len(args) == 1:
1190 if len(args) == 1:
1191 raise UsageError(
1191 raise UsageError(
1192 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1192 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1193 name, codefrom = args[0], " ".join(args[1:])
1193 name, codefrom = args[0], " ".join(args[1:])
1194
1194
1195 #print 'rng',ranges # dbg
1195 #print 'rng',ranges # dbg
1196 try:
1196 try:
1197 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1197 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1198 except (ValueError, TypeError) as e:
1198 except (ValueError, TypeError) as e:
1199 print(e.args[0])
1199 print(e.args[0])
1200 return
1200 return
1201 macro = Macro(lines)
1201 macro = Macro(lines)
1202 self.shell.define_macro(name, macro)
1202 self.shell.define_macro(name, macro)
1203 if not ( 'q' in opts) :
1203 if not ( 'q' in opts) :
1204 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1204 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1205 print('=== Macro contents: ===')
1205 print('=== Macro contents: ===')
1206 print(macro, end=' ')
1206 print(macro, end=' ')
1207
1207
1208 @magic_arguments.magic_arguments()
1208 @magic_arguments.magic_arguments()
1209 @magic_arguments.argument('output', type=str, default='', nargs='?',
1209 @magic_arguments.argument('output', type=str, default='', nargs='?',
1210 help="""The name of the variable in which to store output.
1210 help="""The name of the variable in which to store output.
1211 This is a utils.io.CapturedIO object with stdout/err attributes
1211 This is a utils.io.CapturedIO object with stdout/err attributes
1212 for the text of the captured output.
1212 for the text of the captured output.
1213
1213
1214 CapturedOutput also has a show() method for displaying the output,
1214 CapturedOutput also has a show() method for displaying the output,
1215 and __call__ as well, so you can use that to quickly display the
1215 and __call__ as well, so you can use that to quickly display the
1216 output.
1216 output.
1217
1217
1218 If unspecified, captured output is discarded.
1218 If unspecified, captured output is discarded.
1219 """
1219 """
1220 )
1220 )
1221 @magic_arguments.argument('--no-stderr', action="store_true",
1221 @magic_arguments.argument('--no-stderr', action="store_true",
1222 help="""Don't capture stderr."""
1222 help="""Don't capture stderr."""
1223 )
1223 )
1224 @magic_arguments.argument('--no-stdout', action="store_true",
1224 @magic_arguments.argument('--no-stdout', action="store_true",
1225 help="""Don't capture stdout."""
1225 help="""Don't capture stdout."""
1226 )
1226 )
1227 @magic_arguments.argument('--no-display', action="store_true",
1227 @magic_arguments.argument('--no-display', action="store_true",
1228 help="""Don't capture IPython's rich display."""
1228 help="""Don't capture IPython's rich display."""
1229 )
1229 )
1230 @cell_magic
1230 @cell_magic
1231 def capture(self, line, cell):
1231 def capture(self, line, cell):
1232 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1232 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1233 args = magic_arguments.parse_argstring(self.capture, line)
1233 args = magic_arguments.parse_argstring(self.capture, line)
1234 out = not args.no_stdout
1234 out = not args.no_stdout
1235 err = not args.no_stderr
1235 err = not args.no_stderr
1236 disp = not args.no_display
1236 disp = not args.no_display
1237 with capture_output(out, err, disp) as io:
1237 with capture_output(out, err, disp) as io:
1238 self.shell.run_cell(cell)
1238 self.shell.run_cell(cell)
1239 if args.output:
1239 if args.output:
1240 self.shell.user_ns[args.output] = io
1240 self.shell.user_ns[args.output] = io
1241
1241
1242 def parse_breakpoint(text, current_file):
1242 def parse_breakpoint(text, current_file):
1243 '''Returns (file, line) for file:line and (current_file, line) for line'''
1243 '''Returns (file, line) for file:line and (current_file, line) for line'''
1244 colon = text.find(':')
1244 colon = text.find(':')
1245 if colon == -1:
1245 if colon == -1:
1246 return current_file, int(text)
1246 return current_file, int(text)
1247 else:
1247 else:
1248 return text[:colon], int(text[colon+1:])
1248 return text[:colon], int(text[colon+1:])
1249
1249
1250 def _format_time(timespan, precision=3):
1250 def _format_time(timespan, precision=3):
1251 """Formats the timespan in a human readable form"""
1251 """Formats the timespan in a human readable form"""
1252 import math
1252 import math
1253
1253
1254 if timespan >= 60.0:
1254 if timespan >= 60.0:
1255 # we have more than a minute, format that in a human readable form
1255 # we have more than a minute, format that in a human readable form
1256 # Idea from http://snipplr.com/view/5713/
1256 # Idea from http://snipplr.com/view/5713/
1257 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1257 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1258 time = []
1258 time = []
1259 leftover = timespan
1259 leftover = timespan
1260 for suffix, length in parts:
1260 for suffix, length in parts:
1261 value = int(leftover / length)
1261 value = int(leftover / length)
1262 if value > 0:
1262 if value > 0:
1263 leftover = leftover % length
1263 leftover = leftover % length
1264 time.append(u'%s%s' % (str(value), suffix))
1264 time.append(u'%s%s' % (str(value), suffix))
1265 if leftover < 1:
1265 if leftover < 1:
1266 break
1266 break
1267 return " ".join(time)
1267 return " ".join(time)
1268
1268
1269
1269
1270 # Unfortunately the unicode 'micro' symbol can cause problems in
1270 # Unfortunately the unicode 'micro' symbol can cause problems in
1271 # certain terminals.
1271 # certain terminals.
1272 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1272 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1273 # Try to prevent crashes by being more secure than it needs to
1273 # Try to prevent crashes by being more secure than it needs to
1274 # E.g. eclipse is able to print a Β΅, but has no sys.stdout.encoding set.
1274 # E.g. eclipse is able to print a Β΅, but has no sys.stdout.encoding set.
1275 units = [u"s", u"ms",u'us',"ns"] # the save value
1275 units = [u"s", u"ms",u'us',"ns"] # the save value
1276 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1276 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1277 try:
1277 try:
1278 u'\xb5'.encode(sys.stdout.encoding)
1278 u'\xb5'.encode(sys.stdout.encoding)
1279 units = [u"s", u"ms",u'\xb5s',"ns"]
1279 units = [u"s", u"ms",u'\xb5s',"ns"]
1280 except:
1280 except:
1281 pass
1281 pass
1282 scaling = [1, 1e3, 1e6, 1e9]
1282 scaling = [1, 1e3, 1e6, 1e9]
1283
1283
1284 if timespan > 0.0:
1284 if timespan > 0.0:
1285 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1285 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1286 else:
1286 else:
1287 order = 3
1287 order = 3
1288 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
1288 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
@@ -1,65 +1,65 b''
1 import __builtin__
2 import sys
1 import sys
3
2
4 from IPython.core.displayhook import DisplayHook
3 from IPython.core.displayhook import DisplayHook
5 from IPython.kernel.inprocess.socket import SocketABC
4 from IPython.kernel.inprocess.socket import SocketABC
6 from IPython.utils.jsonutil import encode_images
5 from IPython.utils.jsonutil import encode_images
6 from IPython.utils.py3compat import builtin_mod
7 from IPython.utils.traitlets import Instance, Dict
7 from IPython.utils.traitlets import Instance, Dict
8 from .session import extract_header, Session
8 from .session import extract_header, Session
9
9
10 class ZMQDisplayHook(object):
10 class ZMQDisplayHook(object):
11 """A simple displayhook that publishes the object's repr over a ZeroMQ
11 """A simple displayhook that publishes the object's repr over a ZeroMQ
12 socket."""
12 socket."""
13 topic=b'pyout'
13 topic=b'pyout'
14
14
15 def __init__(self, session, pub_socket):
15 def __init__(self, session, pub_socket):
16 self.session = session
16 self.session = session
17 self.pub_socket = pub_socket
17 self.pub_socket = pub_socket
18 self.parent_header = {}
18 self.parent_header = {}
19
19
20 def __call__(self, obj):
20 def __call__(self, obj):
21 if obj is None:
21 if obj is None:
22 return
22 return
23
23
24 __builtin__._ = obj
24 builtin_mod._ = obj
25 sys.stdout.flush()
25 sys.stdout.flush()
26 sys.stderr.flush()
26 sys.stderr.flush()
27 msg = self.session.send(self.pub_socket, u'pyout', {u'data':repr(obj)},
27 msg = self.session.send(self.pub_socket, u'pyout', {u'data':repr(obj)},
28 parent=self.parent_header, ident=self.topic)
28 parent=self.parent_header, ident=self.topic)
29
29
30 def set_parent(self, parent):
30 def set_parent(self, parent):
31 self.parent_header = extract_header(parent)
31 self.parent_header = extract_header(parent)
32
32
33
33
34 class ZMQShellDisplayHook(DisplayHook):
34 class ZMQShellDisplayHook(DisplayHook):
35 """A displayhook subclass that publishes data using ZeroMQ. This is intended
35 """A displayhook subclass that publishes data using ZeroMQ. This is intended
36 to work with an InteractiveShell instance. It sends a dict of different
36 to work with an InteractiveShell instance. It sends a dict of different
37 representations of the object."""
37 representations of the object."""
38 topic=None
38 topic=None
39
39
40 session = Instance(Session)
40 session = Instance(Session)
41 pub_socket = Instance(SocketABC)
41 pub_socket = Instance(SocketABC)
42 parent_header = Dict({})
42 parent_header = Dict({})
43
43
44 def set_parent(self, parent):
44 def set_parent(self, parent):
45 """Set the parent for outbound messages."""
45 """Set the parent for outbound messages."""
46 self.parent_header = extract_header(parent)
46 self.parent_header = extract_header(parent)
47
47
48 def start_displayhook(self):
48 def start_displayhook(self):
49 self.msg = self.session.msg(u'pyout', {}, parent=self.parent_header)
49 self.msg = self.session.msg(u'pyout', {}, parent=self.parent_header)
50
50
51 def write_output_prompt(self):
51 def write_output_prompt(self):
52 """Write the output prompt."""
52 """Write the output prompt."""
53 self.msg['content']['execution_count'] = self.prompt_count
53 self.msg['content']['execution_count'] = self.prompt_count
54
54
55 def write_format_data(self, format_dict, md_dict=None):
55 def write_format_data(self, format_dict, md_dict=None):
56 self.msg['content']['data'] = encode_images(format_dict)
56 self.msg['content']['data'] = encode_images(format_dict)
57 self.msg['content']['metadata'] = md_dict
57 self.msg['content']['metadata'] = md_dict
58
58
59 def finish_displayhook(self):
59 def finish_displayhook(self):
60 """Finish up all displayhook activities."""
60 """Finish up all displayhook activities."""
61 sys.stdout.flush()
61 sys.stdout.flush()
62 sys.stderr.flush()
62 sys.stderr.flush()
63 self.session.send(self.pub_socket, self.msg, ident=self.topic)
63 self.session.send(self.pub_socket, self.msg, ident=self.topic)
64 self.msg = None
64 self.msg = None
65
65
@@ -1,789 +1,789 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """An interactive kernel that talks to frontends over 0MQ."""
2 """An interactive kernel that talks to frontends over 0MQ."""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Imports
5 # Imports
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 from __future__ import print_function
7 from __future__ import print_function
8
8
9 # Standard library imports
9 # Standard library imports
10 import __builtin__
11 import sys
10 import sys
12 import time
11 import time
13 import traceback
12 import traceback
14 import logging
13 import logging
15 import uuid
14 import uuid
16
15
17 from datetime import datetime
16 from datetime import datetime
18 from signal import (
17 from signal import (
19 signal, default_int_handler, SIGINT
18 signal, default_int_handler, SIGINT
20 )
19 )
21
20
22 # System library imports
21 # System library imports
23 import zmq
22 import zmq
24 from zmq.eventloop import ioloop
23 from zmq.eventloop import ioloop
25 from zmq.eventloop.zmqstream import ZMQStream
24 from zmq.eventloop.zmqstream import ZMQStream
26
25
27 # Local imports
26 # Local imports
28 from IPython.config.configurable import Configurable
27 from IPython.config.configurable import Configurable
29 from IPython.core.error import StdinNotImplementedError
28 from IPython.core.error import StdinNotImplementedError
30 from IPython.core import release
29 from IPython.core import release
31 from IPython.utils import py3compat
30 from IPython.utils import py3compat
31 from IPython.utils.py3compat import builtin_mod
32 from IPython.utils.jsonutil import json_clean
32 from IPython.utils.jsonutil import json_clean
33 from IPython.utils.traitlets import (
33 from IPython.utils.traitlets import (
34 Any, Instance, Float, Dict, List, Set, Integer, Unicode,
34 Any, Instance, Float, Dict, List, Set, Integer, Unicode,
35 Type
35 Type
36 )
36 )
37
37
38 from .serialize import serialize_object, unpack_apply_message
38 from .serialize import serialize_object, unpack_apply_message
39 from .session import Session
39 from .session import Session
40 from .zmqshell import ZMQInteractiveShell
40 from .zmqshell import ZMQInteractiveShell
41
41
42
42
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44 # Main kernel class
44 # Main kernel class
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 protocol_version = list(release.kernel_protocol_version_info)
47 protocol_version = list(release.kernel_protocol_version_info)
48 ipython_version = list(release.version_info)
48 ipython_version = list(release.version_info)
49 language_version = list(sys.version_info[:3])
49 language_version = list(sys.version_info[:3])
50
50
51
51
52 class Kernel(Configurable):
52 class Kernel(Configurable):
53
53
54 #---------------------------------------------------------------------------
54 #---------------------------------------------------------------------------
55 # Kernel interface
55 # Kernel interface
56 #---------------------------------------------------------------------------
56 #---------------------------------------------------------------------------
57
57
58 # attribute to override with a GUI
58 # attribute to override with a GUI
59 eventloop = Any(None)
59 eventloop = Any(None)
60 def _eventloop_changed(self, name, old, new):
60 def _eventloop_changed(self, name, old, new):
61 """schedule call to eventloop from IOLoop"""
61 """schedule call to eventloop from IOLoop"""
62 loop = ioloop.IOLoop.instance()
62 loop = ioloop.IOLoop.instance()
63 loop.add_timeout(time.time()+0.1, self.enter_eventloop)
63 loop.add_timeout(time.time()+0.1, self.enter_eventloop)
64
64
65 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
65 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
66 shell_class = Type(ZMQInteractiveShell)
66 shell_class = Type(ZMQInteractiveShell)
67
67
68 session = Instance(Session)
68 session = Instance(Session)
69 profile_dir = Instance('IPython.core.profiledir.ProfileDir')
69 profile_dir = Instance('IPython.core.profiledir.ProfileDir')
70 shell_streams = List()
70 shell_streams = List()
71 control_stream = Instance(ZMQStream)
71 control_stream = Instance(ZMQStream)
72 iopub_socket = Instance(zmq.Socket)
72 iopub_socket = Instance(zmq.Socket)
73 stdin_socket = Instance(zmq.Socket)
73 stdin_socket = Instance(zmq.Socket)
74 log = Instance(logging.Logger)
74 log = Instance(logging.Logger)
75
75
76 user_module = Any()
76 user_module = Any()
77 def _user_module_changed(self, name, old, new):
77 def _user_module_changed(self, name, old, new):
78 if self.shell is not None:
78 if self.shell is not None:
79 self.shell.user_module = new
79 self.shell.user_module = new
80
80
81 user_ns = Instance(dict, args=None, allow_none=True)
81 user_ns = Instance(dict, args=None, allow_none=True)
82 def _user_ns_changed(self, name, old, new):
82 def _user_ns_changed(self, name, old, new):
83 if self.shell is not None:
83 if self.shell is not None:
84 self.shell.user_ns = new
84 self.shell.user_ns = new
85 self.shell.init_user_ns()
85 self.shell.init_user_ns()
86
86
87 # identities:
87 # identities:
88 int_id = Integer(-1)
88 int_id = Integer(-1)
89 ident = Unicode()
89 ident = Unicode()
90
90
91 def _ident_default(self):
91 def _ident_default(self):
92 return unicode(uuid.uuid4())
92 return unicode(uuid.uuid4())
93
93
94
94
95 # Private interface
95 # Private interface
96
96
97 # Time to sleep after flushing the stdout/err buffers in each execute
97 # Time to sleep after flushing the stdout/err buffers in each execute
98 # cycle. While this introduces a hard limit on the minimal latency of the
98 # cycle. While this introduces a hard limit on the minimal latency of the
99 # execute cycle, it helps prevent output synchronization problems for
99 # execute cycle, it helps prevent output synchronization problems for
100 # clients.
100 # clients.
101 # Units are in seconds. The minimum zmq latency on local host is probably
101 # Units are in seconds. The minimum zmq latency on local host is probably
102 # ~150 microseconds, set this to 500us for now. We may need to increase it
102 # ~150 microseconds, set this to 500us for now. We may need to increase it
103 # a little if it's not enough after more interactive testing.
103 # a little if it's not enough after more interactive testing.
104 _execute_sleep = Float(0.0005, config=True)
104 _execute_sleep = Float(0.0005, config=True)
105
105
106 # Frequency of the kernel's event loop.
106 # Frequency of the kernel's event loop.
107 # Units are in seconds, kernel subclasses for GUI toolkits may need to
107 # Units are in seconds, kernel subclasses for GUI toolkits may need to
108 # adapt to milliseconds.
108 # adapt to milliseconds.
109 _poll_interval = Float(0.05, config=True)
109 _poll_interval = Float(0.05, config=True)
110
110
111 # If the shutdown was requested over the network, we leave here the
111 # If the shutdown was requested over the network, we leave here the
112 # necessary reply message so it can be sent by our registered atexit
112 # necessary reply message so it can be sent by our registered atexit
113 # handler. This ensures that the reply is only sent to clients truly at
113 # handler. This ensures that the reply is only sent to clients truly at
114 # the end of our shutdown process (which happens after the underlying
114 # the end of our shutdown process (which happens after the underlying
115 # IPython shell's own shutdown).
115 # IPython shell's own shutdown).
116 _shutdown_message = None
116 _shutdown_message = None
117
117
118 # This is a dict of port number that the kernel is listening on. It is set
118 # This is a dict of port number that the kernel is listening on. It is set
119 # by record_ports and used by connect_request.
119 # by record_ports and used by connect_request.
120 _recorded_ports = Dict()
120 _recorded_ports = Dict()
121
121
122 # A reference to the Python builtin 'raw_input' function.
122 # A reference to the Python builtin 'raw_input' function.
123 # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
123 # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
124 _sys_raw_input = Any()
124 _sys_raw_input = Any()
125 _sys_eval_input = Any()
125 _sys_eval_input = Any()
126
126
127 # set of aborted msg_ids
127 # set of aborted msg_ids
128 aborted = Set()
128 aborted = Set()
129
129
130
130
131 def __init__(self, **kwargs):
131 def __init__(self, **kwargs):
132 super(Kernel, self).__init__(**kwargs)
132 super(Kernel, self).__init__(**kwargs)
133
133
134 # Initialize the InteractiveShell subclass
134 # Initialize the InteractiveShell subclass
135 self.shell = self.shell_class.instance(parent=self,
135 self.shell = self.shell_class.instance(parent=self,
136 profile_dir = self.profile_dir,
136 profile_dir = self.profile_dir,
137 user_module = self.user_module,
137 user_module = self.user_module,
138 user_ns = self.user_ns,
138 user_ns = self.user_ns,
139 kernel = self,
139 kernel = self,
140 )
140 )
141 self.shell.displayhook.session = self.session
141 self.shell.displayhook.session = self.session
142 self.shell.displayhook.pub_socket = self.iopub_socket
142 self.shell.displayhook.pub_socket = self.iopub_socket
143 self.shell.displayhook.topic = self._topic('pyout')
143 self.shell.displayhook.topic = self._topic('pyout')
144 self.shell.display_pub.session = self.session
144 self.shell.display_pub.session = self.session
145 self.shell.display_pub.pub_socket = self.iopub_socket
145 self.shell.display_pub.pub_socket = self.iopub_socket
146 self.shell.data_pub.session = self.session
146 self.shell.data_pub.session = self.session
147 self.shell.data_pub.pub_socket = self.iopub_socket
147 self.shell.data_pub.pub_socket = self.iopub_socket
148
148
149 # TMP - hack while developing
149 # TMP - hack while developing
150 self.shell._reply_content = None
150 self.shell._reply_content = None
151
151
152 # Build dict of handlers for message types
152 # Build dict of handlers for message types
153 msg_types = [ 'execute_request', 'complete_request',
153 msg_types = [ 'execute_request', 'complete_request',
154 'object_info_request', 'history_request',
154 'object_info_request', 'history_request',
155 'kernel_info_request',
155 'kernel_info_request',
156 'connect_request', 'shutdown_request',
156 'connect_request', 'shutdown_request',
157 'apply_request',
157 'apply_request',
158 ]
158 ]
159 self.shell_handlers = {}
159 self.shell_handlers = {}
160 for msg_type in msg_types:
160 for msg_type in msg_types:
161 self.shell_handlers[msg_type] = getattr(self, msg_type)
161 self.shell_handlers[msg_type] = getattr(self, msg_type)
162
162
163 comm_msg_types = [ 'comm_open', 'comm_msg', 'comm_close' ]
163 comm_msg_types = [ 'comm_open', 'comm_msg', 'comm_close' ]
164 comm_manager = self.shell.comm_manager
164 comm_manager = self.shell.comm_manager
165 for msg_type in comm_msg_types:
165 for msg_type in comm_msg_types:
166 self.shell_handlers[msg_type] = getattr(comm_manager, msg_type)
166 self.shell_handlers[msg_type] = getattr(comm_manager, msg_type)
167
167
168 control_msg_types = msg_types + [ 'clear_request', 'abort_request' ]
168 control_msg_types = msg_types + [ 'clear_request', 'abort_request' ]
169 self.control_handlers = {}
169 self.control_handlers = {}
170 for msg_type in control_msg_types:
170 for msg_type in control_msg_types:
171 self.control_handlers[msg_type] = getattr(self, msg_type)
171 self.control_handlers[msg_type] = getattr(self, msg_type)
172
172
173
173
174 def dispatch_control(self, msg):
174 def dispatch_control(self, msg):
175 """dispatch control requests"""
175 """dispatch control requests"""
176 idents,msg = self.session.feed_identities(msg, copy=False)
176 idents,msg = self.session.feed_identities(msg, copy=False)
177 try:
177 try:
178 msg = self.session.unserialize(msg, content=True, copy=False)
178 msg = self.session.unserialize(msg, content=True, copy=False)
179 except:
179 except:
180 self.log.error("Invalid Control Message", exc_info=True)
180 self.log.error("Invalid Control Message", exc_info=True)
181 return
181 return
182
182
183 self.log.debug("Control received: %s", msg)
183 self.log.debug("Control received: %s", msg)
184
184
185 header = msg['header']
185 header = msg['header']
186 msg_id = header['msg_id']
186 msg_id = header['msg_id']
187 msg_type = header['msg_type']
187 msg_type = header['msg_type']
188
188
189 handler = self.control_handlers.get(msg_type, None)
189 handler = self.control_handlers.get(msg_type, None)
190 if handler is None:
190 if handler is None:
191 self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
191 self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
192 else:
192 else:
193 try:
193 try:
194 handler(self.control_stream, idents, msg)
194 handler(self.control_stream, idents, msg)
195 except Exception:
195 except Exception:
196 self.log.error("Exception in control handler:", exc_info=True)
196 self.log.error("Exception in control handler:", exc_info=True)
197
197
198 def dispatch_shell(self, stream, msg):
198 def dispatch_shell(self, stream, msg):
199 """dispatch shell requests"""
199 """dispatch shell requests"""
200 # flush control requests first
200 # flush control requests first
201 if self.control_stream:
201 if self.control_stream:
202 self.control_stream.flush()
202 self.control_stream.flush()
203
203
204 idents,msg = self.session.feed_identities(msg, copy=False)
204 idents,msg = self.session.feed_identities(msg, copy=False)
205 try:
205 try:
206 msg = self.session.unserialize(msg, content=True, copy=False)
206 msg = self.session.unserialize(msg, content=True, copy=False)
207 except:
207 except:
208 self.log.error("Invalid Message", exc_info=True)
208 self.log.error("Invalid Message", exc_info=True)
209 return
209 return
210
210
211 header = msg['header']
211 header = msg['header']
212 msg_id = header['msg_id']
212 msg_id = header['msg_id']
213 msg_type = msg['header']['msg_type']
213 msg_type = msg['header']['msg_type']
214
214
215 # Print some info about this message and leave a '--->' marker, so it's
215 # Print some info about this message and leave a '--->' marker, so it's
216 # easier to trace visually the message chain when debugging. Each
216 # easier to trace visually the message chain when debugging. Each
217 # handler prints its message at the end.
217 # handler prints its message at the end.
218 self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
218 self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
219 self.log.debug(' Content: %s\n --->\n ', msg['content'])
219 self.log.debug(' Content: %s\n --->\n ', msg['content'])
220
220
221 if msg_id in self.aborted:
221 if msg_id in self.aborted:
222 self.aborted.remove(msg_id)
222 self.aborted.remove(msg_id)
223 # is it safe to assume a msg_id will not be resubmitted?
223 # is it safe to assume a msg_id will not be resubmitted?
224 reply_type = msg_type.split('_')[0] + '_reply'
224 reply_type = msg_type.split('_')[0] + '_reply'
225 status = {'status' : 'aborted'}
225 status = {'status' : 'aborted'}
226 md = {'engine' : self.ident}
226 md = {'engine' : self.ident}
227 md.update(status)
227 md.update(status)
228 reply_msg = self.session.send(stream, reply_type, metadata=md,
228 reply_msg = self.session.send(stream, reply_type, metadata=md,
229 content=status, parent=msg, ident=idents)
229 content=status, parent=msg, ident=idents)
230 return
230 return
231
231
232 handler = self.shell_handlers.get(msg_type, None)
232 handler = self.shell_handlers.get(msg_type, None)
233 if handler is None:
233 if handler is None:
234 self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
234 self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
235 else:
235 else:
236 # ensure default_int_handler during handler call
236 # ensure default_int_handler during handler call
237 sig = signal(SIGINT, default_int_handler)
237 sig = signal(SIGINT, default_int_handler)
238 try:
238 try:
239 handler(stream, idents, msg)
239 handler(stream, idents, msg)
240 except Exception:
240 except Exception:
241 self.log.error("Exception in message handler:", exc_info=True)
241 self.log.error("Exception in message handler:", exc_info=True)
242 finally:
242 finally:
243 signal(SIGINT, sig)
243 signal(SIGINT, sig)
244
244
245 def enter_eventloop(self):
245 def enter_eventloop(self):
246 """enter eventloop"""
246 """enter eventloop"""
247 self.log.info("entering eventloop")
247 self.log.info("entering eventloop")
248 # restore default_int_handler
248 # restore default_int_handler
249 signal(SIGINT, default_int_handler)
249 signal(SIGINT, default_int_handler)
250 while self.eventloop is not None:
250 while self.eventloop is not None:
251 try:
251 try:
252 self.eventloop(self)
252 self.eventloop(self)
253 except KeyboardInterrupt:
253 except KeyboardInterrupt:
254 # Ctrl-C shouldn't crash the kernel
254 # Ctrl-C shouldn't crash the kernel
255 self.log.error("KeyboardInterrupt caught in kernel")
255 self.log.error("KeyboardInterrupt caught in kernel")
256 continue
256 continue
257 else:
257 else:
258 # eventloop exited cleanly, this means we should stop (right?)
258 # eventloop exited cleanly, this means we should stop (right?)
259 self.eventloop = None
259 self.eventloop = None
260 break
260 break
261 self.log.info("exiting eventloop")
261 self.log.info("exiting eventloop")
262
262
263 def start(self):
263 def start(self):
264 """register dispatchers for streams"""
264 """register dispatchers for streams"""
265 self.shell.exit_now = False
265 self.shell.exit_now = False
266 if self.control_stream:
266 if self.control_stream:
267 self.control_stream.on_recv(self.dispatch_control, copy=False)
267 self.control_stream.on_recv(self.dispatch_control, copy=False)
268
268
269 def make_dispatcher(stream):
269 def make_dispatcher(stream):
270 def dispatcher(msg):
270 def dispatcher(msg):
271 return self.dispatch_shell(stream, msg)
271 return self.dispatch_shell(stream, msg)
272 return dispatcher
272 return dispatcher
273
273
274 for s in self.shell_streams:
274 for s in self.shell_streams:
275 s.on_recv(make_dispatcher(s), copy=False)
275 s.on_recv(make_dispatcher(s), copy=False)
276
276
277 # publish idle status
277 # publish idle status
278 self._publish_status('starting')
278 self._publish_status('starting')
279
279
280 def do_one_iteration(self):
280 def do_one_iteration(self):
281 """step eventloop just once"""
281 """step eventloop just once"""
282 if self.control_stream:
282 if self.control_stream:
283 self.control_stream.flush()
283 self.control_stream.flush()
284 for stream in self.shell_streams:
284 for stream in self.shell_streams:
285 # handle at most one request per iteration
285 # handle at most one request per iteration
286 stream.flush(zmq.POLLIN, 1)
286 stream.flush(zmq.POLLIN, 1)
287 stream.flush(zmq.POLLOUT)
287 stream.flush(zmq.POLLOUT)
288
288
289
289
290 def record_ports(self, ports):
290 def record_ports(self, ports):
291 """Record the ports that this kernel is using.
291 """Record the ports that this kernel is using.
292
292
293 The creator of the Kernel instance must call this methods if they
293 The creator of the Kernel instance must call this methods if they
294 want the :meth:`connect_request` method to return the port numbers.
294 want the :meth:`connect_request` method to return the port numbers.
295 """
295 """
296 self._recorded_ports = ports
296 self._recorded_ports = ports
297
297
298 #---------------------------------------------------------------------------
298 #---------------------------------------------------------------------------
299 # Kernel request handlers
299 # Kernel request handlers
300 #---------------------------------------------------------------------------
300 #---------------------------------------------------------------------------
301
301
302 def _make_metadata(self, other=None):
302 def _make_metadata(self, other=None):
303 """init metadata dict, for execute/apply_reply"""
303 """init metadata dict, for execute/apply_reply"""
304 new_md = {
304 new_md = {
305 'dependencies_met' : True,
305 'dependencies_met' : True,
306 'engine' : self.ident,
306 'engine' : self.ident,
307 'started': datetime.now(),
307 'started': datetime.now(),
308 }
308 }
309 if other:
309 if other:
310 new_md.update(other)
310 new_md.update(other)
311 return new_md
311 return new_md
312
312
313 def _publish_pyin(self, code, parent, execution_count):
313 def _publish_pyin(self, code, parent, execution_count):
314 """Publish the code request on the pyin stream."""
314 """Publish the code request on the pyin stream."""
315
315
316 self.session.send(self.iopub_socket, u'pyin',
316 self.session.send(self.iopub_socket, u'pyin',
317 {u'code':code, u'execution_count': execution_count},
317 {u'code':code, u'execution_count': execution_count},
318 parent=parent, ident=self._topic('pyin')
318 parent=parent, ident=self._topic('pyin')
319 )
319 )
320
320
321 def _publish_status(self, status, parent=None):
321 def _publish_status(self, status, parent=None):
322 """send status (busy/idle) on IOPub"""
322 """send status (busy/idle) on IOPub"""
323 self.session.send(self.iopub_socket,
323 self.session.send(self.iopub_socket,
324 u'status',
324 u'status',
325 {u'execution_state': status},
325 {u'execution_state': status},
326 parent=parent,
326 parent=parent,
327 ident=self._topic('status'),
327 ident=self._topic('status'),
328 )
328 )
329
329
330
330
331 def execute_request(self, stream, ident, parent):
331 def execute_request(self, stream, ident, parent):
332 """handle an execute_request"""
332 """handle an execute_request"""
333
333
334 self._publish_status(u'busy', parent)
334 self._publish_status(u'busy', parent)
335
335
336 try:
336 try:
337 content = parent[u'content']
337 content = parent[u'content']
338 code = content[u'code']
338 code = content[u'code']
339 silent = content[u'silent']
339 silent = content[u'silent']
340 store_history = content.get(u'store_history', not silent)
340 store_history = content.get(u'store_history', not silent)
341 except:
341 except:
342 self.log.error("Got bad msg: ")
342 self.log.error("Got bad msg: ")
343 self.log.error("%s", parent)
343 self.log.error("%s", parent)
344 return
344 return
345
345
346 md = self._make_metadata(parent['metadata'])
346 md = self._make_metadata(parent['metadata'])
347
347
348 shell = self.shell # we'll need this a lot here
348 shell = self.shell # we'll need this a lot here
349
349
350 # Replace raw_input. Note that is not sufficient to replace
350 # Replace raw_input. Note that is not sufficient to replace
351 # raw_input in the user namespace.
351 # raw_input in the user namespace.
352 if content.get('allow_stdin', False):
352 if content.get('allow_stdin', False):
353 raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
353 raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
354 input = lambda prompt='': eval(raw_input(prompt))
354 input = lambda prompt='': eval(raw_input(prompt))
355 else:
355 else:
356 raw_input = input = lambda prompt='' : self._no_raw_input()
356 raw_input = input = lambda prompt='' : self._no_raw_input()
357
357
358 if py3compat.PY3:
358 if py3compat.PY3:
359 self._sys_raw_input = __builtin__.input
359 self._sys_raw_input = builtin_mod.input
360 __builtin__.input = raw_input
360 builtin_mod.input = raw_input
361 else:
361 else:
362 self._sys_raw_input = __builtin__.raw_input
362 self._sys_raw_input = builtin_mod.raw_input
363 self._sys_eval_input = __builtin__.input
363 self._sys_eval_input = builtin_mod.input
364 __builtin__.raw_input = raw_input
364 builtin_mod.raw_input = raw_input
365 __builtin__.input = input
365 builtin_mod.input = input
366
366
367 # Set the parent message of the display hook and out streams.
367 # Set the parent message of the display hook and out streams.
368 shell.set_parent(parent)
368 shell.set_parent(parent)
369
369
370 # Re-broadcast our input for the benefit of listening clients, and
370 # Re-broadcast our input for the benefit of listening clients, and
371 # start computing output
371 # start computing output
372 if not silent:
372 if not silent:
373 self._publish_pyin(code, parent, shell.execution_count)
373 self._publish_pyin(code, parent, shell.execution_count)
374
374
375 reply_content = {}
375 reply_content = {}
376 try:
376 try:
377 # FIXME: the shell calls the exception handler itself.
377 # FIXME: the shell calls the exception handler itself.
378 shell.run_cell(code, store_history=store_history, silent=silent)
378 shell.run_cell(code, store_history=store_history, silent=silent)
379 except:
379 except:
380 status = u'error'
380 status = u'error'
381 # FIXME: this code right now isn't being used yet by default,
381 # FIXME: this code right now isn't being used yet by default,
382 # because the run_cell() call above directly fires off exception
382 # because the run_cell() call above directly fires off exception
383 # reporting. This code, therefore, is only active in the scenario
383 # reporting. This code, therefore, is only active in the scenario
384 # where runlines itself has an unhandled exception. We need to
384 # where runlines itself has an unhandled exception. We need to
385 # uniformize this, for all exception construction to come from a
385 # uniformize this, for all exception construction to come from a
386 # single location in the codbase.
386 # single location in the codbase.
387 etype, evalue, tb = sys.exc_info()
387 etype, evalue, tb = sys.exc_info()
388 tb_list = traceback.format_exception(etype, evalue, tb)
388 tb_list = traceback.format_exception(etype, evalue, tb)
389 reply_content.update(shell._showtraceback(etype, evalue, tb_list))
389 reply_content.update(shell._showtraceback(etype, evalue, tb_list))
390 else:
390 else:
391 status = u'ok'
391 status = u'ok'
392 finally:
392 finally:
393 # Restore raw_input.
393 # Restore raw_input.
394 if py3compat.PY3:
394 if py3compat.PY3:
395 __builtin__.input = self._sys_raw_input
395 builtin_mod.input = self._sys_raw_input
396 else:
396 else:
397 __builtin__.raw_input = self._sys_raw_input
397 builtin_mod.raw_input = self._sys_raw_input
398 __builtin__.input = self._sys_eval_input
398 builtin_mod.input = self._sys_eval_input
399
399
400 reply_content[u'status'] = status
400 reply_content[u'status'] = status
401
401
402 # Return the execution counter so clients can display prompts
402 # Return the execution counter so clients can display prompts
403 reply_content['execution_count'] = shell.execution_count - 1
403 reply_content['execution_count'] = shell.execution_count - 1
404
404
405 # FIXME - fish exception info out of shell, possibly left there by
405 # FIXME - fish exception info out of shell, possibly left there by
406 # runlines. We'll need to clean up this logic later.
406 # runlines. We'll need to clean up this logic later.
407 if shell._reply_content is not None:
407 if shell._reply_content is not None:
408 reply_content.update(shell._reply_content)
408 reply_content.update(shell._reply_content)
409 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute')
409 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute')
410 reply_content['engine_info'] = e_info
410 reply_content['engine_info'] = e_info
411 # reset after use
411 # reset after use
412 shell._reply_content = None
412 shell._reply_content = None
413
413
414 if 'traceback' in reply_content:
414 if 'traceback' in reply_content:
415 self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback']))
415 self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback']))
416
416
417
417
418 # At this point, we can tell whether the main code execution succeeded
418 # At this point, we can tell whether the main code execution succeeded
419 # or not. If it did, we proceed to evaluate user_variables/expressions
419 # or not. If it did, we proceed to evaluate user_variables/expressions
420 if reply_content['status'] == 'ok':
420 if reply_content['status'] == 'ok':
421 reply_content[u'user_variables'] = \
421 reply_content[u'user_variables'] = \
422 shell.user_variables(content.get(u'user_variables', []))
422 shell.user_variables(content.get(u'user_variables', []))
423 reply_content[u'user_expressions'] = \
423 reply_content[u'user_expressions'] = \
424 shell.user_expressions(content.get(u'user_expressions', {}))
424 shell.user_expressions(content.get(u'user_expressions', {}))
425 else:
425 else:
426 # If there was an error, don't even try to compute variables or
426 # If there was an error, don't even try to compute variables or
427 # expressions
427 # expressions
428 reply_content[u'user_variables'] = {}
428 reply_content[u'user_variables'] = {}
429 reply_content[u'user_expressions'] = {}
429 reply_content[u'user_expressions'] = {}
430
430
431 # Payloads should be retrieved regardless of outcome, so we can both
431 # Payloads should be retrieved regardless of outcome, so we can both
432 # recover partial output (that could have been generated early in a
432 # recover partial output (that could have been generated early in a
433 # block, before an error) and clear the payload system always.
433 # block, before an error) and clear the payload system always.
434 reply_content[u'payload'] = shell.payload_manager.read_payload()
434 reply_content[u'payload'] = shell.payload_manager.read_payload()
435 # Be agressive about clearing the payload because we don't want
435 # Be agressive about clearing the payload because we don't want
436 # it to sit in memory until the next execute_request comes in.
436 # it to sit in memory until the next execute_request comes in.
437 shell.payload_manager.clear_payload()
437 shell.payload_manager.clear_payload()
438
438
439 # Flush output before sending the reply.
439 # Flush output before sending the reply.
440 sys.stdout.flush()
440 sys.stdout.flush()
441 sys.stderr.flush()
441 sys.stderr.flush()
442 # FIXME: on rare occasions, the flush doesn't seem to make it to the
442 # FIXME: on rare occasions, the flush doesn't seem to make it to the
443 # clients... This seems to mitigate the problem, but we definitely need
443 # clients... This seems to mitigate the problem, but we definitely need
444 # to better understand what's going on.
444 # to better understand what's going on.
445 if self._execute_sleep:
445 if self._execute_sleep:
446 time.sleep(self._execute_sleep)
446 time.sleep(self._execute_sleep)
447
447
448 # Send the reply.
448 # Send the reply.
449 reply_content = json_clean(reply_content)
449 reply_content = json_clean(reply_content)
450
450
451 md['status'] = reply_content['status']
451 md['status'] = reply_content['status']
452 if reply_content['status'] == 'error' and \
452 if reply_content['status'] == 'error' and \
453 reply_content['ename'] == 'UnmetDependency':
453 reply_content['ename'] == 'UnmetDependency':
454 md['dependencies_met'] = False
454 md['dependencies_met'] = False
455
455
456 reply_msg = self.session.send(stream, u'execute_reply',
456 reply_msg = self.session.send(stream, u'execute_reply',
457 reply_content, parent, metadata=md,
457 reply_content, parent, metadata=md,
458 ident=ident)
458 ident=ident)
459
459
460 self.log.debug("%s", reply_msg)
460 self.log.debug("%s", reply_msg)
461
461
462 if not silent and reply_msg['content']['status'] == u'error':
462 if not silent and reply_msg['content']['status'] == u'error':
463 self._abort_queues()
463 self._abort_queues()
464
464
465 self._publish_status(u'idle', parent)
465 self._publish_status(u'idle', parent)
466
466
467 def complete_request(self, stream, ident, parent):
467 def complete_request(self, stream, ident, parent):
468 txt, matches = self._complete(parent)
468 txt, matches = self._complete(parent)
469 matches = {'matches' : matches,
469 matches = {'matches' : matches,
470 'matched_text' : txt,
470 'matched_text' : txt,
471 'status' : 'ok'}
471 'status' : 'ok'}
472 matches = json_clean(matches)
472 matches = json_clean(matches)
473 completion_msg = self.session.send(stream, 'complete_reply',
473 completion_msg = self.session.send(stream, 'complete_reply',
474 matches, parent, ident)
474 matches, parent, ident)
475 self.log.debug("%s", completion_msg)
475 self.log.debug("%s", completion_msg)
476
476
477 def object_info_request(self, stream, ident, parent):
477 def object_info_request(self, stream, ident, parent):
478 content = parent['content']
478 content = parent['content']
479 object_info = self.shell.object_inspect(content['oname'],
479 object_info = self.shell.object_inspect(content['oname'],
480 detail_level = content.get('detail_level', 0)
480 detail_level = content.get('detail_level', 0)
481 )
481 )
482 # Before we send this object over, we scrub it for JSON usage
482 # Before we send this object over, we scrub it for JSON usage
483 oinfo = json_clean(object_info)
483 oinfo = json_clean(object_info)
484 msg = self.session.send(stream, 'object_info_reply',
484 msg = self.session.send(stream, 'object_info_reply',
485 oinfo, parent, ident)
485 oinfo, parent, ident)
486 self.log.debug("%s", msg)
486 self.log.debug("%s", msg)
487
487
488 def history_request(self, stream, ident, parent):
488 def history_request(self, stream, ident, parent):
489 # We need to pull these out, as passing **kwargs doesn't work with
489 # We need to pull these out, as passing **kwargs doesn't work with
490 # unicode keys before Python 2.6.5.
490 # unicode keys before Python 2.6.5.
491 hist_access_type = parent['content']['hist_access_type']
491 hist_access_type = parent['content']['hist_access_type']
492 raw = parent['content']['raw']
492 raw = parent['content']['raw']
493 output = parent['content']['output']
493 output = parent['content']['output']
494 if hist_access_type == 'tail':
494 if hist_access_type == 'tail':
495 n = parent['content']['n']
495 n = parent['content']['n']
496 hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
496 hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
497 include_latest=True)
497 include_latest=True)
498
498
499 elif hist_access_type == 'range':
499 elif hist_access_type == 'range':
500 session = parent['content']['session']
500 session = parent['content']['session']
501 start = parent['content']['start']
501 start = parent['content']['start']
502 stop = parent['content']['stop']
502 stop = parent['content']['stop']
503 hist = self.shell.history_manager.get_range(session, start, stop,
503 hist = self.shell.history_manager.get_range(session, start, stop,
504 raw=raw, output=output)
504 raw=raw, output=output)
505
505
506 elif hist_access_type == 'search':
506 elif hist_access_type == 'search':
507 n = parent['content'].get('n')
507 n = parent['content'].get('n')
508 unique = parent['content'].get('unique', False)
508 unique = parent['content'].get('unique', False)
509 pattern = parent['content']['pattern']
509 pattern = parent['content']['pattern']
510 hist = self.shell.history_manager.search(
510 hist = self.shell.history_manager.search(
511 pattern, raw=raw, output=output, n=n, unique=unique)
511 pattern, raw=raw, output=output, n=n, unique=unique)
512
512
513 else:
513 else:
514 hist = []
514 hist = []
515 hist = list(hist)
515 hist = list(hist)
516 content = {'history' : hist}
516 content = {'history' : hist}
517 content = json_clean(content)
517 content = json_clean(content)
518 msg = self.session.send(stream, 'history_reply',
518 msg = self.session.send(stream, 'history_reply',
519 content, parent, ident)
519 content, parent, ident)
520 self.log.debug("Sending history reply with %i entries", len(hist))
520 self.log.debug("Sending history reply with %i entries", len(hist))
521
521
522 def connect_request(self, stream, ident, parent):
522 def connect_request(self, stream, ident, parent):
523 if self._recorded_ports is not None:
523 if self._recorded_ports is not None:
524 content = self._recorded_ports.copy()
524 content = self._recorded_ports.copy()
525 else:
525 else:
526 content = {}
526 content = {}
527 msg = self.session.send(stream, 'connect_reply',
527 msg = self.session.send(stream, 'connect_reply',
528 content, parent, ident)
528 content, parent, ident)
529 self.log.debug("%s", msg)
529 self.log.debug("%s", msg)
530
530
531 def kernel_info_request(self, stream, ident, parent):
531 def kernel_info_request(self, stream, ident, parent):
532 vinfo = {
532 vinfo = {
533 'protocol_version': protocol_version,
533 'protocol_version': protocol_version,
534 'ipython_version': ipython_version,
534 'ipython_version': ipython_version,
535 'language_version': language_version,
535 'language_version': language_version,
536 'language': 'python',
536 'language': 'python',
537 }
537 }
538 msg = self.session.send(stream, 'kernel_info_reply',
538 msg = self.session.send(stream, 'kernel_info_reply',
539 vinfo, parent, ident)
539 vinfo, parent, ident)
540 self.log.debug("%s", msg)
540 self.log.debug("%s", msg)
541
541
542 def shutdown_request(self, stream, ident, parent):
542 def shutdown_request(self, stream, ident, parent):
543 self.shell.exit_now = True
543 self.shell.exit_now = True
544 content = dict(status='ok')
544 content = dict(status='ok')
545 content.update(parent['content'])
545 content.update(parent['content'])
546 self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
546 self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
547 # same content, but different msg_id for broadcasting on IOPub
547 # same content, but different msg_id for broadcasting on IOPub
548 self._shutdown_message = self.session.msg(u'shutdown_reply',
548 self._shutdown_message = self.session.msg(u'shutdown_reply',
549 content, parent
549 content, parent
550 )
550 )
551
551
552 self._at_shutdown()
552 self._at_shutdown()
553 # call sys.exit after a short delay
553 # call sys.exit after a short delay
554 loop = ioloop.IOLoop.instance()
554 loop = ioloop.IOLoop.instance()
555 loop.add_timeout(time.time()+0.1, loop.stop)
555 loop.add_timeout(time.time()+0.1, loop.stop)
556
556
557 #---------------------------------------------------------------------------
557 #---------------------------------------------------------------------------
558 # Engine methods
558 # Engine methods
559 #---------------------------------------------------------------------------
559 #---------------------------------------------------------------------------
560
560
561 def apply_request(self, stream, ident, parent):
561 def apply_request(self, stream, ident, parent):
562 try:
562 try:
563 content = parent[u'content']
563 content = parent[u'content']
564 bufs = parent[u'buffers']
564 bufs = parent[u'buffers']
565 msg_id = parent['header']['msg_id']
565 msg_id = parent['header']['msg_id']
566 except:
566 except:
567 self.log.error("Got bad msg: %s", parent, exc_info=True)
567 self.log.error("Got bad msg: %s", parent, exc_info=True)
568 return
568 return
569
569
570 self._publish_status(u'busy', parent)
570 self._publish_status(u'busy', parent)
571
571
572 # Set the parent message of the display hook and out streams.
572 # Set the parent message of the display hook and out streams.
573 shell = self.shell
573 shell = self.shell
574 shell.set_parent(parent)
574 shell.set_parent(parent)
575
575
576 # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
576 # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
577 # self.iopub_socket.send(pyin_msg)
577 # self.iopub_socket.send(pyin_msg)
578 # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
578 # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
579 md = self._make_metadata(parent['metadata'])
579 md = self._make_metadata(parent['metadata'])
580 try:
580 try:
581 working = shell.user_ns
581 working = shell.user_ns
582
582
583 prefix = "_"+str(msg_id).replace("-","")+"_"
583 prefix = "_"+str(msg_id).replace("-","")+"_"
584
584
585 f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
585 f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
586
586
587 fname = getattr(f, '__name__', 'f')
587 fname = getattr(f, '__name__', 'f')
588
588
589 fname = prefix+"f"
589 fname = prefix+"f"
590 argname = prefix+"args"
590 argname = prefix+"args"
591 kwargname = prefix+"kwargs"
591 kwargname = prefix+"kwargs"
592 resultname = prefix+"result"
592 resultname = prefix+"result"
593
593
594 ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
594 ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
595 # print ns
595 # print ns
596 working.update(ns)
596 working.update(ns)
597 code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
597 code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
598 try:
598 try:
599 exec(code, shell.user_global_ns, shell.user_ns)
599 exec(code, shell.user_global_ns, shell.user_ns)
600 result = working.get(resultname)
600 result = working.get(resultname)
601 finally:
601 finally:
602 for key in ns.iterkeys():
602 for key in ns.iterkeys():
603 working.pop(key)
603 working.pop(key)
604
604
605 result_buf = serialize_object(result,
605 result_buf = serialize_object(result,
606 buffer_threshold=self.session.buffer_threshold,
606 buffer_threshold=self.session.buffer_threshold,
607 item_threshold=self.session.item_threshold,
607 item_threshold=self.session.item_threshold,
608 )
608 )
609
609
610 except:
610 except:
611 # invoke IPython traceback formatting
611 # invoke IPython traceback formatting
612 shell.showtraceback()
612 shell.showtraceback()
613 # FIXME - fish exception info out of shell, possibly left there by
613 # FIXME - fish exception info out of shell, possibly left there by
614 # run_code. We'll need to clean up this logic later.
614 # run_code. We'll need to clean up this logic later.
615 reply_content = {}
615 reply_content = {}
616 if shell._reply_content is not None:
616 if shell._reply_content is not None:
617 reply_content.update(shell._reply_content)
617 reply_content.update(shell._reply_content)
618 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply')
618 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply')
619 reply_content['engine_info'] = e_info
619 reply_content['engine_info'] = e_info
620 # reset after use
620 # reset after use
621 shell._reply_content = None
621 shell._reply_content = None
622
622
623 self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent,
623 self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent,
624 ident=self._topic('pyerr'))
624 ident=self._topic('pyerr'))
625 self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback']))
625 self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback']))
626 result_buf = []
626 result_buf = []
627
627
628 if reply_content['ename'] == 'UnmetDependency':
628 if reply_content['ename'] == 'UnmetDependency':
629 md['dependencies_met'] = False
629 md['dependencies_met'] = False
630 else:
630 else:
631 reply_content = {'status' : 'ok'}
631 reply_content = {'status' : 'ok'}
632
632
633 # put 'ok'/'error' status in header, for scheduler introspection:
633 # put 'ok'/'error' status in header, for scheduler introspection:
634 md['status'] = reply_content['status']
634 md['status'] = reply_content['status']
635
635
636 # flush i/o
636 # flush i/o
637 sys.stdout.flush()
637 sys.stdout.flush()
638 sys.stderr.flush()
638 sys.stderr.flush()
639
639
640 reply_msg = self.session.send(stream, u'apply_reply', reply_content,
640 reply_msg = self.session.send(stream, u'apply_reply', reply_content,
641 parent=parent, ident=ident,buffers=result_buf, metadata=md)
641 parent=parent, ident=ident,buffers=result_buf, metadata=md)
642
642
643 self._publish_status(u'idle', parent)
643 self._publish_status(u'idle', parent)
644
644
645 #---------------------------------------------------------------------------
645 #---------------------------------------------------------------------------
646 # Control messages
646 # Control messages
647 #---------------------------------------------------------------------------
647 #---------------------------------------------------------------------------
648
648
649 def abort_request(self, stream, ident, parent):
649 def abort_request(self, stream, ident, parent):
650 """abort a specifig msg by id"""
650 """abort a specifig msg by id"""
651 msg_ids = parent['content'].get('msg_ids', None)
651 msg_ids = parent['content'].get('msg_ids', None)
652 if isinstance(msg_ids, basestring):
652 if isinstance(msg_ids, basestring):
653 msg_ids = [msg_ids]
653 msg_ids = [msg_ids]
654 if not msg_ids:
654 if not msg_ids:
655 self.abort_queues()
655 self.abort_queues()
656 for mid in msg_ids:
656 for mid in msg_ids:
657 self.aborted.add(str(mid))
657 self.aborted.add(str(mid))
658
658
659 content = dict(status='ok')
659 content = dict(status='ok')
660 reply_msg = self.session.send(stream, 'abort_reply', content=content,
660 reply_msg = self.session.send(stream, 'abort_reply', content=content,
661 parent=parent, ident=ident)
661 parent=parent, ident=ident)
662 self.log.debug("%s", reply_msg)
662 self.log.debug("%s", reply_msg)
663
663
664 def clear_request(self, stream, idents, parent):
664 def clear_request(self, stream, idents, parent):
665 """Clear our namespace."""
665 """Clear our namespace."""
666 self.shell.reset(False)
666 self.shell.reset(False)
667 msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
667 msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
668 content = dict(status='ok'))
668 content = dict(status='ok'))
669
669
670
670
671 #---------------------------------------------------------------------------
671 #---------------------------------------------------------------------------
672 # Protected interface
672 # Protected interface
673 #---------------------------------------------------------------------------
673 #---------------------------------------------------------------------------
674
674
675 def _wrap_exception(self, method=None):
675 def _wrap_exception(self, method=None):
676 # import here, because _wrap_exception is only used in parallel,
676 # import here, because _wrap_exception is only used in parallel,
677 # and parallel has higher min pyzmq version
677 # and parallel has higher min pyzmq version
678 from IPython.parallel.error import wrap_exception
678 from IPython.parallel.error import wrap_exception
679 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
679 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
680 content = wrap_exception(e_info)
680 content = wrap_exception(e_info)
681 return content
681 return content
682
682
683 def _topic(self, topic):
683 def _topic(self, topic):
684 """prefixed topic for IOPub messages"""
684 """prefixed topic for IOPub messages"""
685 if self.int_id >= 0:
685 if self.int_id >= 0:
686 base = "engine.%i" % self.int_id
686 base = "engine.%i" % self.int_id
687 else:
687 else:
688 base = "kernel.%s" % self.ident
688 base = "kernel.%s" % self.ident
689
689
690 return py3compat.cast_bytes("%s.%s" % (base, topic))
690 return py3compat.cast_bytes("%s.%s" % (base, topic))
691
691
692 def _abort_queues(self):
692 def _abort_queues(self):
693 for stream in self.shell_streams:
693 for stream in self.shell_streams:
694 if stream:
694 if stream:
695 self._abort_queue(stream)
695 self._abort_queue(stream)
696
696
697 def _abort_queue(self, stream):
697 def _abort_queue(self, stream):
698 poller = zmq.Poller()
698 poller = zmq.Poller()
699 poller.register(stream.socket, zmq.POLLIN)
699 poller.register(stream.socket, zmq.POLLIN)
700 while True:
700 while True:
701 idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
701 idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
702 if msg is None:
702 if msg is None:
703 return
703 return
704
704
705 self.log.info("Aborting:")
705 self.log.info("Aborting:")
706 self.log.info("%s", msg)
706 self.log.info("%s", msg)
707 msg_type = msg['header']['msg_type']
707 msg_type = msg['header']['msg_type']
708 reply_type = msg_type.split('_')[0] + '_reply'
708 reply_type = msg_type.split('_')[0] + '_reply'
709
709
710 status = {'status' : 'aborted'}
710 status = {'status' : 'aborted'}
711 md = {'engine' : self.ident}
711 md = {'engine' : self.ident}
712 md.update(status)
712 md.update(status)
713 reply_msg = self.session.send(stream, reply_type, metadata=md,
713 reply_msg = self.session.send(stream, reply_type, metadata=md,
714 content=status, parent=msg, ident=idents)
714 content=status, parent=msg, ident=idents)
715 self.log.debug("%s", reply_msg)
715 self.log.debug("%s", reply_msg)
716 # We need to wait a bit for requests to come in. This can probably
716 # We need to wait a bit for requests to come in. This can probably
717 # be set shorter for true asynchronous clients.
717 # be set shorter for true asynchronous clients.
718 poller.poll(50)
718 poller.poll(50)
719
719
720
720
721 def _no_raw_input(self):
721 def _no_raw_input(self):
722 """Raise StdinNotImplentedError if active frontend doesn't support
722 """Raise StdinNotImplentedError if active frontend doesn't support
723 stdin."""
723 stdin."""
724 raise StdinNotImplementedError("raw_input was called, but this "
724 raise StdinNotImplementedError("raw_input was called, but this "
725 "frontend does not support stdin.")
725 "frontend does not support stdin.")
726
726
727 def _raw_input(self, prompt, ident, parent):
727 def _raw_input(self, prompt, ident, parent):
728 # Flush output before making the request.
728 # Flush output before making the request.
729 sys.stderr.flush()
729 sys.stderr.flush()
730 sys.stdout.flush()
730 sys.stdout.flush()
731 # flush the stdin socket, to purge stale replies
731 # flush the stdin socket, to purge stale replies
732 while True:
732 while True:
733 try:
733 try:
734 self.stdin_socket.recv_multipart(zmq.NOBLOCK)
734 self.stdin_socket.recv_multipart(zmq.NOBLOCK)
735 except zmq.ZMQError as e:
735 except zmq.ZMQError as e:
736 if e.errno == zmq.EAGAIN:
736 if e.errno == zmq.EAGAIN:
737 break
737 break
738 else:
738 else:
739 raise
739 raise
740
740
741 # Send the input request.
741 # Send the input request.
742 content = json_clean(dict(prompt=prompt))
742 content = json_clean(dict(prompt=prompt))
743 self.session.send(self.stdin_socket, u'input_request', content, parent,
743 self.session.send(self.stdin_socket, u'input_request', content, parent,
744 ident=ident)
744 ident=ident)
745
745
746 # Await a response.
746 # Await a response.
747 while True:
747 while True:
748 try:
748 try:
749 ident, reply = self.session.recv(self.stdin_socket, 0)
749 ident, reply = self.session.recv(self.stdin_socket, 0)
750 except Exception:
750 except Exception:
751 self.log.warn("Invalid Message:", exc_info=True)
751 self.log.warn("Invalid Message:", exc_info=True)
752 except KeyboardInterrupt:
752 except KeyboardInterrupt:
753 # re-raise KeyboardInterrupt, to truncate traceback
753 # re-raise KeyboardInterrupt, to truncate traceback
754 raise KeyboardInterrupt
754 raise KeyboardInterrupt
755 else:
755 else:
756 break
756 break
757 try:
757 try:
758 value = py3compat.unicode_to_str(reply['content']['value'])
758 value = py3compat.unicode_to_str(reply['content']['value'])
759 except:
759 except:
760 self.log.error("Got bad raw_input reply: ")
760 self.log.error("Got bad raw_input reply: ")
761 self.log.error("%s", parent)
761 self.log.error("%s", parent)
762 value = ''
762 value = ''
763 if value == '\x04':
763 if value == '\x04':
764 # EOF
764 # EOF
765 raise EOFError
765 raise EOFError
766 return value
766 return value
767
767
768 def _complete(self, msg):
768 def _complete(self, msg):
769 c = msg['content']
769 c = msg['content']
770 try:
770 try:
771 cpos = int(c['cursor_pos'])
771 cpos = int(c['cursor_pos'])
772 except:
772 except:
773 # If we don't get something that we can convert to an integer, at
773 # If we don't get something that we can convert to an integer, at
774 # least attempt the completion guessing the cursor is at the end of
774 # least attempt the completion guessing the cursor is at the end of
775 # the text, if there's any, and otherwise of the line
775 # the text, if there's any, and otherwise of the line
776 cpos = len(c['text'])
776 cpos = len(c['text'])
777 if cpos==0:
777 if cpos==0:
778 cpos = len(c['line'])
778 cpos = len(c['line'])
779 return self.shell.complete(c['text'], c['line'], cpos)
779 return self.shell.complete(c['text'], c['line'], cpos)
780
780
781 def _at_shutdown(self):
781 def _at_shutdown(self):
782 """Actions taken at shutdown by the kernel, called by python's atexit.
782 """Actions taken at shutdown by the kernel, called by python's atexit.
783 """
783 """
784 # io.rprint("Kernel at_shutdown") # dbg
784 # io.rprint("Kernel at_shutdown") # dbg
785 if self._shutdown_message is not None:
785 if self._shutdown_message is not None:
786 self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
786 self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
787 self.log.debug("%s", self._shutdown_message)
787 self.log.debug("%s", self._shutdown_message)
788 [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
788 [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
789
789
@@ -1,338 +1,339 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 A module to change reload() so that it acts recursively.
3 A module to change reload() so that it acts recursively.
4 To enable it type::
4 To enable it type::
5
5
6 import __builtin__, deepreload
6 import __builtin__, deepreload
7 __builtin__.reload = deepreload.reload
7 __builtin__.reload = deepreload.reload
8
8
9 You can then disable it with::
9 You can then disable it with::
10
10
11 __builtin__.reload = deepreload.original_reload
11 __builtin__.reload = deepreload.original_reload
12
12
13 Alternatively, you can add a dreload builtin alongside normal reload with::
13 Alternatively, you can add a dreload builtin alongside normal reload with::
14
14
15 __builtin__.dreload = deepreload.reload
15 __builtin__.dreload = deepreload.reload
16
16
17 This code is almost entirely based on knee.py, which is a Python
17 This code is almost entirely based on knee.py, which is a Python
18 re-implementation of hierarchical module import.
18 re-implementation of hierarchical module import.
19 """
19 """
20 from __future__ import print_function
20 from __future__ import print_function
21 #*****************************************************************************
21 #*****************************************************************************
22 # Copyright (C) 2001 Nathaniel Gray <n8gray@caltech.edu>
22 # Copyright (C) 2001 Nathaniel Gray <n8gray@caltech.edu>
23 #
23 #
24 # Distributed under the terms of the BSD License. The full license is in
24 # Distributed under the terms of the BSD License. The full license is in
25 # the file COPYING, distributed as part of this software.
25 # the file COPYING, distributed as part of this software.
26 #*****************************************************************************
26 #*****************************************************************************
27
27
28 import __builtin__
29 from contextlib import contextmanager
28 from contextlib import contextmanager
30 import imp
29 import imp
31 import sys
30 import sys
32
31
33 from types import ModuleType
32 from types import ModuleType
34 from warnings import warn
33 from warnings import warn
35
34
36 original_import = __builtin__.__import__
35 from IPython.utils.py3compat import builtin_mod, builtin_mod_name
36
37 original_import = builtin_mod.__import__
37
38
38 @contextmanager
39 @contextmanager
39 def replace_import_hook(new_import):
40 def replace_import_hook(new_import):
40 saved_import = __builtin__.__import__
41 saved_import = builtin_mod.__import__
41 __builtin__.__import__ = new_import
42 builtin_mod.__import__ = new_import
42 try:
43 try:
43 yield
44 yield
44 finally:
45 finally:
45 __builtin__.__import__ = saved_import
46 builtin_mod.__import__ = saved_import
46
47
47 def get_parent(globals, level):
48 def get_parent(globals, level):
48 """
49 """
49 parent, name = get_parent(globals, level)
50 parent, name = get_parent(globals, level)
50
51
51 Return the package that an import is being performed in. If globals comes
52 Return the package that an import is being performed in. If globals comes
52 from the module foo.bar.bat (not itself a package), this returns the
53 from the module foo.bar.bat (not itself a package), this returns the
53 sys.modules entry for foo.bar. If globals is from a package's __init__.py,
54 sys.modules entry for foo.bar. If globals is from a package's __init__.py,
54 the package's entry in sys.modules is returned.
55 the package's entry in sys.modules is returned.
55
56
56 If globals doesn't come from a package or a module in a package, or a
57 If globals doesn't come from a package or a module in a package, or a
57 corresponding entry is not found in sys.modules, None is returned.
58 corresponding entry is not found in sys.modules, None is returned.
58 """
59 """
59 orig_level = level
60 orig_level = level
60
61
61 if not level or not isinstance(globals, dict):
62 if not level or not isinstance(globals, dict):
62 return None, ''
63 return None, ''
63
64
64 pkgname = globals.get('__package__', None)
65 pkgname = globals.get('__package__', None)
65
66
66 if pkgname is not None:
67 if pkgname is not None:
67 # __package__ is set, so use it
68 # __package__ is set, so use it
68 if not hasattr(pkgname, 'rindex'):
69 if not hasattr(pkgname, 'rindex'):
69 raise ValueError('__package__ set to non-string')
70 raise ValueError('__package__ set to non-string')
70 if len(pkgname) == 0:
71 if len(pkgname) == 0:
71 if level > 0:
72 if level > 0:
72 raise ValueError('Attempted relative import in non-package')
73 raise ValueError('Attempted relative import in non-package')
73 return None, ''
74 return None, ''
74 name = pkgname
75 name = pkgname
75 else:
76 else:
76 # __package__ not set, so figure it out and set it
77 # __package__ not set, so figure it out and set it
77 if '__name__' not in globals:
78 if '__name__' not in globals:
78 return None, ''
79 return None, ''
79 modname = globals['__name__']
80 modname = globals['__name__']
80
81
81 if '__path__' in globals:
82 if '__path__' in globals:
82 # __path__ is set, so modname is already the package name
83 # __path__ is set, so modname is already the package name
83 globals['__package__'] = name = modname
84 globals['__package__'] = name = modname
84 else:
85 else:
85 # Normal module, so work out the package name if any
86 # Normal module, so work out the package name if any
86 lastdot = modname.rfind('.')
87 lastdot = modname.rfind('.')
87 if lastdot < 0 and level > 0:
88 if lastdot < 0 and level > 0:
88 raise ValueError("Attempted relative import in non-package")
89 raise ValueError("Attempted relative import in non-package")
89 if lastdot < 0:
90 if lastdot < 0:
90 globals['__package__'] = None
91 globals['__package__'] = None
91 return None, ''
92 return None, ''
92 globals['__package__'] = name = modname[:lastdot]
93 globals['__package__'] = name = modname[:lastdot]
93
94
94 dot = len(name)
95 dot = len(name)
95 for x in xrange(level, 1, -1):
96 for x in xrange(level, 1, -1):
96 try:
97 try:
97 dot = name.rindex('.', 0, dot)
98 dot = name.rindex('.', 0, dot)
98 except ValueError:
99 except ValueError:
99 raise ValueError("attempted relative import beyond top-level "
100 raise ValueError("attempted relative import beyond top-level "
100 "package")
101 "package")
101 name = name[:dot]
102 name = name[:dot]
102
103
103 try:
104 try:
104 parent = sys.modules[name]
105 parent = sys.modules[name]
105 except:
106 except:
106 if orig_level < 1:
107 if orig_level < 1:
107 warn("Parent module '%.200s' not found while handling absolute "
108 warn("Parent module '%.200s' not found while handling absolute "
108 "import" % name)
109 "import" % name)
109 parent = None
110 parent = None
110 else:
111 else:
111 raise SystemError("Parent module '%.200s' not loaded, cannot "
112 raise SystemError("Parent module '%.200s' not loaded, cannot "
112 "perform relative import" % name)
113 "perform relative import" % name)
113
114
114 # We expect, but can't guarantee, if parent != None, that:
115 # We expect, but can't guarantee, if parent != None, that:
115 # - parent.__name__ == name
116 # - parent.__name__ == name
116 # - parent.__dict__ is globals
117 # - parent.__dict__ is globals
117 # If this is violated... Who cares?
118 # If this is violated... Who cares?
118 return parent, name
119 return parent, name
119
120
120 def load_next(mod, altmod, name, buf):
121 def load_next(mod, altmod, name, buf):
121 """
122 """
122 mod, name, buf = load_next(mod, altmod, name, buf)
123 mod, name, buf = load_next(mod, altmod, name, buf)
123
124
124 altmod is either None or same as mod
125 altmod is either None or same as mod
125 """
126 """
126
127
127 if len(name) == 0:
128 if len(name) == 0:
128 # completely empty module name should only happen in
129 # completely empty module name should only happen in
129 # 'from . import' (or '__import__("")')
130 # 'from . import' (or '__import__("")')
130 return mod, None, buf
131 return mod, None, buf
131
132
132 dot = name.find('.')
133 dot = name.find('.')
133 if dot == 0:
134 if dot == 0:
134 raise ValueError('Empty module name')
135 raise ValueError('Empty module name')
135
136
136 if dot < 0:
137 if dot < 0:
137 subname = name
138 subname = name
138 next = None
139 next = None
139 else:
140 else:
140 subname = name[:dot]
141 subname = name[:dot]
141 next = name[dot+1:]
142 next = name[dot+1:]
142
143
143 if buf != '':
144 if buf != '':
144 buf += '.'
145 buf += '.'
145 buf += subname
146 buf += subname
146
147
147 result = import_submodule(mod, subname, buf)
148 result = import_submodule(mod, subname, buf)
148 if result is None and mod != altmod:
149 if result is None and mod != altmod:
149 result = import_submodule(altmod, subname, subname)
150 result = import_submodule(altmod, subname, subname)
150 if result is not None:
151 if result is not None:
151 buf = subname
152 buf = subname
152
153
153 if result is None:
154 if result is None:
154 raise ImportError("No module named %.200s" % name)
155 raise ImportError("No module named %.200s" % name)
155
156
156 return result, next, buf
157 return result, next, buf
157
158
158 # Need to keep track of what we've already reloaded to prevent cyclic evil
159 # Need to keep track of what we've already reloaded to prevent cyclic evil
159 found_now = {}
160 found_now = {}
160
161
161 def import_submodule(mod, subname, fullname):
162 def import_submodule(mod, subname, fullname):
162 """m = import_submodule(mod, subname, fullname)"""
163 """m = import_submodule(mod, subname, fullname)"""
163 # Require:
164 # Require:
164 # if mod == None: subname == fullname
165 # if mod == None: subname == fullname
165 # else: mod.__name__ + "." + subname == fullname
166 # else: mod.__name__ + "." + subname == fullname
166
167
167 global found_now
168 global found_now
168 if fullname in found_now and fullname in sys.modules:
169 if fullname in found_now and fullname in sys.modules:
169 m = sys.modules[fullname]
170 m = sys.modules[fullname]
170 else:
171 else:
171 print('Reloading', fullname)
172 print('Reloading', fullname)
172 found_now[fullname] = 1
173 found_now[fullname] = 1
173 oldm = sys.modules.get(fullname, None)
174 oldm = sys.modules.get(fullname, None)
174
175
175 if mod is None:
176 if mod is None:
176 path = None
177 path = None
177 elif hasattr(mod, '__path__'):
178 elif hasattr(mod, '__path__'):
178 path = mod.__path__
179 path = mod.__path__
179 else:
180 else:
180 return None
181 return None
181
182
182 try:
183 try:
183 # This appears to be necessary on Python 3, because imp.find_module()
184 # This appears to be necessary on Python 3, because imp.find_module()
184 # tries to import standard libraries (like io) itself, and we don't
185 # tries to import standard libraries (like io) itself, and we don't
185 # want them to be processed by our deep_import_hook.
186 # want them to be processed by our deep_import_hook.
186 with replace_import_hook(original_import):
187 with replace_import_hook(original_import):
187 fp, filename, stuff = imp.find_module(subname, path)
188 fp, filename, stuff = imp.find_module(subname, path)
188 except ImportError:
189 except ImportError:
189 return None
190 return None
190
191
191 try:
192 try:
192 m = imp.load_module(fullname, fp, filename, stuff)
193 m = imp.load_module(fullname, fp, filename, stuff)
193 except:
194 except:
194 # load_module probably removed name from modules because of
195 # load_module probably removed name from modules because of
195 # the error. Put back the original module object.
196 # the error. Put back the original module object.
196 if oldm:
197 if oldm:
197 sys.modules[fullname] = oldm
198 sys.modules[fullname] = oldm
198 raise
199 raise
199 finally:
200 finally:
200 if fp: fp.close()
201 if fp: fp.close()
201
202
202 add_submodule(mod, m, fullname, subname)
203 add_submodule(mod, m, fullname, subname)
203
204
204 return m
205 return m
205
206
206 def add_submodule(mod, submod, fullname, subname):
207 def add_submodule(mod, submod, fullname, subname):
207 """mod.{subname} = submod"""
208 """mod.{subname} = submod"""
208 if mod is None:
209 if mod is None:
209 return #Nothing to do here.
210 return #Nothing to do here.
210
211
211 if submod is None:
212 if submod is None:
212 submod = sys.modules[fullname]
213 submod = sys.modules[fullname]
213
214
214 setattr(mod, subname, submod)
215 setattr(mod, subname, submod)
215
216
216 return
217 return
217
218
218 def ensure_fromlist(mod, fromlist, buf, recursive):
219 def ensure_fromlist(mod, fromlist, buf, recursive):
219 """Handle 'from module import a, b, c' imports."""
220 """Handle 'from module import a, b, c' imports."""
220 if not hasattr(mod, '__path__'):
221 if not hasattr(mod, '__path__'):
221 return
222 return
222 for item in fromlist:
223 for item in fromlist:
223 if not hasattr(item, 'rindex'):
224 if not hasattr(item, 'rindex'):
224 raise TypeError("Item in ``from list'' not a string")
225 raise TypeError("Item in ``from list'' not a string")
225 if item == '*':
226 if item == '*':
226 if recursive:
227 if recursive:
227 continue # avoid endless recursion
228 continue # avoid endless recursion
228 try:
229 try:
229 all = mod.__all__
230 all = mod.__all__
230 except AttributeError:
231 except AttributeError:
231 pass
232 pass
232 else:
233 else:
233 ret = ensure_fromlist(mod, all, buf, 1)
234 ret = ensure_fromlist(mod, all, buf, 1)
234 if not ret:
235 if not ret:
235 return 0
236 return 0
236 elif not hasattr(mod, item):
237 elif not hasattr(mod, item):
237 import_submodule(mod, item, buf + '.' + item)
238 import_submodule(mod, item, buf + '.' + item)
238
239
239 def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
240 def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
240 """Replacement for __import__()"""
241 """Replacement for __import__()"""
241 parent, buf = get_parent(globals, level)
242 parent, buf = get_parent(globals, level)
242
243
243 head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
244 head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
244
245
245 tail = head
246 tail = head
246 while name:
247 while name:
247 tail, name, buf = load_next(tail, tail, name, buf)
248 tail, name, buf = load_next(tail, tail, name, buf)
248
249
249 # If tail is None, both get_parent and load_next found
250 # If tail is None, both get_parent and load_next found
250 # an empty module name: someone called __import__("") or
251 # an empty module name: someone called __import__("") or
251 # doctored faulty bytecode
252 # doctored faulty bytecode
252 if tail is None:
253 if tail is None:
253 raise ValueError('Empty module name')
254 raise ValueError('Empty module name')
254
255
255 if not fromlist:
256 if not fromlist:
256 return head
257 return head
257
258
258 ensure_fromlist(tail, fromlist, buf, 0)
259 ensure_fromlist(tail, fromlist, buf, 0)
259 return tail
260 return tail
260
261
261 modules_reloading = {}
262 modules_reloading = {}
262
263
263 def deep_reload_hook(m):
264 def deep_reload_hook(m):
264 """Replacement for reload()."""
265 """Replacement for reload()."""
265 if not isinstance(m, ModuleType):
266 if not isinstance(m, ModuleType):
266 raise TypeError("reload() argument must be module")
267 raise TypeError("reload() argument must be module")
267
268
268 name = m.__name__
269 name = m.__name__
269
270
270 if name not in sys.modules:
271 if name not in sys.modules:
271 raise ImportError("reload(): module %.200s not in sys.modules" % name)
272 raise ImportError("reload(): module %.200s not in sys.modules" % name)
272
273
273 global modules_reloading
274 global modules_reloading
274 try:
275 try:
275 return modules_reloading[name]
276 return modules_reloading[name]
276 except:
277 except:
277 modules_reloading[name] = m
278 modules_reloading[name] = m
278
279
279 dot = name.rfind('.')
280 dot = name.rfind('.')
280 if dot < 0:
281 if dot < 0:
281 subname = name
282 subname = name
282 path = None
283 path = None
283 else:
284 else:
284 try:
285 try:
285 parent = sys.modules[name[:dot]]
286 parent = sys.modules[name[:dot]]
286 except KeyError:
287 except KeyError:
287 modules_reloading.clear()
288 modules_reloading.clear()
288 raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
289 raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
289 subname = name[dot+1:]
290 subname = name[dot+1:]
290 path = getattr(parent, "__path__", None)
291 path = getattr(parent, "__path__", None)
291
292
292 try:
293 try:
293 # This appears to be necessary on Python 3, because imp.find_module()
294 # This appears to be necessary on Python 3, because imp.find_module()
294 # tries to import standard libraries (like io) itself, and we don't
295 # tries to import standard libraries (like io) itself, and we don't
295 # want them to be processed by our deep_import_hook.
296 # want them to be processed by our deep_import_hook.
296 with replace_import_hook(original_import):
297 with replace_import_hook(original_import):
297 fp, filename, stuff = imp.find_module(subname, path)
298 fp, filename, stuff = imp.find_module(subname, path)
298 finally:
299 finally:
299 modules_reloading.clear()
300 modules_reloading.clear()
300
301
301 try:
302 try:
302 newm = imp.load_module(name, fp, filename, stuff)
303 newm = imp.load_module(name, fp, filename, stuff)
303 except:
304 except:
304 # load_module probably removed name from modules because of
305 # load_module probably removed name from modules because of
305 # the error. Put back the original module object.
306 # the error. Put back the original module object.
306 sys.modules[name] = m
307 sys.modules[name] = m
307 raise
308 raise
308 finally:
309 finally:
309 if fp: fp.close()
310 if fp: fp.close()
310
311
311 modules_reloading.clear()
312 modules_reloading.clear()
312 return newm
313 return newm
313
314
314 # Save the original hooks
315 # Save the original hooks
315 try:
316 try:
316 original_reload = __builtin__.reload
317 original_reload = builtin_mod.reload
317 except AttributeError:
318 except AttributeError:
318 original_reload = imp.reload # Python 3
319 original_reload = imp.reload # Python 3
319
320
320 # Replacement for reload()
321 # Replacement for reload()
321 def reload(module, exclude=['sys', 'os.path', '__builtin__', '__main__']):
322 def reload(module, exclude=['sys', 'os.path', builtin_mod_name, '__main__']):
322 """Recursively reload all modules used in the given module. Optionally
323 """Recursively reload all modules used in the given module. Optionally
323 takes a list of modules to exclude from reloading. The default exclude
324 takes a list of modules to exclude from reloading. The default exclude
324 list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
325 list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
325 display, exception, and io hooks.
326 display, exception, and io hooks.
326 """
327 """
327 global found_now
328 global found_now
328 for i in exclude:
329 for i in exclude:
329 found_now[i] = 1
330 found_now[i] = 1
330 try:
331 try:
331 with replace_import_hook(deep_import_hook):
332 with replace_import_hook(deep_import_hook):
332 return deep_reload_hook(module)
333 return deep_reload_hook(module)
333 finally:
334 finally:
334 found_now = {}
335 found_now = {}
335
336
336 # Uncomment the following to automatically activate deep reloading whenever
337 # Uncomment the following to automatically activate deep reloading whenever
337 # this module is imported
338 # this module is imported
338 #__builtin__.reload = reload
339 #builtin_mod.reload = reload
@@ -1,52 +1,53 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Test suite for the deepreload module."""
2 """Test suite for the deepreload module."""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Imports
5 # Imports
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7
7
8 import os
8 import os
9
9
10 import nose.tools as nt
10 import nose.tools as nt
11
11
12 from IPython.testing import decorators as dec
12 from IPython.testing import decorators as dec
13 from IPython.utils.py3compat import builtin_mod_name
13 from IPython.utils.syspathcontext import prepended_to_syspath
14 from IPython.utils.syspathcontext import prepended_to_syspath
14 from IPython.utils.tempdir import TemporaryDirectory
15 from IPython.utils.tempdir import TemporaryDirectory
15 from IPython.lib.deepreload import reload as dreload
16 from IPython.lib.deepreload import reload as dreload
16
17
17 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
18 # Test functions begin
19 # Test functions begin
19 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
20
21
21 @dec.skipif_not_numpy
22 @dec.skipif_not_numpy
22 def test_deepreload_numpy():
23 def test_deepreload_numpy():
23 "Test that NumPy can be deep reloaded."
24 "Test that NumPy can be deep reloaded."
24 import numpy
25 import numpy
25 exclude = [
26 exclude = [
26 # Standard exclusions:
27 # Standard exclusions:
27 'sys', 'os.path', '__builtin__', '__main__',
28 'sys', 'os.path', builtin_mod_name, '__main__',
28 # Test-related exclusions:
29 # Test-related exclusions:
29 'unittest', 'UserDict',
30 'unittest', 'UserDict',
30 ]
31 ]
31 dreload(numpy, exclude=exclude)
32 dreload(numpy, exclude=exclude)
32
33
33 def test_deepreload():
34 def test_deepreload():
34 "Test that dreload does deep reloads and skips excluded modules."
35 "Test that dreload does deep reloads and skips excluded modules."
35 with TemporaryDirectory() as tmpdir:
36 with TemporaryDirectory() as tmpdir:
36 with prepended_to_syspath(tmpdir):
37 with prepended_to_syspath(tmpdir):
37 with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
38 with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
38 f.write("class Object(object):\n pass\n")
39 f.write("class Object(object):\n pass\n")
39 with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
40 with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
40 f.write("import A\n")
41 f.write("import A\n")
41 import A
42 import A
42 import B
43 import B
43
44
44 # Test that A is not reloaded.
45 # Test that A is not reloaded.
45 obj = A.Object()
46 obj = A.Object()
46 dreload(B, exclude=['A'])
47 dreload(B, exclude=['A'])
47 nt.assert_true(isinstance(obj, A.Object))
48 nt.assert_true(isinstance(obj, A.Object))
48
49
49 # Test that A is reloaded.
50 # Test that A is reloaded.
50 obj = A.Object()
51 obj = A.Object()
51 dreload(B)
52 dreload(B)
52 nt.assert_false(isinstance(obj, A.Object))
53 nt.assert_false(isinstance(obj, A.Object))
@@ -1,1117 +1,1117 b''
1 """Views of remote engines.
1 """Views of remote engines.
2
2
3 Authors:
3 Authors:
4
4
5 * Min RK
5 * Min RK
6 """
6 """
7 from __future__ import print_function
7 from __future__ import print_function
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2010-2011 The IPython Development Team
9 # Copyright (C) 2010-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 import imp
19 import imp
20 import sys
20 import sys
21 import warnings
21 import warnings
22 from contextlib import contextmanager
22 from contextlib import contextmanager
23 from types import ModuleType
23 from types import ModuleType
24
24
25 import zmq
25 import zmq
26
26
27 from IPython.testing.skipdoctest import skip_doctest
27 from IPython.testing.skipdoctest import skip_doctest
28 from IPython.utils.traitlets import (
28 from IPython.utils.traitlets import (
29 HasTraits, Any, Bool, List, Dict, Set, Instance, CFloat, Integer
29 HasTraits, Any, Bool, List, Dict, Set, Instance, CFloat, Integer
30 )
30 )
31 from IPython.external.decorator import decorator
31 from IPython.external.decorator import decorator
32
32
33 from IPython.parallel import util
33 from IPython.parallel import util
34 from IPython.parallel.controller.dependency import Dependency, dependent
34 from IPython.parallel.controller.dependency import Dependency, dependent
35
35
36 from . import map as Map
36 from . import map as Map
37 from .asyncresult import AsyncResult, AsyncMapResult
37 from .asyncresult import AsyncResult, AsyncMapResult
38 from .remotefunction import ParallelFunction, parallel, remote, getname
38 from .remotefunction import ParallelFunction, parallel, remote, getname
39
39
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Decorators
41 # Decorators
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43
43
44 @decorator
44 @decorator
45 def save_ids(f, self, *args, **kwargs):
45 def save_ids(f, self, *args, **kwargs):
46 """Keep our history and outstanding attributes up to date after a method call."""
46 """Keep our history and outstanding attributes up to date after a method call."""
47 n_previous = len(self.client.history)
47 n_previous = len(self.client.history)
48 try:
48 try:
49 ret = f(self, *args, **kwargs)
49 ret = f(self, *args, **kwargs)
50 finally:
50 finally:
51 nmsgs = len(self.client.history) - n_previous
51 nmsgs = len(self.client.history) - n_previous
52 msg_ids = self.client.history[-nmsgs:]
52 msg_ids = self.client.history[-nmsgs:]
53 self.history.extend(msg_ids)
53 self.history.extend(msg_ids)
54 map(self.outstanding.add, msg_ids)
54 map(self.outstanding.add, msg_ids)
55 return ret
55 return ret
56
56
57 @decorator
57 @decorator
58 def sync_results(f, self, *args, **kwargs):
58 def sync_results(f, self, *args, **kwargs):
59 """sync relevant results from self.client to our results attribute."""
59 """sync relevant results from self.client to our results attribute."""
60 if self._in_sync_results:
60 if self._in_sync_results:
61 return f(self, *args, **kwargs)
61 return f(self, *args, **kwargs)
62 self._in_sync_results = True
62 self._in_sync_results = True
63 try:
63 try:
64 ret = f(self, *args, **kwargs)
64 ret = f(self, *args, **kwargs)
65 finally:
65 finally:
66 self._in_sync_results = False
66 self._in_sync_results = False
67 self._sync_results()
67 self._sync_results()
68 return ret
68 return ret
69
69
70 @decorator
70 @decorator
71 def spin_after(f, self, *args, **kwargs):
71 def spin_after(f, self, *args, **kwargs):
72 """call spin after the method."""
72 """call spin after the method."""
73 ret = f(self, *args, **kwargs)
73 ret = f(self, *args, **kwargs)
74 self.spin()
74 self.spin()
75 return ret
75 return ret
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Classes
78 # Classes
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 @skip_doctest
81 @skip_doctest
82 class View(HasTraits):
82 class View(HasTraits):
83 """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
83 """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
84
84
85 Don't use this class, use subclasses.
85 Don't use this class, use subclasses.
86
86
87 Methods
87 Methods
88 -------
88 -------
89
89
90 spin
90 spin
91 flushes incoming results and registration state changes
91 flushes incoming results and registration state changes
92 control methods spin, and requesting `ids` also ensures up to date
92 control methods spin, and requesting `ids` also ensures up to date
93
93
94 wait
94 wait
95 wait on one or more msg_ids
95 wait on one or more msg_ids
96
96
97 execution methods
97 execution methods
98 apply
98 apply
99 legacy: execute, run
99 legacy: execute, run
100
100
101 data movement
101 data movement
102 push, pull, scatter, gather
102 push, pull, scatter, gather
103
103
104 query methods
104 query methods
105 get_result, queue_status, purge_results, result_status
105 get_result, queue_status, purge_results, result_status
106
106
107 control methods
107 control methods
108 abort, shutdown
108 abort, shutdown
109
109
110 """
110 """
111 # flags
111 # flags
112 block=Bool(False)
112 block=Bool(False)
113 track=Bool(True)
113 track=Bool(True)
114 targets = Any()
114 targets = Any()
115
115
116 history=List()
116 history=List()
117 outstanding = Set()
117 outstanding = Set()
118 results = Dict()
118 results = Dict()
119 client = Instance('IPython.parallel.Client')
119 client = Instance('IPython.parallel.Client')
120
120
121 _socket = Instance('zmq.Socket')
121 _socket = Instance('zmq.Socket')
122 _flag_names = List(['targets', 'block', 'track'])
122 _flag_names = List(['targets', 'block', 'track'])
123 _in_sync_results = Bool(False)
123 _in_sync_results = Bool(False)
124 _targets = Any()
124 _targets = Any()
125 _idents = Any()
125 _idents = Any()
126
126
127 def __init__(self, client=None, socket=None, **flags):
127 def __init__(self, client=None, socket=None, **flags):
128 super(View, self).__init__(client=client, _socket=socket)
128 super(View, self).__init__(client=client, _socket=socket)
129 self.results = client.results
129 self.results = client.results
130 self.block = client.block
130 self.block = client.block
131
131
132 self.set_flags(**flags)
132 self.set_flags(**flags)
133
133
134 assert not self.__class__ is View, "Don't use base View objects, use subclasses"
134 assert not self.__class__ is View, "Don't use base View objects, use subclasses"
135
135
136 def __repr__(self):
136 def __repr__(self):
137 strtargets = str(self.targets)
137 strtargets = str(self.targets)
138 if len(strtargets) > 16:
138 if len(strtargets) > 16:
139 strtargets = strtargets[:12]+'...]'
139 strtargets = strtargets[:12]+'...]'
140 return "<%s %s>"%(self.__class__.__name__, strtargets)
140 return "<%s %s>"%(self.__class__.__name__, strtargets)
141
141
142 def __len__(self):
142 def __len__(self):
143 if isinstance(self.targets, list):
143 if isinstance(self.targets, list):
144 return len(self.targets)
144 return len(self.targets)
145 elif isinstance(self.targets, int):
145 elif isinstance(self.targets, int):
146 return 1
146 return 1
147 else:
147 else:
148 return len(self.client)
148 return len(self.client)
149
149
150 def set_flags(self, **kwargs):
150 def set_flags(self, **kwargs):
151 """set my attribute flags by keyword.
151 """set my attribute flags by keyword.
152
152
153 Views determine behavior with a few attributes (`block`, `track`, etc.).
153 Views determine behavior with a few attributes (`block`, `track`, etc.).
154 These attributes can be set all at once by name with this method.
154 These attributes can be set all at once by name with this method.
155
155
156 Parameters
156 Parameters
157 ----------
157 ----------
158
158
159 block : bool
159 block : bool
160 whether to wait for results
160 whether to wait for results
161 track : bool
161 track : bool
162 whether to create a MessageTracker to allow the user to
162 whether to create a MessageTracker to allow the user to
163 safely edit after arrays and buffers during non-copying
163 safely edit after arrays and buffers during non-copying
164 sends.
164 sends.
165 """
165 """
166 for name, value in kwargs.iteritems():
166 for name, value in kwargs.iteritems():
167 if name not in self._flag_names:
167 if name not in self._flag_names:
168 raise KeyError("Invalid name: %r"%name)
168 raise KeyError("Invalid name: %r"%name)
169 else:
169 else:
170 setattr(self, name, value)
170 setattr(self, name, value)
171
171
172 @contextmanager
172 @contextmanager
173 def temp_flags(self, **kwargs):
173 def temp_flags(self, **kwargs):
174 """temporarily set flags, for use in `with` statements.
174 """temporarily set flags, for use in `with` statements.
175
175
176 See set_flags for permanent setting of flags
176 See set_flags for permanent setting of flags
177
177
178 Examples
178 Examples
179 --------
179 --------
180
180
181 >>> view.track=False
181 >>> view.track=False
182 ...
182 ...
183 >>> with view.temp_flags(track=True):
183 >>> with view.temp_flags(track=True):
184 ... ar = view.apply(dostuff, my_big_array)
184 ... ar = view.apply(dostuff, my_big_array)
185 ... ar.tracker.wait() # wait for send to finish
185 ... ar.tracker.wait() # wait for send to finish
186 >>> view.track
186 >>> view.track
187 False
187 False
188
188
189 """
189 """
190 # preflight: save flags, and set temporaries
190 # preflight: save flags, and set temporaries
191 saved_flags = {}
191 saved_flags = {}
192 for f in self._flag_names:
192 for f in self._flag_names:
193 saved_flags[f] = getattr(self, f)
193 saved_flags[f] = getattr(self, f)
194 self.set_flags(**kwargs)
194 self.set_flags(**kwargs)
195 # yield to the with-statement block
195 # yield to the with-statement block
196 try:
196 try:
197 yield
197 yield
198 finally:
198 finally:
199 # postflight: restore saved flags
199 # postflight: restore saved flags
200 self.set_flags(**saved_flags)
200 self.set_flags(**saved_flags)
201
201
202
202
203 #----------------------------------------------------------------
203 #----------------------------------------------------------------
204 # apply
204 # apply
205 #----------------------------------------------------------------
205 #----------------------------------------------------------------
206
206
207 def _sync_results(self):
207 def _sync_results(self):
208 """to be called by @sync_results decorator
208 """to be called by @sync_results decorator
209
209
210 after submitting any tasks.
210 after submitting any tasks.
211 """
211 """
212 delta = self.outstanding.difference(self.client.outstanding)
212 delta = self.outstanding.difference(self.client.outstanding)
213 completed = self.outstanding.intersection(delta)
213 completed = self.outstanding.intersection(delta)
214 self.outstanding = self.outstanding.difference(completed)
214 self.outstanding = self.outstanding.difference(completed)
215
215
216 @sync_results
216 @sync_results
217 @save_ids
217 @save_ids
218 def _really_apply(self, f, args, kwargs, block=None, **options):
218 def _really_apply(self, f, args, kwargs, block=None, **options):
219 """wrapper for client.send_apply_request"""
219 """wrapper for client.send_apply_request"""
220 raise NotImplementedError("Implement in subclasses")
220 raise NotImplementedError("Implement in subclasses")
221
221
222 def apply(self, f, *args, **kwargs):
222 def apply(self, f, *args, **kwargs):
223 """calls f(*args, **kwargs) on remote engines, returning the result.
223 """calls f(*args, **kwargs) on remote engines, returning the result.
224
224
225 This method sets all apply flags via this View's attributes.
225 This method sets all apply flags via this View's attributes.
226
226
227 if self.block is False:
227 if self.block is False:
228 returns AsyncResult
228 returns AsyncResult
229 else:
229 else:
230 returns actual result of f(*args, **kwargs)
230 returns actual result of f(*args, **kwargs)
231 """
231 """
232 return self._really_apply(f, args, kwargs)
232 return self._really_apply(f, args, kwargs)
233
233
234 def apply_async(self, f, *args, **kwargs):
234 def apply_async(self, f, *args, **kwargs):
235 """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
235 """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
236
236
237 returns AsyncResult
237 returns AsyncResult
238 """
238 """
239 return self._really_apply(f, args, kwargs, block=False)
239 return self._really_apply(f, args, kwargs, block=False)
240
240
241 @spin_after
241 @spin_after
242 def apply_sync(self, f, *args, **kwargs):
242 def apply_sync(self, f, *args, **kwargs):
243 """calls f(*args, **kwargs) on remote engines in a blocking manner,
243 """calls f(*args, **kwargs) on remote engines in a blocking manner,
244 returning the result.
244 returning the result.
245
245
246 returns: actual result of f(*args, **kwargs)
246 returns: actual result of f(*args, **kwargs)
247 """
247 """
248 return self._really_apply(f, args, kwargs, block=True)
248 return self._really_apply(f, args, kwargs, block=True)
249
249
250 #----------------------------------------------------------------
250 #----------------------------------------------------------------
251 # wrappers for client and control methods
251 # wrappers for client and control methods
252 #----------------------------------------------------------------
252 #----------------------------------------------------------------
253 @sync_results
253 @sync_results
254 def spin(self):
254 def spin(self):
255 """spin the client, and sync"""
255 """spin the client, and sync"""
256 self.client.spin()
256 self.client.spin()
257
257
258 @sync_results
258 @sync_results
259 def wait(self, jobs=None, timeout=-1):
259 def wait(self, jobs=None, timeout=-1):
260 """waits on one or more `jobs`, for up to `timeout` seconds.
260 """waits on one or more `jobs`, for up to `timeout` seconds.
261
261
262 Parameters
262 Parameters
263 ----------
263 ----------
264
264
265 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
265 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
266 ints are indices to self.history
266 ints are indices to self.history
267 strs are msg_ids
267 strs are msg_ids
268 default: wait on all outstanding messages
268 default: wait on all outstanding messages
269 timeout : float
269 timeout : float
270 a time in seconds, after which to give up.
270 a time in seconds, after which to give up.
271 default is -1, which means no timeout
271 default is -1, which means no timeout
272
272
273 Returns
273 Returns
274 -------
274 -------
275
275
276 True : when all msg_ids are done
276 True : when all msg_ids are done
277 False : timeout reached, some msg_ids still outstanding
277 False : timeout reached, some msg_ids still outstanding
278 """
278 """
279 if jobs is None:
279 if jobs is None:
280 jobs = self.history
280 jobs = self.history
281 return self.client.wait(jobs, timeout)
281 return self.client.wait(jobs, timeout)
282
282
283 def abort(self, jobs=None, targets=None, block=None):
283 def abort(self, jobs=None, targets=None, block=None):
284 """Abort jobs on my engines.
284 """Abort jobs on my engines.
285
285
286 Parameters
286 Parameters
287 ----------
287 ----------
288
288
289 jobs : None, str, list of strs, optional
289 jobs : None, str, list of strs, optional
290 if None: abort all jobs.
290 if None: abort all jobs.
291 else: abort specific msg_id(s).
291 else: abort specific msg_id(s).
292 """
292 """
293 block = block if block is not None else self.block
293 block = block if block is not None else self.block
294 targets = targets if targets is not None else self.targets
294 targets = targets if targets is not None else self.targets
295 jobs = jobs if jobs is not None else list(self.outstanding)
295 jobs = jobs if jobs is not None else list(self.outstanding)
296
296
297 return self.client.abort(jobs=jobs, targets=targets, block=block)
297 return self.client.abort(jobs=jobs, targets=targets, block=block)
298
298
299 def queue_status(self, targets=None, verbose=False):
299 def queue_status(self, targets=None, verbose=False):
300 """Fetch the Queue status of my engines"""
300 """Fetch the Queue status of my engines"""
301 targets = targets if targets is not None else self.targets
301 targets = targets if targets is not None else self.targets
302 return self.client.queue_status(targets=targets, verbose=verbose)
302 return self.client.queue_status(targets=targets, verbose=verbose)
303
303
304 def purge_results(self, jobs=[], targets=[]):
304 def purge_results(self, jobs=[], targets=[]):
305 """Instruct the controller to forget specific results."""
305 """Instruct the controller to forget specific results."""
306 if targets is None or targets == 'all':
306 if targets is None or targets == 'all':
307 targets = self.targets
307 targets = self.targets
308 return self.client.purge_results(jobs=jobs, targets=targets)
308 return self.client.purge_results(jobs=jobs, targets=targets)
309
309
310 def shutdown(self, targets=None, restart=False, hub=False, block=None):
310 def shutdown(self, targets=None, restart=False, hub=False, block=None):
311 """Terminates one or more engine processes, optionally including the hub.
311 """Terminates one or more engine processes, optionally including the hub.
312 """
312 """
313 block = self.block if block is None else block
313 block = self.block if block is None else block
314 if targets is None or targets == 'all':
314 if targets is None or targets == 'all':
315 targets = self.targets
315 targets = self.targets
316 return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
316 return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
317
317
318 @spin_after
318 @spin_after
319 def get_result(self, indices_or_msg_ids=None):
319 def get_result(self, indices_or_msg_ids=None):
320 """return one or more results, specified by history index or msg_id.
320 """return one or more results, specified by history index or msg_id.
321
321
322 See client.get_result for details.
322 See client.get_result for details.
323
323
324 """
324 """
325
325
326 if indices_or_msg_ids is None:
326 if indices_or_msg_ids is None:
327 indices_or_msg_ids = -1
327 indices_or_msg_ids = -1
328 if isinstance(indices_or_msg_ids, int):
328 if isinstance(indices_or_msg_ids, int):
329 indices_or_msg_ids = self.history[indices_or_msg_ids]
329 indices_or_msg_ids = self.history[indices_or_msg_ids]
330 elif isinstance(indices_or_msg_ids, (list,tuple,set)):
330 elif isinstance(indices_or_msg_ids, (list,tuple,set)):
331 indices_or_msg_ids = list(indices_or_msg_ids)
331 indices_or_msg_ids = list(indices_or_msg_ids)
332 for i,index in enumerate(indices_or_msg_ids):
332 for i,index in enumerate(indices_or_msg_ids):
333 if isinstance(index, int):
333 if isinstance(index, int):
334 indices_or_msg_ids[i] = self.history[index]
334 indices_or_msg_ids[i] = self.history[index]
335 return self.client.get_result(indices_or_msg_ids)
335 return self.client.get_result(indices_or_msg_ids)
336
336
337 #-------------------------------------------------------------------
337 #-------------------------------------------------------------------
338 # Map
338 # Map
339 #-------------------------------------------------------------------
339 #-------------------------------------------------------------------
340
340
341 @sync_results
341 @sync_results
342 def map(self, f, *sequences, **kwargs):
342 def map(self, f, *sequences, **kwargs):
343 """override in subclasses"""
343 """override in subclasses"""
344 raise NotImplementedError
344 raise NotImplementedError
345
345
346 def map_async(self, f, *sequences, **kwargs):
346 def map_async(self, f, *sequences, **kwargs):
347 """Parallel version of builtin `map`, using this view's engines.
347 """Parallel version of builtin `map`, using this view's engines.
348
348
349 This is equivalent to map(...block=False)
349 This is equivalent to map(...block=False)
350
350
351 See `self.map` for details.
351 See `self.map` for details.
352 """
352 """
353 if 'block' in kwargs:
353 if 'block' in kwargs:
354 raise TypeError("map_async doesn't take a `block` keyword argument.")
354 raise TypeError("map_async doesn't take a `block` keyword argument.")
355 kwargs['block'] = False
355 kwargs['block'] = False
356 return self.map(f,*sequences,**kwargs)
356 return self.map(f,*sequences,**kwargs)
357
357
358 def map_sync(self, f, *sequences, **kwargs):
358 def map_sync(self, f, *sequences, **kwargs):
359 """Parallel version of builtin `map`, using this view's engines.
359 """Parallel version of builtin `map`, using this view's engines.
360
360
361 This is equivalent to map(...block=True)
361 This is equivalent to map(...block=True)
362
362
363 See `self.map` for details.
363 See `self.map` for details.
364 """
364 """
365 if 'block' in kwargs:
365 if 'block' in kwargs:
366 raise TypeError("map_sync doesn't take a `block` keyword argument.")
366 raise TypeError("map_sync doesn't take a `block` keyword argument.")
367 kwargs['block'] = True
367 kwargs['block'] = True
368 return self.map(f,*sequences,**kwargs)
368 return self.map(f,*sequences,**kwargs)
369
369
370 def imap(self, f, *sequences, **kwargs):
370 def imap(self, f, *sequences, **kwargs):
371 """Parallel version of `itertools.imap`.
371 """Parallel version of `itertools.imap`.
372
372
373 See `self.map` for details.
373 See `self.map` for details.
374
374
375 """
375 """
376
376
377 return iter(self.map_async(f,*sequences, **kwargs))
377 return iter(self.map_async(f,*sequences, **kwargs))
378
378
379 #-------------------------------------------------------------------
379 #-------------------------------------------------------------------
380 # Decorators
380 # Decorators
381 #-------------------------------------------------------------------
381 #-------------------------------------------------------------------
382
382
383 def remote(self, block=None, **flags):
383 def remote(self, block=None, **flags):
384 """Decorator for making a RemoteFunction"""
384 """Decorator for making a RemoteFunction"""
385 block = self.block if block is None else block
385 block = self.block if block is None else block
386 return remote(self, block=block, **flags)
386 return remote(self, block=block, **flags)
387
387
388 def parallel(self, dist='b', block=None, **flags):
388 def parallel(self, dist='b', block=None, **flags):
389 """Decorator for making a ParallelFunction"""
389 """Decorator for making a ParallelFunction"""
390 block = self.block if block is None else block
390 block = self.block if block is None else block
391 return parallel(self, dist=dist, block=block, **flags)
391 return parallel(self, dist=dist, block=block, **flags)
392
392
393 @skip_doctest
393 @skip_doctest
394 class DirectView(View):
394 class DirectView(View):
395 """Direct Multiplexer View of one or more engines.
395 """Direct Multiplexer View of one or more engines.
396
396
397 These are created via indexed access to a client:
397 These are created via indexed access to a client:
398
398
399 >>> dv_1 = client[1]
399 >>> dv_1 = client[1]
400 >>> dv_all = client[:]
400 >>> dv_all = client[:]
401 >>> dv_even = client[::2]
401 >>> dv_even = client[::2]
402 >>> dv_some = client[1:3]
402 >>> dv_some = client[1:3]
403
403
404 This object provides dictionary access to engine namespaces:
404 This object provides dictionary access to engine namespaces:
405
405
406 # push a=5:
406 # push a=5:
407 >>> dv['a'] = 5
407 >>> dv['a'] = 5
408 # pull 'foo':
408 # pull 'foo':
409 >>> db['foo']
409 >>> db['foo']
410
410
411 """
411 """
412
412
413 def __init__(self, client=None, socket=None, targets=None):
413 def __init__(self, client=None, socket=None, targets=None):
414 super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
414 super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
415
415
416 @property
416 @property
417 def importer(self):
417 def importer(self):
418 """sync_imports(local=True) as a property.
418 """sync_imports(local=True) as a property.
419
419
420 See sync_imports for details.
420 See sync_imports for details.
421
421
422 """
422 """
423 return self.sync_imports(True)
423 return self.sync_imports(True)
424
424
425 @contextmanager
425 @contextmanager
426 def sync_imports(self, local=True, quiet=False):
426 def sync_imports(self, local=True, quiet=False):
427 """Context Manager for performing simultaneous local and remote imports.
427 """Context Manager for performing simultaneous local and remote imports.
428
428
429 'import x as y' will *not* work. The 'as y' part will simply be ignored.
429 'import x as y' will *not* work. The 'as y' part will simply be ignored.
430
430
431 If `local=True`, then the package will also be imported locally.
431 If `local=True`, then the package will also be imported locally.
432
432
433 If `quiet=True`, no output will be produced when attempting remote
433 If `quiet=True`, no output will be produced when attempting remote
434 imports.
434 imports.
435
435
436 Note that remote-only (`local=False`) imports have not been implemented.
436 Note that remote-only (`local=False`) imports have not been implemented.
437
437
438 >>> with view.sync_imports():
438 >>> with view.sync_imports():
439 ... from numpy import recarray
439 ... from numpy import recarray
440 importing recarray from numpy on engine(s)
440 importing recarray from numpy on engine(s)
441
441
442 """
442 """
443 import __builtin__
443 from IPython.utils.py3compat import builtin_mod
444 local_import = __builtin__.__import__
444 local_import = builtin_mod.__import__
445 modules = set()
445 modules = set()
446 results = []
446 results = []
447 @util.interactive
447 @util.interactive
448 def remote_import(name, fromlist, level):
448 def remote_import(name, fromlist, level):
449 """the function to be passed to apply, that actually performs the import
449 """the function to be passed to apply, that actually performs the import
450 on the engine, and loads up the user namespace.
450 on the engine, and loads up the user namespace.
451 """
451 """
452 import sys
452 import sys
453 user_ns = globals()
453 user_ns = globals()
454 mod = __import__(name, fromlist=fromlist, level=level)
454 mod = __import__(name, fromlist=fromlist, level=level)
455 if fromlist:
455 if fromlist:
456 for key in fromlist:
456 for key in fromlist:
457 user_ns[key] = getattr(mod, key)
457 user_ns[key] = getattr(mod, key)
458 else:
458 else:
459 user_ns[name] = sys.modules[name]
459 user_ns[name] = sys.modules[name]
460
460
461 def view_import(name, globals={}, locals={}, fromlist=[], level=0):
461 def view_import(name, globals={}, locals={}, fromlist=[], level=0):
462 """the drop-in replacement for __import__, that optionally imports
462 """the drop-in replacement for __import__, that optionally imports
463 locally as well.
463 locally as well.
464 """
464 """
465 # don't override nested imports
465 # don't override nested imports
466 save_import = __builtin__.__import__
466 save_import = builtin_mod.__import__
467 __builtin__.__import__ = local_import
467 builtin_mod.__import__ = local_import
468
468
469 if imp.lock_held():
469 if imp.lock_held():
470 # this is a side-effect import, don't do it remotely, or even
470 # this is a side-effect import, don't do it remotely, or even
471 # ignore the local effects
471 # ignore the local effects
472 return local_import(name, globals, locals, fromlist, level)
472 return local_import(name, globals, locals, fromlist, level)
473
473
474 imp.acquire_lock()
474 imp.acquire_lock()
475 if local:
475 if local:
476 mod = local_import(name, globals, locals, fromlist, level)
476 mod = local_import(name, globals, locals, fromlist, level)
477 else:
477 else:
478 raise NotImplementedError("remote-only imports not yet implemented")
478 raise NotImplementedError("remote-only imports not yet implemented")
479 imp.release_lock()
479 imp.release_lock()
480
480
481 key = name+':'+','.join(fromlist or [])
481 key = name+':'+','.join(fromlist or [])
482 if level <= 0 and key not in modules:
482 if level <= 0 and key not in modules:
483 modules.add(key)
483 modules.add(key)
484 if not quiet:
484 if not quiet:
485 if fromlist:
485 if fromlist:
486 print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
486 print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
487 else:
487 else:
488 print("importing %s on engine(s)"%name)
488 print("importing %s on engine(s)"%name)
489 results.append(self.apply_async(remote_import, name, fromlist, level))
489 results.append(self.apply_async(remote_import, name, fromlist, level))
490 # restore override
490 # restore override
491 __builtin__.__import__ = save_import
491 builtin_mod.__import__ = save_import
492
492
493 return mod
493 return mod
494
494
495 # override __import__
495 # override __import__
496 __builtin__.__import__ = view_import
496 builtin_mod.__import__ = view_import
497 try:
497 try:
498 # enter the block
498 # enter the block
499 yield
499 yield
500 except ImportError:
500 except ImportError:
501 if local:
501 if local:
502 raise
502 raise
503 else:
503 else:
504 # ignore import errors if not doing local imports
504 # ignore import errors if not doing local imports
505 pass
505 pass
506 finally:
506 finally:
507 # always restore __import__
507 # always restore __import__
508 __builtin__.__import__ = local_import
508 builtin_mod.__import__ = local_import
509
509
510 for r in results:
510 for r in results:
511 # raise possible remote ImportErrors here
511 # raise possible remote ImportErrors here
512 r.get()
512 r.get()
513
513
514
514
515 @sync_results
515 @sync_results
516 @save_ids
516 @save_ids
517 def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
517 def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
518 """calls f(*args, **kwargs) on remote engines, returning the result.
518 """calls f(*args, **kwargs) on remote engines, returning the result.
519
519
520 This method sets all of `apply`'s flags via this View's attributes.
520 This method sets all of `apply`'s flags via this View's attributes.
521
521
522 Parameters
522 Parameters
523 ----------
523 ----------
524
524
525 f : callable
525 f : callable
526
526
527 args : list [default: empty]
527 args : list [default: empty]
528
528
529 kwargs : dict [default: empty]
529 kwargs : dict [default: empty]
530
530
531 targets : target list [default: self.targets]
531 targets : target list [default: self.targets]
532 where to run
532 where to run
533 block : bool [default: self.block]
533 block : bool [default: self.block]
534 whether to block
534 whether to block
535 track : bool [default: self.track]
535 track : bool [default: self.track]
536 whether to ask zmq to track the message, for safe non-copying sends
536 whether to ask zmq to track the message, for safe non-copying sends
537
537
538 Returns
538 Returns
539 -------
539 -------
540
540
541 if self.block is False:
541 if self.block is False:
542 returns AsyncResult
542 returns AsyncResult
543 else:
543 else:
544 returns actual result of f(*args, **kwargs) on the engine(s)
544 returns actual result of f(*args, **kwargs) on the engine(s)
545 This will be a list of self.targets is also a list (even length 1), or
545 This will be a list of self.targets is also a list (even length 1), or
546 the single result if self.targets is an integer engine id
546 the single result if self.targets is an integer engine id
547 """
547 """
548 args = [] if args is None else args
548 args = [] if args is None else args
549 kwargs = {} if kwargs is None else kwargs
549 kwargs = {} if kwargs is None else kwargs
550 block = self.block if block is None else block
550 block = self.block if block is None else block
551 track = self.track if track is None else track
551 track = self.track if track is None else track
552 targets = self.targets if targets is None else targets
552 targets = self.targets if targets is None else targets
553
553
554 _idents, _targets = self.client._build_targets(targets)
554 _idents, _targets = self.client._build_targets(targets)
555 msg_ids = []
555 msg_ids = []
556 trackers = []
556 trackers = []
557 for ident in _idents:
557 for ident in _idents:
558 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
558 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
559 ident=ident)
559 ident=ident)
560 if track:
560 if track:
561 trackers.append(msg['tracker'])
561 trackers.append(msg['tracker'])
562 msg_ids.append(msg['header']['msg_id'])
562 msg_ids.append(msg['header']['msg_id'])
563 if isinstance(targets, int):
563 if isinstance(targets, int):
564 msg_ids = msg_ids[0]
564 msg_ids = msg_ids[0]
565 tracker = None if track is False else zmq.MessageTracker(*trackers)
565 tracker = None if track is False else zmq.MessageTracker(*trackers)
566 ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets, tracker=tracker)
566 ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets, tracker=tracker)
567 if block:
567 if block:
568 try:
568 try:
569 return ar.get()
569 return ar.get()
570 except KeyboardInterrupt:
570 except KeyboardInterrupt:
571 pass
571 pass
572 return ar
572 return ar
573
573
574
574
575 @sync_results
575 @sync_results
576 def map(self, f, *sequences, **kwargs):
576 def map(self, f, *sequences, **kwargs):
577 """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
577 """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
578
578
579 Parallel version of builtin `map`, using this View's `targets`.
579 Parallel version of builtin `map`, using this View's `targets`.
580
580
581 There will be one task per target, so work will be chunked
581 There will be one task per target, so work will be chunked
582 if the sequences are longer than `targets`.
582 if the sequences are longer than `targets`.
583
583
584 Results can be iterated as they are ready, but will become available in chunks.
584 Results can be iterated as they are ready, but will become available in chunks.
585
585
586 Parameters
586 Parameters
587 ----------
587 ----------
588
588
589 f : callable
589 f : callable
590 function to be mapped
590 function to be mapped
591 *sequences: one or more sequences of matching length
591 *sequences: one or more sequences of matching length
592 the sequences to be distributed and passed to `f`
592 the sequences to be distributed and passed to `f`
593 block : bool
593 block : bool
594 whether to wait for the result or not [default self.block]
594 whether to wait for the result or not [default self.block]
595
595
596 Returns
596 Returns
597 -------
597 -------
598
598
599 if block=False:
599 if block=False:
600 AsyncMapResult
600 AsyncMapResult
601 An object like AsyncResult, but which reassembles the sequence of results
601 An object like AsyncResult, but which reassembles the sequence of results
602 into a single list. AsyncMapResults can be iterated through before all
602 into a single list. AsyncMapResults can be iterated through before all
603 results are complete.
603 results are complete.
604 else:
604 else:
605 list
605 list
606 the result of map(f,*sequences)
606 the result of map(f,*sequences)
607 """
607 """
608
608
609 block = kwargs.pop('block', self.block)
609 block = kwargs.pop('block', self.block)
610 for k in kwargs.keys():
610 for k in kwargs.keys():
611 if k not in ['block', 'track']:
611 if k not in ['block', 'track']:
612 raise TypeError("invalid keyword arg, %r"%k)
612 raise TypeError("invalid keyword arg, %r"%k)
613
613
614 assert len(sequences) > 0, "must have some sequences to map onto!"
614 assert len(sequences) > 0, "must have some sequences to map onto!"
615 pf = ParallelFunction(self, f, block=block, **kwargs)
615 pf = ParallelFunction(self, f, block=block, **kwargs)
616 return pf.map(*sequences)
616 return pf.map(*sequences)
617
617
618 @sync_results
618 @sync_results
619 @save_ids
619 @save_ids
620 def execute(self, code, silent=True, targets=None, block=None):
620 def execute(self, code, silent=True, targets=None, block=None):
621 """Executes `code` on `targets` in blocking or nonblocking manner.
621 """Executes `code` on `targets` in blocking or nonblocking manner.
622
622
623 ``execute`` is always `bound` (affects engine namespace)
623 ``execute`` is always `bound` (affects engine namespace)
624
624
625 Parameters
625 Parameters
626 ----------
626 ----------
627
627
628 code : str
628 code : str
629 the code string to be executed
629 the code string to be executed
630 block : bool
630 block : bool
631 whether or not to wait until done to return
631 whether or not to wait until done to return
632 default: self.block
632 default: self.block
633 """
633 """
634 block = self.block if block is None else block
634 block = self.block if block is None else block
635 targets = self.targets if targets is None else targets
635 targets = self.targets if targets is None else targets
636
636
637 _idents, _targets = self.client._build_targets(targets)
637 _idents, _targets = self.client._build_targets(targets)
638 msg_ids = []
638 msg_ids = []
639 trackers = []
639 trackers = []
640 for ident in _idents:
640 for ident in _idents:
641 msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)
641 msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)
642 msg_ids.append(msg['header']['msg_id'])
642 msg_ids.append(msg['header']['msg_id'])
643 if isinstance(targets, int):
643 if isinstance(targets, int):
644 msg_ids = msg_ids[0]
644 msg_ids = msg_ids[0]
645 ar = AsyncResult(self.client, msg_ids, fname='execute', targets=_targets)
645 ar = AsyncResult(self.client, msg_ids, fname='execute', targets=_targets)
646 if block:
646 if block:
647 try:
647 try:
648 ar.get()
648 ar.get()
649 except KeyboardInterrupt:
649 except KeyboardInterrupt:
650 pass
650 pass
651 return ar
651 return ar
652
652
653 def run(self, filename, targets=None, block=None):
653 def run(self, filename, targets=None, block=None):
654 """Execute contents of `filename` on my engine(s).
654 """Execute contents of `filename` on my engine(s).
655
655
656 This simply reads the contents of the file and calls `execute`.
656 This simply reads the contents of the file and calls `execute`.
657
657
658 Parameters
658 Parameters
659 ----------
659 ----------
660
660
661 filename : str
661 filename : str
662 The path to the file
662 The path to the file
663 targets : int/str/list of ints/strs
663 targets : int/str/list of ints/strs
664 the engines on which to execute
664 the engines on which to execute
665 default : all
665 default : all
666 block : bool
666 block : bool
667 whether or not to wait until done
667 whether or not to wait until done
668 default: self.block
668 default: self.block
669
669
670 """
670 """
671 with open(filename, 'r') as f:
671 with open(filename, 'r') as f:
672 # add newline in case of trailing indented whitespace
672 # add newline in case of trailing indented whitespace
673 # which will cause SyntaxError
673 # which will cause SyntaxError
674 code = f.read()+'\n'
674 code = f.read()+'\n'
675 return self.execute(code, block=block, targets=targets)
675 return self.execute(code, block=block, targets=targets)
676
676
677 def update(self, ns):
677 def update(self, ns):
678 """update remote namespace with dict `ns`
678 """update remote namespace with dict `ns`
679
679
680 See `push` for details.
680 See `push` for details.
681 """
681 """
682 return self.push(ns, block=self.block, track=self.track)
682 return self.push(ns, block=self.block, track=self.track)
683
683
684 def push(self, ns, targets=None, block=None, track=None):
684 def push(self, ns, targets=None, block=None, track=None):
685 """update remote namespace with dict `ns`
685 """update remote namespace with dict `ns`
686
686
687 Parameters
687 Parameters
688 ----------
688 ----------
689
689
690 ns : dict
690 ns : dict
691 dict of keys with which to update engine namespace(s)
691 dict of keys with which to update engine namespace(s)
692 block : bool [default : self.block]
692 block : bool [default : self.block]
693 whether to wait to be notified of engine receipt
693 whether to wait to be notified of engine receipt
694
694
695 """
695 """
696
696
697 block = block if block is not None else self.block
697 block = block if block is not None else self.block
698 track = track if track is not None else self.track
698 track = track if track is not None else self.track
699 targets = targets if targets is not None else self.targets
699 targets = targets if targets is not None else self.targets
700 # applier = self.apply_sync if block else self.apply_async
700 # applier = self.apply_sync if block else self.apply_async
701 if not isinstance(ns, dict):
701 if not isinstance(ns, dict):
702 raise TypeError("Must be a dict, not %s"%type(ns))
702 raise TypeError("Must be a dict, not %s"%type(ns))
703 return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
703 return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
704
704
705 def get(self, key_s):
705 def get(self, key_s):
706 """get object(s) by `key_s` from remote namespace
706 """get object(s) by `key_s` from remote namespace
707
707
708 see `pull` for details.
708 see `pull` for details.
709 """
709 """
710 # block = block if block is not None else self.block
710 # block = block if block is not None else self.block
711 return self.pull(key_s, block=True)
711 return self.pull(key_s, block=True)
712
712
713 def pull(self, names, targets=None, block=None):
713 def pull(self, names, targets=None, block=None):
714 """get object(s) by `name` from remote namespace
714 """get object(s) by `name` from remote namespace
715
715
716 will return one object if it is a key.
716 will return one object if it is a key.
717 can also take a list of keys, in which case it will return a list of objects.
717 can also take a list of keys, in which case it will return a list of objects.
718 """
718 """
719 block = block if block is not None else self.block
719 block = block if block is not None else self.block
720 targets = targets if targets is not None else self.targets
720 targets = targets if targets is not None else self.targets
721 applier = self.apply_sync if block else self.apply_async
721 applier = self.apply_sync if block else self.apply_async
722 if isinstance(names, basestring):
722 if isinstance(names, basestring):
723 pass
723 pass
724 elif isinstance(names, (list,tuple,set)):
724 elif isinstance(names, (list,tuple,set)):
725 for key in names:
725 for key in names:
726 if not isinstance(key, basestring):
726 if not isinstance(key, basestring):
727 raise TypeError("keys must be str, not type %r"%type(key))
727 raise TypeError("keys must be str, not type %r"%type(key))
728 else:
728 else:
729 raise TypeError("names must be strs, not %r"%names)
729 raise TypeError("names must be strs, not %r"%names)
730 return self._really_apply(util._pull, (names,), block=block, targets=targets)
730 return self._really_apply(util._pull, (names,), block=block, targets=targets)
731
731
732 def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
732 def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
733 """
733 """
734 Partition a Python sequence and send the partitions to a set of engines.
734 Partition a Python sequence and send the partitions to a set of engines.
735 """
735 """
736 block = block if block is not None else self.block
736 block = block if block is not None else self.block
737 track = track if track is not None else self.track
737 track = track if track is not None else self.track
738 targets = targets if targets is not None else self.targets
738 targets = targets if targets is not None else self.targets
739
739
740 # construct integer ID list:
740 # construct integer ID list:
741 targets = self.client._build_targets(targets)[1]
741 targets = self.client._build_targets(targets)[1]
742
742
743 mapObject = Map.dists[dist]()
743 mapObject = Map.dists[dist]()
744 nparts = len(targets)
744 nparts = len(targets)
745 msg_ids = []
745 msg_ids = []
746 trackers = []
746 trackers = []
747 for index, engineid in enumerate(targets):
747 for index, engineid in enumerate(targets):
748 partition = mapObject.getPartition(seq, index, nparts)
748 partition = mapObject.getPartition(seq, index, nparts)
749 if flatten and len(partition) == 1:
749 if flatten and len(partition) == 1:
750 ns = {key: partition[0]}
750 ns = {key: partition[0]}
751 else:
751 else:
752 ns = {key: partition}
752 ns = {key: partition}
753 r = self.push(ns, block=False, track=track, targets=engineid)
753 r = self.push(ns, block=False, track=track, targets=engineid)
754 msg_ids.extend(r.msg_ids)
754 msg_ids.extend(r.msg_ids)
755 if track:
755 if track:
756 trackers.append(r._tracker)
756 trackers.append(r._tracker)
757
757
758 if track:
758 if track:
759 tracker = zmq.MessageTracker(*trackers)
759 tracker = zmq.MessageTracker(*trackers)
760 else:
760 else:
761 tracker = None
761 tracker = None
762
762
763 r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
763 r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
764 if block:
764 if block:
765 r.wait()
765 r.wait()
766 else:
766 else:
767 return r
767 return r
768
768
769 @sync_results
769 @sync_results
770 @save_ids
770 @save_ids
771 def gather(self, key, dist='b', targets=None, block=None):
771 def gather(self, key, dist='b', targets=None, block=None):
772 """
772 """
773 Gather a partitioned sequence on a set of engines as a single local seq.
773 Gather a partitioned sequence on a set of engines as a single local seq.
774 """
774 """
775 block = block if block is not None else self.block
775 block = block if block is not None else self.block
776 targets = targets if targets is not None else self.targets
776 targets = targets if targets is not None else self.targets
777 mapObject = Map.dists[dist]()
777 mapObject = Map.dists[dist]()
778 msg_ids = []
778 msg_ids = []
779
779
780 # construct integer ID list:
780 # construct integer ID list:
781 targets = self.client._build_targets(targets)[1]
781 targets = self.client._build_targets(targets)[1]
782
782
783 for index, engineid in enumerate(targets):
783 for index, engineid in enumerate(targets):
784 msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
784 msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
785
785
786 r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
786 r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
787
787
788 if block:
788 if block:
789 try:
789 try:
790 return r.get()
790 return r.get()
791 except KeyboardInterrupt:
791 except KeyboardInterrupt:
792 pass
792 pass
793 return r
793 return r
794
794
795 def __getitem__(self, key):
795 def __getitem__(self, key):
796 return self.get(key)
796 return self.get(key)
797
797
798 def __setitem__(self,key, value):
798 def __setitem__(self,key, value):
799 self.update({key:value})
799 self.update({key:value})
800
800
801 def clear(self, targets=None, block=None):
801 def clear(self, targets=None, block=None):
802 """Clear the remote namespaces on my engines."""
802 """Clear the remote namespaces on my engines."""
803 block = block if block is not None else self.block
803 block = block if block is not None else self.block
804 targets = targets if targets is not None else self.targets
804 targets = targets if targets is not None else self.targets
805 return self.client.clear(targets=targets, block=block)
805 return self.client.clear(targets=targets, block=block)
806
806
807 #----------------------------------------
807 #----------------------------------------
808 # activate for %px, %autopx, etc. magics
808 # activate for %px, %autopx, etc. magics
809 #----------------------------------------
809 #----------------------------------------
810
810
811 def activate(self, suffix=''):
811 def activate(self, suffix=''):
812 """Activate IPython magics associated with this View
812 """Activate IPython magics associated with this View
813
813
814 Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`
814 Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`
815
815
816 Parameters
816 Parameters
817 ----------
817 ----------
818
818
819 suffix: str [default: '']
819 suffix: str [default: '']
820 The suffix, if any, for the magics. This allows you to have
820 The suffix, if any, for the magics. This allows you to have
821 multiple views associated with parallel magics at the same time.
821 multiple views associated with parallel magics at the same time.
822
822
823 e.g. ``rc[::2].activate(suffix='_even')`` will give you
823 e.g. ``rc[::2].activate(suffix='_even')`` will give you
824 the magics ``%px_even``, ``%pxresult_even``, etc. for running magics
824 the magics ``%px_even``, ``%pxresult_even``, etc. for running magics
825 on the even engines.
825 on the even engines.
826 """
826 """
827
827
828 from IPython.parallel.client.magics import ParallelMagics
828 from IPython.parallel.client.magics import ParallelMagics
829
829
830 try:
830 try:
831 # This is injected into __builtins__.
831 # This is injected into __builtins__.
832 ip = get_ipython()
832 ip = get_ipython()
833 except NameError:
833 except NameError:
834 print("The IPython parallel magics (%px, etc.) only work within IPython.")
834 print("The IPython parallel magics (%px, etc.) only work within IPython.")
835 return
835 return
836
836
837 M = ParallelMagics(ip, self, suffix)
837 M = ParallelMagics(ip, self, suffix)
838 ip.magics_manager.register(M)
838 ip.magics_manager.register(M)
839
839
840
840
841 @skip_doctest
841 @skip_doctest
842 class LoadBalancedView(View):
842 class LoadBalancedView(View):
843 """An load-balancing View that only executes via the Task scheduler.
843 """An load-balancing View that only executes via the Task scheduler.
844
844
845 Load-balanced views can be created with the client's `view` method:
845 Load-balanced views can be created with the client's `view` method:
846
846
847 >>> v = client.load_balanced_view()
847 >>> v = client.load_balanced_view()
848
848
849 or targets can be specified, to restrict the potential destinations:
849 or targets can be specified, to restrict the potential destinations:
850
850
851 >>> v = client.client.load_balanced_view([1,3])
851 >>> v = client.client.load_balanced_view([1,3])
852
852
853 which would restrict loadbalancing to between engines 1 and 3.
853 which would restrict loadbalancing to between engines 1 and 3.
854
854
855 """
855 """
856
856
857 follow=Any()
857 follow=Any()
858 after=Any()
858 after=Any()
859 timeout=CFloat()
859 timeout=CFloat()
860 retries = Integer(0)
860 retries = Integer(0)
861
861
862 _task_scheme = Any()
862 _task_scheme = Any()
863 _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
863 _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
864
864
865 def __init__(self, client=None, socket=None, **flags):
865 def __init__(self, client=None, socket=None, **flags):
866 super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
866 super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
867 self._task_scheme=client._task_scheme
867 self._task_scheme=client._task_scheme
868
868
869 def _validate_dependency(self, dep):
869 def _validate_dependency(self, dep):
870 """validate a dependency.
870 """validate a dependency.
871
871
872 For use in `set_flags`.
872 For use in `set_flags`.
873 """
873 """
874 if dep is None or isinstance(dep, (basestring, AsyncResult, Dependency)):
874 if dep is None or isinstance(dep, (basestring, AsyncResult, Dependency)):
875 return True
875 return True
876 elif isinstance(dep, (list,set, tuple)):
876 elif isinstance(dep, (list,set, tuple)):
877 for d in dep:
877 for d in dep:
878 if not isinstance(d, (basestring, AsyncResult)):
878 if not isinstance(d, (basestring, AsyncResult)):
879 return False
879 return False
880 elif isinstance(dep, dict):
880 elif isinstance(dep, dict):
881 if set(dep.keys()) != set(Dependency().as_dict().keys()):
881 if set(dep.keys()) != set(Dependency().as_dict().keys()):
882 return False
882 return False
883 if not isinstance(dep['msg_ids'], list):
883 if not isinstance(dep['msg_ids'], list):
884 return False
884 return False
885 for d in dep['msg_ids']:
885 for d in dep['msg_ids']:
886 if not isinstance(d, basestring):
886 if not isinstance(d, basestring):
887 return False
887 return False
888 else:
888 else:
889 return False
889 return False
890
890
891 return True
891 return True
892
892
893 def _render_dependency(self, dep):
893 def _render_dependency(self, dep):
894 """helper for building jsonable dependencies from various input forms."""
894 """helper for building jsonable dependencies from various input forms."""
895 if isinstance(dep, Dependency):
895 if isinstance(dep, Dependency):
896 return dep.as_dict()
896 return dep.as_dict()
897 elif isinstance(dep, AsyncResult):
897 elif isinstance(dep, AsyncResult):
898 return dep.msg_ids
898 return dep.msg_ids
899 elif dep is None:
899 elif dep is None:
900 return []
900 return []
901 else:
901 else:
902 # pass to Dependency constructor
902 # pass to Dependency constructor
903 return list(Dependency(dep))
903 return list(Dependency(dep))
904
904
905 def set_flags(self, **kwargs):
905 def set_flags(self, **kwargs):
906 """set my attribute flags by keyword.
906 """set my attribute flags by keyword.
907
907
908 A View is a wrapper for the Client's apply method, but with attributes
908 A View is a wrapper for the Client's apply method, but with attributes
909 that specify keyword arguments, those attributes can be set by keyword
909 that specify keyword arguments, those attributes can be set by keyword
910 argument with this method.
910 argument with this method.
911
911
912 Parameters
912 Parameters
913 ----------
913 ----------
914
914
915 block : bool
915 block : bool
916 whether to wait for results
916 whether to wait for results
917 track : bool
917 track : bool
918 whether to create a MessageTracker to allow the user to
918 whether to create a MessageTracker to allow the user to
919 safely edit after arrays and buffers during non-copying
919 safely edit after arrays and buffers during non-copying
920 sends.
920 sends.
921
921
922 after : Dependency or collection of msg_ids
922 after : Dependency or collection of msg_ids
923 Only for load-balanced execution (targets=None)
923 Only for load-balanced execution (targets=None)
924 Specify a list of msg_ids as a time-based dependency.
924 Specify a list of msg_ids as a time-based dependency.
925 This job will only be run *after* the dependencies
925 This job will only be run *after* the dependencies
926 have been met.
926 have been met.
927
927
928 follow : Dependency or collection of msg_ids
928 follow : Dependency or collection of msg_ids
929 Only for load-balanced execution (targets=None)
929 Only for load-balanced execution (targets=None)
930 Specify a list of msg_ids as a location-based dependency.
930 Specify a list of msg_ids as a location-based dependency.
931 This job will only be run on an engine where this dependency
931 This job will only be run on an engine where this dependency
932 is met.
932 is met.
933
933
934 timeout : float/int or None
934 timeout : float/int or None
935 Only for load-balanced execution (targets=None)
935 Only for load-balanced execution (targets=None)
936 Specify an amount of time (in seconds) for the scheduler to
936 Specify an amount of time (in seconds) for the scheduler to
937 wait for dependencies to be met before failing with a
937 wait for dependencies to be met before failing with a
938 DependencyTimeout.
938 DependencyTimeout.
939
939
940 retries : int
940 retries : int
941 Number of times a task will be retried on failure.
941 Number of times a task will be retried on failure.
942 """
942 """
943
943
944 super(LoadBalancedView, self).set_flags(**kwargs)
944 super(LoadBalancedView, self).set_flags(**kwargs)
945 for name in ('follow', 'after'):
945 for name in ('follow', 'after'):
946 if name in kwargs:
946 if name in kwargs:
947 value = kwargs[name]
947 value = kwargs[name]
948 if self._validate_dependency(value):
948 if self._validate_dependency(value):
949 setattr(self, name, value)
949 setattr(self, name, value)
950 else:
950 else:
951 raise ValueError("Invalid dependency: %r"%value)
951 raise ValueError("Invalid dependency: %r"%value)
952 if 'timeout' in kwargs:
952 if 'timeout' in kwargs:
953 t = kwargs['timeout']
953 t = kwargs['timeout']
954 if not isinstance(t, (int, long, float, type(None))):
954 if not isinstance(t, (int, long, float, type(None))):
955 raise TypeError("Invalid type for timeout: %r"%type(t))
955 raise TypeError("Invalid type for timeout: %r"%type(t))
956 if t is not None:
956 if t is not None:
957 if t < 0:
957 if t < 0:
958 raise ValueError("Invalid timeout: %s"%t)
958 raise ValueError("Invalid timeout: %s"%t)
959 self.timeout = t
959 self.timeout = t
960
960
961 @sync_results
961 @sync_results
962 @save_ids
962 @save_ids
963 def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
963 def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
964 after=None, follow=None, timeout=None,
964 after=None, follow=None, timeout=None,
965 targets=None, retries=None):
965 targets=None, retries=None):
966 """calls f(*args, **kwargs) on a remote engine, returning the result.
966 """calls f(*args, **kwargs) on a remote engine, returning the result.
967
967
968 This method temporarily sets all of `apply`'s flags for a single call.
968 This method temporarily sets all of `apply`'s flags for a single call.
969
969
970 Parameters
970 Parameters
971 ----------
971 ----------
972
972
973 f : callable
973 f : callable
974
974
975 args : list [default: empty]
975 args : list [default: empty]
976
976
977 kwargs : dict [default: empty]
977 kwargs : dict [default: empty]
978
978
979 block : bool [default: self.block]
979 block : bool [default: self.block]
980 whether to block
980 whether to block
981 track : bool [default: self.track]
981 track : bool [default: self.track]
982 whether to ask zmq to track the message, for safe non-copying sends
982 whether to ask zmq to track the message, for safe non-copying sends
983
983
984 !!!!!! TODO: THE REST HERE !!!!
984 !!!!!! TODO: THE REST HERE !!!!
985
985
986 Returns
986 Returns
987 -------
987 -------
988
988
989 if self.block is False:
989 if self.block is False:
990 returns AsyncResult
990 returns AsyncResult
991 else:
991 else:
992 returns actual result of f(*args, **kwargs) on the engine(s)
992 returns actual result of f(*args, **kwargs) on the engine(s)
993 This will be a list of self.targets is also a list (even length 1), or
993 This will be a list of self.targets is also a list (even length 1), or
994 the single result if self.targets is an integer engine id
994 the single result if self.targets is an integer engine id
995 """
995 """
996
996
997 # validate whether we can run
997 # validate whether we can run
998 if self._socket.closed:
998 if self._socket.closed:
999 msg = "Task farming is disabled"
999 msg = "Task farming is disabled"
1000 if self._task_scheme == 'pure':
1000 if self._task_scheme == 'pure':
1001 msg += " because the pure ZMQ scheduler cannot handle"
1001 msg += " because the pure ZMQ scheduler cannot handle"
1002 msg += " disappearing engines."
1002 msg += " disappearing engines."
1003 raise RuntimeError(msg)
1003 raise RuntimeError(msg)
1004
1004
1005 if self._task_scheme == 'pure':
1005 if self._task_scheme == 'pure':
1006 # pure zmq scheme doesn't support extra features
1006 # pure zmq scheme doesn't support extra features
1007 msg = "Pure ZMQ scheduler doesn't support the following flags:"
1007 msg = "Pure ZMQ scheduler doesn't support the following flags:"
1008 "follow, after, retries, targets, timeout"
1008 "follow, after, retries, targets, timeout"
1009 if (follow or after or retries or targets or timeout):
1009 if (follow or after or retries or targets or timeout):
1010 # hard fail on Scheduler flags
1010 # hard fail on Scheduler flags
1011 raise RuntimeError(msg)
1011 raise RuntimeError(msg)
1012 if isinstance(f, dependent):
1012 if isinstance(f, dependent):
1013 # soft warn on functional dependencies
1013 # soft warn on functional dependencies
1014 warnings.warn(msg, RuntimeWarning)
1014 warnings.warn(msg, RuntimeWarning)
1015
1015
1016 # build args
1016 # build args
1017 args = [] if args is None else args
1017 args = [] if args is None else args
1018 kwargs = {} if kwargs is None else kwargs
1018 kwargs = {} if kwargs is None else kwargs
1019 block = self.block if block is None else block
1019 block = self.block if block is None else block
1020 track = self.track if track is None else track
1020 track = self.track if track is None else track
1021 after = self.after if after is None else after
1021 after = self.after if after is None else after
1022 retries = self.retries if retries is None else retries
1022 retries = self.retries if retries is None else retries
1023 follow = self.follow if follow is None else follow
1023 follow = self.follow if follow is None else follow
1024 timeout = self.timeout if timeout is None else timeout
1024 timeout = self.timeout if timeout is None else timeout
1025 targets = self.targets if targets is None else targets
1025 targets = self.targets if targets is None else targets
1026
1026
1027 if not isinstance(retries, int):
1027 if not isinstance(retries, int):
1028 raise TypeError('retries must be int, not %r'%type(retries))
1028 raise TypeError('retries must be int, not %r'%type(retries))
1029
1029
1030 if targets is None:
1030 if targets is None:
1031 idents = []
1031 idents = []
1032 else:
1032 else:
1033 idents = self.client._build_targets(targets)[0]
1033 idents = self.client._build_targets(targets)[0]
1034 # ensure *not* bytes
1034 # ensure *not* bytes
1035 idents = [ ident.decode() for ident in idents ]
1035 idents = [ ident.decode() for ident in idents ]
1036
1036
1037 after = self._render_dependency(after)
1037 after = self._render_dependency(after)
1038 follow = self._render_dependency(follow)
1038 follow = self._render_dependency(follow)
1039 metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)
1039 metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)
1040
1040
1041 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
1041 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
1042 metadata=metadata)
1042 metadata=metadata)
1043 tracker = None if track is False else msg['tracker']
1043 tracker = None if track is False else msg['tracker']
1044
1044
1045 ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker)
1045 ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker)
1046
1046
1047 if block:
1047 if block:
1048 try:
1048 try:
1049 return ar.get()
1049 return ar.get()
1050 except KeyboardInterrupt:
1050 except KeyboardInterrupt:
1051 pass
1051 pass
1052 return ar
1052 return ar
1053
1053
1054 @sync_results
1054 @sync_results
1055 @save_ids
1055 @save_ids
1056 def map(self, f, *sequences, **kwargs):
1056 def map(self, f, *sequences, **kwargs):
1057 """view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult
1057 """view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult
1058
1058
1059 Parallel version of builtin `map`, load-balanced by this View.
1059 Parallel version of builtin `map`, load-balanced by this View.
1060
1060
1061 `block`, and `chunksize` can be specified by keyword only.
1061 `block`, and `chunksize` can be specified by keyword only.
1062
1062
1063 Each `chunksize` elements will be a separate task, and will be
1063 Each `chunksize` elements will be a separate task, and will be
1064 load-balanced. This lets individual elements be available for iteration
1064 load-balanced. This lets individual elements be available for iteration
1065 as soon as they arrive.
1065 as soon as they arrive.
1066
1066
1067 Parameters
1067 Parameters
1068 ----------
1068 ----------
1069
1069
1070 f : callable
1070 f : callable
1071 function to be mapped
1071 function to be mapped
1072 *sequences: one or more sequences of matching length
1072 *sequences: one or more sequences of matching length
1073 the sequences to be distributed and passed to `f`
1073 the sequences to be distributed and passed to `f`
1074 block : bool [default self.block]
1074 block : bool [default self.block]
1075 whether to wait for the result or not
1075 whether to wait for the result or not
1076 track : bool
1076 track : bool
1077 whether to create a MessageTracker to allow the user to
1077 whether to create a MessageTracker to allow the user to
1078 safely edit after arrays and buffers during non-copying
1078 safely edit after arrays and buffers during non-copying
1079 sends.
1079 sends.
1080 chunksize : int [default 1]
1080 chunksize : int [default 1]
1081 how many elements should be in each task.
1081 how many elements should be in each task.
1082 ordered : bool [default True]
1082 ordered : bool [default True]
1083 Whether the results should be gathered as they arrive, or enforce
1083 Whether the results should be gathered as they arrive, or enforce
1084 the order of submission.
1084 the order of submission.
1085
1085
1086 Only applies when iterating through AsyncMapResult as results arrive.
1086 Only applies when iterating through AsyncMapResult as results arrive.
1087 Has no effect when block=True.
1087 Has no effect when block=True.
1088
1088
1089 Returns
1089 Returns
1090 -------
1090 -------
1091
1091
1092 if block=False:
1092 if block=False:
1093 AsyncMapResult
1093 AsyncMapResult
1094 An object like AsyncResult, but which reassembles the sequence of results
1094 An object like AsyncResult, but which reassembles the sequence of results
1095 into a single list. AsyncMapResults can be iterated through before all
1095 into a single list. AsyncMapResults can be iterated through before all
1096 results are complete.
1096 results are complete.
1097 else:
1097 else:
1098 the result of map(f,*sequences)
1098 the result of map(f,*sequences)
1099
1099
1100 """
1100 """
1101
1101
1102 # default
1102 # default
1103 block = kwargs.get('block', self.block)
1103 block = kwargs.get('block', self.block)
1104 chunksize = kwargs.get('chunksize', 1)
1104 chunksize = kwargs.get('chunksize', 1)
1105 ordered = kwargs.get('ordered', True)
1105 ordered = kwargs.get('ordered', True)
1106
1106
1107 keyset = set(kwargs.keys())
1107 keyset = set(kwargs.keys())
1108 extra_keys = keyset.difference_update(set(['block', 'chunksize']))
1108 extra_keys = keyset.difference_update(set(['block', 'chunksize']))
1109 if extra_keys:
1109 if extra_keys:
1110 raise TypeError("Invalid kwargs: %s"%list(extra_keys))
1110 raise TypeError("Invalid kwargs: %s"%list(extra_keys))
1111
1111
1112 assert len(sequences) > 0, "must have some sequences to map onto!"
1112 assert len(sequences) > 0, "must have some sequences to map onto!"
1113
1113
1114 pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
1114 pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
1115 return pf.map(*sequences)
1115 return pf.map(*sequences)
1116
1116
1117 __all__ = ['LoadBalancedView', 'DirectView']
1117 __all__ = ['LoadBalancedView', 'DirectView']
@@ -1,176 +1,176 b''
1 """Global IPython app to support test running.
1 """Global IPython app to support test running.
2
2
3 We must start our own ipython object and heavily muck with it so that all the
3 We must start our own ipython object and heavily muck with it so that all the
4 modifications IPython makes to system behavior don't send the doctest machinery
4 modifications IPython makes to system behavior don't send the doctest machinery
5 into a fit. This code should be considered a gross hack, but it gets the job
5 into a fit. This code should be considered a gross hack, but it gets the job
6 done.
6 done.
7 """
7 """
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9 from __future__ import print_function
9 from __future__ import print_function
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Copyright (C) 2009-2011 The IPython Development Team
12 # Copyright (C) 2009-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 # stdlib
22 # stdlib
23 import __builtin__ as builtin_mod
24 import os
23 import os
25 import sys
24 import sys
26
25
27 # our own
26 # our own
28 from . import tools
27 from . import tools
29
28
30 from IPython.core import page
29 from IPython.core import page
31 from IPython.utils import io
30 from IPython.utils import io
32 from IPython.utils import py3compat
31 from IPython.utils import py3compat
32 from IPython.utils.py3compat import builtin_mod
33 from IPython.terminal.interactiveshell import TerminalInteractiveShell
33 from IPython.terminal.interactiveshell import TerminalInteractiveShell
34
34
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36 # Functions
36 # Functions
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38
38
39 class StreamProxy(io.IOStream):
39 class StreamProxy(io.IOStream):
40 """Proxy for sys.stdout/err. This will request the stream *at call time*
40 """Proxy for sys.stdout/err. This will request the stream *at call time*
41 allowing for nose's Capture plugin's redirection of sys.stdout/err.
41 allowing for nose's Capture plugin's redirection of sys.stdout/err.
42
42
43 Parameters
43 Parameters
44 ----------
44 ----------
45 name : str
45 name : str
46 The name of the stream. This will be requested anew at every call
46 The name of the stream. This will be requested anew at every call
47 """
47 """
48
48
49 def __init__(self, name):
49 def __init__(self, name):
50 self.name=name
50 self.name=name
51
51
52 @property
52 @property
53 def stream(self):
53 def stream(self):
54 return getattr(sys, self.name)
54 return getattr(sys, self.name)
55
55
56 def flush(self):
56 def flush(self):
57 self.stream.flush()
57 self.stream.flush()
58
58
59 # Hack to modify the %run command so we can sync the user's namespace with the
59 # Hack to modify the %run command so we can sync the user's namespace with the
60 # test globals. Once we move over to a clean magic system, this will be done
60 # test globals. Once we move over to a clean magic system, this will be done
61 # with much less ugliness.
61 # with much less ugliness.
62
62
63 class py_file_finder(object):
63 class py_file_finder(object):
64 def __init__(self,test_filename):
64 def __init__(self,test_filename):
65 self.test_filename = test_filename
65 self.test_filename = test_filename
66
66
67 def __call__(self,name,win32=False):
67 def __call__(self,name,win32=False):
68 from IPython.utils.path import get_py_filename
68 from IPython.utils.path import get_py_filename
69 try:
69 try:
70 return get_py_filename(name,win32=win32)
70 return get_py_filename(name,win32=win32)
71 except IOError:
71 except IOError:
72 test_dir = os.path.dirname(self.test_filename)
72 test_dir = os.path.dirname(self.test_filename)
73 new_path = os.path.join(test_dir,name)
73 new_path = os.path.join(test_dir,name)
74 return get_py_filename(new_path,win32=win32)
74 return get_py_filename(new_path,win32=win32)
75
75
76
76
77 def _run_ns_sync(self,arg_s,runner=None):
77 def _run_ns_sync(self,arg_s,runner=None):
78 """Modified version of %run that syncs testing namespaces.
78 """Modified version of %run that syncs testing namespaces.
79
79
80 This is strictly needed for running doctests that call %run.
80 This is strictly needed for running doctests that call %run.
81 """
81 """
82 #print('in run_ns_sync', arg_s, file=sys.stderr) # dbg
82 #print('in run_ns_sync', arg_s, file=sys.stderr) # dbg
83 finder = py_file_finder(arg_s)
83 finder = py_file_finder(arg_s)
84 return get_ipython().magic_run_ori(arg_s, runner, finder)
84 return get_ipython().magic_run_ori(arg_s, runner, finder)
85
85
86
86
87 def get_ipython():
87 def get_ipython():
88 # This will get replaced by the real thing once we start IPython below
88 # This will get replaced by the real thing once we start IPython below
89 return start_ipython()
89 return start_ipython()
90
90
91
91
92 # A couple of methods to override those in the running IPython to interact
92 # A couple of methods to override those in the running IPython to interact
93 # better with doctest (doctest captures on raw stdout, so we need to direct
93 # better with doctest (doctest captures on raw stdout, so we need to direct
94 # various types of output there otherwise it will miss them).
94 # various types of output there otherwise it will miss them).
95
95
96 def xsys(self, cmd):
96 def xsys(self, cmd):
97 """Replace the default system call with a capturing one for doctest.
97 """Replace the default system call with a capturing one for doctest.
98 """
98 """
99 # We use getoutput, but we need to strip it because pexpect captures
99 # We use getoutput, but we need to strip it because pexpect captures
100 # the trailing newline differently from commands.getoutput
100 # the trailing newline differently from commands.getoutput
101 print(self.getoutput(cmd, split=False, depth=1).rstrip(), end='', file=sys.stdout)
101 print(self.getoutput(cmd, split=False, depth=1).rstrip(), end='', file=sys.stdout)
102 sys.stdout.flush()
102 sys.stdout.flush()
103
103
104
104
105 def _showtraceback(self, etype, evalue, stb):
105 def _showtraceback(self, etype, evalue, stb):
106 """Print the traceback purely on stdout for doctest to capture it.
106 """Print the traceback purely on stdout for doctest to capture it.
107 """
107 """
108 print(self.InteractiveTB.stb2text(stb), file=sys.stdout)
108 print(self.InteractiveTB.stb2text(stb), file=sys.stdout)
109
109
110
110
111 def start_ipython():
111 def start_ipython():
112 """Start a global IPython shell, which we need for IPython-specific syntax.
112 """Start a global IPython shell, which we need for IPython-specific syntax.
113 """
113 """
114 global get_ipython
114 global get_ipython
115
115
116 # This function should only ever run once!
116 # This function should only ever run once!
117 if hasattr(start_ipython, 'already_called'):
117 if hasattr(start_ipython, 'already_called'):
118 return
118 return
119 start_ipython.already_called = True
119 start_ipython.already_called = True
120
120
121 # Store certain global objects that IPython modifies
121 # Store certain global objects that IPython modifies
122 _displayhook = sys.displayhook
122 _displayhook = sys.displayhook
123 _excepthook = sys.excepthook
123 _excepthook = sys.excepthook
124 _main = sys.modules.get('__main__')
124 _main = sys.modules.get('__main__')
125
125
126 # Create custom argv and namespaces for our IPython to be test-friendly
126 # Create custom argv and namespaces for our IPython to be test-friendly
127 config = tools.default_config()
127 config = tools.default_config()
128
128
129 # Create and initialize our test-friendly IPython instance.
129 # Create and initialize our test-friendly IPython instance.
130 shell = TerminalInteractiveShell.instance(config=config,
130 shell = TerminalInteractiveShell.instance(config=config,
131 )
131 )
132
132
133 # A few more tweaks needed for playing nicely with doctests...
133 # A few more tweaks needed for playing nicely with doctests...
134
134
135 # remove history file
135 # remove history file
136 shell.tempfiles.append(config.HistoryManager.hist_file)
136 shell.tempfiles.append(config.HistoryManager.hist_file)
137
137
138 # These traps are normally only active for interactive use, set them
138 # These traps are normally only active for interactive use, set them
139 # permanently since we'll be mocking interactive sessions.
139 # permanently since we'll be mocking interactive sessions.
140 shell.builtin_trap.activate()
140 shell.builtin_trap.activate()
141
141
142 # Modify the IPython system call with one that uses getoutput, so that we
142 # Modify the IPython system call with one that uses getoutput, so that we
143 # can capture subcommands and print them to Python's stdout, otherwise the
143 # can capture subcommands and print them to Python's stdout, otherwise the
144 # doctest machinery would miss them.
144 # doctest machinery would miss them.
145 shell.system = py3compat.MethodType(xsys, shell)
145 shell.system = py3compat.MethodType(xsys, shell)
146
146
147 shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
147 shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
148
148
149 # IPython is ready, now clean up some global state...
149 # IPython is ready, now clean up some global state...
150
150
151 # Deactivate the various python system hooks added by ipython for
151 # Deactivate the various python system hooks added by ipython for
152 # interactive convenience so we don't confuse the doctest system
152 # interactive convenience so we don't confuse the doctest system
153 sys.modules['__main__'] = _main
153 sys.modules['__main__'] = _main
154 sys.displayhook = _displayhook
154 sys.displayhook = _displayhook
155 sys.excepthook = _excepthook
155 sys.excepthook = _excepthook
156
156
157 # So that ipython magics and aliases can be doctested (they work by making
157 # So that ipython magics and aliases can be doctested (they work by making
158 # a call into a global _ip object). Also make the top-level get_ipython
158 # a call into a global _ip object). Also make the top-level get_ipython
159 # now return this without recursively calling here again.
159 # now return this without recursively calling here again.
160 _ip = shell
160 _ip = shell
161 get_ipython = _ip.get_ipython
161 get_ipython = _ip.get_ipython
162 builtin_mod._ip = _ip
162 builtin_mod._ip = _ip
163 builtin_mod.get_ipython = get_ipython
163 builtin_mod.get_ipython = get_ipython
164
164
165 # To avoid extra IPython messages during testing, suppress io.stdout/stderr
165 # To avoid extra IPython messages during testing, suppress io.stdout/stderr
166 io.stdout = StreamProxy('stdout')
166 io.stdout = StreamProxy('stdout')
167 io.stderr = StreamProxy('stderr')
167 io.stderr = StreamProxy('stderr')
168
168
169 # Override paging, so we don't require user interaction during the tests.
169 # Override paging, so we don't require user interaction during the tests.
170 def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
170 def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
171 print(strng)
171 print(strng)
172
172
173 page.orig_page = page.page
173 page.orig_page = page.page
174 page.page = nopage
174 page.page = nopage
175
175
176 return _ip
176 return _ip
@@ -1,761 +1,761 b''
1 """Nose Plugin that supports IPython doctests.
1 """Nose Plugin that supports IPython doctests.
2
2
3 Limitations:
3 Limitations:
4
4
5 - When generating examples for use as doctests, make sure that you have
5 - When generating examples for use as doctests, make sure that you have
6 pretty-printing OFF. This can be done either by setting the
6 pretty-printing OFF. This can be done either by setting the
7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 by interactively disabling it with %Pprint. This is required so that IPython
8 by interactively disabling it with %Pprint. This is required so that IPython
9 output matches that of normal Python, which is used by doctest for internal
9 output matches that of normal Python, which is used by doctest for internal
10 execution.
10 execution.
11
11
12 - Do not rely on specific prompt numbers for results (such as using
12 - Do not rely on specific prompt numbers for results (such as using
13 '_34==True', for example). For IPython tests run via an external process the
13 '_34==True', for example). For IPython tests run via an external process the
14 prompt numbers may be different, and IPython tests run as normal python code
14 prompt numbers may be different, and IPython tests run as normal python code
15 won't even have these special _NN variables set at all.
15 won't even have these special _NN variables set at all.
16 """
16 """
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Module imports
19 # Module imports
20
20
21 # From the standard library
21 # From the standard library
22 import __builtin__ as builtin_mod
23 import commands
22 import commands
24 import doctest
23 import doctest
25 import inspect
24 import inspect
26 import logging
25 import logging
27 import os
26 import os
28 import re
27 import re
29 import sys
28 import sys
30 import traceback
29 import traceback
31 import unittest
30 import unittest
32
31
33 from inspect import getmodule
32 from inspect import getmodule
34 from StringIO import StringIO
33 from StringIO import StringIO
35
34
36 # We are overriding the default doctest runner, so we need to import a few
35 # We are overriding the default doctest runner, so we need to import a few
37 # things from doctest directly
36 # things from doctest directly
38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
37 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 _unittest_reportflags, DocTestRunner,
38 _unittest_reportflags, DocTestRunner,
40 _extract_future_flags, pdb, _OutputRedirectingPdb,
39 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 _exception_traceback,
40 _exception_traceback,
42 linecache)
41 linecache)
43
42
44 # Third-party modules
43 # Third-party modules
45 import nose.core
44 import nose.core
46
45
47 from nose.plugins import doctests, Plugin
46 from nose.plugins import doctests, Plugin
48 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
47 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
49
48
50 # Our own imports
49 # Our own imports
50 from IPython.utils.py3compat import builtin_mod
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Module globals and other constants
53 # Module globals and other constants
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
57
57
58
58
59 #-----------------------------------------------------------------------------
59 #-----------------------------------------------------------------------------
60 # Classes and functions
60 # Classes and functions
61 #-----------------------------------------------------------------------------
61 #-----------------------------------------------------------------------------
62
62
63 def is_extension_module(filename):
63 def is_extension_module(filename):
64 """Return whether the given filename is an extension module.
64 """Return whether the given filename is an extension module.
65
65
66 This simply checks that the extension is either .so or .pyd.
66 This simply checks that the extension is either .so or .pyd.
67 """
67 """
68 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
68 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
69
69
70
70
71 class DocTestSkip(object):
71 class DocTestSkip(object):
72 """Object wrapper for doctests to be skipped."""
72 """Object wrapper for doctests to be skipped."""
73
73
74 ds_skip = """Doctest to skip.
74 ds_skip = """Doctest to skip.
75 >>> 1 #doctest: +SKIP
75 >>> 1 #doctest: +SKIP
76 """
76 """
77
77
78 def __init__(self,obj):
78 def __init__(self,obj):
79 self.obj = obj
79 self.obj = obj
80
80
81 def __getattribute__(self,key):
81 def __getattribute__(self,key):
82 if key == '__doc__':
82 if key == '__doc__':
83 return DocTestSkip.ds_skip
83 return DocTestSkip.ds_skip
84 else:
84 else:
85 return getattr(object.__getattribute__(self,'obj'),key)
85 return getattr(object.__getattribute__(self,'obj'),key)
86
86
87 # Modified version of the one in the stdlib, that fixes a python bug (doctests
87 # Modified version of the one in the stdlib, that fixes a python bug (doctests
88 # not found in extension modules, http://bugs.python.org/issue3158)
88 # not found in extension modules, http://bugs.python.org/issue3158)
89 class DocTestFinder(doctest.DocTestFinder):
89 class DocTestFinder(doctest.DocTestFinder):
90
90
91 def _from_module(self, module, object):
91 def _from_module(self, module, object):
92 """
92 """
93 Return true if the given object is defined in the given
93 Return true if the given object is defined in the given
94 module.
94 module.
95 """
95 """
96 if module is None:
96 if module is None:
97 return True
97 return True
98 elif inspect.isfunction(object):
98 elif inspect.isfunction(object):
99 return module.__dict__ is object.func_globals
99 return module.__dict__ is object.func_globals
100 elif inspect.isbuiltin(object):
100 elif inspect.isbuiltin(object):
101 return module.__name__ == object.__module__
101 return module.__name__ == object.__module__
102 elif inspect.isclass(object):
102 elif inspect.isclass(object):
103 return module.__name__ == object.__module__
103 return module.__name__ == object.__module__
104 elif inspect.ismethod(object):
104 elif inspect.ismethod(object):
105 # This one may be a bug in cython that fails to correctly set the
105 # This one may be a bug in cython that fails to correctly set the
106 # __module__ attribute of methods, but since the same error is easy
106 # __module__ attribute of methods, but since the same error is easy
107 # to make by extension code writers, having this safety in place
107 # to make by extension code writers, having this safety in place
108 # isn't such a bad idea
108 # isn't such a bad idea
109 return module.__name__ == object.im_class.__module__
109 return module.__name__ == object.im_class.__module__
110 elif inspect.getmodule(object) is not None:
110 elif inspect.getmodule(object) is not None:
111 return module is inspect.getmodule(object)
111 return module is inspect.getmodule(object)
112 elif hasattr(object, '__module__'):
112 elif hasattr(object, '__module__'):
113 return module.__name__ == object.__module__
113 return module.__name__ == object.__module__
114 elif isinstance(object, property):
114 elif isinstance(object, property):
115 return True # [XX] no way not be sure.
115 return True # [XX] no way not be sure.
116 else:
116 else:
117 raise ValueError("object must be a class or function")
117 raise ValueError("object must be a class or function")
118
118
119 def _find(self, tests, obj, name, module, source_lines, globs, seen):
119 def _find(self, tests, obj, name, module, source_lines, globs, seen):
120 """
120 """
121 Find tests for the given object and any contained objects, and
121 Find tests for the given object and any contained objects, and
122 add them to `tests`.
122 add them to `tests`.
123 """
123 """
124 #print '_find for:', obj, name, module # dbg
124 #print '_find for:', obj, name, module # dbg
125 if hasattr(obj,"skip_doctest"):
125 if hasattr(obj,"skip_doctest"):
126 #print 'SKIPPING DOCTEST FOR:',obj # dbg
126 #print 'SKIPPING DOCTEST FOR:',obj # dbg
127 obj = DocTestSkip(obj)
127 obj = DocTestSkip(obj)
128
128
129 doctest.DocTestFinder._find(self,tests, obj, name, module,
129 doctest.DocTestFinder._find(self,tests, obj, name, module,
130 source_lines, globs, seen)
130 source_lines, globs, seen)
131
131
132 # Below we re-run pieces of the above method with manual modifications,
132 # Below we re-run pieces of the above method with manual modifications,
133 # because the original code is buggy and fails to correctly identify
133 # because the original code is buggy and fails to correctly identify
134 # doctests in extension modules.
134 # doctests in extension modules.
135
135
136 # Local shorthands
136 # Local shorthands
137 from inspect import isroutine, isclass, ismodule
137 from inspect import isroutine, isclass, ismodule
138
138
139 # Look for tests in a module's contained objects.
139 # Look for tests in a module's contained objects.
140 if inspect.ismodule(obj) and self._recurse:
140 if inspect.ismodule(obj) and self._recurse:
141 for valname, val in obj.__dict__.items():
141 for valname, val in obj.__dict__.items():
142 valname1 = '%s.%s' % (name, valname)
142 valname1 = '%s.%s' % (name, valname)
143 if ( (isroutine(val) or isclass(val))
143 if ( (isroutine(val) or isclass(val))
144 and self._from_module(module, val) ):
144 and self._from_module(module, val) ):
145
145
146 self._find(tests, val, valname1, module, source_lines,
146 self._find(tests, val, valname1, module, source_lines,
147 globs, seen)
147 globs, seen)
148
148
149 # Look for tests in a class's contained objects.
149 # Look for tests in a class's contained objects.
150 if inspect.isclass(obj) and self._recurse:
150 if inspect.isclass(obj) and self._recurse:
151 #print 'RECURSE into class:',obj # dbg
151 #print 'RECURSE into class:',obj # dbg
152 for valname, val in obj.__dict__.items():
152 for valname, val in obj.__dict__.items():
153 # Special handling for staticmethod/classmethod.
153 # Special handling for staticmethod/classmethod.
154 if isinstance(val, staticmethod):
154 if isinstance(val, staticmethod):
155 val = getattr(obj, valname)
155 val = getattr(obj, valname)
156 if isinstance(val, classmethod):
156 if isinstance(val, classmethod):
157 val = getattr(obj, valname).im_func
157 val = getattr(obj, valname).im_func
158
158
159 # Recurse to methods, properties, and nested classes.
159 # Recurse to methods, properties, and nested classes.
160 if ((inspect.isfunction(val) or inspect.isclass(val) or
160 if ((inspect.isfunction(val) or inspect.isclass(val) or
161 inspect.ismethod(val) or
161 inspect.ismethod(val) or
162 isinstance(val, property)) and
162 isinstance(val, property)) and
163 self._from_module(module, val)):
163 self._from_module(module, val)):
164 valname = '%s.%s' % (name, valname)
164 valname = '%s.%s' % (name, valname)
165 self._find(tests, val, valname, module, source_lines,
165 self._find(tests, val, valname, module, source_lines,
166 globs, seen)
166 globs, seen)
167
167
168
168
169 class IPDoctestOutputChecker(doctest.OutputChecker):
169 class IPDoctestOutputChecker(doctest.OutputChecker):
170 """Second-chance checker with support for random tests.
170 """Second-chance checker with support for random tests.
171
171
172 If the default comparison doesn't pass, this checker looks in the expected
172 If the default comparison doesn't pass, this checker looks in the expected
173 output string for flags that tell us to ignore the output.
173 output string for flags that tell us to ignore the output.
174 """
174 """
175
175
176 random_re = re.compile(r'#\s*random\s+')
176 random_re = re.compile(r'#\s*random\s+')
177
177
178 def check_output(self, want, got, optionflags):
178 def check_output(self, want, got, optionflags):
179 """Check output, accepting special markers embedded in the output.
179 """Check output, accepting special markers embedded in the output.
180
180
181 If the output didn't pass the default validation but the special string
181 If the output didn't pass the default validation but the special string
182 '#random' is included, we accept it."""
182 '#random' is included, we accept it."""
183
183
184 # Let the original tester verify first, in case people have valid tests
184 # Let the original tester verify first, in case people have valid tests
185 # that happen to have a comment saying '#random' embedded in.
185 # that happen to have a comment saying '#random' embedded in.
186 ret = doctest.OutputChecker.check_output(self, want, got,
186 ret = doctest.OutputChecker.check_output(self, want, got,
187 optionflags)
187 optionflags)
188 if not ret and self.random_re.search(want):
188 if not ret and self.random_re.search(want):
189 #print >> sys.stderr, 'RANDOM OK:',want # dbg
189 #print >> sys.stderr, 'RANDOM OK:',want # dbg
190 return True
190 return True
191
191
192 return ret
192 return ret
193
193
194
194
195 class DocTestCase(doctests.DocTestCase):
195 class DocTestCase(doctests.DocTestCase):
196 """Proxy for DocTestCase: provides an address() method that
196 """Proxy for DocTestCase: provides an address() method that
197 returns the correct address for the doctest case. Otherwise
197 returns the correct address for the doctest case. Otherwise
198 acts as a proxy to the test case. To provide hints for address(),
198 acts as a proxy to the test case. To provide hints for address(),
199 an obj may also be passed -- this will be used as the test object
199 an obj may also be passed -- this will be used as the test object
200 for purposes of determining the test address, if it is provided.
200 for purposes of determining the test address, if it is provided.
201 """
201 """
202
202
203 # Note: this method was taken from numpy's nosetester module.
203 # Note: this method was taken from numpy's nosetester module.
204
204
205 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
205 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
206 # its constructor that blocks non-default arguments from being passed
206 # its constructor that blocks non-default arguments from being passed
207 # down into doctest.DocTestCase
207 # down into doctest.DocTestCase
208
208
209 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
209 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
210 checker=None, obj=None, result_var='_'):
210 checker=None, obj=None, result_var='_'):
211 self._result_var = result_var
211 self._result_var = result_var
212 doctests.DocTestCase.__init__(self, test,
212 doctests.DocTestCase.__init__(self, test,
213 optionflags=optionflags,
213 optionflags=optionflags,
214 setUp=setUp, tearDown=tearDown,
214 setUp=setUp, tearDown=tearDown,
215 checker=checker)
215 checker=checker)
216 # Now we must actually copy the original constructor from the stdlib
216 # Now we must actually copy the original constructor from the stdlib
217 # doctest class, because we can't call it directly and a bug in nose
217 # doctest class, because we can't call it directly and a bug in nose
218 # means it never gets passed the right arguments.
218 # means it never gets passed the right arguments.
219
219
220 self._dt_optionflags = optionflags
220 self._dt_optionflags = optionflags
221 self._dt_checker = checker
221 self._dt_checker = checker
222 self._dt_test = test
222 self._dt_test = test
223 self._dt_test_globs_ori = test.globs
223 self._dt_test_globs_ori = test.globs
224 self._dt_setUp = setUp
224 self._dt_setUp = setUp
225 self._dt_tearDown = tearDown
225 self._dt_tearDown = tearDown
226
226
227 # XXX - store this runner once in the object!
227 # XXX - store this runner once in the object!
228 runner = IPDocTestRunner(optionflags=optionflags,
228 runner = IPDocTestRunner(optionflags=optionflags,
229 checker=checker, verbose=False)
229 checker=checker, verbose=False)
230 self._dt_runner = runner
230 self._dt_runner = runner
231
231
232
232
233 # Each doctest should remember the directory it was loaded from, so
233 # Each doctest should remember the directory it was loaded from, so
234 # things like %run work without too many contortions
234 # things like %run work without too many contortions
235 self._ori_dir = os.path.dirname(test.filename)
235 self._ori_dir = os.path.dirname(test.filename)
236
236
237 # Modified runTest from the default stdlib
237 # Modified runTest from the default stdlib
238 def runTest(self):
238 def runTest(self):
239 test = self._dt_test
239 test = self._dt_test
240 runner = self._dt_runner
240 runner = self._dt_runner
241
241
242 old = sys.stdout
242 old = sys.stdout
243 new = StringIO()
243 new = StringIO()
244 optionflags = self._dt_optionflags
244 optionflags = self._dt_optionflags
245
245
246 if not (optionflags & REPORTING_FLAGS):
246 if not (optionflags & REPORTING_FLAGS):
247 # The option flags don't include any reporting flags,
247 # The option flags don't include any reporting flags,
248 # so add the default reporting flags
248 # so add the default reporting flags
249 optionflags |= _unittest_reportflags
249 optionflags |= _unittest_reportflags
250
250
251 try:
251 try:
252 # Save our current directory and switch out to the one where the
252 # Save our current directory and switch out to the one where the
253 # test was originally created, in case another doctest did a
253 # test was originally created, in case another doctest did a
254 # directory change. We'll restore this in the finally clause.
254 # directory change. We'll restore this in the finally clause.
255 curdir = os.getcwdu()
255 curdir = os.getcwdu()
256 #print 'runTest in dir:', self._ori_dir # dbg
256 #print 'runTest in dir:', self._ori_dir # dbg
257 os.chdir(self._ori_dir)
257 os.chdir(self._ori_dir)
258
258
259 runner.DIVIDER = "-"*70
259 runner.DIVIDER = "-"*70
260 failures, tries = runner.run(test,out=new.write,
260 failures, tries = runner.run(test,out=new.write,
261 clear_globs=False)
261 clear_globs=False)
262 finally:
262 finally:
263 sys.stdout = old
263 sys.stdout = old
264 os.chdir(curdir)
264 os.chdir(curdir)
265
265
266 if failures:
266 if failures:
267 raise self.failureException(self.format_failure(new.getvalue()))
267 raise self.failureException(self.format_failure(new.getvalue()))
268
268
269 def setUp(self):
269 def setUp(self):
270 """Modified test setup that syncs with ipython namespace"""
270 """Modified test setup that syncs with ipython namespace"""
271 #print "setUp test", self._dt_test.examples # dbg
271 #print "setUp test", self._dt_test.examples # dbg
272 if isinstance(self._dt_test.examples[0], IPExample):
272 if isinstance(self._dt_test.examples[0], IPExample):
273 # for IPython examples *only*, we swap the globals with the ipython
273 # for IPython examples *only*, we swap the globals with the ipython
274 # namespace, after updating it with the globals (which doctest
274 # namespace, after updating it with the globals (which doctest
275 # fills with the necessary info from the module being tested).
275 # fills with the necessary info from the module being tested).
276 self.user_ns_orig = {}
276 self.user_ns_orig = {}
277 self.user_ns_orig.update(_ip.user_ns)
277 self.user_ns_orig.update(_ip.user_ns)
278 _ip.user_ns.update(self._dt_test.globs)
278 _ip.user_ns.update(self._dt_test.globs)
279 # We must remove the _ key in the namespace, so that Python's
279 # We must remove the _ key in the namespace, so that Python's
280 # doctest code sets it naturally
280 # doctest code sets it naturally
281 _ip.user_ns.pop('_', None)
281 _ip.user_ns.pop('_', None)
282 _ip.user_ns['__builtins__'] = builtin_mod
282 _ip.user_ns['__builtins__'] = builtin_mod
283 self._dt_test.globs = _ip.user_ns
283 self._dt_test.globs = _ip.user_ns
284
284
285 super(DocTestCase, self).setUp()
285 super(DocTestCase, self).setUp()
286
286
287 def tearDown(self):
287 def tearDown(self):
288
288
289 # Undo the test.globs reassignment we made, so that the parent class
289 # Undo the test.globs reassignment we made, so that the parent class
290 # teardown doesn't destroy the ipython namespace
290 # teardown doesn't destroy the ipython namespace
291 if isinstance(self._dt_test.examples[0], IPExample):
291 if isinstance(self._dt_test.examples[0], IPExample):
292 self._dt_test.globs = self._dt_test_globs_ori
292 self._dt_test.globs = self._dt_test_globs_ori
293 _ip.user_ns.clear()
293 _ip.user_ns.clear()
294 _ip.user_ns.update(self.user_ns_orig)
294 _ip.user_ns.update(self.user_ns_orig)
295
295
296 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
296 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
297 # it does look like one to me: its tearDown method tries to run
297 # it does look like one to me: its tearDown method tries to run
298 #
298 #
299 # delattr(__builtin__, self._result_var)
299 # delattr(builtin_mod, self._result_var)
300 #
300 #
301 # without checking that the attribute really is there; it implicitly
301 # without checking that the attribute really is there; it implicitly
302 # assumes it should have been set via displayhook. But if the
302 # assumes it should have been set via displayhook. But if the
303 # displayhook was never called, this doesn't necessarily happen. I
303 # displayhook was never called, this doesn't necessarily happen. I
304 # haven't been able to find a little self-contained example outside of
304 # haven't been able to find a little self-contained example outside of
305 # ipython that would show the problem so I can report it to the nose
305 # ipython that would show the problem so I can report it to the nose
306 # team, but it does happen a lot in our code.
306 # team, but it does happen a lot in our code.
307 #
307 #
308 # So here, we just protect as narrowly as possible by trapping an
308 # So here, we just protect as narrowly as possible by trapping an
309 # attribute error whose message would be the name of self._result_var,
309 # attribute error whose message would be the name of self._result_var,
310 # and letting any other error propagate.
310 # and letting any other error propagate.
311 try:
311 try:
312 super(DocTestCase, self).tearDown()
312 super(DocTestCase, self).tearDown()
313 except AttributeError as exc:
313 except AttributeError as exc:
314 if exc.args[0] != self._result_var:
314 if exc.args[0] != self._result_var:
315 raise
315 raise
316
316
317
317
318 # A simple subclassing of the original with a different class name, so we can
318 # A simple subclassing of the original with a different class name, so we can
319 # distinguish and treat differently IPython examples from pure python ones.
319 # distinguish and treat differently IPython examples from pure python ones.
320 class IPExample(doctest.Example): pass
320 class IPExample(doctest.Example): pass
321
321
322
322
323 class IPExternalExample(doctest.Example):
323 class IPExternalExample(doctest.Example):
324 """Doctest examples to be run in an external process."""
324 """Doctest examples to be run in an external process."""
325
325
326 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
326 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
327 options=None):
327 options=None):
328 # Parent constructor
328 # Parent constructor
329 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
329 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
330
330
331 # An EXTRA newline is needed to prevent pexpect hangs
331 # An EXTRA newline is needed to prevent pexpect hangs
332 self.source += '\n'
332 self.source += '\n'
333
333
334
334
335 class IPDocTestParser(doctest.DocTestParser):
335 class IPDocTestParser(doctest.DocTestParser):
336 """
336 """
337 A class used to parse strings containing doctest examples.
337 A class used to parse strings containing doctest examples.
338
338
339 Note: This is a version modified to properly recognize IPython input and
339 Note: This is a version modified to properly recognize IPython input and
340 convert any IPython examples into valid Python ones.
340 convert any IPython examples into valid Python ones.
341 """
341 """
342 # This regular expression is used to find doctest examples in a
342 # This regular expression is used to find doctest examples in a
343 # string. It defines three groups: `source` is the source code
343 # string. It defines three groups: `source` is the source code
344 # (including leading indentation and prompts); `indent` is the
344 # (including leading indentation and prompts); `indent` is the
345 # indentation of the first (PS1) line of the source code; and
345 # indentation of the first (PS1) line of the source code; and
346 # `want` is the expected output (including leading indentation).
346 # `want` is the expected output (including leading indentation).
347
347
348 # Classic Python prompts or default IPython ones
348 # Classic Python prompts or default IPython ones
349 _PS1_PY = r'>>>'
349 _PS1_PY = r'>>>'
350 _PS2_PY = r'\.\.\.'
350 _PS2_PY = r'\.\.\.'
351
351
352 _PS1_IP = r'In\ \[\d+\]:'
352 _PS1_IP = r'In\ \[\d+\]:'
353 _PS2_IP = r'\ \ \ \.\.\.+:'
353 _PS2_IP = r'\ \ \ \.\.\.+:'
354
354
355 _RE_TPL = r'''
355 _RE_TPL = r'''
356 # Source consists of a PS1 line followed by zero or more PS2 lines.
356 # Source consists of a PS1 line followed by zero or more PS2 lines.
357 (?P<source>
357 (?P<source>
358 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
358 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
359 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
359 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
360 \n? # a newline
360 \n? # a newline
361 # Want consists of any non-blank lines that do not start with PS1.
361 # Want consists of any non-blank lines that do not start with PS1.
362 (?P<want> (?:(?![ ]*$) # Not a blank line
362 (?P<want> (?:(?![ ]*$) # Not a blank line
363 (?![ ]*%s) # Not a line starting with PS1
363 (?![ ]*%s) # Not a line starting with PS1
364 (?![ ]*%s) # Not a line starting with PS2
364 (?![ ]*%s) # Not a line starting with PS2
365 .*$\n? # But any other line
365 .*$\n? # But any other line
366 )*)
366 )*)
367 '''
367 '''
368
368
369 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
369 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
370 re.MULTILINE | re.VERBOSE)
370 re.MULTILINE | re.VERBOSE)
371
371
372 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
372 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
373 re.MULTILINE | re.VERBOSE)
373 re.MULTILINE | re.VERBOSE)
374
374
375 # Mark a test as being fully random. In this case, we simply append the
375 # Mark a test as being fully random. In this case, we simply append the
376 # random marker ('#random') to each individual example's output. This way
376 # random marker ('#random') to each individual example's output. This way
377 # we don't need to modify any other code.
377 # we don't need to modify any other code.
378 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
378 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
379
379
380 # Mark tests to be executed in an external process - currently unsupported.
380 # Mark tests to be executed in an external process - currently unsupported.
381 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
381 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
382
382
383 def ip2py(self,source):
383 def ip2py(self,source):
384 """Convert input IPython source into valid Python."""
384 """Convert input IPython source into valid Python."""
385 block = _ip.input_transformer_manager.transform_cell(source)
385 block = _ip.input_transformer_manager.transform_cell(source)
386 if len(block.splitlines()) == 1:
386 if len(block.splitlines()) == 1:
387 return _ip.prefilter(block)
387 return _ip.prefilter(block)
388 else:
388 else:
389 return block
389 return block
390
390
391 def parse(self, string, name='<string>'):
391 def parse(self, string, name='<string>'):
392 """
392 """
393 Divide the given string into examples and intervening text,
393 Divide the given string into examples and intervening text,
394 and return them as a list of alternating Examples and strings.
394 and return them as a list of alternating Examples and strings.
395 Line numbers for the Examples are 0-based. The optional
395 Line numbers for the Examples are 0-based. The optional
396 argument `name` is a name identifying this string, and is only
396 argument `name` is a name identifying this string, and is only
397 used for error messages.
397 used for error messages.
398 """
398 """
399
399
400 #print 'Parse string:\n',string # dbg
400 #print 'Parse string:\n',string # dbg
401
401
402 string = string.expandtabs()
402 string = string.expandtabs()
403 # If all lines begin with the same indentation, then strip it.
403 # If all lines begin with the same indentation, then strip it.
404 min_indent = self._min_indent(string)
404 min_indent = self._min_indent(string)
405 if min_indent > 0:
405 if min_indent > 0:
406 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
406 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
407
407
408 output = []
408 output = []
409 charno, lineno = 0, 0
409 charno, lineno = 0, 0
410
410
411 # We make 'all random' tests by adding the '# random' mark to every
411 # We make 'all random' tests by adding the '# random' mark to every
412 # block of output in the test.
412 # block of output in the test.
413 if self._RANDOM_TEST.search(string):
413 if self._RANDOM_TEST.search(string):
414 random_marker = '\n# random'
414 random_marker = '\n# random'
415 else:
415 else:
416 random_marker = ''
416 random_marker = ''
417
417
418 # Whether to convert the input from ipython to python syntax
418 # Whether to convert the input from ipython to python syntax
419 ip2py = False
419 ip2py = False
420 # Find all doctest examples in the string. First, try them as Python
420 # Find all doctest examples in the string. First, try them as Python
421 # examples, then as IPython ones
421 # examples, then as IPython ones
422 terms = list(self._EXAMPLE_RE_PY.finditer(string))
422 terms = list(self._EXAMPLE_RE_PY.finditer(string))
423 if terms:
423 if terms:
424 # Normal Python example
424 # Normal Python example
425 #print '-'*70 # dbg
425 #print '-'*70 # dbg
426 #print 'PyExample, Source:\n',string # dbg
426 #print 'PyExample, Source:\n',string # dbg
427 #print '-'*70 # dbg
427 #print '-'*70 # dbg
428 Example = doctest.Example
428 Example = doctest.Example
429 else:
429 else:
430 # It's an ipython example. Note that IPExamples are run
430 # It's an ipython example. Note that IPExamples are run
431 # in-process, so their syntax must be turned into valid python.
431 # in-process, so their syntax must be turned into valid python.
432 # IPExternalExamples are run out-of-process (via pexpect) so they
432 # IPExternalExamples are run out-of-process (via pexpect) so they
433 # don't need any filtering (a real ipython will be executing them).
433 # don't need any filtering (a real ipython will be executing them).
434 terms = list(self._EXAMPLE_RE_IP.finditer(string))
434 terms = list(self._EXAMPLE_RE_IP.finditer(string))
435 if self._EXTERNAL_IP.search(string):
435 if self._EXTERNAL_IP.search(string):
436 #print '-'*70 # dbg
436 #print '-'*70 # dbg
437 #print 'IPExternalExample, Source:\n',string # dbg
437 #print 'IPExternalExample, Source:\n',string # dbg
438 #print '-'*70 # dbg
438 #print '-'*70 # dbg
439 Example = IPExternalExample
439 Example = IPExternalExample
440 else:
440 else:
441 #print '-'*70 # dbg
441 #print '-'*70 # dbg
442 #print 'IPExample, Source:\n',string # dbg
442 #print 'IPExample, Source:\n',string # dbg
443 #print '-'*70 # dbg
443 #print '-'*70 # dbg
444 Example = IPExample
444 Example = IPExample
445 ip2py = True
445 ip2py = True
446
446
447 for m in terms:
447 for m in terms:
448 # Add the pre-example text to `output`.
448 # Add the pre-example text to `output`.
449 output.append(string[charno:m.start()])
449 output.append(string[charno:m.start()])
450 # Update lineno (lines before this example)
450 # Update lineno (lines before this example)
451 lineno += string.count('\n', charno, m.start())
451 lineno += string.count('\n', charno, m.start())
452 # Extract info from the regexp match.
452 # Extract info from the regexp match.
453 (source, options, want, exc_msg) = \
453 (source, options, want, exc_msg) = \
454 self._parse_example(m, name, lineno,ip2py)
454 self._parse_example(m, name, lineno,ip2py)
455
455
456 # Append the random-output marker (it defaults to empty in most
456 # Append the random-output marker (it defaults to empty in most
457 # cases, it's only non-empty for 'all-random' tests):
457 # cases, it's only non-empty for 'all-random' tests):
458 want += random_marker
458 want += random_marker
459
459
460 if Example is IPExternalExample:
460 if Example is IPExternalExample:
461 options[doctest.NORMALIZE_WHITESPACE] = True
461 options[doctest.NORMALIZE_WHITESPACE] = True
462 want += '\n'
462 want += '\n'
463
463
464 # Create an Example, and add it to the list.
464 # Create an Example, and add it to the list.
465 if not self._IS_BLANK_OR_COMMENT(source):
465 if not self._IS_BLANK_OR_COMMENT(source):
466 output.append(Example(source, want, exc_msg,
466 output.append(Example(source, want, exc_msg,
467 lineno=lineno,
467 lineno=lineno,
468 indent=min_indent+len(m.group('indent')),
468 indent=min_indent+len(m.group('indent')),
469 options=options))
469 options=options))
470 # Update lineno (lines inside this example)
470 # Update lineno (lines inside this example)
471 lineno += string.count('\n', m.start(), m.end())
471 lineno += string.count('\n', m.start(), m.end())
472 # Update charno.
472 # Update charno.
473 charno = m.end()
473 charno = m.end()
474 # Add any remaining post-example text to `output`.
474 # Add any remaining post-example text to `output`.
475 output.append(string[charno:])
475 output.append(string[charno:])
476 return output
476 return output
477
477
478 def _parse_example(self, m, name, lineno,ip2py=False):
478 def _parse_example(self, m, name, lineno,ip2py=False):
479 """
479 """
480 Given a regular expression match from `_EXAMPLE_RE` (`m`),
480 Given a regular expression match from `_EXAMPLE_RE` (`m`),
481 return a pair `(source, want)`, where `source` is the matched
481 return a pair `(source, want)`, where `source` is the matched
482 example's source code (with prompts and indentation stripped);
482 example's source code (with prompts and indentation stripped);
483 and `want` is the example's expected output (with indentation
483 and `want` is the example's expected output (with indentation
484 stripped).
484 stripped).
485
485
486 `name` is the string's name, and `lineno` is the line number
486 `name` is the string's name, and `lineno` is the line number
487 where the example starts; both are used for error messages.
487 where the example starts; both are used for error messages.
488
488
489 Optional:
489 Optional:
490 `ip2py`: if true, filter the input via IPython to convert the syntax
490 `ip2py`: if true, filter the input via IPython to convert the syntax
491 into valid python.
491 into valid python.
492 """
492 """
493
493
494 # Get the example's indentation level.
494 # Get the example's indentation level.
495 indent = len(m.group('indent'))
495 indent = len(m.group('indent'))
496
496
497 # Divide source into lines; check that they're properly
497 # Divide source into lines; check that they're properly
498 # indented; and then strip their indentation & prompts.
498 # indented; and then strip their indentation & prompts.
499 source_lines = m.group('source').split('\n')
499 source_lines = m.group('source').split('\n')
500
500
501 # We're using variable-length input prompts
501 # We're using variable-length input prompts
502 ps1 = m.group('ps1')
502 ps1 = m.group('ps1')
503 ps2 = m.group('ps2')
503 ps2 = m.group('ps2')
504 ps1_len = len(ps1)
504 ps1_len = len(ps1)
505
505
506 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
506 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
507 if ps2:
507 if ps2:
508 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
508 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
509
509
510 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
510 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
511
511
512 if ip2py:
512 if ip2py:
513 # Convert source input from IPython into valid Python syntax
513 # Convert source input from IPython into valid Python syntax
514 source = self.ip2py(source)
514 source = self.ip2py(source)
515
515
516 # Divide want into lines; check that it's properly indented; and
516 # Divide want into lines; check that it's properly indented; and
517 # then strip the indentation. Spaces before the last newline should
517 # then strip the indentation. Spaces before the last newline should
518 # be preserved, so plain rstrip() isn't good enough.
518 # be preserved, so plain rstrip() isn't good enough.
519 want = m.group('want')
519 want = m.group('want')
520 want_lines = want.split('\n')
520 want_lines = want.split('\n')
521 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
521 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
522 del want_lines[-1] # forget final newline & spaces after it
522 del want_lines[-1] # forget final newline & spaces after it
523 self._check_prefix(want_lines, ' '*indent, name,
523 self._check_prefix(want_lines, ' '*indent, name,
524 lineno + len(source_lines))
524 lineno + len(source_lines))
525
525
526 # Remove ipython output prompt that might be present in the first line
526 # Remove ipython output prompt that might be present in the first line
527 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
527 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
528
528
529 want = '\n'.join([wl[indent:] for wl in want_lines])
529 want = '\n'.join([wl[indent:] for wl in want_lines])
530
530
531 # If `want` contains a traceback message, then extract it.
531 # If `want` contains a traceback message, then extract it.
532 m = self._EXCEPTION_RE.match(want)
532 m = self._EXCEPTION_RE.match(want)
533 if m:
533 if m:
534 exc_msg = m.group('msg')
534 exc_msg = m.group('msg')
535 else:
535 else:
536 exc_msg = None
536 exc_msg = None
537
537
538 # Extract options from the source.
538 # Extract options from the source.
539 options = self._find_options(source, name, lineno)
539 options = self._find_options(source, name, lineno)
540
540
541 return source, options, want, exc_msg
541 return source, options, want, exc_msg
542
542
543 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
543 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
544 """
544 """
545 Given the lines of a source string (including prompts and
545 Given the lines of a source string (including prompts and
546 leading indentation), check to make sure that every prompt is
546 leading indentation), check to make sure that every prompt is
547 followed by a space character. If any line is not followed by
547 followed by a space character. If any line is not followed by
548 a space character, then raise ValueError.
548 a space character, then raise ValueError.
549
549
550 Note: IPython-modified version which takes the input prompt length as a
550 Note: IPython-modified version which takes the input prompt length as a
551 parameter, so that prompts of variable length can be dealt with.
551 parameter, so that prompts of variable length can be dealt with.
552 """
552 """
553 space_idx = indent+ps1_len
553 space_idx = indent+ps1_len
554 min_len = space_idx+1
554 min_len = space_idx+1
555 for i, line in enumerate(lines):
555 for i, line in enumerate(lines):
556 if len(line) >= min_len and line[space_idx] != ' ':
556 if len(line) >= min_len and line[space_idx] != ' ':
557 raise ValueError('line %r of the docstring for %s '
557 raise ValueError('line %r of the docstring for %s '
558 'lacks blank after %s: %r' %
558 'lacks blank after %s: %r' %
559 (lineno+i+1, name,
559 (lineno+i+1, name,
560 line[indent:space_idx], line))
560 line[indent:space_idx], line))
561
561
562
562
563 SKIP = doctest.register_optionflag('SKIP')
563 SKIP = doctest.register_optionflag('SKIP')
564
564
565
565
566 class IPDocTestRunner(doctest.DocTestRunner,object):
566 class IPDocTestRunner(doctest.DocTestRunner,object):
567 """Test runner that synchronizes the IPython namespace with test globals.
567 """Test runner that synchronizes the IPython namespace with test globals.
568 """
568 """
569
569
570 def run(self, test, compileflags=None, out=None, clear_globs=True):
570 def run(self, test, compileflags=None, out=None, clear_globs=True):
571
571
572 # Hack: ipython needs access to the execution context of the example,
572 # Hack: ipython needs access to the execution context of the example,
573 # so that it can propagate user variables loaded by %run into
573 # so that it can propagate user variables loaded by %run into
574 # test.globs. We put them here into our modified %run as a function
574 # test.globs. We put them here into our modified %run as a function
575 # attribute. Our new %run will then only make the namespace update
575 # attribute. Our new %run will then only make the namespace update
576 # when called (rather than unconconditionally updating test.globs here
576 # when called (rather than unconconditionally updating test.globs here
577 # for all examples, most of which won't be calling %run anyway).
577 # for all examples, most of which won't be calling %run anyway).
578 #_ip._ipdoctest_test_globs = test.globs
578 #_ip._ipdoctest_test_globs = test.globs
579 #_ip._ipdoctest_test_filename = test.filename
579 #_ip._ipdoctest_test_filename = test.filename
580
580
581 test.globs.update(_ip.user_ns)
581 test.globs.update(_ip.user_ns)
582
582
583 return super(IPDocTestRunner,self).run(test,
583 return super(IPDocTestRunner,self).run(test,
584 compileflags,out,clear_globs)
584 compileflags,out,clear_globs)
585
585
586
586
587 class DocFileCase(doctest.DocFileCase):
587 class DocFileCase(doctest.DocFileCase):
588 """Overrides to provide filename
588 """Overrides to provide filename
589 """
589 """
590 def address(self):
590 def address(self):
591 return (self._dt_test.filename, None, None)
591 return (self._dt_test.filename, None, None)
592
592
593
593
594 class ExtensionDoctest(doctests.Doctest):
594 class ExtensionDoctest(doctests.Doctest):
595 """Nose Plugin that supports doctests in extension modules.
595 """Nose Plugin that supports doctests in extension modules.
596 """
596 """
597 name = 'extdoctest' # call nosetests with --with-extdoctest
597 name = 'extdoctest' # call nosetests with --with-extdoctest
598 enabled = True
598 enabled = True
599
599
600 def options(self, parser, env=os.environ):
600 def options(self, parser, env=os.environ):
601 Plugin.options(self, parser, env)
601 Plugin.options(self, parser, env)
602 parser.add_option('--doctest-tests', action='store_true',
602 parser.add_option('--doctest-tests', action='store_true',
603 dest='doctest_tests',
603 dest='doctest_tests',
604 default=env.get('NOSE_DOCTEST_TESTS',True),
604 default=env.get('NOSE_DOCTEST_TESTS',True),
605 help="Also look for doctests in test modules. "
605 help="Also look for doctests in test modules. "
606 "Note that classes, methods and functions should "
606 "Note that classes, methods and functions should "
607 "have either doctests or non-doctest tests, "
607 "have either doctests or non-doctest tests, "
608 "not both. [NOSE_DOCTEST_TESTS]")
608 "not both. [NOSE_DOCTEST_TESTS]")
609 parser.add_option('--doctest-extension', action="append",
609 parser.add_option('--doctest-extension', action="append",
610 dest="doctestExtension",
610 dest="doctestExtension",
611 help="Also look for doctests in files with "
611 help="Also look for doctests in files with "
612 "this extension [NOSE_DOCTEST_EXTENSION]")
612 "this extension [NOSE_DOCTEST_EXTENSION]")
613 # Set the default as a list, if given in env; otherwise
613 # Set the default as a list, if given in env; otherwise
614 # an additional value set on the command line will cause
614 # an additional value set on the command line will cause
615 # an error.
615 # an error.
616 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
616 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
617 if env_setting is not None:
617 if env_setting is not None:
618 parser.set_defaults(doctestExtension=tolist(env_setting))
618 parser.set_defaults(doctestExtension=tolist(env_setting))
619
619
620
620
621 def configure(self, options, config):
621 def configure(self, options, config):
622 Plugin.configure(self, options, config)
622 Plugin.configure(self, options, config)
623 # Pull standard doctest plugin out of config; we will do doctesting
623 # Pull standard doctest plugin out of config; we will do doctesting
624 config.plugins.plugins = [p for p in config.plugins.plugins
624 config.plugins.plugins = [p for p in config.plugins.plugins
625 if p.name != 'doctest']
625 if p.name != 'doctest']
626 self.doctest_tests = options.doctest_tests
626 self.doctest_tests = options.doctest_tests
627 self.extension = tolist(options.doctestExtension)
627 self.extension = tolist(options.doctestExtension)
628
628
629 self.parser = doctest.DocTestParser()
629 self.parser = doctest.DocTestParser()
630 self.finder = DocTestFinder()
630 self.finder = DocTestFinder()
631 self.checker = IPDoctestOutputChecker()
631 self.checker = IPDoctestOutputChecker()
632 self.globs = None
632 self.globs = None
633 self.extraglobs = None
633 self.extraglobs = None
634
634
635
635
636 def loadTestsFromExtensionModule(self,filename):
636 def loadTestsFromExtensionModule(self,filename):
637 bpath,mod = os.path.split(filename)
637 bpath,mod = os.path.split(filename)
638 modname = os.path.splitext(mod)[0]
638 modname = os.path.splitext(mod)[0]
639 try:
639 try:
640 sys.path.append(bpath)
640 sys.path.append(bpath)
641 module = __import__(modname)
641 module = __import__(modname)
642 tests = list(self.loadTestsFromModule(module))
642 tests = list(self.loadTestsFromModule(module))
643 finally:
643 finally:
644 sys.path.pop()
644 sys.path.pop()
645 return tests
645 return tests
646
646
647 # NOTE: the method below is almost a copy of the original one in nose, with
647 # NOTE: the method below is almost a copy of the original one in nose, with
648 # a few modifications to control output checking.
648 # a few modifications to control output checking.
649
649
650 def loadTestsFromModule(self, module):
650 def loadTestsFromModule(self, module):
651 #print '*** ipdoctest - lTM',module # dbg
651 #print '*** ipdoctest - lTM',module # dbg
652
652
653 if not self.matches(module.__name__):
653 if not self.matches(module.__name__):
654 log.debug("Doctest doesn't want module %s", module)
654 log.debug("Doctest doesn't want module %s", module)
655 return
655 return
656
656
657 tests = self.finder.find(module,globs=self.globs,
657 tests = self.finder.find(module,globs=self.globs,
658 extraglobs=self.extraglobs)
658 extraglobs=self.extraglobs)
659 if not tests:
659 if not tests:
660 return
660 return
661
661
662 # always use whitespace and ellipsis options
662 # always use whitespace and ellipsis options
663 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
663 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
664
664
665 tests.sort()
665 tests.sort()
666 module_file = module.__file__
666 module_file = module.__file__
667 if module_file[-4:] in ('.pyc', '.pyo'):
667 if module_file[-4:] in ('.pyc', '.pyo'):
668 module_file = module_file[:-1]
668 module_file = module_file[:-1]
669 for test in tests:
669 for test in tests:
670 if not test.examples:
670 if not test.examples:
671 continue
671 continue
672 if not test.filename:
672 if not test.filename:
673 test.filename = module_file
673 test.filename = module_file
674
674
675 yield DocTestCase(test,
675 yield DocTestCase(test,
676 optionflags=optionflags,
676 optionflags=optionflags,
677 checker=self.checker)
677 checker=self.checker)
678
678
679
679
680 def loadTestsFromFile(self, filename):
680 def loadTestsFromFile(self, filename):
681 #print "ipdoctest - from file", filename # dbg
681 #print "ipdoctest - from file", filename # dbg
682 if is_extension_module(filename):
682 if is_extension_module(filename):
683 for t in self.loadTestsFromExtensionModule(filename):
683 for t in self.loadTestsFromExtensionModule(filename):
684 yield t
684 yield t
685 else:
685 else:
686 if self.extension and anyp(filename.endswith, self.extension):
686 if self.extension and anyp(filename.endswith, self.extension):
687 name = os.path.basename(filename)
687 name = os.path.basename(filename)
688 dh = open(filename)
688 dh = open(filename)
689 try:
689 try:
690 doc = dh.read()
690 doc = dh.read()
691 finally:
691 finally:
692 dh.close()
692 dh.close()
693 test = self.parser.get_doctest(
693 test = self.parser.get_doctest(
694 doc, globs={'__file__': filename}, name=name,
694 doc, globs={'__file__': filename}, name=name,
695 filename=filename, lineno=0)
695 filename=filename, lineno=0)
696 if test.examples:
696 if test.examples:
697 #print 'FileCase:',test.examples # dbg
697 #print 'FileCase:',test.examples # dbg
698 yield DocFileCase(test)
698 yield DocFileCase(test)
699 else:
699 else:
700 yield False # no tests to load
700 yield False # no tests to load
701
701
702
702
703 class IPythonDoctest(ExtensionDoctest):
703 class IPythonDoctest(ExtensionDoctest):
704 """Nose Plugin that supports doctests in extension modules.
704 """Nose Plugin that supports doctests in extension modules.
705 """
705 """
706 name = 'ipdoctest' # call nosetests with --with-ipdoctest
706 name = 'ipdoctest' # call nosetests with --with-ipdoctest
707 enabled = True
707 enabled = True
708
708
709 def makeTest(self, obj, parent):
709 def makeTest(self, obj, parent):
710 """Look for doctests in the given object, which will be a
710 """Look for doctests in the given object, which will be a
711 function, method or class.
711 function, method or class.
712 """
712 """
713 #print 'Plugin analyzing:', obj, parent # dbg
713 #print 'Plugin analyzing:', obj, parent # dbg
714 # always use whitespace and ellipsis options
714 # always use whitespace and ellipsis options
715 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
715 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
716
716
717 doctests = self.finder.find(obj, module=getmodule(parent))
717 doctests = self.finder.find(obj, module=getmodule(parent))
718 if doctests:
718 if doctests:
719 for test in doctests:
719 for test in doctests:
720 if len(test.examples) == 0:
720 if len(test.examples) == 0:
721 continue
721 continue
722
722
723 yield DocTestCase(test, obj=obj,
723 yield DocTestCase(test, obj=obj,
724 optionflags=optionflags,
724 optionflags=optionflags,
725 checker=self.checker)
725 checker=self.checker)
726
726
727 def options(self, parser, env=os.environ):
727 def options(self, parser, env=os.environ):
728 #print "Options for nose plugin:", self.name # dbg
728 #print "Options for nose plugin:", self.name # dbg
729 Plugin.options(self, parser, env)
729 Plugin.options(self, parser, env)
730 parser.add_option('--ipdoctest-tests', action='store_true',
730 parser.add_option('--ipdoctest-tests', action='store_true',
731 dest='ipdoctest_tests',
731 dest='ipdoctest_tests',
732 default=env.get('NOSE_IPDOCTEST_TESTS',True),
732 default=env.get('NOSE_IPDOCTEST_TESTS',True),
733 help="Also look for doctests in test modules. "
733 help="Also look for doctests in test modules. "
734 "Note that classes, methods and functions should "
734 "Note that classes, methods and functions should "
735 "have either doctests or non-doctest tests, "
735 "have either doctests or non-doctest tests, "
736 "not both. [NOSE_IPDOCTEST_TESTS]")
736 "not both. [NOSE_IPDOCTEST_TESTS]")
737 parser.add_option('--ipdoctest-extension', action="append",
737 parser.add_option('--ipdoctest-extension', action="append",
738 dest="ipdoctest_extension",
738 dest="ipdoctest_extension",
739 help="Also look for doctests in files with "
739 help="Also look for doctests in files with "
740 "this extension [NOSE_IPDOCTEST_EXTENSION]")
740 "this extension [NOSE_IPDOCTEST_EXTENSION]")
741 # Set the default as a list, if given in env; otherwise
741 # Set the default as a list, if given in env; otherwise
742 # an additional value set on the command line will cause
742 # an additional value set on the command line will cause
743 # an error.
743 # an error.
744 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
744 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
745 if env_setting is not None:
745 if env_setting is not None:
746 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
746 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
747
747
748 def configure(self, options, config):
748 def configure(self, options, config):
749 #print "Configuring nose plugin:", self.name # dbg
749 #print "Configuring nose plugin:", self.name # dbg
750 Plugin.configure(self, options, config)
750 Plugin.configure(self, options, config)
751 # Pull standard doctest plugin out of config; we will do doctesting
751 # Pull standard doctest plugin out of config; we will do doctesting
752 config.plugins.plugins = [p for p in config.plugins.plugins
752 config.plugins.plugins = [p for p in config.plugins.plugins
753 if p.name != 'doctest']
753 if p.name != 'doctest']
754 self.doctest_tests = options.ipdoctest_tests
754 self.doctest_tests = options.ipdoctest_tests
755 self.extension = tolist(options.ipdoctest_extension)
755 self.extension = tolist(options.ipdoctest_extension)
756
756
757 self.parser = IPDocTestParser()
757 self.parser = IPDocTestParser()
758 self.finder = DocTestFinder(parser=self.parser)
758 self.finder = DocTestFinder(parser=self.parser)
759 self.checker = IPDoctestOutputChecker()
759 self.checker = IPDoctestOutputChecker()
760 self.globs = None
760 self.globs = None
761 self.extraglobs = None
761 self.extraglobs = None
@@ -1,206 +1,207 b''
1 # coding: utf-8
1 # coding: utf-8
2 """Compatibility tricks for Python 3. Mainly to do with unicode."""
2 """Compatibility tricks for Python 3. Mainly to do with unicode."""
3 import __builtin__
4 import functools
3 import functools
5 import sys
4 import sys
6 import re
5 import re
7 import types
6 import types
8
7
9 from .encoding import DEFAULT_ENCODING
8 from .encoding import DEFAULT_ENCODING
10
9
11 orig_open = open
10 orig_open = open
12
11
13 def no_code(x, encoding=None):
12 def no_code(x, encoding=None):
14 return x
13 return x
15
14
16 def decode(s, encoding=None):
15 def decode(s, encoding=None):
17 encoding = encoding or DEFAULT_ENCODING
16 encoding = encoding or DEFAULT_ENCODING
18 return s.decode(encoding, "replace")
17 return s.decode(encoding, "replace")
19
18
20 def encode(u, encoding=None):
19 def encode(u, encoding=None):
21 encoding = encoding or DEFAULT_ENCODING
20 encoding = encoding or DEFAULT_ENCODING
22 return u.encode(encoding, "replace")
21 return u.encode(encoding, "replace")
23
22
24
23
25 def cast_unicode(s, encoding=None):
24 def cast_unicode(s, encoding=None):
26 if isinstance(s, bytes):
25 if isinstance(s, bytes):
27 return decode(s, encoding)
26 return decode(s, encoding)
28 return s
27 return s
29
28
30 def cast_bytes(s, encoding=None):
29 def cast_bytes(s, encoding=None):
31 if not isinstance(s, bytes):
30 if not isinstance(s, bytes):
32 return encode(s, encoding)
31 return encode(s, encoding)
33 return s
32 return s
34
33
35 def _modify_str_or_docstring(str_change_func):
34 def _modify_str_or_docstring(str_change_func):
36 @functools.wraps(str_change_func)
35 @functools.wraps(str_change_func)
37 def wrapper(func_or_str):
36 def wrapper(func_or_str):
38 if isinstance(func_or_str, basestring):
37 if isinstance(func_or_str, basestring):
39 func = None
38 func = None
40 doc = func_or_str
39 doc = func_or_str
41 else:
40 else:
42 func = func_or_str
41 func = func_or_str
43 doc = func.__doc__
42 doc = func.__doc__
44
43
45 doc = str_change_func(doc)
44 doc = str_change_func(doc)
46
45
47 if func:
46 if func:
48 func.__doc__ = doc
47 func.__doc__ = doc
49 return func
48 return func
50 return doc
49 return doc
51 return wrapper
50 return wrapper
52
51
53 def safe_unicode(e):
52 def safe_unicode(e):
54 """unicode(e) with various fallbacks. Used for exceptions, which may not be
53 """unicode(e) with various fallbacks. Used for exceptions, which may not be
55 safe to call unicode() on.
54 safe to call unicode() on.
56 """
55 """
57 try:
56 try:
58 return unicode(e)
57 return unicode(e)
59 except UnicodeError:
58 except UnicodeError:
60 pass
59 pass
61
60
62 try:
61 try:
63 return str_to_unicode(str(e))
62 return str_to_unicode(str(e))
64 except UnicodeError:
63 except UnicodeError:
65 pass
64 pass
66
65
67 try:
66 try:
68 return str_to_unicode(repr(e))
67 return str_to_unicode(repr(e))
69 except UnicodeError:
68 except UnicodeError:
70 pass
69 pass
71
70
72 return u'Unrecoverably corrupt evalue'
71 return u'Unrecoverably corrupt evalue'
73
72
74 if sys.version_info[0] >= 3:
73 if sys.version_info[0] >= 3:
75 PY3 = True
74 PY3 = True
76
75
77 input = input
76 input = input
78 builtin_mod_name = "builtins"
77 builtin_mod_name = "builtins"
78 import builtins as builtin_mod
79
79
80 str_to_unicode = no_code
80 str_to_unicode = no_code
81 unicode_to_str = no_code
81 unicode_to_str = no_code
82 str_to_bytes = encode
82 str_to_bytes = encode
83 bytes_to_str = decode
83 bytes_to_str = decode
84 cast_bytes_py2 = no_code
84 cast_bytes_py2 = no_code
85
85
86 string_types = (str,)
86 string_types = (str,)
87 unicode_type = str
87 unicode_type = str
88
88
89 def isidentifier(s, dotted=False):
89 def isidentifier(s, dotted=False):
90 if dotted:
90 if dotted:
91 return all(isidentifier(a) for a in s.split("."))
91 return all(isidentifier(a) for a in s.split("."))
92 return s.isidentifier()
92 return s.isidentifier()
93
93
94 open = orig_open
94 open = orig_open
95
95
96 MethodType = types.MethodType
96 MethodType = types.MethodType
97
97
98 def execfile(fname, glob, loc=None):
98 def execfile(fname, glob, loc=None):
99 loc = loc if (loc is not None) else glob
99 loc = loc if (loc is not None) else glob
100 with open(fname, 'rb') as f:
100 with open(fname, 'rb') as f:
101 exec(compile(f.read(), fname, 'exec'), glob, loc)
101 exec(compile(f.read(), fname, 'exec'), glob, loc)
102
102
103 # Refactor print statements in doctests.
103 # Refactor print statements in doctests.
104 _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
104 _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
105 def _print_statement_sub(match):
105 def _print_statement_sub(match):
106 expr = match.groups('expr')
106 expr = match.groups('expr')
107 return "print(%s)" % expr
107 return "print(%s)" % expr
108
108
109 @_modify_str_or_docstring
109 @_modify_str_or_docstring
110 def doctest_refactor_print(doc):
110 def doctest_refactor_print(doc):
111 """Refactor 'print x' statements in a doctest to print(x) style. 2to3
111 """Refactor 'print x' statements in a doctest to print(x) style. 2to3
112 unfortunately doesn't pick up on our doctests.
112 unfortunately doesn't pick up on our doctests.
113
113
114 Can accept a string or a function, so it can be used as a decorator."""
114 Can accept a string or a function, so it can be used as a decorator."""
115 return _print_statement_re.sub(_print_statement_sub, doc)
115 return _print_statement_re.sub(_print_statement_sub, doc)
116
116
117 # Abstract u'abc' syntax:
117 # Abstract u'abc' syntax:
118 @_modify_str_or_docstring
118 @_modify_str_or_docstring
119 def u_format(s):
119 def u_format(s):
120 """"{u}'abc'" --> "'abc'" (Python 3)
120 """"{u}'abc'" --> "'abc'" (Python 3)
121
121
122 Accepts a string or a function, so it can be used as a decorator."""
122 Accepts a string or a function, so it can be used as a decorator."""
123 return s.format(u='')
123 return s.format(u='')
124
124
125 else:
125 else:
126 PY3 = False
126 PY3 = False
127
127
128 input = raw_input
128 input = raw_input
129 builtin_mod_name = "__builtin__"
129 builtin_mod_name = "__builtin__"
130 import __builtin__ as builtin_mod
130
131
131 str_to_unicode = decode
132 str_to_unicode = decode
132 unicode_to_str = encode
133 unicode_to_str = encode
133 str_to_bytes = no_code
134 str_to_bytes = no_code
134 bytes_to_str = no_code
135 bytes_to_str = no_code
135 cast_bytes_py2 = cast_bytes
136 cast_bytes_py2 = cast_bytes
136
137
137 string_types = (str, unicode)
138 string_types = (str, unicode)
138 unicode_type = unicode
139 unicode_type = unicode
139
140
140 import re
141 import re
141 _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
142 _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
142 def isidentifier(s, dotted=False):
143 def isidentifier(s, dotted=False):
143 if dotted:
144 if dotted:
144 return all(isidentifier(a) for a in s.split("."))
145 return all(isidentifier(a) for a in s.split("."))
145 return bool(_name_re.match(s))
146 return bool(_name_re.match(s))
146
147
147 class open(object):
148 class open(object):
148 """Wrapper providing key part of Python 3 open() interface."""
149 """Wrapper providing key part of Python 3 open() interface."""
149 def __init__(self, fname, mode="r", encoding="utf-8"):
150 def __init__(self, fname, mode="r", encoding="utf-8"):
150 self.f = orig_open(fname, mode)
151 self.f = orig_open(fname, mode)
151 self.enc = encoding
152 self.enc = encoding
152
153
153 def write(self, s):
154 def write(self, s):
154 return self.f.write(s.encode(self.enc))
155 return self.f.write(s.encode(self.enc))
155
156
156 def read(self, size=-1):
157 def read(self, size=-1):
157 return self.f.read(size).decode(self.enc)
158 return self.f.read(size).decode(self.enc)
158
159
159 def close(self):
160 def close(self):
160 return self.f.close()
161 return self.f.close()
161
162
162 def __enter__(self):
163 def __enter__(self):
163 return self
164 return self
164
165
165 def __exit__(self, etype, value, traceback):
166 def __exit__(self, etype, value, traceback):
166 self.f.close()
167 self.f.close()
167
168
168 def MethodType(func, instance):
169 def MethodType(func, instance):
169 return types.MethodType(func, instance, type(instance))
170 return types.MethodType(func, instance, type(instance))
170
171
171 # don't override system execfile on 2.x:
172 # don't override system execfile on 2.x:
172 execfile = execfile
173 execfile = execfile
173
174
174 def doctest_refactor_print(func_or_str):
175 def doctest_refactor_print(func_or_str):
175 return func_or_str
176 return func_or_str
176
177
177
178
178 # Abstract u'abc' syntax:
179 # Abstract u'abc' syntax:
179 @_modify_str_or_docstring
180 @_modify_str_or_docstring
180 def u_format(s):
181 def u_format(s):
181 """"{u}'abc'" --> "u'abc'" (Python 2)
182 """"{u}'abc'" --> "u'abc'" (Python 2)
182
183
183 Accepts a string or a function, so it can be used as a decorator."""
184 Accepts a string or a function, so it can be used as a decorator."""
184 return s.format(u='u')
185 return s.format(u='u')
185
186
186 if sys.platform == 'win32':
187 if sys.platform == 'win32':
187 def execfile(fname, glob=None, loc=None):
188 def execfile(fname, glob=None, loc=None):
188 loc = loc if (loc is not None) else glob
189 loc = loc if (loc is not None) else glob
189 # The rstrip() is necessary b/c trailing whitespace in files will
190 # The rstrip() is necessary b/c trailing whitespace in files will
190 # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
191 # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
191 # but we still support 2.6). See issue 1027.
192 # but we still support 2.6). See issue 1027.
192 scripttext = __builtin__.open(fname).read().rstrip() + '\n'
193 scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
193 # compile converts unicode filename to str assuming
194 # compile converts unicode filename to str assuming
194 # ascii. Let's do the conversion before calling compile
195 # ascii. Let's do the conversion before calling compile
195 if isinstance(fname, unicode):
196 if isinstance(fname, unicode):
196 filename = unicode_to_str(fname)
197 filename = unicode_to_str(fname)
197 else:
198 else:
198 filename = fname
199 filename = fname
199 exec(compile(scripttext, filename, 'exec'), glob, loc)
200 exec(compile(scripttext, filename, 'exec'), glob, loc)
200 else:
201 else:
201 def execfile(fname, *where):
202 def execfile(fname, *where):
202 if isinstance(fname, unicode):
203 if isinstance(fname, unicode):
203 filename = fname.encode(sys.getfilesystemencoding())
204 filename = fname.encode(sys.getfilesystemencoding())
204 else:
205 else:
205 filename = fname
206 filename = fname
206 __builtin__.execfile(filename, *where)
207 builtin_mod.execfile(filename, *where)
General Comments 0
You need to be logged in to leave comments. Login now