Improvements to docs formatting.

IPython/core/completer.py

@@ -1,979 +1,979 b''
 """Word completion for IPython.
 
 This module is a fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3, but we need a lot more
 functionality specific to IPython, so this module will continue to live as an
 IPython-specific utility.
 
 Original rlcompleter documentation:
 
 This requires the latest extension to the readline module (the
 completes keywords, built-ins and globals in __main__; when completing
 NAME.NAME..., it evaluates (!) the expression up to the last dot and
 completes its attributes.
 
 It's very cool to do "import string" type "string.", hit the
 completion key (twice), and see the list of names defined by the
 string module!
 
 Tip: to use the tab key as the completion key, call
 
     readline.parse_and_bind("tab: complete")
 
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
 generally cause the completion to fail).  This is a feature -- since
 readline sets the tty device in raw (or cbreak) mode, printing a
 traceback wouldn't work well without some complicated hoopla to save,
 reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
 application defined code to be executed if an object with a
-__getattr__ hook is found.  Since it is the responsibility of the
+  ``__getattr__`` hook is found.  Since it is the responsibility of the
 application (or the user) to enable this feature, I consider this an
 acceptable risk.  More complicated expressions (e.g. function calls or
 indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
 raw_input(), and thus these also benefit/suffer from the completer
 features.  Clearly an interactive application can benefit by
 specifying its own completer function and using raw_input() for all
 its input.
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
 """
 
 #*****************************************************************************
 #
 # Since this file is essentially a minimally modified copy of the rlcompleter
 # module which is part of the standard Python distribution, I assume that the
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
 #       Copyright (C) 2008 IPython Development Team
 #       Copyright (C) 2001 Fernando Perez. <[e-mail]>
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import __main__
 import glob
 import inspect
 import itertools
 import keyword
 import os
 import re
 import sys
 
 from IPython.config.configurable import Configurable
 from IPython.core.error import TryNext
 from IPython.core.inputsplitter import ESC_MAGIC
 from IPython.utils import generics
 from IPython.utils import io
 from IPython.utils.dir2 import dir2
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import CBool, Enum
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Public API
 __all__ = ['Completer','IPCompleter']
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 #-----------------------------------------------------------------------------
 # Main functions and classes
 #-----------------------------------------------------------------------------
 
 def has_open_quotes(s):
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
     the string is odd.
 
     Returns
     -------
     If there is an open quote, the quote character is returned.  Else, return
     False.
     """
     # We check " first, then ', so complex cases with nested quotes will get
     # the " to take precedence.
     if s.count('"') % 2:
         return '"'
     elif s.count("'") % 2:
         return "'"
     else:
         return False
 
 
 def protect_filename(s):
     """Escape a string to protect certain characters."""
 
     return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
                     for ch in s])
 
 def expand_user(path):
     """Expand '~'-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
     computing completions, and you wish to return the completions with the
     original '~' instead of its expanded value.
 
     Parameters
     ----------
     path : str
       String to be expanded.  If no ~ is present, the output is the same as the
       input.
 
     Returns
     -------
     newpath : str
       Result of ~ expansion in the input path.
     tilde_expand : bool
       Whether any expansion was performed or not.
     tilde_val : str
       The value that ~ was replaced with.
     """
     # Default values
     tilde_expand = False
     tilde_val = ''
     newpath = path
 
     if path.startswith('~'):
         tilde_expand = True
         rest = len(path)-1
         newpath = os.path.expanduser(path)
         if rest:
             tilde_val = newpath[:-rest]
         else:
             tilde_val = newpath
 
     return newpath, tilde_expand, tilde_val
 
 
 def compress_user(path, tilde_expand, tilde_val):
     """Does the opposite of expand_user, with its outputs.
     """
     if tilde_expand:
         return path.replace(tilde_val, '~')
     else:
         return path
 
 
 class Bunch(object): pass
 
 
 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
 GREEDY_DELIMS = ' =\r\n'
 
 
 class CompletionSplitter(object):
     """An object to split an input line in a manner similar to readline.
 
     By having our own implementation, we can expose readline-like completion in
     a uniform manner to all frontends.  This object only needs to be given the
     line of text to be split and the cursor position on said line, and it
     returns the 'word' to be completed on at the cursor after splitting the
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
     setting the `delims` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
 
     # A string of delimiter characters.  The default value makes sense for
     # IPython's most typical usage patterns.
     _delims = DELIMS
 
     # The expression (a normal string) to be compiled into a regular expression
     # for actual splitting.  We store it as an attribute mostly for ease of
     # debugging, since this type of code can be so tricky to debug.
     _delim_expr = None
 
     # The regular expression that does the actual splitting
     _delim_re = None
 
     def __init__(self, delims=None):
         delims = CompletionSplitter._delims if delims is None else delims
         self.delims = delims
 
     @property
     def delims(self):
         """Return the string of delimiter characters."""
         return self._delims
 
     @delims.setter
     def delims(self, delims):
         """Set the delimiters for line splitting."""
         expr = '[' + ''.join('\\'+ c for c in delims) + ']'
         self._delim_re = re.compile(expr)
         self._delims = delims
         self._delim_expr = expr
 
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
         l = line if cursor_pos is None else line[:cursor_pos]
         return self._delim_re.split(l)[-1]
 
 
 class Completer(Configurable):
 
     greedy = CBool(False, config=True,
         help="""Activate greedy completion
 
         This will enable completion on elements of lists, results of function calls, etc.,
         but can be unsafe because the code is actually evaluated on TAB.
         """
     )
     
 
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
         Completer(namespace=ns,global_namespace=ns2) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
 
         Completer instances should be used as the completion mechanism of
         readline via the set_completer() call:
 
         readline.set_completer(Completer(my_namespace).complete)
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = 1
         else:
             self.use_main_ns = 0
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
         super(Completer, self).__init__(**kwargs)
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         #print 'Completer->global_matches, txt=%r' % text # dbg
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [keyword.kwlist,
                     __builtin__.__dict__.keys(),
                     self.namespace.keys(),
                     self.global_namespace.keys()]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
 
         #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
     
         if m:
             expr, attr = m.group(1, 3)
         elif self.greedy:
             m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
             if not m2:
                 return []
             expr, attr = m2.group(1,2)
         else:
             return []
     
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else: 
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
         except Exception:
             # Silence errors from completion function
             #raise # dbg
             pass
         # Build match list to return
         n = len(attr)
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
     except:
         return []
     
     return [w for w in words if isinstance(w, basestring)]
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     def _greedy_changed(self, name, old, new):
         """update the splitter and readline delims when greedy is changed"""
         if new:
             self.splitter.delims = GREEDY_DELIMS
         else:
             self.splitter.delims = DELIMS
 
         if self.readline:
             self.readline.set_completer_delims(self.splitter.delims)
     
     merge_completions = CBool(True, config=True,
         help="""Whether to merge completion results into a single list
         
         If False, only the completion results from the first non-empty
         completer will be returned.
         """
     )
     omit__names = Enum((0,1,2), default_value=2, config=True,
         help="""Instruct the completer to omit private method names
         
         Specifically, when completing on ``object.<tab>``.
         
         When 2 [default]: all names that start with '_' will be excluded.
         
         When 1: all 'magic' names (``__foo__``) will be excluded.
         
         When 0: nothing will be excluded.
         """
     )
     limit_to__all__ = CBool(default_value=False, config=True,
         help="""Instruct the completer to use __all__ for the completion
         
         Specifically, when completing on ``object.<tab>``.
         
         When True: only those names in obj.__all__ will be included.
         
         When False [default]: the __all__ attribute is ignored 
         """
     )
 
     def __init__(self, shell=None, namespace=None, global_namespace=None,
                  alias_table=None, use_readline=True,
                  config=None, **kwargs):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
         via readline.set_completer().
 
         Inputs:
 
         - shell: a pointer to the ipython shell itself.  This is needed
         because this completer knows about magic functions, and those can
         only be accessed via the ipython instance.
 
         - namespace: an optional dict where completions are performed.
 
         - global_namespace: secondary optional dict for completions, to
         handle cases (such as IPython embedded inside functions) where
         both Python scopes are visible.
 
         - If alias_table is supplied, it should be a dictionary of aliases
         to complete.
 
         use_readline : bool, optional
           If true, use the readline library.  This completer can still function
           without readline, though in that case callers must provide some extra
           information on each call about the current line."""
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
         # Readline configuration, only used by the rlcompleter method.
         if use_readline:
             # We store the right version of readline so that later code
             import IPython.utils.rlineimpl as readline
             self.readline = readline
         else:
             self.readline = None
 
         # _greedy_changed() depends on splitter and readline being defined:
         Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
                             config=config, **kwargs)
 
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
 
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
 
         #regexp to parse docstring for function signature
         self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
         #use this if positional argument name is also needed
         #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
 
         # All active matcher routines for completion
         self.matchers = [self.python_matches,
                          self.file_matches,
                          self.magic_matches,
                          self.alias_matches,
                          self.python_func_kw_matches,
                          ]
 
     def all_completions(self, text):
         """
         Wrapper around the complete method for the benefit of emacs
         and pydb.
         """
         return self.complete(text)[1]
 
     def _clean_glob(self,text):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self,text):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
     def file_matches(self, text):
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         #io.rprint('Completer->file_matches: <%r>' % text) # dbg
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
 
         text_until_cursor = self.text_until_cursor
         # track strings with open quotes
         open_quotes = has_open_quotes(text_until_cursor)
 
         if '(' in text_until_cursor or '[' in text_until_cursor:
             lsplit = text
         else:
             try:
                 # arg_split ~ shlex.split, but with unicode bugs fixed by us
                 lsplit = arg_split(text_until_cursor)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if open_quotes:
                     lsplit = text_until_cursor.split(open_quotes)[-1]
                 else:
                     return []
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
         if not open_quotes and lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped name
             has_protectables = True
             text0,text = text,lsplit
         else:
             has_protectables = False
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         # Compute the matches from the filesystem
         m0 = self.clean_glob(text.replace('\\',''))
 
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 +
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names at all (and we _shouldn't_, as it
                 # would cause bugs when the filesystem call is made).
                 matches = m0
             else:
                 matches = [text_prefix +
                            protect_filename(f) for f in m0]
 
         #io.rprint('mm', matches)  # dbg
 
         # Mark directories in input list by appending '/' to their names.
         matches = [x+'/' if os.path.isdir(x) else x for x in matches]
         return matches
 
     def magic_matches(self, text):
         """Match magics"""
         #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
         line_magics = lsm['line']
         cell_magics = lsm['cell']
         pre = self.magic_escape
         pre2 = pre+pre
         
         # Completion logic:
         # - user gives %%: only do cell magics
         # - user gives %: do both line and cell magics
         # - no prefix: do both
         # In other words, line magics are skipped if the user gives %% explicitly
         bare_text = text.lstrip(pre)
         comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]
         if not text.startswith(pre2):
             comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
         return comp
 
     def alias_matches(self, text):
         """Match internal system aliases"""
         #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg
 
         # if we are not in the first 'item', alias matching
         # doesn't make sense - unless we are starting with 'sudo' command.
         main_text = self.text_until_cursor.lstrip()
         if ' ' in main_text and not main_text.startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
         if text == '':
             return aliases
         else:
             return [a for a in aliases if a.startswith(text)]
 
     def python_matches(self,text):
         """Match attributes or global python names"""
         
         #io.rprint('Completer->python_matches, txt=%r' % text) # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\._.*?',txt) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
 
         return matches
 
     def _default_arguments_from_docstring(self, doc):
         """Parse the first line of docstring for call signature.
 
         Docstring should be of the form 'min(iterable[, key=func])\n'.
         It can also parse cython docstring of the form
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
         """
         if doc is None:
             return []
 
         #care only the firstline
         line = doc.lstrip().splitlines()[0]
 
         #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
         sig = self.docstring_sig_re.search(line)
         if sig is None:
             return []
         # iterable[, key=func]' -> ['iterable[' ,' key=func]']
         sig = sig.groups()[0].split(',')
         ret = []
         for s in sig:
             #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
             ret += self.docstring_kwd_re.findall(s)
         return ret
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
         call_obj = obj
         ret = []
         if inspect.isbuiltin(obj):
             pass
         elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             if inspect.isclass(obj):
                 #for cython embededsignature=True the constructor docstring
                 #belongs to the object itself not __init__
                 ret += self._default_arguments_from_docstring(
                             getattr(obj, '__doc__', ''))
                 # for classes, check for __init__,__new__
                 call_obj = (getattr(obj, '__init__', None) or
                        getattr(obj, '__new__', None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 call_obj = obj.__call__
 
         ret += self._default_arguments_from_docstring(
                  getattr(call_obj, '__doc__', ''))
 
         try:
             args,_,_1,defaults = inspect.getargspec(call_obj)
             if defaults:
                 ret+=args[-len(defaults):]
         except TypeError:
             pass
 
         return list(set(ret))
 
     def python_func_kw_matches(self,text):
         """Match named parameters (kwargs) of the last open function"""
         
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?(?<!\\)' |    # single quoted strings or
                 ".*?(?<!\\)" |    # double quoted strings or
                 \w+          |    # identifier
                 \S                # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
         tokens.reverse()
         iterTokens = iter(tokens); openPar = 0
 
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
 
         while True:
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
                 break
         # lookup the candidate callable matches either using global_matches
         # or attr_matches for dotted names
         if len(ids) == 1:
             callableMatches = self.global_matches(ids[0])
         else:
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
             try:
                 namedArgs = self._default_arguments(eval(callableMatch,
                                                         self.namespace))
             except:
                 continue
 
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         return argMatches
 
     def dispatch_custom_completer(self, text):
         #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
         line = self.line_buffer
         if not line.strip():
             return None
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
         event = Bunch()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         event.text_until_cursor = self.text_until_cursor
 
         #print "\ncustom:{%s]\n" % event # dbg
 
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
                 self.magic_escape + cmd)
         else:
             try_magic = []
 
         for c in itertools.chain(self.custom_completers.s_matches(cmd),
                  try_magic,
                  self.custom_completers.flat_matches(self.text_until_cursor)):
             #print "try",c # dbg
             try:
                 res = c(event)
                 if res:
                     # first, try case sensitive match
                     withcase = [r for r in res if r.startswith(text)]
                     if withcase:
                         return withcase
                     # if none, then case insensitive ones are ok too
                     text_low = text.lower()
                     return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
 
         return None
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None):
         """Find completions for the given text and line context.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         Note that both the text and the line_buffer are optional, but at least
         one of them must be given.
 
         Parameters
         ----------
           text : string, optional
             Text to perform the completion on.  If not given, the line buffer
             is split using the instance's CompletionSplitter object.
 
           line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
             buffer via readline.  This keyword allows clients which are
             requesting for text completions in non-readline contexts to inform
             the completer of the entire text.
 
           cursor_pos : int, optional
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
 
         Returns
         -------
         text : str
           Text that was actually used in the completion.
 
         matches : list
           A list of completion matches.
         """
         #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
 
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
             cursor_pos = len(line_buffer) if text is None else len(text)
 
         # if text is either None or an empty string, rely on the line buffer
         if not text:
             text = self.splitter.split_line(line_buffer, cursor_pos)
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
         #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
 
         # Start with a clean slate of completions
         self.matches[:] = []
         custom_res = self.dispatch_custom_completer(text)
         if custom_res is not None:
             # did custom completers produce something?
             self.matches = custom_res
         else:
             # Extend the list of completions with the results of each
             # matcher, so we return results to the user from all
             # namespaces.
             if self.merge_completions:
                 self.matches = []
                 for matcher in self.matchers:
                     try:
                         self.matches.extend(matcher(text))
                     except:
                         # Show the ugly traceback if the matcher causes an
                         # exception, but do NOT crash the kernel!
                         sys.excepthook(*sys.exc_info())
             else:
                 for matcher in self.matchers:
                     self.matches = matcher(text)
                     if self.matches:
                         break
         # FIXME: we should extend our api to return a dict with completions for
         # different types of objects.  The rlcomplete() method could then
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other evironments.
         self.matches = sorted(set(self.matches))
         #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
         return text, self.matches
 
     def rlcomplete(self, text, state):
         """Return the state-th possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         Parameters
         ----------
           text : string
             Text to perform the completion on.
 
           state : int
             Counter used by readline.
         """
         if state==0:
 
             self.line_buffer = line_buffer = self.readline.get_line_buffer()
             cursor_pos = self.readline.get_endidx()
 
             #io.rprint("\nRLCOMPLETE: %r %r %r" %
             #          (text, line_buffer, cursor_pos) ) # dbg
 
             # if there is only a tab on a line with only whitespace, instead of
             # the mostly useless 'do you want to see all million completions'
             # message, just do the right thing and give the user his tab!
             # Incidentally, this enables pasting of tabbed text from an editor
             # (as long as autoindent is off).
 
             # It should be noted that at least pyreadline still shows file
             # completions - is there a way around it?
 
             # don't apply this on 'dumb' terminals, such as emacs buffers, so
             # we don't interfere with their own tab-completion mechanism.
             if not (self.dumb_terminal or line_buffer.strip()):
                 self.readline.insert_text('\t')
                 sys.stdout.flush()
                 return None
 
             # Note: debugging exceptions that may occur in completion is very
             # tricky, because readline unconditionally silences them.  So if
             # during development you suspect a bug in the completion code, turn
             # this flag on temporarily by uncommenting the second form (don't
             # flip the value in the first line, as the '# dbg' marker can be
             # automatically detected and is used elsewhere).
             DEBUG = False
             #DEBUG = True # dbg
             if DEBUG:
                 try:
                     self.complete(text, line_buffer, cursor_pos)
                 except:
                     import traceback; traceback.print_exc()
             else:
                 # The normal production version is here
 
                 # This method computes the self.matches array
                 self.complete(text, line_buffer, cursor_pos)
 
         try:
             return self.matches[state]
         except IndexError:
             return None

IPython/core/debugger.py

@@ -1,579 +1,584 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 http://www.python.org/2.2.3/license.html"""
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import bdb
 import functools
 import linecache
 import sys
 
 from IPython import get_ipython
 from IPython.utils import PyColorize, ulinecache
 from IPython.utils import coloransi, io, py3compat
 from IPython.core.excolors import exception_colors
 from IPython.testing.skipdoctest import skip_doctest
 
 # See if we can use pydb.
 has_pydb = False
 prompt = 'ipdb> '
 #We have to check this directly from sys.argv, config struct not yet available
 if '--pydb' in sys.argv:
     try:
         import pydb
         if hasattr(pydb.pydb, "runl") and pydb.version>'1.17':
             # Version 1.17 is broken, and that's what ships with Ubuntu Edgy, so we
             # better protect against it.
             has_pydb = True
     except ImportError:
         print("Pydb (http://bashdb.sourceforge.net/pydb/) does not seem to be available")
 
 if has_pydb:
     from pydb import Pdb as OldPdb
     #print "Using pydb for %run -d and post-mortem" #dbg
     prompt = 'ipydb> '
 else:
     from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     @skip_doctest
     def __init__(self,colors=None):
         """Create a local debugger instance.
 
-        :Parameters:
+        Parameters
+        ----------
 
-          - `colors` (None): a string containing the name of the color scheme to
-        use, it must be one of IPython's valid color schemes.  If not given, the
-        function will default to the current IPython scheme when running inside
-        IPython, and to 'NoColor' otherwise.
+        colors : str, optional
+            The name of the color scheme to use, it must be one of IPython's
+            valid color schemes.  If not given, the function will default to
+            the current IPython scheme when running inside IPython, and to
+            'NoColor' otherwise.
 
-        Usage example:
+        Examples
+        --------
+        ::
 
         from IPython.core.debugger import Tracer; debug_here = Tracer()
 
-        ... later in your code
+        Later in your code::
+        
         debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
 
         ip = get_ipython()
         if ip is None:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
             try:
                 # Limited tab completion support
                 import readline
                 readline.parse_and_bind('tab: complete')
             except ImportError:
                 pass
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             from repr import aRepr
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
     def __init__(self,color_scheme='NoColor',completekey=None,
                  stdin=None, stdout=None):
 
         # Parent constructor:
         if has_pydb and completekey is None:
             OldPdb.__init__(self,stdin=stdin,stdout=io.stdout)
         else:
             OldPdb.__init__(self,completekey,stdin,stdout)
 
         self.prompt = prompt # The default prompt is '(Pdb)'
 
         # IPython changes...
         self.is_pydb = has_pydb
 
         self.shell = get_ipython()
 
         if self.shell is None:
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
 
         if self.is_pydb:
 
             # interactiveshell.py's ipalias seems to want pdb's checkline
             # which located in pydb.fn
             import pydb.fns
             self.checkline = lambda filename, lineno: \
                              pydb.fns.checkline(self, filename, lineno)
 
             self.curframe = None
             self.do_restart = self.new_do_restart
 
             self.old_all_completions = self.shell.Completer.all_completions
             self.shell.Completer.all_completions=self.all_completions
 
             self.do_list = decorate_fn_with_doc(self.list_command_pydb,
                                                 OldPdb.do_list)
             self.do_l     = self.do_list
             self.do_frame = decorate_fn_with_doc(self.new_do_frame,
                                                  OldPdb.do_frame)
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         self.set_colors(color_scheme)
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser()
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
 
     def interaction(self, frame, traceback):
         self.shell.set_completer_frame(frame)
         while True:
             try:
                 OldPdb.interaction(self, frame, traceback)
             except KeyboardInterrupt:
                 self.shell.write("\nKeyboardInterrupt\n")
             else:
                 break
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
         self.shell.set_completer_frame(self.curframe)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def postloop(self):
         self.shell.set_completer_frame(None)
 
     def print_stack_trace(self):
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context = 5)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno,prompt_prefix='\n-> ',
                           context = 3):
         #frame, lineno = frame_lineno
         print(self.format_stack_entry(frame_lineno, '', context), file=io.stdout)
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
         import repr
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += repr.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = repr.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = ulinecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
         scheme = self.color_scheme_table.active_scheme_name
         new_line, err = self.parser.format2(line, 'str', scheme)
         if not err: line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                  marker = '> '
             elif pad == 1:
                  marker = '>'
             else:
                  marker = ''
             num = '%s%s' % (marker, str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
 
         return line
 
     def list_command_pydb(self, arg):
         """List command to use if we have a newer pydb installed"""
         filename, first, last = OldPdb.parse_list_cmd(self, arg)
         if filename is not None:
             self.print_list_lines(filename, first, last)
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = ulinecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src), file=io.stdout)
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
 
     def checkline(self, filename, lineno):
         """Check whether specified line seems to be executable.
 
         Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
         line or EOF). Warning: testing is not comprehensive.
         """
         #######################################################################
         # XXX Hack!  Use python-2.5 compatible code for this call, because with
         # all of our changes, we've drifted from the pdb api in 2.6.  For now,
         # changing:
         #
         #line = linecache.getline(filename, lineno, self.curframe.f_globals)
         # to:
         #
         line = linecache.getline(filename, lineno)
         #
         # does the trick.  But in reality, we need to fix this by reconciling
         # our updates with the new Pdb APIs in Python 2.6.
         #
         # End hack.  The rest of this method is copied verbatim from 2.6 pdb.py
         #######################################################################
 
         if not line:
             print('End of file', file=self.stdout)
             return 0
         line = line.strip()
         # Don't allow setting breakpoint at a blank line
         if (not line or (line[0] == '#') or
              (line[:3] == '"""') or line[:3] == "'''"):
             print('*** Blank or comment', file=self.stdout)
             return 0
         return lineno

IPython/core/interactiveshell.py

@@ -1,3150 +1,3151 b''
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 from __future__ import print_function
 
 import __builtin__ as builtin_mod
 import __future__
 import abc
 import ast
 import atexit
 import functools
 import os
 import re
 import runpy
 import sys
 import tempfile
 import types
 from io import open as io_open
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.core import debugger, oinspect
 from IPython.core import magic
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import shadowns
 from IPython.core import ultratb
 from IPython.core.alias import AliasManager, AliasError
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.profiledir import ProfileDir
 from IPython.core.prompts import PromptManager
 from IPython.lib.latextools import LaTeXTool
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils import openpy
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
 from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import (format_screen, LSString, SList,
                                 DollarFormatter)
 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error
 import IPython.core.hooks
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 @undoc
 def no_op(*a, **kw): pass
 
 @undoc
 class NoOpContext(object):
     def __enter__(self): pass
     def __exit__(self, type, value, traceback): pass
 no_op_context = NoOpContext()
 
 class SpaceInInput(Exception): pass
 
 @undoc
 class Bunch: pass
 
 
 def get_default_colors():
     if sys.platform=='darwin':
         return "LightBG"
     elif os.name=='nt':
         return 'Linux'
     else:
         return 'Linux'
 
 
 class SeparateUnicode(Unicode):
     """A Unicode subclass to validate separate_in, separate_out, etc.
 
     This is a Unicode based trait that converts '0'->'' and '\\n'->'\n'.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateUnicode, self).validate(obj, value)
 
 
 class ReadlineNoRecord(object):
     """Context manager to execute some code, then reload readline history
     so that interactive input to the code doesn't appear when pressing up."""
     def __init__(self, shell):
         self.shell = shell
         self._nested_level = 0
 
     def __enter__(self):
         if self._nested_level == 0:
             try:
                 self.orig_length = self.current_length()
                 self.readline_tail = self.get_readline_tail()
             except (AttributeError, IndexError):   # Can fail with pyreadline
                 self.orig_length, self.readline_tail = 999999, []
         self._nested_level += 1
 
     def __exit__(self, type, value, traceback):
         self._nested_level -= 1
         if self._nested_level == 0:
             # Try clipping the end if it's got longer
             try:
                 e = self.current_length() - self.orig_length
                 if e > 0:
                     for _ in range(e):
                         self.shell.readline.remove_history_item(self.orig_length)
 
                 # If it still doesn't match, just reload readline history.
                 if self.current_length() != self.orig_length \
                     or self.get_readline_tail() != self.readline_tail:
                     self.shell.refill_readline_hist()
             except (AttributeError, IndexError):
                 pass
         # Returning False will cause exceptions to propagate
         return False
 
     def current_length(self):
         return self.shell.readline.get_current_history_length()
 
     def get_readline_tail(self, n=10):
         """Get the last n items in readline history."""
         end = self.shell.readline.get_current_history_length() + 1
         start = max(end-n, 1)
         ghi = self.shell.readline.get_history_item
         return [ghi(x) for x in range(start, end)]
 
 
 @undoc
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
     pass
 
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
     
     ast_transformers = List([], config=True, help=
         """
         A list of ast.NodeTransformer subclass instances, which will be applied
         to user input before code is run.
         """
     )
 
     autocall = Enum((0,1,2), default_value=0, config=True, help=
         """
         Make IPython automatically call any callable object even if you didn't
         type explicit parentheses. For example, 'str 43' becomes 'str(43)'
         automatically. The value can be '0' to disable the feature, '1' for
         'smart' autocall, where it is not applied if there are no more
         arguments on the line, and '2' for 'full' autocall, where all callable
         objects are automatically called (even if no arguments are present).
         """
     )
     # TODO: remove all autoindent logic and put into frontends.
     # We can't do this yet because even runlines uses the autoindent.
     autoindent = CBool(True, config=True, help=
         """
         Autoindent IPython code entered interactively.
         """
     )
     automagic = CBool(True, config=True, help=
         """
         Enable magic commands to be called without the leading %.
         """
     )
     cache_size = Integer(1000, config=True, help=
         """
         Set the size of the output cache.  The default is 1000, you can
         change it permanently in your config file.  Setting it to 0 completely
         disables the caching system, and the minimum value accepted is 20 (if
         you provide a value less than 20, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
         time re-flushing a too small cache than working
         """
     )
     color_info = CBool(True, config=True, help=
         """
         Use colors for displaying information about objects. Because this
         information is passed through a pager (like 'less'), and some pagers
         get confused with color codes, this capability can be turned off.
         """
     )
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
                              default_value=get_default_colors(), config=True,
         help="Set the color scheme (NoColor, Linux, or LightBG)."
     )
     colors_force = CBool(False, help=
         """
         Force use of ANSI color codes, regardless of OS and readline
         availability.
         """
         # FIXME: This is essentially a hack to allow ZMQShell to show colors
         # without readline on Win32. When the ZMQ formatting system is
         # refactored, this should be removed.
     )
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True, help=
         """
         Enable deep (recursive) reloading by default. IPython can use the
         deep_reload module which reloads changes in modules recursively (it
         replaces the reload() function, so you don't need to change anything to
         use it). deep_reload() forces a full reload of modules whose code may
         have changed, which the default reload() function does not.  When
         deep_reload is off, IPython will use the normal reload(), but
         deep_reload will still be available as dreload().
         """
     )
     disable_failing_post_execute = CBool(False, config=True,
         help="Don't call post-execute functions that have failed in the past."
     )
     display_formatter = Instance(DisplayFormatter)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     data_pub_class = None
 
     exit_now = CBool(False)
     exiter = Instance(ExitAutocall)
     def _exiter_default(self):
         return ExitAutocall(self)
     # Monotonically increasing execution counter
     execution_count = Integer(1)
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
 
     # Input splitter, to transform input line by line and detect when a block
     # is ready to be executed.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {'line_input_checker': True})
     
     # This InputSplitter instance is used to transform completed cells before
     # running them. It allows cell magics to contain blank lines.
     input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                                          (), {'line_input_checker': False})
     
     logstart = CBool(False, config=True, help=
         """
         Start logging to the default log file.
         """
     )
     logfile = Unicode('', config=True, help=
         """
         The name of the logfile to use.
         """
     )
     logappend = Unicode('', config=True, help=
         """
         Start logging to the given file in append mode.
         """
     )
     object_info_string_level = Enum((0,1,2), default_value=0,
                                     config=True)
     pdb = CBool(False, config=True, help=
         """
         Automatically call the pdb debugger after every exception.
         """
     )
     multiline_history = CBool(sys.platform != 'win32', config=True,
         help="Save multi-line entries as one entry in readline history"
     )
 
     # deprecated prompt traits:
     
     prompt_in1 = Unicode('In [\\#]: ', config=True,
         help="Deprecated, use PromptManager.in_template")
     prompt_in2 = Unicode('   .\\D.: ', config=True,
         help="Deprecated, use PromptManager.in2_template")
     prompt_out = Unicode('Out[\\#]: ', config=True,
         help="Deprecated, use PromptManager.out_template")
     prompts_pad_left = CBool(True, config=True,
         help="Deprecated, use PromptManager.justify")
     
     def _prompt_trait_changed(self, name, old, new):
         table = {
             'prompt_in1' : 'in_template',
             'prompt_in2' : 'in2_template',
             'prompt_out' : 'out_template',
             'prompts_pad_left' : 'justify',
         }
         warn("InteractiveShell.{name} is deprecated, use PromptManager.{newname}".format(
                 name=name, newname=table[name])
         )
         # protect against weird cases where self.config may not exist:
         if self.config is not None:
             # propagate to corresponding PromptManager trait
             setattr(self.config.PromptManager, table[name], new)
     
     _prompt_in1_changed = _prompt_trait_changed
     _prompt_in2_changed = _prompt_trait_changed
     _prompt_out_changed = _prompt_trait_changed
     _prompt_pad_left_changed = _prompt_trait_changed
     
     show_rewritten_input = CBool(True, config=True,
         help="Show rewritten input, e.g. for autocall."
     )
     
     quiet = CBool(False, config=True)
 
     history_length = Integer(10000, config=True)
 
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
     readline_remove_delims = Unicode('-/~', config=True)
     readline_delims = Unicode() # set by init_readline()
     # don't use \M- bindings by default, because they
     # conflict with 8-bit encodings. See gh-58,gh-88
     readline_parse_and_bind = List([
             'tab: complete',
             '"\C-l": clear-screen',
             'set show-all-if-ambiguous on',
             '"\C-o": tab-insert',
             '"\C-r": reverse-search-history',
             '"\C-s": forward-search-history',
             '"\C-p": history-search-backward',
             '"\C-n": history-search-forward',
             '"\e[A": history-search-backward',
             '"\e[B": history-search-forward',
             '"\C-k": kill-line',
             '"\C-u": unix-line-discard',
         ], allow_none=False, config=True)
 
     ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
                                   default_value='last_expr', config=True, 
                                   help="""
         'all', 'last', 'last_expr' or 'none', specifying which nodes should be
         run interactively (displaying output from expressions).""")
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateUnicode('\n', config=True)
     separate_out = SeparateUnicode('', config=True)
     separate_out2 = SeparateUnicode('', config=True)
     wildcards_case_sensitive = CBool(True, config=True)
     xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
                             default_value='Context', config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
     display_trap = Instance('IPython.core.display_trap.DisplayTrap')
     extension_manager = Instance('IPython.core.extensions.ExtensionManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
     history_manager = Instance('IPython.core.history.HistoryManager')
     magics_manager = Instance('IPython.core.magic.MagicsManager')
 
     profile_dir = Instance('IPython.core.application.ProfileDir')
     @property
     def profile(self):
         if self.profile_dir is not None:
             name = os.path.basename(self.profile_dir.location)
             return name.replace('profile_','')
 
 
     # Private interface
     _post_execute = Instance(dict)
 
     # Tracks any GUI loop loaded for pylab
     pylab_gui_select = None
 
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
         self.configurables = [self]
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
         
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
         self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_pushd_popd_magic()
         # self.init_traceback_handlers use to be here, but we moved it below
         # because it and init_io have to come after init_readline.
         self.init_user_ns()
         self.init_logger()
         self.init_alias()
         self.init_builtins()
 
         # The following was in post_config_initialization
         self.init_inspector()
         # init_readline() must come before init_io(), because init_io uses
         # readline related things.
         self.init_readline()
         # We save this here in case user code replaces raw_input, but it needs
         # to be after init_readline(), because PyPy's readline works by replacing
         # raw_input.
         if py3compat.PY3:
             self.raw_input_original = input
         else:
             self.raw_input_original = raw_input
         # init_completer must come after init_readline, because it needs to
         # know whether readline is present or not system-wide to configure the
         # completers, since the completion machinery can now operate
         # independently of readline (e.g. over the network)
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_display_formatter()
         self.init_display_pub()
         self.init_data_pub()
         self.init_displayhook()
         self.init_latextool()
         self.init_magics()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
         self.init_payload()
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
 
     def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0o777)
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if value != 0 and not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             return
 
         self.ipython_dir = get_ipython_dir()
 
     def init_profile_dir(self, profile_dir):
         if profile_dir is not None:
             self.profile_dir = profile_dir
             return
         self.profile_dir =\
             ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = CachingCompiler()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwdu()
 
         # Indentation management
         self.indent_current_nsp = 0
 
         # Dict to track post-execution functions that have been registered
         self._post_execute = {}
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
     def init_syntax_highlighting(self):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.colors)
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         self.home_dir = get_home_dir()
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                              logmode='rotate')
 
     def init_logstart(self):
         """Initialize logging in case it was requested at the command line.
         """
         if self.logappend:
             self.magic('logstart %s append' % self.logappend)
         elif self.logfile:
             self.magic('logstart %s' % self.logfile)
         elif self.logstart:
             self.magic('logstart')
 
     def init_builtins(self):
         # A single, static flag that we set to True.  Its presence indicates
         # that an IPython shell has been created, and we make no attempts at
         # removing on exit or representing the existence of more than one
         # IPython at a time.
         builtin_mod.__dict__['__IPYTHON__'] = True
 
         # In 0.11 we introduced '__IPYTHON__active' as an integer we'd try to
         # manage on enter/exit, but with all our shells it's virtually
         # impossible to get all the cases right.  We're leaving the name in for
         # those who adapted their codes to check for this flag, but will
         # eventually remove it after a few more releases.
         builtin_mod.__dict__['__IPYTHON__active'] = \
                                           'Deprecated, check for __IPYTHON__'
 
         self.builtin_trap = BuiltinTrap(shell=self)
 
     def init_inspector(self):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor',
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because io holds onto
         # references to the underlying streams.
         if (sys.platform == 'win32' or sys.platform == 'cli') and self.has_readline:
             io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
         else:
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_prompts(self):
         self.prompt_manager = PromptManager(shell=self, parent=self)
         self.configurables.append(self.prompt_manager)
         # Set system prompts, so that scripts can decide if they are running
         # interactively.
         sys.ps1 = 'In : '
         sys.ps2 = '...: '
         sys.ps3 = 'Out: '
 
     def init_display_formatter(self):
         self.display_formatter = DisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
         self.display_pub = self.display_pub_class(parent=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):
         if not self.data_pub_class:
             self.data_pub = None
             return
         self.data_pub = self.data_pub_class(parent=self)
         self.configurables.append(self.data_pub)
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             parent=self,
             shell=self,
             cache_size=self.cache_size,
         )
         self.configurables.append(self.displayhook)
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_latextool(self):
         """Configure LaTeXTool."""
         cfg = LaTeXTool.instance(parent=self)
         if cfg not in self.configurables:
             self.configurables.append(cfg)
 
     def init_virtualenv(self):
         """Add a virtualenv to sys.path so the user can import modules from it.
         This isn't perfect: it doesn't use the Python interpreter with which the
         virtualenv was built, and it ignores the --no-site-packages option. A
         warning will appear suggesting the user installs IPython in the
         virtualenv, but for many cases, it probably works well enough.
         
         Adapted from code snippets online.
         
         http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
         """
         if 'VIRTUAL_ENV' not in os.environ:
             # Not in a virtualenv
             return
         
         if sys.executable.startswith(os.environ['VIRTUAL_ENV']):
             # Running properly in the virtualenv, don't need to do anything
             return
         
         warn("Attempting to work in a virtualenv. If you encounter problems, please "
              "install IPython inside the virtualenv.")
         if sys.platform == "win32":
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages') 
         else:
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
                        'python%d.%d' % sys.version_info[:2], 'site-packages')
         
         import site
         sys.path.insert(0, virtual_env)
         site.addsitedir(virtual_env)
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {}
         self._orig_sys_module_state['stdin'] = sys.stdin
         self._orig_sys_module_state['stdout'] = sys.stdout
         self._orig_sys_module_state['stderr'] = sys.stderr
         self._orig_sys_module_state['excepthook'] = sys.excepthook
         self._orig_sys_modules_main_name = self.user_module.__name__
         self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
             for k, v in self._orig_sys_module_state.iteritems():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         if self._orig_sys_modules_main_mod is not None:
             sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100)
 
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
 
         f = types.MethodType(hook,self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
 
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     def register_post_execute(self, func):
         """Register a function for calling after code execution.
         """
         if not callable(func):
             raise ValueError('argument %s must be callable' % func)
         self._post_execute[func] = True
     
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self, filename):
         """Return a new 'main' module object for user code execution.
         
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
         
         When scripts are executed via %run, we must keep a reference to their
         __main__ module (a FakeModule instance) around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the script. This way, for multiple executions of the
         same script we only keep one copy of the namespace (the last one),
         thus preventing memory leaks from old references while allowing the
         objects from the last execution to be accessible.
         """
         filename = os.path.abspath(filename)
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
             main_mod = self._main_mod_cache[filename] = FakeModule()
         else:
             init_fakemod_dict(main_mod)
         
         return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: m = _ip.new_main_mod(IPython.__file__)
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
         self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError('new call_pdb value must be boolean')
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pydb/pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         # use pydb if available
         if debugger.has_pydb:
             from pydb import pm
         else:
             # fallback to our internal debugger
             pm = lambda : self.InteractiveTB.debugger(force=True)
 
         with self.readline_no_record:
             pm()
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
     default_user_namespaces = True
 
     def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return a properly built module and dict as needed by
         # the rest of the code, and can also be used by extension writers to
         # generate properly initialized namespaces.
         if (user_ns is not None) or (user_module is not None):
             self.default_user_namespaces = False
         self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
         self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so docetst and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         #
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         #
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user_global':self.user_module.__dict__,
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
     
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
 
     def prepare_user_module(self, user_module=None, user_ns=None):
         """Prepare the module and namespace in which user code will be run.
         
         When IPython is started normally, both parameters are None: a new module
         is created automatically, and its __dict__ used as the namespace.
         
         If only user_module is provided, its __dict__ is used as the namespace.
         If only user_ns is provided, a dummy module is created, and user_ns
         becomes the global namespace. If both are provided (as they may be
         when embedding), user_ns is the local namespace, and user_module
         provides the global namespace.
 
         Parameters
         ----------
         user_module : module, optional
             The current user module in which IPython is being run. If None,
             a clean module will be created.
         user_ns : dict, optional
             A namespace in which to run interactive commands.
 
         Returns
         -------
         A tuple of user_module and user_ns, each properly initialised.
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
             
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
         
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
         
         if user_ns is None:
             user_ns = user_module.__dict__
 
         return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
         main_name = self.user_module.__name__
         sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         therm.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so theye really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = dict()
         
         # Put 'help' in the user namespace
         try:
             from site import _Helper
             ns['help'] = _Helper()
         except ImportError:
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
         ns['_dh'] = self.history_manager.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.history_manager.input_hist_parsed
         ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
         
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
     
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
         IPython might store a user-created object.
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
         return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True):
         """Clear all internal namespaces, and attempt to release references to
         user objects.
 
         If new_session is True, a new history session will be opened.
         """
         # Clear histories
         self.history_manager.reset(new_session)
         # Reset counter used to index all histories
         if new_session:
             self.execution_count = 1
 
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         if self.user_ns is not self.user_global_ns:
             self.user_ns.clear()
         ns = self.user_global_ns
         drop_keys = set(ns.keys())
         drop_keys.discard('__builtin__')
         drop_keys.discard('__builtins__')
         drop_keys.discard('__name__')
         for k in drop_keys:
             del ns[k]
         
         self.user_ns_hidden.clear()
         
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
 
         # Restore the default and user aliases
         self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
         # Flush the private list of module references kept for script
         # execution protection
         self.clear_main_mod_cache()
 
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
 
         Parameters
         ----------
         varname : str
             The name of the variable to delete.
         by_name : bool
             If True, delete variables with the given name in each
             namespace. If False (default), find the variable in the user
             namespace, and delete references to it.
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
         
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
                     del ns[varname]
                 except KeyError:
                     pass
         else:                         # Delete by object
             try:
                 obj = self.user_ns[varname]
             except KeyError:
                 raise NameError("name '%s' is not defined" % varname)
             # Also check in output history
             ns_refs.append(self.history_manager.output_hist)
             for ns in ns_refs:
                 to_delete = [n for n, o in ns.iteritems() if o is obj]
                 for name in to_delete:
                     del ns[name]
 
             # displayhook keeps extra references, but not in a dictionary
             for name in ('_', '__', '___'):
                 if getattr(self.displayhook, name) is obj:
                     setattr(self.displayhook, name, None)
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
 
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (basestring, list, tuple)):
             if isinstance(variables, basestring):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
 
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
             for name in vdict:
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
     
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
         
         This is intended for use by extensions: variables that they've added can
         be taken back out if they are unloaded, without removing any that the
         user has overwritten.
         
         Parameters
         ----------
         variables : dict
           A dictionary mapping object names (as strings) to the objects.
         """
         for name, obj in variables.iteritems():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
                 self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         if not oname.startswith(ESC_MAGIC) and \
             not oname.startswith(ESC_MAGIC2) and \
             not py3compat.isidentifier(oname, dotted=True):
             return dict(found=False)
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ('Alias', self.alias_manager.alias_table),
                            ]
             alias_ns = self.alias_manager.alias_table
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
         ismagic = False; isalias = False; parent = None
 
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
         if (oname == 'print' and not py3compat.PY3 and not \
             (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = True
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             obj = None
             if oname.startswith(ESC_MAGIC2):
                 oname = oname.lstrip(ESC_MAGIC2)
                 obj = self.find_cell_magic(oname)
             elif oname.startswith(ESC_MAGIC):
                 oname = oname.lstrip(ESC_MAGIC)
                 obj = self.find_line_magic(oname)
             else:
                 # search without prefix, so run? will find %run?
                 obj = self.find_line_magic(oname)
                 if obj is None:
                     obj = self.find_cell_magic(oname)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
     def _ofind_property(self, oname, info):
         """Second part of object finding, to look for property details."""
         if info.found:
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__')
                     # The object belongs to a class instance.
                     try:
                         target = getattr(target, path[-1])
                         # The class defines the object.
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
 
         # We return either the new info or the unmodified input if the object
         # hadn't been found
         return info
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         inf = Struct(self._ofind(oname, namespaces))
         return Struct(self._ofind_property(oname, inf))
 
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
         info = self._object_find(oname, namespaces)
         if info.found:
             pmethod = getattr(self.inspector, meth)
             formatter = format_screen if info.ismagic else None
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj, oname, formatter, info, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
             print('Object `%s` not found.' % oname)
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname, detail_level=0):
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector.info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 return oinspect.object_info(name=oname, found=False)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
         self.history_manager = HistoryManager(shell=self, parent=self)
         self.configurables.append(self.history_manager)
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
 
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
                                    check_cache=check_linecache_ipython)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         run_code() method).
 
         Parameters
         ----------
 
         exc_tuple : tuple of exception classes
             A *tuple* of exception classes, for which to call the defined
             handler.  It is very important that you use a tuple, and NOT A
             LIST here, because of the way Python's except statement works.  If
             you only want to trap a single exception, use a singleton tuple::
 
                 exc_tuple == (MyCustomException,)
 
         handler : callable
             handler must have the following signature::
 
                 def my_handler(self, etype, value, tb, tb_offset=None):
                     ...
                     return structured_traceback
 
             Your handler must return a structured traceback (a list of strings),
             or None.
 
             This will be made into an instance method (via types.MethodType)
             of IPython itself, and it will be called if any of the exceptions
             listed in the exc_tuple are caught. If the handler is None, an
             internal basic one is used, which just prints basic info.
 
             To protect IPython from crashes, if your handler ever raises an
             exception or returns an invalid result, it will be immediately
             disabled.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb,tb_offset=None):
             print('*** Simple custom exception handler ***')
             print('Exception type :',etype)
             print('Exception value:',value)
             print('Traceback      :',tb)
             #print 'Source code    :','\n'.join(self.buffer)
         
         def validate_stb(stb):
             """validate structured traceback return type
             
             return type of CustomTB *should* be a list of strings, but allow
             single strings or None, which are harmless.
             
             This function will *always* return a list of strings,
             and will raise a TypeError if stb is inappropriate.
             """
             msg = "CustomTB must return list of strings, not %r" % stb
             if stb is None:
                 return []
             elif isinstance(stb, basestring):
                 return [stb]
             elif not isinstance(stb, list):
                 raise TypeError(msg)
             # it's a list
             for line in stb:
                 # check every element
                 if not isinstance(line, basestring):
                     raise TypeError(msg)
             return stb
 
         if handler is None:
             wrapped = dummy_handler
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
                 
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
                 try:
                     stb = handler(self,etype,value,tb,tb_offset=tb_offset)
                     return validate_stb(stb)
                 except:
                     # clear custom handler immediately
                     self.set_custom_exc((), None)
                     print("Custom TB Handler failed, unregistering", file=io.stderr)
                     # show the exception in handler first
                     stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
                     print(self.InteractiveTB.stb2text(stb), file=io.stdout)
                     print("The original exception:", file=io.stdout)
                     stb = self.InteractiveTB.structured_traceback(
                                             (etype,value,tb), tb_offset=tb_offset
                     )
                 return stb
 
         self.CustomTB = types.MethodType(wrapped,self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
         
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
         
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
             etype, value, tb = sys.exc_info()
         else:
             etype, value, tb = exc_tuple
 
         if etype is None:
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
         
         if etype is None:
             raise ValueError("No exception to find")
         
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
         # like pdb depend on their existence, so let's set them.  If we
         # find problems in the field, we'll need to revisit their use.
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
         
         return etype, value, tb
     
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
         
         These are special exceptions that shouldn't show a traceback.
         """
         self.write_err("UsageError: %s" % exc)
     
     def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
                       exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         try:
             try:
                 etype, value, tb = self._get_exc_info(exc_tuple)
             except ValueError:
                 self.write_err('No traceback available to show.\n')
                 return
             
             if issubclass(etype, SyntaxError):
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases with imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 self.show_usage_error(value)
             else:
                 if exception_only:
                     stb = ['An exception has occurred, use %tb to see '
                            'the full traceback.\n']
                     stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                      value))
                 else:
                     try:
                         # Exception classes can customise their traceback - we
                         # use this in IPython.parallel for exceptions occurring
                         # in the engines. This should return a list of strings.
                         stb = value._render_traceback_()
                     except Exception:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                             value, tb, tb_offset=tb_offset)
 
                     self._showtraceback(etype, value, stb)
                     if self.call_pdb:
                         # drop into debugger
                         self.debugger(force=True)
                     return
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
 
         except KeyboardInterrupt:
             self.write_err("\nKeyboardInterrupt\n")
 
     def _showtraceback(self, etype, evalue, stb):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         print(self.InteractiveTB.stb2text(stb), file=io.stdout)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = self._get_exc_info()
 
         if filename and issubclass(etype, SyntaxError):
             try:
                 value.filename = filename
             except:
                 # Not the format we expect; leave it alone
                 pass
         
         stb = self.SyntaxTB.structured_traceback(etype, value, [])
         self._showtraceback(etype, value, stb)
 
     # This is overridden in TerminalInteractiveShell to show a message about
     # the %paste magic.
     def showindentationerror(self):
         """Called by run_cell when there's an IndentationError in code entered
         at the prompt.
 
         This is overridden in TerminalInteractiveShell to show a message about
         the %paste magic."""
         self.showsyntaxerror()
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
         if self.readline_use:
             import IPython.utils.rlineimpl as readline
 
         self.rl_next_input = None
         self.rl_do_indent = False
 
         if not self.readline_use or not readline.have_readline:
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
             self.readline_no_record = no_op_context
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             if self.readline_use:
                 warn('Readline services not available or not loaded.')
         else:
             self.has_readline = True
             self.readline = readline
             sys.modules['readline'] = readline
 
             # Platform-specific configuration
             if os.name == 'nt':
                 # FIXME - check with Frederick to see if we can harmonize
                 # naming conventions with pyreadline to avoid this
                 # platform-dependent check
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             # Or if libedit is used, load editrc.
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 inputrc_name = '.inputrc'
                 if readline.uses_libedit:
                     inputrc_name = '.editrc'
                 inputrc_name = os.path.join(self.home_dir, inputrc_name)
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
 
             # Configure readline according to user's prefs
             # This is only done if GNU readline is being used.  If libedit
             # is being used (as on Leopard) the readline config is
             # not run as the syntax for libedit is different.
             if not readline.uses_libedit:
                 for rlcommand in self.readline_parse_and_bind:
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
             # Remove some chars from the delimiters list.  If we encounter
             # unicode chars, discard them.
             delims = readline.get_completer_delims()
             if not py3compat.PY3:
                 delims = delims.encode("ascii", "ignore")
             for d in self.readline_remove_delims:
                 delims = delims.replace(d, "")
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # Store these so we can restore them if something like rpy2 modifies
             # them.
             self.readline_delims = delims
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(self.history_length)
 
             self.refill_readline_hist()
             self.readline_no_record = ReadlineNoRecord(self)
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
     def refill_readline_hist(self):
         # Load the last 1000 lines from history
         self.readline.clear_history()
         stdin_encoding = sys.stdin.encoding or "utf-8"
         last_cell = u""
         for _, _, cell in self.history_manager.get_tail(1000,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 try:
                     if self.multiline_history:
                           self.readline.add_history(py3compat.unicode_to_str(cell,
                                                                     stdin_encoding))
                     else:
                         for line in cell.splitlines():
                             self.readline.add_history(py3compat.unicode_to_str(line,
                                                                     stdin_encoding))
                     last_cell = cell
                 
                 except TypeError:
                     # The history DB can get corrupted so it returns strings
                     # containing null bytes, which readline objects to.
                     continue
 
     @skip_doctest
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
 
         Requires readline.
 
         Example::
 
             In [1]: _ip.set_next_input("Hello Word")
             In [2]: Hello Word_  # cursor is here
         """
         self.rl_next_input = py3compat.cast_bytes_py2(s)
 
     # Maybe move this to the terminal subclass?
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         if self.rl_do_indent:
             self.readline.insert_text(self._indent_current_str())
         if self.rl_next_input is not None:
             self.readline.insert_text(self.rl_next_input)
             self.rl_next_input = None
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.input_splitter.indent_spaces * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programatically (such as in test suites) or out-of-prcess
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                 magic_run_completer, cd_completer, reset_completer)
 
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
                                      alias_table=self.alias_manager.alias_table,
                                      use_readline=self.has_readline,
                                      parent=self,
                                      )
         self.configurables.append(self.Completer)
 
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
         # Only configure readline if we truly are using readline.  IPython can
         # do tab-completion over the network, in GUIs, etc, where readline
         # itself may be absent
         if self.has_readline:
             self.set_readline_completer()
 
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = types.MethodType(completer,self.Completer)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_readline_completer(self):
         """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.rlcomplete)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         from IPython.core import magics as m
         self.magics_manager = magic.MagicsManager(shell=self,
                                    parent=self,
                                    user_magics=m.UserMagics(self))
         self.configurables.append(self.magics_manager)
 
         # Expose as public API from the magics manager
         self.register_magics = self.magics_manager.register
         self.define_magic = self.magics_manager.define_magic
 
         self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
             m.ConfigMagics, m.DeprecatedMagics, m.DisplayMagics, m.ExecutionMagics,
             m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
             m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
         )
 
         # Register Magic Aliases
         mman = self.magics_manager
         # FIXME: magic aliases should be defined by the Magics classes
         # or in MagicsManager, not here
         mman.register_alias('ed', 'edit')
         mman.register_alias('hist', 'history')
         mman.register_alias('rep', 'recall')
         mman.register_alias('SVG', 'svg', 'cell')
         mman.register_alias('HTML', 'html', 'cell')
         mman.register_alias('file', 'writefile', 'cell')
 
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.magic('colors %s' % self.colors)
     
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
         self.magics_manager.register_function(func, 
                                   magic_kind=magic_kind, magic_name=magic_name)
 
     def run_line_magic(self, magic_name, line):
         """Execute the given line magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the input line as a single string.
         """
         fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
             extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
                                     'did you mean that instead?)' % magic_name )
             error(etpl % (magic_name, extra))
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             magic_arg_s = self.var_expand(line, stack_depth)
             # Put magic args in a list so we can call with f(*a) syntax
             args = [magic_arg_s]
             kwargs = {}
             # Grab local namespace if we need it:
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
             with self.builtin_trap:
                 result = fn(*args,**kwargs)
             return result
 
     def run_cell_magic(self, magic_name, line, cell):
         """Execute the given cell magic.
         
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the first input line as a single string.
 
         cell : str
           The body of the cell as a (possibly multiline) string.
         """
         fn = self.find_cell_magic(magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
             extra = '' if lm is None else (' (But line magic `%{0}` exists, '
                             'did you mean that instead?)'.format(magic_name))
             error(etpl.format(magic_name, extra))
         elif cell == '':
             message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
             if self.find_line_magic(magic_name) is not None:
                 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
             raise UsageError(message)
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             magic_arg_s = self.var_expand(line, stack_depth)
             with self.builtin_trap:
                 result = fn(magic_arg_s, cell)
             return result
 
     def find_line_magic(self, magic_name):
         """Find and return a line magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['line'].get(magic_name)
 
     def find_cell_magic(self, magic_name):
         """Find and return a cell magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['cell'].get(magic_name)
 
     def find_magic(self, magic_name, magic_kind='line'):
         """Find and return a magic of the given type by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics[magic_kind].get(magic_name)
 
     def magic(self, arg_s):
         """DEPRECATED. Use run_line_magic() instead.
 
         Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         # TODO: should we issue a loud deprecation warning here?
         magic_name, _, magic_arg_s = arg_s.partition(' ')
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
         return self.run_line_magic(magic_name, magic_arg_s)
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new
             Macro object is created by passing the string to it.
         """
 
         from IPython.core import macro
 
         if isinstance(themacro, basestring):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system_piped(self, cmd):
         """Call the given cmd in a subprocess, piping stdout/err
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.  Should not be a command that expects input
           other than simple text.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             # We do not support backgrounding processes because we either use
             # pexpect or pipes to read from.  Users can always just call
             # os.system() or use ip.system=ip.system_raw
             # if they really want a background process.
             raise OSError("Background processes not supported.")
 
         # we explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
 
     def system_raw(self, cmd):
         """Call the given cmd in a subprocess using os.system
 
         Parameters
         ----------
         cmd : str
           Command to execute.
         """
         cmd = self.var_expand(cmd, depth=1)
         # protect os.system from UNC paths on Windows, which it can't handle:
         if sys.platform == 'win32':
             from IPython.utils._process_win32 import AvoidUNCPath
             with AvoidUNCPath() as path:
                 if path is not None:
                     cmd = '"pushd %s &&"%s' % (path, cmd)
                 cmd = py3compat.unicode_to_str(cmd)
                 ec = os.system(cmd)
         else:
             cmd = py3compat.unicode_to_str(cmd)
             ec = os.system(cmd)
             # The high byte is the exit code, the low byte is a signal number
             # that we discard for now. See the docs for os.wait()
             if ec > 255:
                 ec >>= 8
         
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = ec
 
     # use piped system by default, because it is better behaved
     system = system_piped
 
     def getoutput(self, cmd, split=True, depth=0):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
         depth : int, optional
           How many frames above the caller are the local variables which should
           be expanded in the command string? The default (0) assumes that the
           expansion variables are in the stack frame calling this function.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=depth+1))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
         self.ns_table['alias'] = self.alias_manager.alias_table,
 
     #-------------------------------------------------------------------------
     # Things related to extensions
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, parent=self)
         self.configurables.append(self.extension_manager)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, parent=self)
         self.configurables.append(self.prefilter_manager)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
 
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         if not self.show_rewritten_input:
             return
         
         rw = self.prompt_manager.render('rewrite') + cmd
 
         try:
             # plain ascii works better w/ pyreadline, on some machines, so
             # we use it and only print uncolored rewrite if we have unicode
             rw = str(rw)
             print(rw, file=io.stdout)
         except UnicodeEncodeError:
             print("------> " + cmd)
 
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _user_obj_error(self):
         """return simple exception dict
         
         for use in user_variables / expressions
         """
         
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
         
         exc_info = {
             u'status' : 'error',
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         return exc_info
     
     def _format_user_obj(self, obj):
         """format a user object to display dict
         
         for use in user_expressions / variables
         """
         
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
             'data' : data,
             'metadata' : md,
         }
         return value
     
     def user_variables(self, names):
         """Get a list of variable names from the user's namespace.
 
         Parameters
         ----------
         names : list of strings
           A list of names of variables to be read from the user namespace.
 
         Returns
         -------
         A dict, keyed by the input names and with the rich mime-type repr(s) of each value.
         Each element will be a sub-dict of the same form as a display_data message.
         """
         out = {}
         user_ns = self.user_ns
         
         for varname in names:
             try:
                 value = self._format_user_obj(user_ns[varname])
             except:
                 value = self._user_obj_error()
             out[varname] = value
         return out
 
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
 
         Returns
         -------
         A dict, keyed like the input expressions dict, with the rich mime-typed
         display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
         
         for key, expr in expressions.iteritems():
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
             except:
                 value = self._user_obj_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with self.builtin_trap:
             exec cmd in self.user_global_ns, self.user_ns
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with self.builtin_trap:
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         raise_exceptions : bool (False)
             If True raise exceptions everywhere. Meant for testing.
 
         """
         kw.setdefault('exit_ignore', False)
         kw.setdefault('raise_exceptions', False)
 
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 py3compat.execfile(fname,*where)
             except SystemExit as status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if kw['raise_exceptions']:
                     raise
                 if status.code and not kw['exit_ignore']:
                     self.showtraceback(exception_only=True)
             except:
                 if kw['raise_exceptions']:
                     raise
                 self.showtraceback()
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy extension.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 with open(fname) as thefile:
                     # self.run_cell currently captures all exceptions
                     # raised in user code.  It would be nice if there were
                     # versions of runlines, execfile that did raise, so
                     # we could catch the errors.
                     self.run_cell(thefile.read(), store_history=False, shell_futures=False)
             except:
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def safe_run_module(self, mod_name, where):
         """A safe version of runpy.run_module().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.
 
         `SystemExit` exceptions with status code 0 or None are ignored.
 
         Parameters
         ----------
         mod_name : string
             The name of the module to be executed.
         where : dict
             The globals namespace.
         """
         try:
             try:
                 where.update(
                     runpy.run_module(str(mod_name), run_name="__main__",
                                      alter_sys=True)
                     )
             except SystemExit as status:
                 if status.code:
                     raise
         except:
             self.showtraceback()
             warn('Unknown failure executing module: <%s>' % mod_name)
 
     def _run_cached_cell_magic(self, magic_name, line):
         """Special method to call a cell magic with the data stored in self.
         """
         cell = self._current_cell_magic_body
         self._current_cell_magic_body = None
         return self.run_cell_magic(magic_name, line, cell)
 
     def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
         """Run a complete IPython cell.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
         """
         if (not raw_cell) or raw_cell.isspace():
             return
         
         if silent:
             store_history = False
 
         self.input_transformer_manager.push(raw_cell)
         cell = self.input_transformer_manager.source_reset()
 
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
         compiler = self.compile if shell_futures else CachingCompiler()
 
         with self.builtin_trap:
             prefilter_failed = False
             if len(cell.splitlines()) == 1:
                 try:
                     # use prefilter_lines to handle trailing newlines
                     # restore trailing newline for ast.parse
                     cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
                 except AliasError as e:
                     error(e)
                     prefilter_failed = True
                 except Exception:
                     # don't allow prefilter errors to crash IPython
                     self.showtraceback()
                     prefilter_failed = True
 
             # Store raw and processed history
             if store_history:
                 self.history_manager.store_inputs(self.execution_count,
                                                   cell, raw_cell)
             if not silent:
                 self.logger.log(cell, raw_cell)
 
             if not prefilter_failed:
                 # don't run if prefilter failed
                 cell_name = self.compile.cache(cell, self.execution_count)
 
                 with self.display_trap:
                     try:
                         code_ast = compiler.ast_parse(cell, filename=cell_name)
                     except IndentationError:
                         self.showindentationerror()
                         if store_history:
                             self.execution_count += 1
                         return None
                     except (OverflowError, SyntaxError, ValueError, TypeError,
                             MemoryError):
                         self.showsyntaxerror()
                         if store_history:
                             self.execution_count += 1
                         return None
                     
                     code_ast = self.transform_ast(code_ast)
                     
                     interactivity = "none" if silent else self.ast_node_interactivity
                     self.run_ast_nodes(code_ast.body, cell_name,
                                        interactivity=interactivity, compiler=compiler)
                     
                     # Execute any registered post-execution functions.
                     # unless we are silent
                     post_exec = [] if silent else self._post_execute.iteritems()
                     
                     for func, status in post_exec:
                         if self.disable_failing_post_execute and not status:
                             continue
                         try:
                             func()
                         except KeyboardInterrupt:
                             print("\nKeyboardInterrupt", file=io.stderr)
                         except Exception:
                             # register as failing:
                             self._post_execute[func] = False
                             self.showtraceback()
                             print('\n'.join([
                                 "post-execution function %r produced an error." % func,
                                 "If this problem persists, you can disable failing post-exec functions with:",
                                 "",
                                 "    get_ipython().disable_failing_post_execute = True"
                             ]), file=io.stderr)
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
     
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
         
         Parameters
         ----------
         node : ast.Node
           The root node to be transformed. Typically called with the ast.Module
           produced by parsing user input.
         
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
         may also modify the passed object, so don't rely on references to the
         original AST.
         """
         for transformer in self.ast_transformers:
             try:
                 node = transformer.visit(node)
             except Exception:
                 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
                 self.ast_transformers.remove(transformer)
         
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
                 
 
     def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
                         compiler=compile):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
 
         Parameters
         ----------
         nodelist : list
           A sequence of AST nodes to run.
         cell_name : str
           Will be passed to the compiler as the filename of the cell. Typically
           the value returned by ip.compile.cache(cell).
         interactivity : str
           'all', 'last', 'last_expr' or 'none', specifying which nodes should be
           run interactively (displaying output from expressions). 'last_expr'
           will run the last node interactively only if it is an expression (i.e.
           expressions in loops or other blocks are not displayed. Other values
           for this parameter will raise a ValueError.
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
         """
         if not nodelist:
             return
 
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
             else:
                 interactivity = "none"
 
         if interactivity == 'none':
             to_run_exec, to_run_interactive = nodelist, []
         elif interactivity == 'last':
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         else:
             raise ValueError("Interactivity was %r" % interactivity)
 
         exec_count = self.execution_count
 
         try:
             for i, node in enumerate(to_run_exec):
                 mod = ast.Module([node])
                 code = compiler(mod, cell_name, "exec")
                 if self.run_code(code):
                     return True
 
             for i, node in enumerate(to_run_interactive):
                 mod = ast.Interactive([node])
                 code = compiler(mod, cell_name, "single")
                 if self.run_code(code):
                     return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
                 print()
 
         except:
             # It's possible to have exceptions raised here, typically by
             # compilation of odd code (such as a naked 'return' outside a
             # function) that did parse but isn't valid. Typically the exception
             # is a SyntaxError, but it's safest just to catch anything and show
             # the user a traceback.
 
             # We do only one try/except outside the loop to minimize the impact
             # on runtime, and also because if any node in the node list is
             # broken, we should stop execution completely.
             self.showtraceback()
 
         return False
 
     def run_code(self, code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Parameters
         ----------
         code_obj : code object
           A compiled code object, to be executed
 
         Returns
         -------
         False : successful execution.
         True : an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_run_code_hook()
                 #rprint('Running code', repr(code_obj)) # dbg
                 exec code_obj in self.user_global_ns, self.user_ns
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.showtraceback(exception_only=True)
             warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
         return outflag
 
     # For backwards compatibility
     runcode = run_code
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
     
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
         
         This takes the following steps:
         
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
         
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         """
         from IPython.core import pylabtools as pt
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
     
         if gui != 'inline':
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
                 self.pylab_gui_select = gui
             # Otherwise if they are different
             elif gui != self.pylab_gui_select:
                 print ('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
         
         pt.activate_matplotlib(backend)
         pt.configure_inline_support(self, backend)
         
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
         
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime.
 
         This turns on support for matplotlib, preloads into the interactive
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
         
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         import_all : optional, bool, default: True
           Whether to do `from numpy import *` and `from pylab import *`
           in addition to module imports.
         welcome_message : deprecated
           This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
         
         gui, backend = self.enable_matplotlib(gui)
         
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
         # user_ns_hidden with this information.
         ns = {}
         import_pylab(ns, import_all)
         # warn about clobbered names
         ignored = set(["__builtins__"])
         both = set(ns).intersection(self.user_ns).difference(ignored)
         clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
         self.user_ns.update(ns)
         self.user_ns_hidden.update(ns)
         return gui, backend, clobbered
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
         ns = self.user_ns.copy()
         ns.update(sys._getframe(depth+1).f_locals)
         try:
             # We have to use .vformat() here, because 'self' is a valid and common
             # name, and expanding **ns for .format() would make it collide with
             # the 'self' argument of the method.
             cmd = formatter.vformat(cmd, args=[], kwargs=ns)
         except Exception:
             # if formatter couldn't format, just let it go untransformed
             pass
         return cmd
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
         filename internally so ipython cleans it up at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
           immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py', prefix)
         self.tempfiles.append(filename)
 
         if data:
             tmp_file = open(filename,'w')
             tmp_file.write(data)
             tmp_file.close()
         return filename
 
     # TODO:  This should be removed when Term is refactored.
     def write(self,data):
         """Write a string to the default output"""
         io.stdout.write(data)
 
     # TODO:  This should be removed when Term is refactored.
     def write_err(self,data):
         """Write a string to the default error output"""
         io.stderr.write(data)
 
     def ask_yes_no(self, prompt, default=None):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default)
 
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : string
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
         Optional Parameters:
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
-        N-M -> include items N..M (closed endpoint)."""
+        N-M -> include items N..M (closed endpoint).
+        """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
     def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
 
         target : str
 
           A string specifying code to retrieve. This will be tried respectively
           as: ranges of input history (see %history for syntax), url,
           correspnding .py file, filename, or an expression evaluating to a
           string or Macro in the user namespace.
 
         raw : bool
           If true (default), retrieve raw history. Has no effect on the other
           retrieval mechanisms.
 
         py_only : bool (default False)
           Only try to fetch python code, do not try alternative methods to decode file
           if unicode fails.
 
         Returns
         -------
         A string of code.
 
         ValueError is raised if nothing is found, and TypeError if it evaluates
         to an object of another type. In each case, .args[0] is a printable
         message.
         """
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
         utarget = unquote_filename(target)
         try:
             if utarget.startswith(('http://', 'https://')):
                 return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
                 from urllib import urlopen  # Deferred import
                 response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % utarget)
 
         potential_target = [target]
         try :
             potential_target.insert(0,get_py_filename(target))
         except IOError:
             pass
 
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
                     return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError :
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :
                             return f.read()
                     raise ValueError(("'%s' seem to be unreadable.") % target)
             elif os.path.isdir(os.path.expanduser(tgt)):
                 raise ValueError("'%s' is a directory, not a regular file." % target)
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception:
             raise ValueError(("'%s' was not found in history, as a file, url, "
                                 "nor in the user namespace.") % target)
         if isinstance(codeobj, basestring):
             return codeobj
         elif isinstance(codeobj, Macro):
             return codeobj.value
 
         raise TypeError("%s is neither a string nor a macro." % target,
                         codeobj)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter
         """
         # Close the history session (this stores the end time and line count)
         # this must be *before* the tempfile cleanup, in case of temporary
         # history db
         self.history_manager.end_session()
 
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset(new_session=False)
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
 class InteractiveShellABC(object):
     """An abstract base class for InteractiveShell."""
     __metaclass__ = abc.ABCMeta
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/magics/execution.py

@@ -1,1231 +1,1248 b''
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import __builtin__ as builtin_mod
 import ast
 import bdb
 import os
 import sys
 import time
 from StringIO import StringIO
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile, pstats
     except ImportError:
         profile = pstats = None
 
 # Our own packages
 from IPython.core import debugger, oinspect
 from IPython.core import magic_arguments
 from IPython.core import page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
                                 line_cell_magic, on_off, needs_local_scope)
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.io import capture_output
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, unquote_filename, shellglob
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
 
     """
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         if profile is None:
             self.prun = self.profile_missing_notice
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     @skip_doctest
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         Usage, in line mode:
           %prun [options] statement
 
         Usage, in cell mode:
           %%prun [options] [statement]
           code...
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
-        -l <limit>: you can place restrictions on what or how much of the
+        -l <limit>
+          you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
-        example, '-l __init__ -l 5' will print only the topmost 5 lines of
+          example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
         information about class constructors.
 
-        -r: return the pstats.Stats object generated by the profiling. This
+        -r
+          return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
-       -s <key>: sort profile by given key. You can provide more than one key
+        -s <key>
+          sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
 
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
+          ============  =====================
                 Valid Arg       Meaning
+          ============  =====================
                   "calls"      call count
                   "cumulative" cumulative time
                   "file"       file name
                   "module"     file name
                   "pcalls"     primitive call count
                   "line"       line number
                   "name"       function name
                   "nfl"        name/file/line
                   "stdname"    standard name
                   "time"       internal time
+          ============  =====================
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
-        -T <filename>: save profile results as shown on screen to a text
+        -T <filename>
+          save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
-        -D <filename>: save (via dump_stats) profile statistics to given
+        -D <filename>
+          save (via dump_stats) profile statistics to given
         filename. This data is in a format understood by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
-        -q: suppress output to the pager.  Best used with -T and/or -D above.
+        -q
+          suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
-        '%run -p [prof_opts] filename.py [args to program]' where prof_opts
+        ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.input_splitter.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             dump_file = unquote_filename(dump_file)
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   repr(dump_file)+'.',sys_exit
         if text_file:
             text_file = unquote_filename(text_file)
             pfile = open(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   repr(text_file)+'.',sys_exit
 
         if 'r' in opts:
             return stats
         else:
             return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @line_cell_magic
     def debug(self, line='', cell=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         else:
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint):
         if breakpoint:
             (filename, bp_line) = breakpoint.split(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
 
     @line_magic
     def tb(self, s):
         """Print the last traceback with the currently active exception mode.
 
         See %xmode for changing exception reporting modes."""
         self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
-        Usage:
+        Usage::
+        
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | file ) [args]
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
-        This is similar to running at a system prompt:\\
-          $ python file args\\
+        This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
-        __name__=='__main__' and sys.argv constructed as indicated. It thus
+        ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
-        *two* back slashes (e.g., '\\\\*') to suppress expansions.
+        *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         Options:
 
-        -n: __name__ is NOT set to '__main__', but to the running file's name
+        -n
+          __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
-        protected by an ' if __name__ == "__main__" ' clause.
+          protected by an ``if __name__ == "__main__"`` clause.
 
-        -i: run the file in IPython's namespace instead of an empty one. This
+        -i
+          run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
-        -e: ignore sys.exit() calls or SystemExit exceptions in the script
+        -e
+          ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
-        -t: print timing information at the end of the run.  IPython will give
+        -t
+          print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
-        If -t is given, an additional -N<N> option can be given, where <N>
+        If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
-            IPython CPU timings (estimated):\\
-              User  :    0.19597 s.\\
-              System:        0.0 s.\\
+            IPython CPU timings (estimated):
+              User  :    0.19597 s.
+              System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
-            IPython CPU timings (estimated):\\
-            Total runs performed: 5\\
-              Times :      Total       Per run\\
-              User  :   0.910862 s,  0.1821724 s.\\
+            IPython CPU timings (estimated):
+            Total runs performed: 5
+              Times :      Total       Per run
+              User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
-        -d: run your program under the control of pdb, the Python debugger.
+        -d
+          run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
-        etc.  Internally, what IPython does is similar to calling:
+          etc.  Internally, what IPython does is similar to calling::
 
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example::
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         Or you can specify a breakpoint in a different file::
 
           %run -d -b myotherfile.py:20 myscript
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without quotes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
-        -p: run program under the control of the Python profiler module (which
+        -p
+          run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
-        -m: specify module name to load instead of script path. Similar to
+        -m
+          specify module name to load instead of script path. Similar to
         the -m option for the python interpreter. Use this option last if you
         want to combine with other %run options. Unlike the python interpreter
         only source modules are allowed no .pyc or .pyo files.
         For example::
 
             %run -m example
 
         will run the example module.
 
-        -G: disable shell-like glob expansion of arguments.
+        -G
+          disable shell-like glob expansion of arguments.
 
         """
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 warn('%r is not a valid modulename on sys.path'%modulename)
                 return
             arg_lst = [modpath] + arg_lst
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n', oinspect.getdoc(self.run)
             return
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
         # protect sys.argv from potential unicode strings on Python 2:
         if not py3compat.PY3:
             sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.user_module
         else:
             # Run in a fresh, empty namespace
             if 'n' in opts:
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename)
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             with self.shell.readline_no_record:
                 if 'p' in opts:
                     stats = self._run_with_profiler(code, opts, code_ns)
                 else:
                     if 'd' in opts:
                         bp_file, bp_line = parse_breakpoint(
                             opts.get('b', ['1'])[0], filename)
                         self._run_with_debugger(
                             code, code_ns, filename, bp_line, bp_file)
                     else:
                         if 'm' in opts:
                             def run():
                                 self.shell.safe_run_module(modulename, prog_ns)
                         else:
                             if runner is None:
                                 runner = self.default_runner
                             if runner is None:
                                 runner = self.shell.safe_execfile
 
                             def run():
                                 runner(filename, prog_ns, prog_ns,
                                        exit_ignore=exit_ignore)
 
                         if 't' in opts:
                             # timed execution
                             try:
                                 nruns = int(opts['N'][0])
                                 if nruns < 1:
                                     error('Number of runs must be >=1')
                                     return
                             except (KeyError):
                                 nruns = 1
                             self._run_with_timing(run, nruns)
                         else:
                             # regular execution
                             run()
 
                 if 'i' in opts:
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the
                     # __name__ key was never set; using pop we don't have to
                     # worry about a possible KeyError.
                     prog_ns.pop('__name__', None)
 
                     with preserve_keys(self.shell.user_ns, '__file__'):
                         self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(self, code, code_ns, filename=None,
                            bp_line=None, bp_file=None):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = debugger.Pdb(self.shell.colors)
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print "NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             deb.run(code, code_ns)
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.time()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print "\nIPython CPU timings (estimated):"
             print "  User   : %10.2f s." % t_usr
             print "  System : %10.2f s." % t_sys
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print "\nIPython CPU timings (estimated):"
             print "Total runs performed:", nruns
             print "  Times  : %10s   %10s" % ('Total', 'Per run')
             print "  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
             print "  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
         twall1 = time.time()
         print "Wall time: %10.2f s." % (twall1 - twall0)
 
     @skip_doctest
     @line_cell_magic
     def timeit(self, line='', cell=None):
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
           %timeit [-n<N> -r<R> [-t|-c]] statement
         or in cell mode:
           %%timeit [-n<N> -r<R> [-t|-c]] setup_code
           code
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen.
 
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
 
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
 
         opts, stmt = self.parse_options(line,'n:r:tcp:',
                                         posix=False, strict=False)
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.input_splitter.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = ast.parse("pass")
             ast_stmt = ast.parse(transform(stmt))
         else:
             ast_setup = ast.parse(transform(stmt))
             ast_stmt = ast.parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         class TimeitTemplateFiller(ast.NodeTransformer):
             "This is quite tightly tied to the template definition above."
             def visit_FunctionDef(self, node):
                 "Fill in the setup statement"
                 self.generic_visit(node)
                 if node.name == "inner":
                     node.body[:1] = ast_setup.body
 
                 return node
 
             def visit_For(self, node):
                 "Fill in the statement to be timed"
                 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
                     node.body = ast_stmt.body
                 return node
 
         timeit_ast = TimeitTemplateFiller().visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
 
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
 
         best = min(timer.repeat(repeat, number)) / number
 
         print u"%d loops, best of %d: %s per loop" % (number, repeat,
                                                           _format_time(best, precision))
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
 
     @skip_doctest
     @needs_local_scope
     @line_cell_magic
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
         
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly 
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit 
         magic for more controll over the measurement.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: %time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: %time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
 
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.input_transformer_manager.transform_cell(cell)
         else:
             expr = self.shell.input_transformer_manager.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = ast.parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
         t0 = clock()
         code = compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             out = eval(code, glob, local_ns)
             end = clock2()
         else:
             st = clock2()
             exec code in glob, local_ns
             end = clock2()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         # On windows cpu_sys is always zero, so no new information to the next print 
         if sys.platform != 'win32':
             print "CPU times: user %s, sys: %s, total: %s" % \
                 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot))
         print "Wall time: %s" % _format_time(wall_time)
         if tc > tc_min:
             print "Compiler : %s" % _format_time(tc)
         if tp > tp_min:
             print "Parser   : %s" % _format_time(tp)
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.iteritems() if\
                                                         isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print e.args[0]
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         if not ( 'q' in opts) : 
             print 'Macro `%s` created. To execute, type its name (without quotes).' % name
             print '=== Macro contents: ==='
             print macro,
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if args.output:
             self.shell.user_ns[args.output] = io
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
     import math
     
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
     
     # Unfortunately the unicode 'micro' symbol can cause problems in
     # certain terminals.  
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = [u"s", u"ms",u'us',"ns"] # the save value   
     if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
         try:
             u'\xb5'.encode(sys.stdout.encoding)
             units = [u"s", u"ms",u'\xb5s',"ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     ret =  u"%.*g %s" % (precision, timespan * scaling[order], units[order])
     return ret

IPython/core/magics/logging.py

@@ -1,170 +1,184 b''
 """Implementation of magic functions for IPython's own logging.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import sys
 
 # Our own packages
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.utils.warn import warn
 from IPython.utils.py3compat import str_to_unicode
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 @magics_class
 class LoggingMagics(Magics):
     """Magics related to all logging machinery."""
 
     @line_magic
     def logstart(self, parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
-        of (note that the modes are given unquoted):\\
-          append: well, that says it.\\
-          backup: rename (if exists) to name~ and start name.\\
-          global: single logfile in your home dir, appended to.\\
-          over  : overwrite existing log.\\
-          rotate: create rotating logs name.1~, name.2~, etc.
+        of (note that the modes are given unquoted):
+
+        append
+            Keep logging at the end of any existing file.
+
+        backup
+            Rename any existing file to name~ and start name.
+
+        global
+            Append to  a single logfile in your home directory.
+
+        over
+            Overwrite any existing log.
+
+        rotate
+            Create rotating logs: name.1~, name.2~, etc.
 
         Options:
 
-          -o: log also IPython's output.  In this mode, all commands which
+          -o
+            log also IPython's output. In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call::
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
-          -r: log 'raw' input.  Normally, IPython's logs contain the processed
+          -r
+            log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           _ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
-          -t: put timestamps before each input line logged (these are put in
-          comments)."""
+          -t
+            put timestamps before each input line logged (these are put in
+            comments).
+        """
 
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipython remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = self.shell.logfile
         if logfname:
             logfname = os.path.expanduser(logfname)
         self.shell.logfile = logfname
 
         loghead = u'# IPython log file\n\n'
         try:
             logger.logstart(logfname, loghead, logmode, log_output, timestamp,
                             log_raw_input)
         except:
             self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.history_manager.input_hist_raw
             else:
                 input_hist = self.shell.history_manager.input_hist_parsed
 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.history_manager.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip() + u'\n')
                     if n in output_hist:
                         log_write(str_to_unicode(repr(output_hist[n])),'output')
             else:
                 logger.log_write(u'\n'.join(input_hist[1:]))
                 logger.log_write(u'\n')
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     @line_magic
     def logstop(self, parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.shell.logger.logstop()
 
     @line_magic
     def logoff(self, parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
 
     @line_magic
     def logon(self, parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
 
         self.shell.logger.switch_log(1)
 
     @line_magic
     def logstate(self, parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()

IPython/core/page.py

@@ -1,348 +1,350 b''
 # encoding: utf-8
 """
 Paging capabilities for IPython.core
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 
 For now this uses ipapi, so it can't be in IPython.utils.  If we can get
 rid of that dependency, we could move it there.
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import os
 import re
 import sys
 import tempfile
 
 from io import UnsupportedOperation
 
 from IPython import get_ipython
 from IPython.core.error import TryNext
 from IPython.utils.data import chop
 from IPython.utils import io
 from IPython.utils.process import system
 from IPython.utils.terminal import get_terminal_size
 from IPython.utils import py3compat
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print(os.linesep.join(screens[0]), file=io.stdout)
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print(last_escape + hunk, file=io.stdout)
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]), file=io.stdout)
 
 def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
     if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
         return screen_lines_def
     
     try:
         import termios
         import curses
     except ImportError:
         return screen_lines_def
     
     # There is a bug in curses, where *sometimes* it fails to properly
     # initialize, and then after the endwin() call is made, the
     # terminal is left in an unusable state.  Rather than trying to
     # check everytime for this (by requesting and comparing termios
     # flags each time), we just save the initial terminal state and
     # unconditionally reset it every time.  It's cheaper than making
     # the checks.
     term_flags = termios.tcgetattr(sys.stdout)
 
     # Curses modifies the stdout buffer size by default, which messes
     # up Python's normal stdout buffering.  This would manifest itself
     # to IPython users as delayed printing on stdout after having used
     # the pager.
     #
     # We can prevent this by manually setting the NCURSES_NO_SETBUF
     # environment variable.  For more details, see:
     # http://bugs.python.org/issue10144
     NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
     os.environ['NCURSES_NO_SETBUF'] = ''
 
     # Proceed with curses initialization
     try:
         scr = curses.initscr()
     except AttributeError:
         # Curses on Solaris may not be complete, so we can't use it there
         return screen_lines_def
     
     screen_lines_real,screen_cols = scr.getmaxyx()
     curses.endwin()
 
     # Restore environment
     if NCURSES_NO_SETBUF is None:
         del os.environ['NCURSES_NO_SETBUF']
     else:
         os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF
 
     # Restore terminal state in case endwin() didn't.
     termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
     # Now we have what we needed: the screen size in rows/columns
     return screen_lines_real
     #print '***Screen size:',screen_lines_real,'lines x',\
     #screen_cols,'columns.' # dbg
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
 
     # first, try the hook
     ip = get_ipython()
     if ip:
         try:
             ip.hooks.show_in_pager(strng)
             return
         except TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print(strng)
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.splitlines()[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         try:
             screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint, file=io.stdout)
             return
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print(str_toprint, file=io.stdout)
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = open(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd, 'w')
                 try:
                     pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(py3compat.cast_bytes_py2(
                         strng, encoding=pager_encoding))
                 finally:
                     retval = pager.close()
             except IOError as msg:  # broken pipe when user quits
                 if msg.args == (32, 'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 
 def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         system(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print('Unable to show file',repr(fname))
 
 
 def get_pager_cmd(pager_cmd=None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one.
     """
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 
 def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 
 # (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         io.stdout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getwch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         io.stdout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
+    
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
+    
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print(header, end=' ')
     if len(str) < width:
         print(str)
         snip = 0
     else:
         whalf = int((width -5)/2)
         print(str[:whalf] + ' <...> ' + str[-whalf:])
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip

IPython/core/ultratb.py

@@ -1,1247 +1,1251 b''
 # -*- coding: utf-8 -*-
 """
 ultratb.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
-Installation instructions for ColorTB:
+Installation instructions for ColorTB::
+
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 * VerboseTB
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
-Note:
+.. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 
-Installation instructions for ColorTB:
+Installation instructions for ColorTB::
+
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
-* Color schemes
+Color schemes
+-------------
+
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from __future__ import unicode_literals
 
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 try:                           # Python 2
     generate_tokens = tokenize.generate_tokens
 except AttributeError:         # Python 3
     generate_tokens = tokenize.tokenize
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import openpy
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
 from IPython.utils.warn import info, error
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with Python >= 2.5
 inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = ulinecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         ipinst = get_ipython()
         if ipinst is not None:
             scheme = ipinst.colors
         else:
             scheme = DEFAULT_SCHEME
 
     _line_format = _parser.format2
 
     for line in lines:
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str', scheme)
         if not err: line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num,
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num,
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(object):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing io.stdout until runtime.  The way
         # things are written now, the io.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
         to io.stdout.  This ensures compatibility with most tools, including
         Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return io.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor', call_pdb=False, ostream=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         value : object
           Data stored in the exception
 
         elist : list
           List of frames, see class docstring for details.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance value is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
         # Note: this code originally read:
 
         ## for line in lines[:-1]:
         ##     out_list.append(" "+line)
         ## out_list.append(lines[-1])
 
         # This means it was indenting everything but the last line by a little
         # bit.  I've disabled this for now, but if we see ugliness somewhre we
         # can restore it.
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal,
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         #from pprint import pformat; print 'LISTTB', pformat(list) # dbg
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = Colors.excName + etype.__name__ + Colors.Normal
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append( py3compat.cast_unicode(stype) + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 #print 'filename is',filename  # dbg
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = ulinecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                         (Colors.normalEm,
                          Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                          Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset-1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal) )
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value, [])
 
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              context=5):
         """Return a nice text document describing the traceback."""
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
                                            pyver, date.rjust(75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
                                      'Traceback (most recent call last)'.\
                                                   rjust(75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         ##self.check_cache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context, tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             if not file:
                 file = '?'
             elif not(file.startswith(str("<")) and file.endswith(str(">"))):
                 # Guess that filenames like <string> aren't real filenames, so
                 # don't call abspath on them.                    
                 try:
                     file = abspath(file)
                 except OSError:
                     # Not sure if this can still happen: abspath now works with
                     # file names like <string>
                     pass
             file = py3compat.cast_unicode(file, util_path.fs_encoding)
             link = tpl_link % file
             args, varargs, varkw, locals = inspect.getargvalues(frame)
 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # This happens in situations like errors inside generator
                     # expressions, where local variables are listed in the
                     # line, but can't be extracted from the frame.  I'm not
                     # 100% sure this isn't actually a bug in inspect itself,
                     # but since there's no info for us to compute with, the
                     # best we can do is report the failure and move on.  Here
                     # we must *not* call any traceback construction again,
                     # because that would mess up use of %debug later on.  So we
                     # simply report the failure and move on.  The only
                     # limitation will be that this frame won't have locals
                     # listed in the call signature.  Quite subtle problem...
                     # I can't think of a good way to validate this in a unit
                     # test, but running a script consisting of:
                     #  dict( (k,v.strip()) for (k,v) in range(10) )
                     # will illustrate the error, if this exception catch is
                     # disabled.
                     call = tpl_call_fail % func
             
             # Don't attempt to tokenize binary files.
             if file.endswith(('.so', '.pyd', '.dll')):
                 frames.append('%s %s\n' % (link,call))
                 continue
             elif file.endswith(('.pyc','.pyo')):
                 # Look up the corresponding source file.
                 file = openpy.source_from_cache(file)
 
             def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 names = []
                 name_cont = False
                 
                 for token_type, token, start, end, line in generate_tokens(linereader):
                     # build composite names
                     if token_type == tokenize.NAME and token not in keyword.kwlist:
                         if name_cont:
                             # Continuation of a dotted name
                             try:
                                 names[-1].append(token)
                             except IndexError:
                                 names.append([token])
                             name_cont = False
                         else:
                             # Regular new names.  We append everything, the caller
                             # will be responsible for pruning the list later.  It's
                             # very tricky to try to prune as we go, b/c composite
                             # names can fool us.  The pruning at the end is easy
                             # to do (or the caller can print a list with repeated
                             # names if so desired.
                             names.append([token])
                     elif token == '.':
                         name_cont = True
                     elif token_type == tokenize.NEWLINE:
                         break
                         
             except (IndexError, UnicodeDecodeError):
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError as msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
 
             # Join composite names (e.g. "dict.fromkeys")
             names = ['.'.join(n) for n in names]
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if name_base in locals:
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if name_base in frame.f_globals:
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _format_traceback_lines(lnum,index,lines,Colors,lvals,
                                             col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, py3compat.cast_unicode(evalue_str))]
         if (not py3compat.PY3) and type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, py3compat.cast_unicode(evalue_str)))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         # vds: >>
         if records:
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
              ipinst = get_ipython()
              if ipinst is not None:
                  ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
 
         # return all our info assembled as a single string
         # return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
         return [head] + frames + [''.join(exception[0])]
 
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self,'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
 
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, context
             )
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
                 self, etype, value, elist, tb_offset, context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <'+mode+'>\n'
                              'Valid modes: '+str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, context=5):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, context)
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self,color_scheme = 'NoColor'):
         ListTB.__init__(self,color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
         ListTB.__call__(self,etype,value,elist)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, d_e):
         (d, e) = d_e
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
 
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         handler(*sys.exc_info())
     print ''
 
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         handler(*sys.exc_info())
     print ''
 

IPython/nbconvert/preprocessors/coalescestreams.py

@@ -1,75 +1,76 b''
 """Module that allows latex output notebooks to be conditioned before
 they are converted.  Exposes a decorator (@cell_preprocessor) in
 addition to the coalesce_streams pre-proccessor.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def cell_preprocessor(function):
     """
     Wrap a function to be executed on all cells of a notebook
     
     Wrapped Parameters
-    ----------
+    ------------------
+    
     cell : NotebookNode cell
         Notebook cell being processed
     resources : dictionary
         Additional resources used in the conversion process.  Allows
         preprocessors to pass variables into the Jinja engine.
     index : int
         Index of the cell being processed
     """
     
     def wrappedfunc(nb, resources):
         for worksheet in nb.worksheets :
             for index, cell in enumerate(worksheet.cells):
                 worksheet.cells[index], resources = function(cell, resources, index)
         return nb, resources
     return wrappedfunc
 
 
 @cell_preprocessor
 def coalesce_streams(cell, resources, index):
     """
     Merge consecutive sequences of stream output into single stream
     to prevent extra newlines inserted at flush calls
     
     Parameters
     ----------
     cell : NotebookNode cell
         Notebook cell being processed
     resources : dictionary
         Additional resources used in the conversion process.  Allows
         transformers to pass variables into the Jinja engine.
     index : int
         Index of the cell being processed
     """
     
     outputs = cell.get('outputs', [])
     if not outputs:
         return cell, resources
     
     last = outputs[0]
     new_outputs = [last]
     
     for output in outputs[1:]:
         if (output.output_type == 'stream' and
             last.output_type == 'stream' and
             last.stream == output.stream
         ):
             last.text += output.text
         else:
             new_outputs.append(output)
             last = output
 
     cell.outputs = new_outputs
     return cell, resources

IPython/utils/_process_win32.py

@@ -1,187 +1,187 b''
 """Windows-specific implementation of process utilities.
 
 This file is only meant to be imported by process.py, not by end-users.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # stdlib
 import os
 import sys
 import ctypes
 
 from ctypes import c_int, POINTER
 from ctypes.wintypes import LPCWSTR, HLOCAL
 from subprocess import STDOUT
 
 # our own imports
 from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
 from . import py3compat
 from .encoding import DEFAULT_ENCODING
 
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
 
 class AvoidUNCPath(object):
     """A context manager to protect command execution from UNC paths.
 
     In the Win32 API, commands can't be invoked with the cwd being a UNC path.
     This context manager temporarily changes directory to the 'C:' drive on
     entering, and restores the original working directory on exit.
 
     The context manager returns the starting working directory *if* it made a
     change and None otherwise, so that users can apply the necessary adjustment
     to their system calls in the event of a change.
 
-    Example
-    -------
+    Examples
+    --------
     ::
         cmd = 'dir'
         with AvoidUNCPath() as path:
             if path is not None:
                 cmd = '"pushd %s &&"%s' % (path, cmd)
             os.system(cmd)
     """
     def __enter__(self):
         self.path = os.getcwdu()
         self.is_unc_path = self.path.startswith(r"\\")
         if self.is_unc_path:
             # change to c drive (as cmd.exe cannot handle UNC addresses)
             os.chdir("C:")
             return self.path
         else:
             # We return None to signal that there was no change in the working
             # directory
             return None
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.is_unc_path:
             os.chdir(self.path)
 
 
 def _find_cmd(cmd):
     """Find the full path to a .bat or .exe using the win32api module."""
     try:
         from win32api import SearchPath
     except ImportError:
         raise ImportError('you need to have pywin32 installed for this to work')
     else:
         PATH = os.environ['PATH']
         extensions = ['.exe', '.com', '.bat', '.py']
         path = None
         for ext in extensions:
             try:
                 path = SearchPath(PATH, cmd, ext)[0]
             except:
                 pass
         if path is None:
             raise OSError("command %r not found" % cmd)
         else:
             return path
 
 
 def _system_body(p):
     """Callback for _system."""
     enc = DEFAULT_ENCODING
     for line in read_no_interrupt(p.stdout).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stdout)
     for line in read_no_interrupt(p.stderr).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stderr)
 
     # Wait to finish for returncode
     return p.wait()
 
 
 def system(cmd):
     """Win32 version of os.system() that works with network shares.
 
     Note that this implementation returns None, as meant for use in IPython.
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     None : we explicitly do NOT return the subprocess status code, as this
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
     # The controller provides interactivity with both
     # stdin and stdout
     #import _process_win32_controller
     #_process_win32_controller.system(cmd)
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         return process_handler(cmd, _system_body)
 
 def getoutput(cmd):
     """Return standard output of executing cmd in a shell.
 
     Accepts the same arguments as os.system().
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     stdout : str
     """
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
 
     if out is None:
         out = b''
     return py3compat.bytes_to_str(out)
 
 try:
     CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
     CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
     CommandLineToArgvW.restype = POINTER(LPCWSTR)
     LocalFree = ctypes.windll.kernel32.LocalFree
     LocalFree.res_type = HLOCAL
     LocalFree.arg_types = [HLOCAL]
     
     def arg_split(commandline, posix=False, strict=True):
         """Split a command line's arguments in a shell-like manner.
 
         This is a special version for windows that use a ctypes call to CommandLineToArgvW
         to do the argv splitting. The posix paramter is ignored.
         
         If strict=False, process_common.arg_split(...strict=False) is used instead.
         """
         #CommandLineToArgvW returns path to executable if called with empty string.
         if commandline.strip() == "":
             return []
         if not strict:
             # not really a cl-arg, fallback on _process_common
             return py_arg_split(commandline, posix=posix, strict=strict)
         argvn = c_int()
         result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
         result_array_type = LPCWSTR * argvn.value
         result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
         retval = LocalFree(result_pointer)
         return result
 except AttributeError:
     arg_split = py_arg_split

IPython/utils/_process_win32_controller.py

@@ -1,574 +1,574 b''
 """Windows-specific implementation of process utilities with direct WinAPI.
 
 This file is meant to be used by process.py
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 # stdlib
 import os, sys, threading
 import ctypes, msvcrt
 
 # Win32 API types needed for the API calls
 from ctypes import POINTER
 from ctypes.wintypes import HANDLE, HLOCAL, LPVOID, WORD, DWORD, BOOL, \
         ULONG, LPCWSTR
 LPDWORD = POINTER(DWORD)
 LPHANDLE = POINTER(HANDLE)
 ULONG_PTR = POINTER(ULONG)
 class SECURITY_ATTRIBUTES(ctypes.Structure):
     _fields_ = [("nLength", DWORD),
                 ("lpSecurityDescriptor", LPVOID),
                 ("bInheritHandle", BOOL)]
 LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
 class STARTUPINFO(ctypes.Structure):
     _fields_ = [("cb", DWORD),
                 ("lpReserved", LPCWSTR),
                 ("lpDesktop", LPCWSTR),
                 ("lpTitle", LPCWSTR),
                 ("dwX", DWORD),
                 ("dwY", DWORD),
                 ("dwXSize", DWORD),
                 ("dwYSize", DWORD),
                 ("dwXCountChars", DWORD),
                 ("dwYCountChars", DWORD),
                 ("dwFillAttribute", DWORD),
                 ("dwFlags", DWORD),
                 ("wShowWindow", WORD),
                 ("cbReserved2", WORD),
                 ("lpReserved2", LPVOID),
                 ("hStdInput", HANDLE),
                 ("hStdOutput", HANDLE),
                 ("hStdError", HANDLE)]
 LPSTARTUPINFO = POINTER(STARTUPINFO)
 class PROCESS_INFORMATION(ctypes.Structure):
     _fields_ = [("hProcess", HANDLE),
                 ("hThread", HANDLE),
                 ("dwProcessId", DWORD),
                 ("dwThreadId", DWORD)]
 LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
 
 # Win32 API constants needed
 ERROR_HANDLE_EOF = 38
 ERROR_BROKEN_PIPE = 109
 ERROR_NO_DATA = 232
 HANDLE_FLAG_INHERIT = 0x0001
 STARTF_USESTDHANDLES = 0x0100
 CREATE_SUSPENDED = 0x0004
 CREATE_NEW_CONSOLE = 0x0010
 CREATE_NO_WINDOW = 0x08000000
 STILL_ACTIVE = 259
 WAIT_TIMEOUT = 0x0102
 WAIT_FAILED = 0xFFFFFFFF
 INFINITE = 0xFFFFFFFF
 DUPLICATE_SAME_ACCESS = 0x00000002
 ENABLE_ECHO_INPUT = 0x0004
 ENABLE_LINE_INPUT = 0x0002
 ENABLE_PROCESSED_INPUT = 0x0001
 
 # Win32 API functions needed
 GetLastError = ctypes.windll.kernel32.GetLastError
 GetLastError.argtypes = []
 GetLastError.restype = DWORD
 
 CreateFile = ctypes.windll.kernel32.CreateFileW
 CreateFile.argtypes = [LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE]
 CreateFile.restype = HANDLE
 
 CreatePipe = ctypes.windll.kernel32.CreatePipe
 CreatePipe.argtypes = [POINTER(HANDLE), POINTER(HANDLE),
         LPSECURITY_ATTRIBUTES, DWORD]
 CreatePipe.restype = BOOL
 
 CreateProcess = ctypes.windll.kernel32.CreateProcessW
 CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
         LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
         LPPROCESS_INFORMATION]
 CreateProcess.restype = BOOL
 
 GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
 GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
 GetExitCodeProcess.restype = BOOL
 
 GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
 GetCurrentProcess.argtypes = []
 GetCurrentProcess.restype = HANDLE
 
 ResumeThread = ctypes.windll.kernel32.ResumeThread
 ResumeThread.argtypes = [HANDLE]
 ResumeThread.restype = DWORD
 
 ReadFile = ctypes.windll.kernel32.ReadFile
 ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
 ReadFile.restype = BOOL
 
 WriteFile = ctypes.windll.kernel32.WriteFile
 WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
 WriteFile.restype = BOOL
 
 GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
 GetConsoleMode.argtypes = [HANDLE, LPDWORD]
 GetConsoleMode.restype = BOOL
 
 SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
 SetConsoleMode.argtypes = [HANDLE, DWORD]
 SetConsoleMode.restype = BOOL
 
 FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
 FlushConsoleInputBuffer.argtypes = [HANDLE]
 FlushConsoleInputBuffer.restype = BOOL
 
 WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
 WaitForSingleObject.argtypes = [HANDLE, DWORD]
 WaitForSingleObject.restype = DWORD
 
 DuplicateHandle = ctypes.windll.kernel32.DuplicateHandle
 DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, LPHANDLE,
         DWORD, BOOL, DWORD]
 DuplicateHandle.restype = BOOL
 
 SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
 SetHandleInformation.argtypes = [HANDLE, DWORD, DWORD]
 SetHandleInformation.restype = BOOL
 
 CloseHandle = ctypes.windll.kernel32.CloseHandle
 CloseHandle.argtypes = [HANDLE]
 CloseHandle.restype = BOOL
 
 CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
 CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(ctypes.c_int)]
 CommandLineToArgvW.restype = POINTER(LPCWSTR)
 
 LocalFree = ctypes.windll.kernel32.LocalFree
 LocalFree.argtypes = [HLOCAL]
 LocalFree.restype = HLOCAL
 
 class AvoidUNCPath(object):
     """A context manager to protect command execution from UNC paths.
 
     In the Win32 API, commands can't be invoked with the cwd being a UNC path.
     This context manager temporarily changes directory to the 'C:' drive on
     entering, and restores the original working directory on exit.
 
     The context manager returns the starting working directory *if* it made a
     change and None otherwise, so that users can apply the necessary adjustment
     to their system calls in the event of a change.
 
-    Example
-    -------
+    Examples
+    --------
     ::
         cmd = 'dir'
         with AvoidUNCPath() as path:
             if path is not None:
                 cmd = '"pushd %s &&"%s' % (path, cmd)
             os.system(cmd)
     """
     def __enter__(self):
         self.path = os.getcwdu()
         self.is_unc_path = self.path.startswith(r"\\")
         if self.is_unc_path:
             # change to c drive (as cmd.exe cannot handle UNC addresses)
             os.chdir("C:")
             return self.path
         else:
             # We return None to signal that there was no change in the working
             # directory
             return None
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.is_unc_path:
             os.chdir(self.path)
 
 
 class Win32ShellCommandController(object):
     """Runs a shell command in a 'with' context.
 
     This implementation is Win32-specific.
 
     Example:
         # Runs the command interactively with default console stdin/stdout
         with ShellCommandController('python -i') as scc:
             scc.run()
 
         # Runs the command using the provided functions for stdin/stdout
         def my_stdout_func(s):
             # print or save the string 's'
             write_to_stdout(s)
         def my_stdin_func():
             # If input is available, return it as a string.
             if input_available():
                 return get_input()
             # If no input available, return None after a short delay to
             # keep from blocking.
             else:
                 time.sleep(0.01)
                 return None
       
         with ShellCommandController('python -i') as scc:
             scc.run(my_stdout_func, my_stdin_func)
     """
 
     def __init__(self, cmd, mergeout = True):
         """Initializes the shell command controller.
 
         The cmd is the program to execute, and mergeout is
         whether to blend stdout and stderr into one output
         in stdout. Merging them together in this fashion more
         reliably keeps stdout and stderr in the correct order
         especially for interactive shell usage.
         """
         self.cmd = cmd
         self.mergeout = mergeout
 
     def __enter__(self):
         cmd = self.cmd
         mergeout = self.mergeout
 
         self.hstdout, self.hstdin, self.hstderr = None, None, None
         self.piProcInfo = None
         try:
             p_hstdout, c_hstdout, p_hstderr, \
                     c_hstderr, p_hstdin, c_hstdin = [None]*6
 
             # SECURITY_ATTRIBUTES with inherit handle set to True
             saAttr = SECURITY_ATTRIBUTES()
             saAttr.nLength = ctypes.sizeof(saAttr)
             saAttr.bInheritHandle = True
             saAttr.lpSecurityDescriptor = None
 
             def create_pipe(uninherit):
                 """Creates a Windows pipe, which consists of two handles.
 
                 The 'uninherit' parameter controls which handle is not
                 inherited by the child process.
                 """
                 handles = HANDLE(), HANDLE()
                 if not CreatePipe(ctypes.byref(handles[0]),
                             ctypes.byref(handles[1]), ctypes.byref(saAttr), 0):
                     raise ctypes.WinError()
                 if not SetHandleInformation(handles[uninherit],
                             HANDLE_FLAG_INHERIT, 0):
                     raise ctypes.WinError()
                 return handles[0].value, handles[1].value
 
             p_hstdout, c_hstdout = create_pipe(uninherit=0)
             # 'mergeout' signals that stdout and stderr should be merged.
             # We do that by using one pipe for both of them.
             if mergeout:
                 c_hstderr = HANDLE()
                 if not DuplicateHandle(GetCurrentProcess(), c_hstdout,
                                 GetCurrentProcess(), ctypes.byref(c_hstderr),
                                 0, True, DUPLICATE_SAME_ACCESS):
                     raise ctypes.WinError()
             else:
                 p_hstderr, c_hstderr = create_pipe(uninherit=0)
             c_hstdin,  p_hstdin  = create_pipe(uninherit=1)
 
             # Create the process object
             piProcInfo = PROCESS_INFORMATION()
             siStartInfo = STARTUPINFO()
             siStartInfo.cb = ctypes.sizeof(siStartInfo)
             siStartInfo.hStdInput = c_hstdin
             siStartInfo.hStdOutput = c_hstdout
             siStartInfo.hStdError = c_hstderr
             siStartInfo.dwFlags = STARTF_USESTDHANDLES
             dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
 
             if not CreateProcess(None,
                     u"cmd.exe /c " + cmd,
                     None, None, True, dwCreationFlags,
                     None, None, ctypes.byref(siStartInfo),
                     ctypes.byref(piProcInfo)):
                 raise ctypes.WinError()
 
             # Close this process's versions of the child handles
             CloseHandle(c_hstdin)
             c_hstdin = None
             CloseHandle(c_hstdout)
             c_hstdout = None
             if c_hstderr != None:
                 CloseHandle(c_hstderr)
                 c_hstderr = None
 
             # Transfer ownership of the parent handles to the object
             self.hstdin = p_hstdin
             p_hstdin = None
             self.hstdout = p_hstdout
             p_hstdout = None
             if not mergeout:
                 self.hstderr = p_hstderr
                 p_hstderr = None
             self.piProcInfo = piProcInfo
 
         finally:
             if p_hstdin:
                 CloseHandle(p_hstdin)
             if c_hstdin:
                 CloseHandle(c_hstdin)
             if p_hstdout:
                 CloseHandle(p_hstdout)
             if c_hstdout:
                 CloseHandle(c_hstdout)
             if p_hstderr:
                 CloseHandle(p_hstderr)
             if c_hstderr:
                 CloseHandle(c_hstderr)
 
         return self
 
     def _stdin_thread(self, handle, hprocess, func, stdout_func):
         exitCode = DWORD()
         bytesWritten = DWORD(0)
         while True:
             #print("stdin thread loop start")
             # Get the input string (may be bytes or unicode)
             data = func()
 
             # None signals to poll whether the process has exited
             if data is None:
                 #print("checking for process completion")
                 if not GetExitCodeProcess(hprocess, ctypes.byref(exitCode)):
                     raise ctypes.WinError()
                 if exitCode.value != STILL_ACTIVE:
                     return
                 # TESTING: Does zero-sized writefile help?
                 if not WriteFile(handle, "", 0,
                         ctypes.byref(bytesWritten), None):
                     raise ctypes.WinError()
                 continue
             #print("\nGot str %s\n" % repr(data), file=sys.stderr)
 
             # Encode the string to the console encoding
             if isinstance(data, unicode): #FIXME: Python3
                 data = data.encode('utf_8')
 
             # What we have now must be a string of bytes
             if not isinstance(data, str): #FIXME: Python3
                 raise RuntimeError("internal stdin function string error")
 
             # An empty string signals EOF
             if len(data) == 0:
                 return
 
             # In a windows console, sometimes the input is echoed,
             # but sometimes not. How do we determine when to do this?
             stdout_func(data)
             # WriteFile may not accept all the data at once.
             # Loop until everything is processed
             while len(data) != 0:
                 #print("Calling writefile")
                 if not WriteFile(handle, data, len(data),
                         ctypes.byref(bytesWritten), None):
                     # This occurs at exit
                     if GetLastError() == ERROR_NO_DATA:
                         return
                     raise ctypes.WinError()
                 #print("Called writefile")
                 data = data[bytesWritten.value:]
 
     def _stdout_thread(self, handle, func):
         # Allocate the output buffer
         data = ctypes.create_string_buffer(4096)
         while True:
             bytesRead = DWORD(0)
             if not ReadFile(handle, data, 4096,
                         ctypes.byref(bytesRead), None):
                 le = GetLastError()
                 if le == ERROR_BROKEN_PIPE:
                     return
                 else:
                     raise ctypes.WinError()
             # FIXME: Python3
             s = data.value[0:bytesRead.value]
             #print("\nv: %s" % repr(s), file=sys.stderr)
             func(s.decode('utf_8', 'replace'))
 
     def run(self, stdout_func = None, stdin_func = None, stderr_func = None):
         """Runs the process, using the provided functions for I/O.
 
         The function stdin_func should return strings whenever a
         character or characters become available.
         The functions stdout_func and stderr_func are called whenever
         something is printed to stdout or stderr, respectively.
         These functions are called from different threads (but not
         concurrently, because of the GIL).
         """
         if stdout_func == None and stdin_func == None and stderr_func == None:
             return self._run_stdio()
 
         if stderr_func != None and self.mergeout:
             raise RuntimeError("Shell command was initiated with "
                     "merged stdin/stdout, but a separate stderr_func "
                     "was provided to the run() method")
 
         # Create a thread for each input/output handle
         stdin_thread = None
         threads = []
         if stdin_func:
             stdin_thread = threading.Thread(target=self._stdin_thread,
                                 args=(self.hstdin, self.piProcInfo.hProcess,
                                 stdin_func, stdout_func))
         threads.append(threading.Thread(target=self._stdout_thread,
                                     args=(self.hstdout, stdout_func)))
         if not self.mergeout:
             if stderr_func == None:
                 stderr_func = stdout_func
             threads.append(threading.Thread(target=self._stdout_thread,
                                         args=(self.hstderr, stderr_func)))
         # Start the I/O threads and the process
         if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
             raise ctypes.WinError()
         if stdin_thread is not None:
             stdin_thread.start()
         for thread in threads:
             thread.start()
         # Wait for the process to complete
         if WaitForSingleObject(self.piProcInfo.hProcess, INFINITE) == \
                     WAIT_FAILED:
             raise ctypes.WinError()
         # Wait for the I/O threads to complete
         for thread in threads:
             thread.join()
 
         # Wait for the stdin thread to complete
         if stdin_thread is not None:
             stdin_thread.join()
 
     def _stdin_raw_nonblock(self):
         """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
         # WARNING: This is experimental, and produces inconsistent results.
         #          It's possible for the handle not to be appropriate for use
         #          with WaitForSingleObject, among other things.
         handle = msvcrt.get_osfhandle(sys.stdin.fileno())
         result = WaitForSingleObject(handle, 100)
         if result == WAIT_FAILED:
             raise ctypes.WinError()
         elif result == WAIT_TIMEOUT:
             print(".", end='')
             return None
         else:
             data = ctypes.create_string_buffer(256)
             bytesRead = DWORD(0)
             print('?', end='')
 
             if not ReadFile(handle, data, 256,
                         ctypes.byref(bytesRead), None):
                 raise ctypes.WinError()
             # This ensures the non-blocking works with an actual console
             # Not checking the error, so the processing will still work with
             # other handle types
             FlushConsoleInputBuffer(handle)
 
             data = data.value
             data = data.replace('\r\n', '\n')
             data = data.replace('\r', '\n')
             print(repr(data) + " ", end='')
             return data
 
     def _stdin_raw_block(self):
         """Use a blocking stdin read"""
         # The big problem with the blocking read is that it doesn't
         # exit when it's supposed to in all contexts. An extra
         # key-press may be required to trigger the exit.
         try:
             data = sys.stdin.read(1)
             data = data.replace('\r', '\n')
             return data
         except WindowsError as we:
             if we.winerror == ERROR_NO_DATA:
                 # This error occurs when the pipe is closed
                 return None
             else:
                 # Otherwise let the error propagate
                 raise we
 
     def _stdout_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stdout)
         sys.stdout.flush()
 
     def _stderr_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stderr)
         sys.stderr.flush()
 
     def _run_stdio(self):
         """Runs the process using the system standard I/O.
 
         IMPORTANT: stdin needs to be asynchronous, so the Python
                    sys.stdin object is not used. Instead,
                    msvcrt.kbhit/getwch are used asynchronously.
         """
         # Disable Line and Echo mode
         #lpMode = DWORD()
         #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
         #if GetConsoleMode(handle, ctypes.byref(lpMode)):
         #    set_console_mode = True
         #    if not SetConsoleMode(handle, lpMode.value &
         #            ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
         #        raise ctypes.WinError()
 
         if self.mergeout:
             return self.run(stdout_func = self._stdout_raw,
                     stdin_func = self._stdin_raw_block)
         else:
             return self.run(stdout_func = self._stdout_raw,
                     stdin_func = self._stdin_raw_block,
                     stderr_func = self._stderr_raw)
 
         # Restore the previous console mode
         #if set_console_mode:
         #    if not SetConsoleMode(handle, lpMode.value):
         #        raise ctypes.WinError()
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.hstdin:
             CloseHandle(self.hstdin)
             self.hstdin = None
         if self.hstdout:
             CloseHandle(self.hstdout)
             self.hstdout = None
         if self.hstderr:
             CloseHandle(self.hstderr)
             self.hstderr = None
         if self.piProcInfo != None:
             CloseHandle(self.piProcInfo.hProcess)
             CloseHandle(self.piProcInfo.hThread)
             self.piProcInfo = None
 
 
 def system(cmd):
     """Win32 version of os.system() that works with network shares.
 
     Note that this implementation returns None, as meant for use in IPython.
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     None : we explicitly do NOT return the subprocess status code, as this
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         with Win32ShellCommandController(cmd) as scc:
             scc.run()
 
 
 if __name__ == "__main__":
     print("Test starting!")
     #system("cmd")
     system("python -i")
     print("Test finished!")

IPython/utils/capture.py

@@ -1,173 +1,173 b''
 # encoding: utf-8
 """
 IO capturing utilities.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 class RichOutput(object):
     def __init__(self, source="", data=None, metadata=None):
         self.source = source
         self.data = data or {}
         self.metadata = metadata or {}
     
     def display(self):
         from IPython.display import publish_display_data
         publish_display_data(self.source, self.data, self.metadata)
     
     def _repr_mime_(self, mime):
         if mime not in self.data:
             return
         data = self.data[mime]
         if mime in self.metadata:
             return data, self.metadata[mime]
         else:
             return data
             
     def _repr_html_(self):
         return self._repr_mime_("text/html")
     
     def _repr_latex_(self):
         return self._repr_mime_("text/latex")
     
     def _repr_json_(self):
         return self._repr_mime_("application/json")
     
     def _repr_javascript_(self):
         return self._repr_mime_("application/javascript")
     
     def _repr_png_(self):
         return self._repr_mime_("image/png")
     
     def _repr_jpeg_(self):
         return self._repr_mime_("image/jpeg")
     
     def _repr_svg_(self):
         return self._repr_mime_("image/svg+xml")
 
 
 class CapturedIO(object):
     """Simple object for containing captured stdout/err and rich display StringIO objects
 
     Each instance `c` has three attributes:
 
-        c.stdout : standard output as a string
-        c.stderr : standard error as a string
-        c.outputs: a list of rich display outputs
+    - ``c.stdout`` : standard output as a string
+    - ``c.stderr`` : standard error as a string
+    - ``c.outputs``: a list of rich display outputs
 
-    Additionally, there's a `c.show()` method which will print all of the
-    above in the same order, and can be invoked simply via `c()`.
+    Additionally, there's a ``c.show()`` method which will print all of the
+    above in the same order, and can be invoked simply via ``c()``.
     """
     
     def __init__(self, stdout, stderr, outputs=None):
         self._stdout = stdout
         self._stderr = stderr
         if outputs is None:
             outputs = []
         self._outputs = outputs
     
     def __str__(self):
         return self.stdout
     
     @property
     def stdout(self):
         "Captured standard output"
         if not self._stdout:
             return ''
         return self._stdout.getvalue()
     
     @property
     def stderr(self):
         "Captured standard error"
         if not self._stderr:
             return ''
         return self._stderr.getvalue()
     
     @property
     def outputs(self):
         """A list of the captured rich display outputs, if any.
         
-        If you have a CapturedIO object `c`, these can be displayed in IPython
-        using:
+        If you have a CapturedIO object ``c``, these can be displayed in IPython
+        using::
 
             from IPython.display import display
             for o in c.outputs:
                 display(o)
         """
         return [ RichOutput(s, d, md) for s, d, md in self._outputs ]
     
     def show(self):
         """write my output to sys.stdout/err as appropriate"""
         sys.stdout.write(self.stdout)
         sys.stderr.write(self.stderr)
         sys.stdout.flush()
         sys.stderr.flush()
         for source, data, metadata in self._outputs:
             RichOutput(source, data, metadata).display()
     
     __call__ = show
 
 
 class capture_output(object):
     """context manager for capturing stdout/err"""
     stdout = True
     stderr = True
     display = True
     
     def __init__(self, stdout=True, stderr=True, display=True):
         self.stdout = stdout
         self.stderr = stderr
         self.display = display
         self.shell = None
     
     def __enter__(self):
         from IPython.core.getipython import get_ipython
         from IPython.core.displaypub import CapturingDisplayPublisher
         
         self.sys_stdout = sys.stdout
         self.sys_stderr = sys.stderr
         
         if self.display:
             self.shell = get_ipython()
             if self.shell is None:
                 self.save_display_pub = None
                 self.display = False
         
         stdout = stderr = outputs = None
         if self.stdout:
             stdout = sys.stdout = StringIO()
         if self.stderr:
             stderr = sys.stderr = StringIO()
         if self.display:
             self.save_display_pub = self.shell.display_pub
             self.shell.display_pub = CapturingDisplayPublisher()
             outputs = self.shell.display_pub.outputs
             
         
         return CapturedIO(stdout, stderr, outputs)
     
     def __exit__(self, exc_type, exc_value, traceback):
         sys.stdout = self.sys_stdout
         sys.stderr = self.sys_stderr
         if self.display and self.shell:
             self.shell.display_pub = self.save_display_pub
 
 

IPython/utils/contexts.py

@@ -1,71 +1,71 b''
 # encoding: utf-8
 """
 Context managers for temporarily updating dictionaries.
 
 Authors:
 
 * Bradley Froehle
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2012  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 class preserve_keys(object):
     """Preserve a set of keys in a dictionary.
 
     Upon entering the context manager the current values of the keys
     will be saved. Upon exiting, the dictionary will be updated to
     restore the original value of the preserved keys. Preserved keys
     which did not exist when entering the context manager will be
     deleted.
 
-    Example
-    -------
+    Examples
+    --------
 
     >>> d = {'a': 1, 'b': 2, 'c': 3}
     >>> with preserve_keys(d, 'b', 'c', 'd'):
     ...     del d['a']
     ...     del d['b']      # will be reset to 2
     ...     d['c'] = None   # will be reset to 3
     ...     d['d'] = 4      # will be deleted
     ...     d['e'] = 5
     ...     print(sorted(d.items()))
     ...
     [('c', None), ('d', 4), ('e', 5)]
     >>> print(sorted(d.items()))
     [('b', 2), ('c', 3), ('e', 5)]
     """
 
     def __init__(self, dictionary, *keys):
         self.dictionary = dictionary
         self.keys = keys
 
     def __enter__(self):
         # Actions to perform upon exiting.
         to_delete = []
         to_update = {}
 
         d = self.dictionary
         for k in self.keys:
             if k in d:
                 to_update[k] = d[k]
             else:
                 to_delete.append(k)
 
         self.to_delete = to_delete
         self.to_update = to_update
 
     def __exit__(self, *exc_info):
         d = self.dictionary
 
         for k in self.to_delete:
             d.pop(k, None)
         d.update(self.to_update)

IPython/utils/sysinfo.py

@@ -1,167 +1,169 b''
 # encoding: utf-8
 """
 Utilities for getting information about IPython and the system it's running in.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import platform
 import pprint
 import sys
 import subprocess
 
 from IPython.core import release
 from IPython.utils import py3compat, _sysinfo, encoding
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def pkg_commit_hash(pkg_path):
     """Get short form of commit hash given directory `pkg_path`
 
     We get the commit hash from (in order of preference):
 
     * IPython.utils._sysinfo.commit
     * git output, if we are in a git repository
 
     If these fail, we return a not-found placeholder tuple
 
     Parameters
     ----------
     pkg_path : str
        directory containing package
        only used for getting commit from active repo
 
     Returns
     -------
     hash_from : str
        Where we got the hash from - description
     hash_str : str
        short form of hash
     """
     # Try and get commit from written commit text file
     if _sysinfo.commit:
         return "installation", _sysinfo.commit
 
     # maybe we are in a repository
     proc = subprocess.Popen('git rev-parse --short HEAD',
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             cwd=pkg_path, shell=True)
     repo_commit, _ = proc.communicate()
     if repo_commit:
         return 'repository', repo_commit.strip()
     return '(none found)', '<not found>'
 
 
 def pkg_info(pkg_path):
     """Return dict describing the context of this package
 
     Parameters
     ----------
     pkg_path : str
        path containing __init__.py for package
 
     Returns
     -------
     context : dict
        with named parameters of interest
     """
     src, hsh = pkg_commit_hash(pkg_path)
     return dict(
         ipython_version=release.version,
         ipython_path=pkg_path,
         codename=release.codename,
         commit_source=src,
         commit_hash=hsh,
         sys_version=sys.version,
         sys_executable=sys.executable,
         sys_platform=sys.platform,
         platform=platform.platform(),
         os_name=os.name,
         default_encoding=encoding.DEFAULT_ENCODING,
         )
 
 
 @py3compat.doctest_refactor_print
 def sys_info():
     """Return useful information about IPython and the system, as a string.
 
-    Example
-    -------
+    Examples
+    --------
+    ::
+    
     In [2]: print sys_info()
     {'commit_hash': '144fdae',      # random
      'commit_source': 'repository',
      'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
      'ipython_version': '0.11.dev',
      'os_name': 'posix',
      'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
      'sys_executable': '/usr/bin/python',
      'sys_platform': 'linux2',
      'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
    """
     p = os.path
     path = p.dirname(p.abspath(p.join(__file__, '..')))
     return pprint.pformat(pkg_info(path))
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
 
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus