Remove unused imports

IPython/core/alias.py

@@ -1,263 +1,262 b''
 # encoding: utf-8
 """
 System command aliases.
 
 Authors:
 
 * Fernando Perez
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import keyword
 import os
 import re
 import sys
 
 from IPython.config.configurable import Configurable
 from IPython.core.splitinput import split_user_input
 
 from IPython.utils.traitlets import List, Instance
-from IPython.utils.autoattr import auto_attr
 from IPython.utils.warn import warn, error
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # This is used as the pattern for calls to split_user_input.
 shell_line_split = re.compile(r'^(\s*)()(\S+)(.*$)')
 
 def default_aliases():
     """Return list of shell aliases to auto-define.
     """
     # Note: the aliases defined here should be safe to use on a kernel
     # regardless of what frontend it is attached to.  Frontends that use a
     # kernel in-process can define additional aliases that will only work in
     # their case.  For example, things like 'less' or 'clear' that manipulate
     # the terminal should NOT be declared here, as they will only work if the
     # kernel is running inside a true terminal, and not over the network.
 
     if os.name == 'posix':
         default_aliases = [('mkdir', 'mkdir'), ('rmdir', 'rmdir'),
                            ('mv', 'mv -i'), ('rm', 'rm -i'), ('cp', 'cp -i'),
                            ('cat', 'cat'),
                            ]
         # Useful set of ls aliases.  The GNU and BSD options are a little
         # different, so we make aliases that provide as similar as possible
         # behavior in ipython, by passing the right flags for each platform
         if sys.platform.startswith('linux'):
             ls_aliases = [('ls', 'ls -F --color'),
                           # long ls
                           ('ll', 'ls -F -o --color'),
                           # ls normal files only
                           ('lf', 'ls -F -o --color %l | grep ^-'),
                           # ls symbolic links
                           ('lk', 'ls -F -o --color %l | grep ^l'),
                           # directories or links to directories,
                           ('ldir', 'ls -F -o --color %l | grep /$'),
                           # things which are executable
                           ('lx', 'ls -F -o --color %l | grep ^-..x'),
                           ]
         else:
             # BSD, OSX, etc.
             ls_aliases = [('ls', 'ls -F'),
                           # long ls
                           ('ll', 'ls -F -l'),
                           # ls normal files only
                           ('lf', 'ls -F -l %l | grep ^-'),
                           # ls symbolic links
                           ('lk', 'ls -F -l %l | grep ^l'),
                           # directories or links to directories,
                           ('ldir', 'ls -F -l %l | grep /$'),
                           # things which are executable
                           ('lx', 'ls -F -l %l | grep ^-..x'),
                           ]
         default_aliases = default_aliases + ls_aliases
     elif os.name in ['nt', 'dos']:
         default_aliases = [('ls', 'dir /on'),
                            ('ddir', 'dir /ad /on'), ('ldir', 'dir /ad /on'),
                            ('mkdir', 'mkdir'), ('rmdir', 'rmdir'),
                            ('echo', 'echo'), ('ren', 'ren'), ('copy', 'copy'),
                            ]
     else:
         default_aliases = []
 
     return default_aliases
 
 
 class AliasError(Exception):
     pass
 
 
 class InvalidAliasError(AliasError):
     pass
 
 #-----------------------------------------------------------------------------
 # Main AliasManager class
 #-----------------------------------------------------------------------------
 
 class AliasManager(Configurable):
 
     default_aliases = List(default_aliases(), config=True)
     user_aliases = List(default_value=[], config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def __init__(self, shell=None, config=None):
         super(AliasManager, self).__init__(shell=shell, config=config)
         self.alias_table = {}
         self.exclude_aliases()
         self.init_aliases()
 
     def __contains__(self, name):
         return name in self.alias_table
 
     @property
     def aliases(self):
         return [(item[0], item[1][1]) for item in self.alias_table.iteritems()]
 
     def exclude_aliases(self):
         # set of things NOT to alias (keywords, builtins and some magics)
         no_alias = set(['cd','popd','pushd','dhist','alias','unalias'])
         no_alias.update(set(keyword.kwlist))
         no_alias.update(set(__builtin__.__dict__.keys()))
         self.no_alias = no_alias
 
     def init_aliases(self):
         # Load default aliases
         for name, cmd in self.default_aliases:
             self.soft_define_alias(name, cmd)
 
         # Load user aliases
         for name, cmd in self.user_aliases:
             self.soft_define_alias(name, cmd)
 
     def clear_aliases(self):
         self.alias_table.clear()
 
     def soft_define_alias(self, name, cmd):
         """Define an alias, but don't raise on an AliasError."""
         try:
             self.define_alias(name, cmd)
         except AliasError as e:
             error("Invalid alias: %s" % e)
 
     def define_alias(self, name, cmd):
         """Define a new alias after validating it.
 
         This will raise an :exc:`AliasError` if there are validation
         problems.
         """
         nargs = self.validate_alias(name, cmd)
         self.alias_table[name] = (nargs, cmd)
 
     def undefine_alias(self, name):
         if name in self.alias_table:
             del self.alias_table[name]
 
     def validate_alias(self, name, cmd):
         """Validate an alias and return the its number of arguments."""
         if name in self.no_alias:
             raise InvalidAliasError("The name %s can't be aliased "
                                     "because it is a keyword or builtin." % name)
         if not (isinstance(cmd, basestring)):
             raise InvalidAliasError("An alias command must be a string, "
                                     "got: %r" % cmd)
         nargs = cmd.count('%s')
         if nargs>0 and cmd.find('%l')>=0:
             raise InvalidAliasError('The %s and %l specifiers are mutually '
                                     'exclusive in alias definitions.')
         return nargs
 
     def call_alias(self, alias, rest=''):
         """Call an alias given its name and the rest of the line."""
         cmd = self.transform_alias(alias, rest)
         try:
             self.shell.system(cmd)
         except:
             self.shell.showtraceback()
 
     def transform_alias(self, alias,rest=''):
         """Transform alias to system command string."""
         nargs, cmd = self.alias_table[alias]
 
         if ' ' in cmd and os.path.isfile(cmd):
             cmd = '"%s"' % cmd
 
         # Expand the %l special to be the user's input line
         if cmd.find('%l') >= 0:
             cmd = cmd.replace('%l', rest)
             rest = ''
         if nargs==0:
             # Simple, argument-less aliases
             cmd = '%s %s' % (cmd, rest)
         else:
             # Handle aliases with positional arguments
             args = rest.split(None, nargs)
             if len(args) < nargs:
                 raise AliasError('Alias <%s> requires %s arguments, %s given.' %
                       (alias, nargs, len(args)))
             cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
         return cmd
 
     def expand_alias(self, line):
         """ Expand an alias in the command line
 
         Returns the provided command line, possibly with the first word
         (command) translated according to alias expansion rules.
 
         [ipython]|16> _ip.expand_aliases("np myfile.txt")
                  <16> 'q:/opt/np/notepad++.exe myfile.txt'
         """
 
         pre,_,fn,rest = split_user_input(line)
         res = pre + self.expand_aliases(fn, rest)
         return res
 
     def expand_aliases(self, fn, rest):
         """Expand multiple levels of aliases:
 
         if:
 
         alias foo bar /tmp
         alias baz foo
 
         then:
 
         baz huhhahhei -> bar /tmp huhhahhei
         """
         line = fn + " " + rest
 
         done = set()
         while 1:
             pre,_,fn,rest = split_user_input(line, shell_line_split)
             if fn in self.alias_table:
                 if fn in done:
                     warn("Cyclic alias definition, repeated '%s'" % fn)
                     return ""
                 done.add(fn)
 
                 l2 = self.transform_alias(fn, rest)
                 if l2 == line:
                     break
                 # ls -> ls -F should not recurse forever
                 if l2.split(None,1)[0] == line.split(None,1)[0]:
                     line = l2
                     break
                 line=l2
             else:
                 break
 
         return line

IPython/core/completer.py

@@ -1,933 +1,932 b''
 """Word completion for IPython.
 
 This module is a fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3, but we need a lot more
 functionality specific to IPython, so this module will continue to live as an
 IPython-specific utility.
 
 Original rlcompleter documentation:
 
 This requires the latest extension to the readline module (the
 completes keywords, built-ins and globals in __main__; when completing
 NAME.NAME..., it evaluates (!) the expression up to the last dot and
 completes its attributes.
 
 It's very cool to do "import string" type "string.", hit the
 completion key (twice), and see the list of names defined by the
 string module!
 
 Tip: to use the tab key as the completion key, call
 
     readline.parse_and_bind("tab: complete")
 
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
 generally cause the completion to fail).  This is a feature -- since
 readline sets the tty device in raw (or cbreak) mode, printing a
 traceback wouldn't work well without some complicated hoopla to save,
 reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
 application defined code to be executed if an object with a
 __getattr__ hook is found.  Since it is the responsibility of the
 application (or the user) to enable this feature, I consider this an
 acceptable risk.  More complicated expressions (e.g. function calls or
 indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
 raw_input(), and thus these also benefit/suffer from the completer
 features.  Clearly an interactive application can benefit by
 specifying its own completer function and using raw_input() for all
 its input.
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
 """
 
 #*****************************************************************************
 #
 # Since this file is essentially a minimally modified copy of the rlcompleter
 # module which is part of the standard Python distribution, I assume that the
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
 #       Copyright (C) 2008 IPython Development Team
 #       Copyright (C) 2001 Fernando Perez. <[e-mail]>
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import __main__
 import glob
 import inspect
 import itertools
 import keyword
 import os
 import re
-import shlex
 import sys
 
 from IPython.config.configurable import Configurable
 from IPython.core.error import TryNext
 from IPython.core.inputsplitter import ESC_MAGIC
 from IPython.utils import generics
 from IPython.utils import io
 from IPython.utils.dir2 import dir2
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import CBool, Enum
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Public API
 __all__ = ['Completer','IPCompleter']
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 #-----------------------------------------------------------------------------
 # Main functions and classes
 #-----------------------------------------------------------------------------
 
 def has_open_quotes(s):
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
     the string is odd.
 
     Returns
     -------
     If there is an open quote, the quote character is returned.  Else, return
     False.
     """
     # We check " first, then ', so complex cases with nested quotes will get
     # the " to take precedence.
     if s.count('"') % 2:
         return '"'
     elif s.count("'") % 2:
         return "'"
     else:
         return False
 
 
 def protect_filename(s):
     """Escape a string to protect certain characters."""
 
     return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
                     for ch in s])
 
 def expand_user(path):
     """Expand '~'-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
     computing completions, and you wish to return the completions with the
     original '~' instead of its expanded value.
 
     Parameters
     ----------
     path : str
       String to be expanded.  If no ~ is present, the output is the same as the
       input.
 
     Returns
     -------
     newpath : str
       Result of ~ expansion in the input path.
     tilde_expand : bool
       Whether any expansion was performed or not.
     tilde_val : str
       The value that ~ was replaced with.
     """
     # Default values
     tilde_expand = False
     tilde_val = ''
     newpath = path
 
     if path.startswith('~'):
         tilde_expand = True
         rest = len(path)-1
         newpath = os.path.expanduser(path)
         if rest:
             tilde_val = newpath[:-rest]
         else:
             tilde_val = newpath
 
     return newpath, tilde_expand, tilde_val
 
 
 def compress_user(path, tilde_expand, tilde_val):
     """Does the opposite of expand_user, with its outputs.
     """
     if tilde_expand:
         return path.replace(tilde_val, '~')
     else:
         return path
 
 
 class Bunch(object): pass
 
 
 DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
 GREEDY_DELIMS = ' =\r\n'
 
 
 class CompletionSplitter(object):
     """An object to split an input line in a manner similar to readline.
 
     By having our own implementation, we can expose readline-like completion in
     a uniform manner to all frontends.  This object only needs to be given the
     line of text to be split and the cursor position on said line, and it
     returns the 'word' to be completed on at the cursor after splitting the
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
     setting the `delims` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
 
     # A string of delimiter characters.  The default value makes sense for
     # IPython's most typical usage patterns.
     _delims = DELIMS
 
     # The expression (a normal string) to be compiled into a regular expression
     # for actual splitting.  We store it as an attribute mostly for ease of
     # debugging, since this type of code can be so tricky to debug.
     _delim_expr = None
 
     # The regular expression that does the actual splitting
     _delim_re = None
 
     def __init__(self, delims=None):
         delims = CompletionSplitter._delims if delims is None else delims
         self.delims = delims
 
     @property
     def delims(self):
         """Return the string of delimiter characters."""
         return self._delims
 
     @delims.setter
     def delims(self, delims):
         """Set the delimiters for line splitting."""
         expr = '[' + ''.join('\\'+ c for c in delims) + ']'
         self._delim_re = re.compile(expr)
         self._delims = delims
         self._delim_expr = expr
 
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
         l = line if cursor_pos is None else line[:cursor_pos]
         return self._delim_re.split(l)[-1]
 
 
 class Completer(Configurable):
 
     greedy = CBool(False, config=True,
         help="""Activate greedy completion
 
         This will enable completion on elements of lists, results of function calls, etc.,
         but can be unsafe because the code is actually evaluated on TAB.
         """
     )
     
 
     def __init__(self, namespace=None, global_namespace=None, config=None, **kwargs):
         """Create a new completer for the command line.
 
         Completer(namespace=ns,global_namespace=ns2) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
 
         Completer instances should be used as the completion mechanism of
         readline via the set_completer() call:
 
         readline.set_completer(Completer(my_namespace).complete)
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = 1
         else:
             self.use_main_ns = 0
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
         super(Completer, self).__init__(config=config, **kwargs)
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         #print 'Completer->global_matches, txt=%r' % text # dbg
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [keyword.kwlist,
                     __builtin__.__dict__.keys(),
                     self.namespace.keys(),
                     self.global_namespace.keys()]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
 
         #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
     
         if m:
             expr, attr = m.group(1, 3)
         elif self.greedy:
             m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
             if not m2:
                 return []
             expr, attr = m2.group(1,2)
         else:
             return []
     
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else: 
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
         except Exception:
             # Silence errors from completion function
             #raise # dbg
             pass
         # Build match list to return
         n = len(attr)
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
     except:
         return []
     
     return [w for w in words if isinstance(w, basestring)]
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     def _greedy_changed(self, name, old, new):
         """update the splitter and readline delims when greedy is changed"""
         if new:
             self.splitter.delims = GREEDY_DELIMS
         else:
             self.splitter.delims = DELIMS
 
         if self.readline:
             self.readline.set_completer_delims(self.splitter.delims)
     
     merge_completions = CBool(True, config=True,
         help="""Whether to merge completion results into a single list
         
         If False, only the completion results from the first non-empty
         completer will be returned.
         """
     )
     omit__names = Enum((0,1,2), default_value=2, config=True,
         help="""Instruct the completer to omit private method names
         
         Specifically, when completing on ``object.<tab>``.
         
         When 2 [default]: all names that start with '_' will be excluded.
         
         When 1: all 'magic' names (``__foo__``) will be excluded.
         
         When 0: nothing will be excluded.
         """
     )
     limit_to__all__ = CBool(default_value=False, config=True,
         help="""Instruct the completer to use __all__ for the completion
         
         Specifically, when completing on ``object.<tab>``.
         
         When True: only those names in obj.__all__ will be included.
         
         When False [default]: the __all__ attribute is ignored 
         """
     )
 
     def __init__(self, shell=None, namespace=None, global_namespace=None,
                  alias_table=None, use_readline=True,
                  config=None, **kwargs):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
         via readline.set_completer().
 
         Inputs:
 
         - shell: a pointer to the ipython shell itself.  This is needed
         because this completer knows about magic functions, and those can
         only be accessed via the ipython instance.
 
         - namespace: an optional dict where completions are performed.
 
         - global_namespace: secondary optional dict for completions, to
         handle cases (such as IPython embedded inside functions) where
         both Python scopes are visible.
 
         - If alias_table is supplied, it should be a dictionary of aliases
         to complete.
 
         use_readline : bool, optional
           If true, use the readline library.  This completer can still function
           without readline, though in that case callers must provide some extra
           information on each call about the current line."""
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
         # Readline configuration, only used by the rlcompleter method.
         if use_readline:
             # We store the right version of readline so that later code
             import IPython.utils.rlineimpl as readline
             self.readline = readline
         else:
             self.readline = None
 
         # _greedy_changed() depends on splitter and readline being defined:
         Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
                             config=config, **kwargs)
 
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
 
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
 
         # All active matcher routines for completion
         self.matchers = [self.python_matches,
                          self.file_matches,
                          self.magic_matches,
                          self.alias_matches,
                          self.python_func_kw_matches,
                          ]
 
     def all_completions(self, text):
         """
         Wrapper around the complete method for the benefit of emacs
         and pydb.
         """
         return self.complete(text)[1]
 
     def _clean_glob(self,text):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self,text):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
     def file_matches(self, text):
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         #io.rprint('Completer->file_matches: <%r>' % text) # dbg
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
 
         text_until_cursor = self.text_until_cursor
         # track strings with open quotes
         open_quotes = has_open_quotes(text_until_cursor)
 
         if '(' in text_until_cursor or '[' in text_until_cursor:
             lsplit = text
         else:
             try:
                 # arg_split ~ shlex.split, but with unicode bugs fixed by us
                 lsplit = arg_split(text_until_cursor)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if open_quotes:
                     lsplit = text_until_cursor.split(open_quotes)[-1]
                 else:
                     return []
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
         if not open_quotes and lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped name
             has_protectables = True
             text0,text = text,lsplit
         else:
             has_protectables = False
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         # Compute the matches from the filesystem
         m0 = self.clean_glob(text.replace('\\',''))
 
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 +
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names at all (and we _shouldn't_, as it
                 # would cause bugs when the filesystem call is made).
                 matches = m0
             else:
                 matches = [text_prefix +
                            protect_filename(f) for f in m0]
 
         #io.rprint('mm', matches)  # dbg
 
         # Mark directories in input list by appending '/' to their names.
         matches = [x+'/' if os.path.isdir(x) else x for x in matches]
         return matches
 
     def magic_matches(self, text):
         """Match magics"""
         #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
         line_magics = lsm['line']
         cell_magics = lsm['cell']
         pre = self.magic_escape
         pre2 = pre+pre
         
         # Completion logic:
         # - user gives %%: only do cell magics
         # - user gives %: do both line and cell magics
         # - no prefix: do both
         # In other words, line magics are skipped if the user gives %% explicitly
         bare_text = text.lstrip(pre)
         comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]
         if not text.startswith(pre2):
             comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
         return comp
 
     def alias_matches(self, text):
         """Match internal system aliases"""
         #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg
 
         # if we are not in the first 'item', alias matching
         # doesn't make sense - unless we are starting with 'sudo' command.
         main_text = self.text_until_cursor.lstrip()
         if ' ' in main_text and not main_text.startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
         if text == '':
             return aliases
         else:
             return [a for a in aliases if a.startswith(text)]
 
     def python_matches(self,text):
         """Match attributes or global python names"""
         
         #io.rprint('Completer->python_matches, txt=%r' % text) # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\._.*?',txt) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
 
         return matches
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
 
         if not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             # for classes, check for __init__,__new__
             if inspect.isclass(obj):
                 obj = (getattr(obj,'__init__',None) or
                        getattr(obj,'__new__',None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 obj = obj.__call__
             # XXX: is there a way to handle the builtins ?
         try:
             args,_,_1,defaults = inspect.getargspec(obj)
             if defaults:
                 return args[-len(defaults):]
         except TypeError: pass
         return []
 
     def python_func_kw_matches(self,text):
         """Match named parameters (kwargs) of the last open function"""
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?(?<!\\)' |    # single quoted strings or
                 ".*?(?<!\\)" |    # double quoted strings or
                 \w+          |    # identifier
                 \S                # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
         tokens.reverse()
         iterTokens = iter(tokens); openPar = 0
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
         while True:
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
                 break
         # lookup the candidate callable matches either using global_matches
         # or attr_matches for dotted names
         if len(ids) == 1:
             callableMatches = self.global_matches(ids[0])
         else:
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
             try:
                 namedArgs = self._default_arguments(eval(callableMatch,
                                                          self.namespace))
             except:
                 continue
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         return argMatches
 
     def dispatch_custom_completer(self, text):
         #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
         line = self.line_buffer
         if not line.strip():
             return None
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
         event = Bunch()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         event.text_until_cursor = self.text_until_cursor
 
         #print "\ncustom:{%s]\n" % event # dbg
 
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
                 self.magic_escape + cmd)
         else:
             try_magic = []
 
         for c in itertools.chain(self.custom_completers.s_matches(cmd),
                  try_magic,
                  self.custom_completers.flat_matches(self.text_until_cursor)):
             #print "try",c # dbg
             try:
                 res = c(event)
                 if res:
                     # first, try case sensitive match
                     withcase = [r for r in res if r.startswith(text)]
                     if withcase:
                         return withcase
                     # if none, then case insensitive ones are ok too
                     text_low = text.lower()
                     return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
 
         return None
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None):
         """Find completions for the given text and line context.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         Note that both the text and the line_buffer are optional, but at least
         one of them must be given.
 
         Parameters
         ----------
           text : string, optional
             Text to perform the completion on.  If not given, the line buffer
             is split using the instance's CompletionSplitter object.
 
           line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
             buffer via readline.  This keyword allows clients which are
             requesting for text completions in non-readline contexts to inform
             the completer of the entire text.
 
           cursor_pos : int, optional
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
 
         Returns
         -------
         text : str
           Text that was actually used in the completion.
 
         matches : list
           A list of completion matches.
         """
         #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
 
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
             cursor_pos = len(line_buffer) if text is None else len(text)
 
         # if text is either None or an empty string, rely on the line buffer
         if not text:
             text = self.splitter.split_line(line_buffer, cursor_pos)
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
         #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
 
         # Start with a clean slate of completions
         self.matches[:] = []
         custom_res = self.dispatch_custom_completer(text)
         if custom_res is not None:
             # did custom completers produce something?
             self.matches = custom_res
         else:
             # Extend the list of completions with the results of each
             # matcher, so we return results to the user from all
             # namespaces.
             if self.merge_completions:
                 self.matches = []
                 for matcher in self.matchers:
                     try:
                         self.matches.extend(matcher(text))
                     except:
                         # Show the ugly traceback if the matcher causes an
                         # exception, but do NOT crash the kernel!
                         sys.excepthook(*sys.exc_info())
             else:
                 for matcher in self.matchers:
                     self.matches = matcher(text)
                     if self.matches:
                         break
         # FIXME: we should extend our api to return a dict with completions for
         # different types of objects.  The rlcomplete() method could then
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other evironments.
         self.matches = sorted(set(self.matches))
         #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
         return text, self.matches
 
     def rlcomplete(self, text, state):
         """Return the state-th possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         Parameters
         ----------
           text : string
             Text to perform the completion on.
 
           state : int
             Counter used by readline.
         """
         if state==0:
 
             self.line_buffer = line_buffer = self.readline.get_line_buffer()
             cursor_pos = self.readline.get_endidx()
 
             #io.rprint("\nRLCOMPLETE: %r %r %r" %
             #          (text, line_buffer, cursor_pos) ) # dbg
 
             # if there is only a tab on a line with only whitespace, instead of
             # the mostly useless 'do you want to see all million completions'
             # message, just do the right thing and give the user his tab!
             # Incidentally, this enables pasting of tabbed text from an editor
             # (as long as autoindent is off).
 
             # It should be noted that at least pyreadline still shows file
             # completions - is there a way around it?
 
             # don't apply this on 'dumb' terminals, such as emacs buffers, so
             # we don't interfere with their own tab-completion mechanism.
             if not (self.dumb_terminal or line_buffer.strip()):
                 self.readline.insert_text('\t')
                 sys.stdout.flush()
                 return None
 
             # Note: debugging exceptions that may occur in completion is very
             # tricky, because readline unconditionally silences them.  So if
             # during development you suspect a bug in the completion code, turn
             # this flag on temporarily by uncommenting the second form (don't
             # flip the value in the first line, as the '# dbg' marker can be
             # automatically detected and is used elsewhere).
             DEBUG = False
             #DEBUG = True # dbg
             if DEBUG:
                 try:
                     self.complete(text, line_buffer, cursor_pos)
                 except:
                     import traceback; traceback.print_exc()
             else:
                 # The normal production version is here
 
                 # This method computes the self.matches array
                 self.complete(text, line_buffer, cursor_pos)
 
         try:
             return self.matches[state]
         except IndexError:
             return None

IPython/core/completerlib.py

@@ -1,333 +1,332 b''
 """Implementations for various useful completers.
 
 These are all loaded by default by IPython.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import glob
 import imp
 import inspect
 import os
 import re
 import sys
 
 # Third-party imports
 from time import time
 from zipimport import zipimporter
 
 # Our own imports
 from IPython.core.completer import expand_user, compress_user
 from IPython.core.error import TryNext
-from IPython.utils import py3compat
 from IPython.utils._process_common import arg_split
 
 # FIXME: this should be pulled in with the right call via the component system
 from IPython.core.ipapi import get as get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # Time in seconds after which the rootmodules will be stored permanently in the
 # ipython ip.db database (kept in the user's .ipython dir).
 TIMEOUT_STORAGE = 2
 
 # Time in seconds after which we give up
 TIMEOUT_GIVEUP = 20
 
 # Regular expression for the python import statement
 import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
                        r'(?P<package>[/\\]__init__)?'
                        r'(?P<suffix>%s)$' %
                        r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))
 
 # RE for the ipython %run command (python + ipython scripts)
 magic_run_re = re.compile(r'.*(\.ipy|\.py[w]?)$')
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 def module_list(path):
     """
     Return the list containing the names of the modules available in the given
     folder.
     """
     # sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
     if path == '':
         path = '.'
 
     # A few local constants to be used in loops below
     pjoin = os.path.join
 
     if os.path.isdir(path):
         # Build a list of all files in the directory and all files
         # in its subdirectories. For performance reasons, do not
         # recurse more than one level into subdirectories.
         files = []
         for root, dirs, nondirs in os.walk(path):
             subdir = root[len(path)+1:]
             if subdir:
                 files.extend(pjoin(subdir, f) for f in nondirs)
                 dirs[:] = [] # Do not recurse into additional subdirectories.
             else:
                 files.extend(nondirs)
 
     else:
         try:
             files = list(zipimporter(path)._files.keys())
         except:
             files = []
 
     # Build a list of modules which match the import_re regex.
     modules = []
     for f in files:
         m = import_re.match(f)
         if m:
             modules.append(m.group('name'))
     return list(set(modules))
 
 def get_root_modules():
     """
     Returns a list containing the names of all the modules available in the
     folders of the pythonpath.
     """
     ip = get_ipython()
 
     if 'rootmodules' in ip.db:
         return ip.db['rootmodules']
 
     t = time()
     store = False
     modules = list(sys.builtin_module_names)
     for path in sys.path:
         modules += module_list(path)
         if time() - t >= TIMEOUT_STORAGE and not store:
             store = True
             print("\nCaching the list of root modules, please wait!")
             print("(This will only be done once - type '%rehashx' to "
                   "reset cache!)\n")
             sys.stdout.flush()
         if time() - t > TIMEOUT_GIVEUP:
             print("This is taking too long, we give up.\n")
             ip.db['rootmodules'] = []
             return []
 
     modules = set(modules)
     if '__init__' in modules:
         modules.remove('__init__')
     modules = list(modules)
     if store:
         ip.db['rootmodules'] = modules
     return modules
 
 
 def is_importable(module, attr, only_modules):
     if only_modules:
         return inspect.ismodule(getattr(module, attr))
     else:
         return not(attr[:2] == '__' and attr[-2:] == '__')
 
 
 def try_import(mod, only_modules=False):
     try:
         m = __import__(mod)
     except:
         return []
     mods = mod.split('.')
     for module in mods[1:]:
         m = getattr(m, module)
 
     m_is_init = hasattr(m, '__file__') and '__init__' in m.__file__
 
     completions = []
     if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
         completions.extend( [attr for attr in dir(m) if
                              is_importable(m, attr, only_modules)])
 
     completions.extend(getattr(m, '__all__', []))
     if m_is_init:
         completions.extend(module_list(os.path.dirname(m.__file__)))
     completions = set(completions)
     if '__init__' in completions:
         completions.remove('__init__')
     return list(completions)
 
 
 #-----------------------------------------------------------------------------
 # Completion-related functions.
 #-----------------------------------------------------------------------------
 
 def quick_completer(cmd, completions):
     """ Easily create a trivial completer for a command.
 
     Takes either a list of completions, or all completions in string (that will
     be split on whitespace).
 
     Example::
 
         [d:\ipython]|1> import ipy_completers
         [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
         [d:\ipython]|3> foo b<TAB>
         bar baz
         [d:\ipython]|3> foo ba
     """
 
     if isinstance(completions, basestring):
         completions = completions.split()
 
     def do_complete(self, event):
         return completions
 
     get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
 
 def module_completion(line):
     """
     Returns a list containing the completion possibilities for an import line.
 
     The line looks like this :
     'import xml.d'
     'from xml.dom import'
     """
 
     words = line.split(' ')
     nwords = len(words)
 
     # from whatever <tab> -> 'import '
     if nwords == 3 and words[0] == 'from':
         return ['import ']
 
     # 'from xy<tab>' or 'import xy<tab>'
     if nwords < 3 and (words[0] in ['import','from']) :
         if nwords == 1:
             return get_root_modules()
         mod = words[1].split('.')
         if len(mod) < 2:
             return get_root_modules()
         completion_list = try_import('.'.join(mod[:-1]), True)
         return ['.'.join(mod[:-1] + [el]) for el in completion_list]
 
     # 'from xyz import abc<tab>'
     if nwords >= 3 and words[0] == 'from':
         mod = words[1]
         return try_import(mod)
 
 #-----------------------------------------------------------------------------
 # Completers
 #-----------------------------------------------------------------------------
 # These all have the func(self, event) signature to be used as custom
 # completers
 
 def module_completer(self,event):
     """Give completions after user has typed 'import ...' or 'from ...'"""
 
     # This works in all versions of python.  While 2.5 has
     # pkgutil.walk_packages(), that particular routine is fairly dangerous,
     # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
     # of possibly problematic side effects.
     # This search the folders in the sys.path for available modules.
 
     return module_completion(event.line)
 
 # FIXME: there's a lot of logic common to the run, cd and builtin file
 # completers, that is currently reimplemented in each.
 
 def magic_run_completer(self, event):
     """Complete files that end in .py or .ipy for the %run command.
     """
     comps = arg_split(event.line, strict=False)
     relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
 
     #print("\nev=", event)  # dbg
     #print("rp=", relpath)  # dbg
     #print('comps=', comps)  # dbg
 
     lglob = glob.glob
     isdir = os.path.isdir
     relpath, tilde_expand, tilde_val = expand_user(relpath)
 
     dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
 
     # Find if the user has already typed the first filename, after which we
     # should complete on all files, since after the first one other files may
     # be arguments to the input script.
 
     if filter(magic_run_re.match, comps):
         pys =  [f.replace('\\','/') for f in lglob('*')]
     else:
         pys =  [f.replace('\\','/')
                 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
                 lglob(relpath + '*.pyw')]
     #print('run comp:', dirs+pys) # dbg
     return [compress_user(p, tilde_expand, tilde_val) for p in dirs+pys]
 
 
 def cd_completer(self, event):
     """Completer function for cd, which only returns directories."""
     ip = get_ipython()
     relpath = event.symbol
 
     #print(event) # dbg
     if event.line.endswith('-b') or ' -b ' in event.line:
         # return only bookmark completions
         bkms = self.db.get('bookmarks', None)
         if bkms:
             return bkms.keys()
         else:
             return []
 
     if event.symbol == '-':
         width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
         # jump in directory history by number
         fmt = '-%0' + width_dh +'d [%s]'
         ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
         if len(ents) > 1:
             return ents
         return []
 
     if event.symbol.startswith('--'):
         return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
 
     # Expand ~ in path and normalize directory separators.
     relpath, tilde_expand, tilde_val = expand_user(relpath)
     relpath = relpath.replace('\\','/')
 
     found = []
     for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
               if os.path.isdir(f)]:
         if ' ' in d:
             # we don't want to deal with any of that, complex code
             # for this is elsewhere
             raise TryNext
 
         found.append(d)
 
     if not found:
         if os.path.isdir(relpath):
             return [compress_user(relpath, tilde_expand, tilde_val)]
 
         # if no completions so far, try bookmarks
         bks = self.db.get('bookmarks',{}).iterkeys()
         bkmatches = [s for s in bks if s.startswith(event.symbol)]
         if bkmatches:
             return bkmatches
 
         raise TryNext
 
     return [compress_user(p, tilde_expand, tilde_val) for p in found]
 
 def reset_completer(self, event):
     "A completer for %reset magic"
     return '-f -s in out array dhist'.split()

IPython/core/debugger.py

@@ -1,566 +1,566 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 http://www.python.org/2.2.3/license.html"""
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import bdb
 import functools
 import linecache
 import sys
 
 from IPython.utils import PyColorize, ulinecache
 from IPython.core import ipapi
-from IPython.utils import coloransi, io, openpy, py3compat
+from IPython.utils import coloransi, io, py3compat
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.
 has_pydb = False
 prompt = 'ipdb> '
 #We have to check this directly from sys.argv, config struct not yet available
 if '--pydb' in sys.argv:
     try:
         import pydb
         if hasattr(pydb.pydb, "runl") and pydb.version>'1.17':
             # Version 1.17 is broken, and that's what ships with Ubuntu Edgy, so we
             # better protect against it.
             has_pydb = True
     except ImportError:
         print("Pydb (http://bashdb.sourceforge.net/pydb/) does not seem to be available")
 
 if has_pydb:
     from pydb import Pdb as OldPdb
     #print "Using pydb for %run -d and post-mortem" #dbg
     prompt = 'ipydb> '
 else:
     from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     def __init__(self,colors=None):
         """Create a local debugger instance.
 
         :Parameters:
 
           - `colors` (None): a string containing the name of the color scheme to
         use, it must be one of IPython's valid color schemes.  If not given, the
         function will default to the current IPython scheme when running inside
         IPython, and to 'NoColor' otherwise.
 
         Usage example:
 
         from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         ... later in your code
         debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
 
         try:
             ip = get_ipython()
         except NameError:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
             try:
                 # Limited tab completion support
                 import readline
                 readline.parse_and_bind('tab: complete')
             except ImportError:
                 pass
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             from repr import aRepr
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
     def __init__(self,color_scheme='NoColor',completekey=None,
                  stdin=None, stdout=None):
 
         # Parent constructor:
         if has_pydb and completekey is None:
             OldPdb.__init__(self,stdin=stdin,stdout=io.stdout)
         else:
             OldPdb.__init__(self,completekey,stdin,stdout)
 
         self.prompt = prompt # The default prompt is '(Pdb)'
 
         # IPython changes...
         self.is_pydb = has_pydb
 
         self.shell = ipapi.get()
 
         if self.is_pydb:
 
             # interactiveshell.py's ipalias seems to want pdb's checkline
             # which located in pydb.fn
             import pydb.fns
             self.checkline = lambda filename, lineno: \
                              pydb.fns.checkline(self, filename, lineno)
 
             self.curframe = None
             self.do_restart = self.new_do_restart
 
             self.old_all_completions = self.shell.Completer.all_completions
             self.shell.Completer.all_completions=self.all_completions
 
             self.do_list = decorate_fn_with_doc(self.list_command_pydb,
                                                 OldPdb.do_list)
             self.do_l     = self.do_list
             self.do_frame = decorate_fn_with_doc(self.new_do_frame,
                                                  OldPdb.do_frame)
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         self.set_colors(color_scheme)
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser()
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
 
     def interaction(self, frame, traceback):
         self.shell.set_completer_frame(frame)
         OldPdb.interaction(self, frame, traceback)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
         self.shell.set_completer_frame(self.curframe)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
         self.shell.set_completer_frame(self.curframe)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def postloop(self):
         self.shell.set_completer_frame(None)
 
     def print_stack_trace(self):
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context = 5)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno,prompt_prefix='\n-> ',
                           context = 3):
         #frame, lineno = frame_lineno
         print(self.format_stack_entry(frame_lineno, '', context), file=io.stdout)
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
         import repr
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += repr.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = repr.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = ulinecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
         scheme = self.color_scheme_table.active_scheme_name
         new_line, err = self.parser.format2(line, 'str', scheme)
         if not err: line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                  marker = '> '
             elif pad == 1:
                  marker = '>'
             else:
                  marker = ''
             num = '%s%s' % (marker, str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
             line = tpl_line % (bp_mark_color + bp_mark, num, line)
 
         return line
 
     def list_command_pydb(self, arg):
         """List command to use if we have a newer pydb installed"""
         filename, first, last = OldPdb.parse_list_cmd(self, arg)
         if filename is not None:
             self.print_list_lines(filename, first, last)
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = ulinecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src), file=io.stdout)
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
 
     def checkline(self, filename, lineno):
         """Check whether specified line seems to be executable.
 
         Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
         line or EOF). Warning: testing is not comprehensive.
         """
         #######################################################################
         # XXX Hack!  Use python-2.5 compatible code for this call, because with
         # all of our changes, we've drifted from the pdb api in 2.6.  For now,
         # changing:
         #
         #line = linecache.getline(filename, lineno, self.curframe.f_globals)
         # to:
         #
         line = linecache.getline(filename, lineno)
         #
         # does the trick.  But in reality, we need to fix this by reconciling
         # our updates with the new Pdb APIs in Python 2.6.
         #
         # End hack.  The rest of this method is copied verbatim from 2.6 pdb.py
         #######################################################################
 
         if not line:
             print('End of file', file=self.stdout)
             return 0
         line = line.strip()
         # Don't allow setting breakpoint at a blank line
         if (not line or (line[0] == '#') or
              (line[:3] == '"""') or line[:3] == "'''"):
             print('*** Blank or comment', file=self.stdout)
             return 0
         return lineno

IPython/core/displayhook.py

@@ -1,275 +1,275 b''
 # -*- coding: utf-8 -*-
 """Displayhook for IPython.
 
 This defines a callable class that IPython uses for `sys.displayhook`.
 
 Authors:
 
 * Fernando Perez
 * Brian Granger
 * Robert Kern
 """
 
 #-----------------------------------------------------------------------------
 #       Copyright (C) 2008-2011 The IPython Development Team
 #       Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import __builtin__
 
 import sys
 
 
 from IPython.config.configurable import Configurable
 from IPython.utils import io
-from IPython.utils.traitlets import Instance, List
+from IPython.utils.traitlets import Instance
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
 # Main displayhook class
 #-----------------------------------------------------------------------------
 
 # TODO: Move the various attributes (cache_size, [others now moved]). Some
 # of these are also attributes of InteractiveShell. They should be on ONE object
 # only and the other objects should ask that one object for their values.
 
 class DisplayHook(Configurable):
     """The custom IPython displayhook to replace sys.displayhook.
 
     This class does many things, but the basic idea is that it is a callable
     that gets called anytime user code returns a value.
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def __init__(self, shell=None, cache_size=1000, config=None):
         super(DisplayHook, self).__init__(shell=shell, config=config)
 
         cache_size_min = 3
         if cache_size <= 0:
             self.do_full_cache = 0
             cache_size = 0
         elif cache_size < cache_size_min:
             self.do_full_cache = 0
             cache_size = 0
             warn('caching was disabled (min value for cache size is %s).' %
                  cache_size_min,level=3)
         else:
             self.do_full_cache = 1
 
         self.cache_size = cache_size
 
         # we need a reference to the user-level namespace
         self.shell = shell
         
         self._,self.__,self.___ = '','',''
 
         # these are deliberately global:
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.shell.user_ns.update(to_user_ns)
 
     @property
     def prompt_count(self):
         return self.shell.execution_count
 
     #-------------------------------------------------------------------------
     # Methods used in __call__. Override these methods to modify the behavior
     # of the displayhook.
     #-------------------------------------------------------------------------
 
     def check_for_underscore(self):
         """Check if the user has set the '_' variable by hand."""
         # If something injected a '_' variable in __builtin__, delete
         # ipython's automatic one so we don't clobber that.  gettext() in
         # particular uses _, so we need to stay away from it.
         if '_' in __builtin__.__dict__:
             try:
                 del self.shell.user_ns['_']
             except KeyError:
                 pass
 
     def quiet(self):
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         try:
             cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
             if cell.rstrip().endswith(';'):
                 return True
         except IndexError:
             # some uses of ipshellembed may fail here
             pass
         return False
 
     def start_displayhook(self):
         """Start the displayhook, initializing resources."""
         pass
 
     def write_output_prompt(self):
         """Write the output prompt.
 
         The default implementation simply writes the prompt to
         ``io.stdout``.
         """
         # Use write, not print which adds an extra space.
         io.stdout.write(self.shell.separate_out)
         outprompt = self.shell.prompt_manager.render('out')
         if self.do_full_cache:
             io.stdout.write(outprompt)
 
     def compute_format_data(self, result):
         """Compute format data of the object to be displayed.
 
         The format data is a generalization of the :func:`repr` of an object.
         In the default implementation the format data is a :class:`dict` of
         key value pair where the keys are valid MIME types and the values
         are JSON'able data structure containing the raw data for that MIME
         type. It is up to frontends to determine pick a MIME to to use and
         display that data in an appropriate manner.
 
         This method only computes the format data for the object and should
         NOT actually print or write that to a stream.
 
         Parameters
         ----------
         result : object
             The Python object passed to the display hook, whose format will be
             computed.
 
         Returns
         -------
         format_data : dict
             A :class:`dict` whose keys are valid MIME types and values are
             JSON'able raw data for that MIME type. It is recommended that
             all return values of this should always include the "text/plain"
             MIME type representation of the object.
         """
         return self.shell.display_formatter.format(result)
 
     def write_format_data(self, format_dict):
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text
         representation of the object to ``io.stdout``. Subclasses should
         override this method to send the entire `format_dict` to the
         frontends.
 
         Parameters
         ----------
         format_dict : dict
             The format dict for the object passed to `sys.displayhook`.
         """
         # We want to print because we want to always make sure we have a
         # newline, even if all the prompt separators are ''. This is the
         # standard IPython behavior.
         result_repr = format_dict['text/plain']
         if '\n' in result_repr:
             # So that multi-line strings line up with the left column of
             # the screen, instead of having the output prompt mess up
             # their first line.
             # We use the prompt template instead of the expanded prompt
             # because the expansion may add ANSI escapes that will interfere
             # with our ability to determine whether or not we should add
             # a newline.
             prompt_template = self.shell.prompt_manager.out_template
             if prompt_template and not prompt_template.endswith('\n'):
                 # But avoid extraneous empty lines.
                 result_repr = '\n' + result_repr
 
         print(result_repr, file=io.stdout)
 
     def update_user_ns(self, result):
         """Update user_ns with various things like _, __, _1, etc."""
 
         # Avoid recursive reference when displaying _oh/Out
         if result is not self.shell.user_ns['_oh']:
             if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                 warn('Output cache limit (currently '+
                       repr(self.cache_size)+' entries) hit.\n'
                      'Flushing cache and resetting history counter...\n'
                      'The only history variables available will be _,__,___ and _1\n'
                      'with the current result.')
 
                 self.flush()
             # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
             # we cause buggy behavior for things like gettext).
 
             if '_' not in __builtin__.__dict__:
                 self.___ = self.__
                 self.__ = self._
                 self._ = result
                 self.shell.push({'_':self._,
                                  '__':self.__,
                                 '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_'+repr(self.prompt_count)
                 to_main[new_result] = result
                 self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def log_output(self, format_dict):
         """Log the output."""
         if self.shell.logger.log_output:
             self.shell.logger.log_write(format_dict['text/plain'], 'output')
         self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                     format_dict['text/plain']
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
         io.stdout.write(self.shell.separate_out2)
         io.stdout.flush()
 
     def __call__(self, result=None):
         """Printing with history cache management.
 
         This is invoked everytime the interpreter needs to print, and is
         activated by setting the variable sys.displayhook to it.
         """
         self.check_for_underscore()
         if result is not None and not self.quiet():
             self.start_displayhook()
             self.write_output_prompt()
             format_dict = self.compute_format_data(result)
             self.write_format_data(format_dict)
             self.update_user_ns(result)
             self.log_output(format_dict)
             self.finish_displayhook()
 
     def flush(self):
         if not self.do_full_cache:
             raise ValueError("You shouldn't have reached the cache flush "
                              "if full caching is not enabled!")
         # delete auto-generated vars from global namespace
 
         for n in range(1,self.prompt_count + 1):
             key = '_'+repr(n)
             try:
                 del self.shell.user_ns[key]
             except: pass
         # In some embedded circumstances, the user_ns doesn't have the
         # '_oh' key set up.
         oh = self.shell.user_ns.get('_oh', None)
         if oh is not None:
             oh.clear()
 
         # Release our own references to objects:
         self._, self.__, self.___ = '', '', ''
 
         if '_' not in __builtin__.__dict__:
             self.shell.user_ns.update({'_':None,'__':None, '___':None})
         import gc
         # TODO: Is this really needed?
         # IronPython blocks here forever
         if sys.platform != "cli":
             gc.collect()
 

IPython/core/macro.py

@@ -1,60 +1,59 b''
 """Support for interactive macros in IPython"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2005 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 import re
-import sys
 
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 
 coding_declaration = re.compile(r"#\s*coding[:=]\s*([-\w.]+)")
 
 class Macro(object):
     """Simple class to store the value of macros as strings.
 
     Macro is just a callable that executes a string of IPython
     input when called.
     
     Args to macro are available in _margv list if you need them.
     """
 
     def __init__(self,code):
         """store the macro value, as a single string which can be executed"""
         lines = []
         enc = None
         for line in code.splitlines():
             coding_match = coding_declaration.match(line)
             if coding_match:
                 enc = coding_match.group(1)
             else:
                 lines.append(line)
         code = "\n".join(lines)
         if isinstance(code, bytes):
             code = code.decode(enc or DEFAULT_ENCODING)
         self.value = code + '\n'
     
     def __str__(self):
         return py3compat.unicode_to_str(self.value)
     
     def __unicode__(self):
         return self.value
 
     def __repr__(self):
         return 'IPython.macro.Macro(%s)' % repr(self.value)
     
     def __getstate__(self):
         """ needed for safe pickling via %store """
         return {'value': self.value}
     
     def __add__(self, other):
         if isinstance(other, Macro):
             return Macro(self.value + other.value)
         elif isinstance(other, basestring):
             return Macro(self.value + other)
         raise TypeError

IPython/core/magic.py

@@ -1,683 +1,683 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008 The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # Stdlib
 import os
 import re
 import sys
 import types
 from getopt import getopt, GetoptError
 
 # Our own
 from IPython.config.configurable import Configurable
 from IPython.core import oinspect
 from IPython.core.error import UsageError
 from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
 from IPython.external.decorator import decorator
 from IPython.utils.ipstruct import Struct
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent
 from IPython.utils.traitlets import Bool, Dict, Instance, MetaHasTraits
-from IPython.utils.warn import error, warn
+from IPython.utils.warn import error
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # A dict we'll use for each class that has magics, used as temporary storage to
 # pass information between the @line/cell_magic method decorators and the
 # @magics_class class decorator, because the method decorators have no
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
 magics = dict(line={}, cell={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
 magic_escapes = dict(line=ESC_MAGIC, cell=ESC_MAGIC2)
 
 #-----------------------------------------------------------------------------
 # Utility classes and functions
 #-----------------------------------------------------------------------------
 
 class Bunch: pass
 
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 
 def compress_dhist(dh):
     """Compress a directory history into a new one with at most 20 entries.
 
     Return a new list made from the first and last 10 elements of dhist after
     removal of duplicates.
     """
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail
 
 
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
 #-----------------------------------------------------------------------------
 # Class and method decorators for registering magics
 #-----------------------------------------------------------------------------
 
 def magics_class(cls):
     """Class decorator for all subclasses of the main Magics class.
 
     Any class that subclasses Magics *must* also apply this decorator, to
     ensure that all the methods that have been decorated as line/cell magics
     get correctly registered in the class instance.  This is necessary because
     when method decorators run, the class does not exist yet, so they
     temporarily store their information into a module global.  Application of
     this class decorator copies that global data to the class instance and
     clears the global.
 
     Obviously, this mechanism is not thread-safe, which means that the
     *creation* of subclasses of Magic should only be done in a single-thread
     context.  Instantiation of the classes has no restrictions.  Given that
     these classes are typically created at IPython startup time and before user
     application code becomes active, in practice this should not pose any
     problems.
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
                       cell = magics['cell'])
     magics['line'] = {}
     magics['cell'] = {}
     return cls
 
 
 def record_magic(dct, magic_kind, magic_name, func):
     """Utility function to store a function as a magic of a specific kind.
 
     Parameters
     ----------
     dct : dict
       A dictionary with 'line' and 'cell' subdicts.
 
     magic_kind : str
       Kind of magic to be stored.
 
     magic_name : str
       Key to store the magic as.
 
     func : function
       Callable object to store.
     """
     if magic_kind == 'line_cell':
         dct['line'][magic_name] = dct['cell'][magic_name] = func
     else:
         dct[magic_kind][magic_name] = func
 
 
 def validate_type(magic_kind):
     """Ensure that the given magic_kind is valid.
 
     Check that the given magic_kind is one of the accepted spec types (stored
     in the global `magic_spec`), raise ValueError otherwise.
     """
     if magic_kind not in magic_spec:
         raise ValueError('magic_kind must be one of %s, %s given' %
                          magic_kinds, magic_kind)
 
 
 # The docstrings for the decorator below will be fairly similar for the two
 # types (method and function), so we generate them here once and reuse the
 # templates below.
 _docstring_template = \
 """Decorate the given {0} as {1} magic.
 
 The decorator can be used with or without arguments, as follows.
 
 i) without arguments: it will create a {1} magic named as the {0} being
 decorated::
 
     @deco
     def foo(...)
 
 will create a {1} magic named `foo`.
 
 ii) with one string argument: which will be used as the actual name of the
 resulting magic::
 
     @deco('bar')
     def foo(...)
 
 will create a {1} magic named `bar`.
 """
 
 # These two are decorator factories.  While they are conceptually very similar,
 # there are enough differences in the details that it's simpler to have them
 # written as completely standalone functions rather than trying to share code
 # and make a single one with convoluted logic.
 
 def _method_magic_marker(magic_kind):
     """Decorator factory for methods in Magics subclasses.
     """
 
     validate_type(magic_kind)
 
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.func_name
             retval = decorator(call, func)
             record_magic(magics, magic_kind, name, name)
         elif isinstance(arg, basestring):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 record_magic(magics, magic_kind, name, func.func_name)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                             "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     magic_deco.__doc__ = _docstring_template.format('method', magic_kind)
     return magic_deco
 
 
 def _function_magic_marker(magic_kind):
     """Decorator factory for standalone functions.
     """
     validate_type(magic_kind)
     
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         # Find get_ipython() in the caller's namespace
         caller = sys._getframe(1)
         for ns in ['f_locals', 'f_globals', 'f_builtins']:
             get_ipython = getattr(caller, ns).get('get_ipython')
             if get_ipython is not None:
                 break
         else:
             raise NameError('Decorator can only run in context where '
                             '`get_ipython` exists')
 
         ip = get_ipython()
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.func_name
             ip.register_magic_function(func, magic_kind, name)
             retval = decorator(call, func)
         elif isinstance(arg, basestring):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 ip.register_magic_function(func, magic_kind, name)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                              "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     ds = _docstring_template.format('function', magic_kind)
 
     ds += dedent("""
     Note: this decorator can only be used in a context where IPython is already
     active, so that the `get_ipython()` call succeeds.  You can therefore use
     it in your startup files loaded after IPython initializes, but *not* in the
     IPython configuration file itself, which is executed before IPython is
     fully up and running.  Any file located in the `startup` subdirectory of
     your configuration profile will be OK in this sense.
     """)
     
     magic_deco.__doc__ = ds
     return magic_deco
 
 
 # Create the actual decorators for public use
 
 # These three are used to decorate methods in class definitions
 line_magic = _method_magic_marker('line')
 cell_magic = _method_magic_marker('cell')
 line_cell_magic = _method_magic_marker('line_cell')
 
 # These three decorate standalone functions and perform the decoration
 # immediately.  They can only run where get_ipython() works
 register_line_magic = _function_magic_marker('line')
 register_cell_magic = _function_magic_marker('cell')
 register_line_cell_magic = _function_magic_marker('line_cell')
 
 #-----------------------------------------------------------------------------
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
     # Non-configurable class attributes
 
     # A two-level dict, first keyed by magic type, then by magic function, and
     # holding the actual callable object as value.  This is the dict used for
     # magic function dispatch
     magics = Dict
 
     # A registry of the original objects that we've been given holding magics.
     registry = Dict
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     auto_magic = Bool(True, config=True, help=
         "Automatically call line magics without requiring explicit % prefix")
     
     _auto_status = [
         'Automagic is OFF, % prefix IS needed for line magics.',
         'Automagic is ON, % prefix IS NOT needed for line magics.']
 
     user_magics = Instance('IPython.core.magics.UserMagics')
 
     def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
         self.magics = dict(line={}, cell={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
 
     def auto_status(self):
         """Return descriptive string with automagic status."""
         return self._auto_status[self.auto_magic]
     
     def lsmagic_info(self):
         magic_list = []
         for m_type in self.magics :
             for m_name,mgc in self.magics[m_type].items():
                 try :
                     magic_list.append({'name':m_name,'type':m_type,'class':mgc.im_class.__name__})
                 except AttributeError :
                     magic_list.append({'name':m_name,'type':m_type,'class':'Other'})
         return magic_list
 
     def lsmagic(self):
         """Return a dict of currently available magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support.  Each value is a list of names.
         """
         return self.magics
 
     def lsmagic_docs(self, brief=False, missing=''):
         """Return dict of documentation of magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support. Each value is a dict keyed by magic
         name whose value is the function docstring. If a docstring is
         unavailable, the value of `missing` is used instead.
 
         If brief is True, only the first line of each docstring will be returned.
         """
         docs = {}
         for m_type in self.magics:
             m_docs = {}
             for m_name, m_func in self.magics[m_type].iteritems():
                 if m_func.__doc__:
                     if brief:
                         m_docs[m_name] = m_func.__doc__.split('\n', 1)[0]
                     else:
                         m_docs[m_name] = m_func.__doc__.rstrip()
                 else:
                     m_docs[m_name] = missing
             docs[m_type] = m_docs
         return docs
 
     def register(self, *magic_objects):
         """Register one or more instances of Magics.
 
         Take one or more classes or instances of classes that subclass the main 
         `core.Magic` class, and register them with IPython to use the magic
         functions they provide.  The registration process will then ensure that
         any methods that have decorated to provide line and/or cell magics will
         be recognized with the `%x`/`%%x` syntax as a line/cell magic
         respectively.
 
         If classes are given, they will be instantiated with the default
         constructor.  If your classes need a custom constructor, you should
         instanitate them first and pass the instance.
 
         The provided arguments can be an arbitrary mix of classes and instances.
 
         Parameters
         ----------
         magic_objects : one or more classes or instances
         """
         # Start by validating them to ensure they have all had their magic
         # methods registered at the instance level
         for m in magic_objects:
             if not m.registered:
                 raise ValueError("Class of magics %r was constructed without "
                                  "the @register_magics class decorator")
             if type(m) in (type, MetaHasTraits):
                 # If we're given an uninstantiated class
                 m = m(shell=self.shell)
 
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
             for mtype in magic_kinds:
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
         """Expose a standalone function as magic function for IPython.
 
         This will create an IPython magic (line, cell or both) from a
         standalone function.  The functions should have the following
         signatures: 
 
         * For line magics: `def f(line)`
         * For cell magics: `def f(line, cell)`
         * For a function that does both: `def f(line, cell=None)`
 
         In the latter case, the function will be called with `cell==None` when
         invoked as `%f`, and with cell as a string when invoked as `%%f`.
 
         Parameters
         ----------
         func : callable
           Function to be registered as a magic.
 
         magic_kind : str
           Kind of magic, one of 'line', 'cell' or 'line_cell'
 
         magic_name : optional str
           If given, the name the magic will have in the IPython namespace.  By
           default, the name of the function itself is used.
         """
 
         # Create the new method in the user_magics and register it in the
         # global table
         validate_type(magic_kind)
         magic_name = func.func_name if magic_name is None else magic_name
         setattr(self.user_magics, magic_name, func)
         record_magic(self.magics, magic_kind, magic_name, func)
 
     def define_magic(self, name, func):
         """[Deprecated] Expose own function as magic function for IPython.
 
         Example::
 
             def foo_impl(self, parameter_s=''):
                 'My very own magic!. (Use docstrings, IPython reads them).'
                 print 'Magic function. Passed parameter is between < >:'
                 print '<%s>' % parameter_s
                 print 'The self object is:', self
 
             ip.define_magic('foo',foo_impl)
         """
         meth = types.MethodType(func, self.user_magics)
         setattr(self.user_magics, name, meth)
         record_magic(self.magics, 'line', name, meth)
 
     def register_alias(self, alias_name, magic_name, magic_kind='line'):
         """Register an alias to a magic function.
 
         The alias is an instance of :class:`MagicAlias`, which holds the
         name and kind of the magic it should call. Binding is done at
         call time, so if the underlying magic function is changed the alias
         will call the new function.
 
         Parameters
         ----------
         alias_name : str
           The name of the magic to be registered.
 
         magic_name : str
           The name of an existing magic.
 
         magic_kind : str
           Kind of magic, one of 'line' or 'cell'
         """
 
         # `validate_type` is too permissive, as it allows 'line_cell'
         # which we do not handle.
         if magic_kind not in magic_kinds:
             raise ValueError('magic_kind must be one of %s, %s given' %
                              magic_kinds, magic_kind)
 
         alias = MagicAlias(self.shell, magic_name, magic_kind)
         setattr(self.user_magics, alias_name, alias)
         record_magic(self.magics, magic_kind, alias_name, alias)
 
 # Key base class that provides the central functionality for magics.
 
 class Magics(object):
     """Base class for implementing magic functions.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     Classes providing magic functions need to subclass this class, and they
     MUST:
 
     - Use the method decorators `@line_magic` and `@cell_magic` to decorate
     individual methods as magic functions, AND
 
     - Use the class decorator `@magics_class` to ensure that the magic
     methods are properly registered at the instance level upon instance
     initialization.
 
     See :mod:`magic_functions` for examples of actual implementation classes.
     """
     # Dict holding all command-line options for each magic.
     options_table = None
     # Dict for the mapping of magic names to methods, set by class decorator
     magics = None
     # Flag to check that the class decorator was properly applied
     registered = False
     # Instance of IPython shell
     shell = None
 
     def __init__(self, shell):
         if not(self.__class__.registered):
             raise ValueError('Magics subclass without registration - '
                              'did you forget to apply @magics_class?')
         self.shell = shell
         self.options_table = {}
         # The method decorators are run when the instance doesn't exist yet, so
         # they can only record the names of the methods they are supposed to
         # grab.  Only now, that the instance exists, can we create the proper
         # mapping to bound methods.  So we read the info off the original names
         # table and replace each method name by the actual bound method.
         # But we mustn't clobber the *class* mapping, in case of multiple instances.
         class_magics = self.magics
         self.magics = {}
         for mtype in magic_kinds:
             tab = self.magics[mtype] = {}
             cls_tab = class_magics[mtype]
             for magic_name, meth_name in cls_tab.iteritems():
                 if isinstance(meth_name, basestring):
                     # it's a method name, grab it
                     tab[magic_name] = getattr(self, meth_name)
                 else:
                     # it's the real thing
                     tab[magic_name] = meth_name
 
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print oinspect.getdoc(func)
 
     def format_latex(self, strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self, arg_str, opt_str, *long_opts, **kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
 
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
 
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError('incorrect mode given: %s' % mode)
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
         strict = kw.get('strict', True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str, posix, strict)
             # Do regular option processing
             try:
                 opts,args = getopt(argv, opt_str, long_opts)
             except GetoptError as e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
 
     def default_option(self, fn, optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
 class MagicAlias(object):
     """An alias to another magic function.
 
     An alias is determined by its magic name and magic kind. Lookup
     is done at call time, so if the underlying magic changes the alias
     will call the new function.
 
     Use the :meth:`MagicsManager.register_alias` method or the
     `%alias_magic` magic function to create and register a new alias.
     """
     def __init__(self, shell, magic_name, magic_kind):
         self.shell = shell
         self.magic_name = magic_name
         self.magic_kind = magic_kind
 
         self.pretty_target = '%s%s' % (magic_escapes[self.magic_kind], self.magic_name)
         self.__doc__ = "Alias for `%s`." % self.pretty_target
 
         self._in_call = False
 
     def __call__(self, *args, **kwargs):
         """Call the magic alias."""
         fn = self.shell.find_magic(self.magic_name, self.magic_kind)
         if fn is None:
             raise UsageError("Magic `%s` not found." % self.pretty_target)
 
         # Protect against infinite recursion.
         if self._in_call:
             raise UsageError("Infinite recursion detected; "
                              "magic aliases cannot call themselves.")
         self._in_call = True
         try:
             return fn(*args, **kwargs)
         finally:
             self._in_call = False

IPython/core/magics/code.py

@@ -1,559 +1,558 b''
 """Implementation of code management magic functions.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import inspect
 import io
 import os
 import re
 import sys
 
 # Our own packages
 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.core.oinspect import find_file, find_source_lines
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.io import file_read
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 # Used for exception handling in magic_edit
 class MacroToEdit(ValueError): pass
 
 ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
 
 class InteractivelyDefined(Exception):
     """Exception for interactively defined variable in magic_edit"""
     def __init__(self, index):
         self.index = index
 
 
 @magics_class
 class CodeMagics(Magics):
     """Magics related to code management (loading, saving, editing, ...)."""
 
     @line_magic
     def save(self, parameter_s=''):
         """Save a set of lines or a macro to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
           
           -f: force overwrite.  If file exists, %save will prompt for overwrite
           unless -f is given.
 
           -a: append to the file instead of overwriting it.
 
         This function uses the same syntax as %history for input ranges,
         then saves the lines to the filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files.
 
         If `-r` option is used, the default extension is `.ipy`.
         """
 
         opts,args = self.parse_options(parameter_s,'fra',mode='list')
         if not args:
             raise UsageError('Missing filename.')
         raw = 'r' in opts
         force = 'f' in opts
         append = 'a' in opts
         mode = 'a' if append else 'w'
         ext = u'.ipy' if raw else u'.py'
         fname, codefrom = unquote_filename(args[0]), " ".join(args[1:])
         if not fname.endswith((u'.py',u'.ipy')):
             fname += ext
         file_exists = os.path.isfile(fname)
         if file_exists and not force and not append:
             try:
                 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
             except StdinNotImplementedError:
                 print "File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s)
                 return
             if not overwrite :
                 print 'Operation cancelled.'
                 return
         try:
             cmds = self.shell.find_user_code(codefrom,raw)
         except (TypeError, ValueError) as e:
             print e.args[0]
             return
         out = py3compat.cast_unicode(cmds)
         with io.open(fname, mode, encoding="utf-8") as f:
             if not file_exists or not append:
                 f.write(u"# coding: utf-8\n")
             f.write(out)
             # make sure we end on a newline
             if not out.endswith(u'\n'):
                 f.write(u'\n')
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     @line_magic
     def pastebin(self, parameter_s=''):
         """Upload code to Github's Gist paste bin, returning the URL.
 
         Usage:\\
           %pastebin [-d "Custom description"] 1-7
 
         The argument can be an input history range, a filename, or the name of a
         string or macro.
 
         Options:
 
           -d: Pass a custom description for the gist. The default will say
               "Pasted from IPython".
         """
         opts, args = self.parse_options(parameter_s, 'd:')
 
         try:
             code = self.shell.find_user_code(args)
         except (ValueError, TypeError) as e:
             print e.args[0]
             return
 
         from urllib2 import urlopen  # Deferred import
         import json
         post_data = json.dumps({
           "description": opts.get('d', "Pasted from IPython"),
           "public": True,
           "files": {
             "file1.py": {
               "content": code
             }
           }
         }).encode('utf-8')
 
         response = urlopen("https://api.github.com/gists", post_data)
         response_data = json.loads(response.read().decode('utf-8'))
         return response_data['html_url']
 
     @line_magic
     def loadpy(self, arg_s):
         """Alias of `%load`
 
         `%loadpy` has gained some flexibility and droped the requirement of a `.py`
         extension. So it has been renamed simply into %load. You can look at
         `%load`'s docstring for more info.
         """
         self.load(arg_s)
 
     @line_magic
     def load(self, arg_s):
         """Load code into the current frontend.
 
         Usage:\\
           %load [options] source
 
           where source can be a filename, URL, input history range or macro
 
         Options:
         --------
           -y : Don't ask confirmation for loading source above 200 000 characters.
 
         This magic command can either take a local filename, a URL, an history
         range (see %history) or a macro as argument, it will prompt for
         confirmation before loading source with more than 200 000 characters, unless
         -y flag is passed or if the frontend does not support raw_input::
 
         %load myscript.py
         %load 7-27
         %load myMacro
         %load http://www.example.com/myscript.py
         """
         opts,args = self.parse_options(arg_s,'y')
         if not args:
             raise UsageError('Missing filename, URL, input history range, '
                              'or macro.')
 
         contents = self.shell.find_user_code(args)
         l = len(contents)
 
         # 200 000 is ~ 2500 full 80 caracter lines
         # so in average, more than 5000 lines
         if l > 200000 and 'y' not in opts:
             try:
                 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
                 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
             except StdinNotImplementedError:
                 #asume yes if raw input not implemented
                 ans = True
 
             if ans is False :
                 print 'Operation cancelled.'
                 return
 
         self.shell.set_next_input(contents)
 
     @staticmethod
     def _find_edit_target(shell, args, opts, last_call):
         """Utility method used by magic_edit to find what to edit."""
 
         def make_filename(arg):
             "Make a filename from the given args"
             arg = unquote_filename(arg)
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 # If it ends with .py but doesn't already exist, assume we want
                 # a new file.
                 if arg.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # Set a few locals from the options for convenience:
         opts_prev = 'p' in opts
         opts_raw = 'r' in opts
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_prev:
             args = '_%s' % last_call[0]
             if args not in shell.user_ns:
                 args = last_call[1]
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = True
 
         data = ''
 
         # First, see if the arguments should be a filename.
         filename = make_filename(args)
         if filename:
             use_temp = False
         elif args:
             # Mode where user specifies ranges of lines, like in %macro.
             data = shell.extract_input_lines(args, opts_raw)
             if not data:
                 try:
                     # Load the parameter given as a variable. If not a string,
                     # process it as an object instead (below)
 
                     #print '*** args',args,'type',type(args)  # dbg
                     data = eval(args, shell.user_ns)
                     if not isinstance(data, basestring):
                         raise DataIsObject
 
                 except (NameError,SyntaxError):
                     # given argument is not a variable, try as a filename
                     filename = make_filename(args)
                     if filename is None:
                         warn("Argument given (%s) can't be found as a variable "
                              "or as a filename." % args)
                         return (None, None, None)
                     use_temp = False
 
                 except DataIsObject:
                     # macros have a special edit function
                     if isinstance(data, Macro):
                         raise MacroToEdit(data)
 
                     # For objects, try to edit the file where they are defined
                     filename = find_file(data)
                     if filename:
                         if 'fakemodule' in filename.lower() and \
                             inspect.isclass(data):
                             # class created by %edit? Try to find source
                             # by looking for method definitions instead, the
                             # __module__ in those classes is FakeModule.
                             attrs = [getattr(data, aname) for aname in dir(data)]
                             for attr in attrs:
                                 if not inspect.ismethod(attr):
                                     continue
                                 filename = find_file(attr)
                                 if filename and \
                                   'fakemodule' not in filename.lower():
                                     # change the attribute to be the edit
                                     # target instead
                                     data = attr
                                     break
                         
                         m = ipython_input_pat.match(os.path.basename(filename))
                         if m:
                             raise InteractivelyDefined(int(m.groups()[0]))
                         
                         datafile = 1
                     if filename is None:
                         filename = make_filename(args)
                         datafile = 1
                         if filename is not None:
                             # only warn about this if we get a real name
                             warn('Could not find file where `%s` is defined.\n'
                              'Opening a file named `%s`' % (args, filename))
                     # Now, make sure we can actually read the source (if it was
                     # in a temp file it's gone by now).
                     if datafile:
                         if lineno is None:
                             lineno = find_source_lines(data)
                         if lineno is None:
                             filename = make_filename(args)
                             if filename is None:
                                 warn('The file where `%s` was defined '
                                      'cannot be read or found.' % data)
                                 return (None, None, None)
                     use_temp = False
 
         if use_temp:
             filename = shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = shell.displayhook.prompt_count
             if not opts_prev:
                 last_call[1] = args
         except:
             pass
 
 
         return filename, lineno, use_temp
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     @skip_doctest
     @line_magic
     def edit(self, parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook. The default version of this hook is
         set to call the editor specified by your $EDITOR environment variable.
         If this isn't found, it will default to vi under Linux/Unix and to
         notepad under Windows. See the end of this docstring for how to change
         the editor hook.
 
         You can also set the value of this editor via the
         ``TerminalInteractiveShell.editor`` option in your configuration file.
         This is useful if you wish to use a different editor from your typical
         default with IPython (and for Windows users who typically don't set
         environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
 
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
 
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilities exist:
 
         - If the argument is a filename, IPython will load that into the
           editor. It will execute its contents with execfile() when you exit,
           loading any code in the file into your interactive namespace.
 
         - The arguments are ranges of input history,  e.g. "7 ~1/4-6".
           The syntax is the same as in the %history magic.
 
         - If the argument is a string variable, its contents are loaded
           into the editor. You can thus edit any string which contains
           python code (including the result of previous edits).
 
         - If the argument is the name of an object (other than a string),
           IPython will try to locate the file where it was defined and open the
           editor at the point where it is defined. You can use `%edit function`
           to load an editor exactly at the point where 'function' is defined,
           edit it and have the file be executed automatically.
 
         - If the object is a macro (see %macro for details), this opens up your
           specified editor with a temporary file containing the macro's data.
           Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor::
 
           In [1]: edit
           Editing... done. Executing edited code...
           Out[1]: 'def foo():\\n    print "foo() was defined in an editing
           session"\\n'
 
         We can then call the function foo()::
 
           In [2]: foo()
           foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined::
 
           In [3]: edit foo
           Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version::
 
           In [4]: foo()
           foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor::
 
           In [5]: edit
           Editing... done. Executing edited code...
           hello
           Out[5]: "print 'hello'\\n"
 
         Now we call it again with the previous output (stored in _)::
 
           In [6]: edit _
           Editing... done. Executing edited code...
           hello world
           Out[6]: "print 'hello world'\\n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8])::
 
           In [7]: edit _8
           Editing... done. Executing edited code...
           hello again
           Out[7]: "print 'hello again'\\n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.core.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         opts,args = self.parse_options(parameter_s,'prxn:')
 
         try:
             filename, lineno, is_temp = self._find_edit_target(self.shell, 
                                                        args, opts, last_call)
         except MacroToEdit as e:
             self._edit_macro(args, e.args[0])
             return
         except InteractivelyDefined as e:
             print "Editing In[%i]" % e.index
             args = str(e.index)
             filename, lineno, is_temp = self._find_edit_target(self.shell, 
                                                        args, opts, last_call)
         if filename is None:
             # nothing was found, warnings have already been issued,
             # just give up.
             return
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         try:
             # Quote filenames that may have spaces in them
             if ' ' in filename:
                 filename = "'%s'" % filename
             self.shell.hooks.editor(filename,lineno)
         except TryNext:
             warn('Could not open editor')
             return
 
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
 
         if 'x' in opts:  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             with preserve_keys(self.shell.user_ns, '__file__'):
                 if not is_temp:
                     self.shell.user_ns['__file__'] = filename
                 if 'r' in opts:    # Untranslated IPython code
                     self.shell.run_cell(file_read(filename),
                                         store_history=False)
                 else:
                     self.shell.safe_execfile(filename, self.shell.user_ns,
                                              self.shell.user_ns)
 
         if is_temp:
             try:
                 return open(filename).read()
             except IOError as msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()

IPython/core/magics/display.py

@@ -1,47 +1,47 b''
 """Simple magics for display formats"""
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Our own packages
 from IPython.core.display import display, Javascript, Latex, SVG
 from IPython.core.magic import  (
-    Magics, magics_class, line_magic, cell_magic
+    Magics, magics_class, cell_magic
 )
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 @magics_class
 class DisplayMagics(Magics):
     """Magics for displaying various output types with literals
     
     Defines javascript/latex cell magics for writing blocks in those languages,
     to be rendered in the frontend.
     """
     
     @cell_magic
     def javascript(self, line, cell):
         """Run the cell block of Javascript code"""
         display(Javascript(cell))
         
     
     @cell_magic
     def latex(self, line, cell):
         """Render the cell as a block of latex"""
         display(Latex(cell))
 
     @cell_magic
     def svg(self, line, cell):
         """Render the cell as an SVG literal"""
         display(SVG(cell))

IPython/core/magics/history.py

@@ -1,319 +1,318 b''
 """Implementation of magic functions related to History.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012, IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import os
 from io import open as io_open
-from IPython.external.argparse import Action
 
 # Our own packages
 from IPython.core.error import StdinNotImplementedError
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.core.magic_arguments import (argument, magic_arguments,
                                           parse_argstring)
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import io
 
 #-----------------------------------------------------------------------------
 # Magics class implementation
 #-----------------------------------------------------------------------------
 
 
 _unspecified = object()
 
 
 @magics_class
 class HistoryMagics(Magics):
 
     @magic_arguments()
     @argument(
         '-n', dest='print_nums', action='store_true', default=False,
         help="""
         print line numbers for each input.
         This feature is only available if numbered prompts are in use.
         """)
     @argument(
         '-o', dest='get_output', action='store_true', default=False,
         help="also print outputs for each input.")
     @argument(
         '-p', dest='pyprompts', action='store_true', default=False,
         help="""
         print classic '>>>' python prompts before each input.
         This is useful for making documentation, and in conjunction
         with -o, for producing doctest-ready output.
         """)
     @argument(
         '-t', dest='raw', action='store_false', default=True,
         help="""
         print the 'translated' history, as IPython understands it.
         IPython filters your input and converts it all into valid Python
         source before executing it (things like magics or aliases are turned
         into function calls, for example). With this option, you'll see the
         native history instead of the user-entered version: '%%cd /' will be
         seen as 'get_ipython().magic("%%cd /")' instead of '%%cd /'.
         """)
     @argument(
         '-f', dest='filename',
         help="""
         FILENAME: instead of printing the output to the screen, redirect
         it to the given file.  The file is always overwritten, though *when
         it can*, IPython asks for confirmation first. In particular, running
         the command 'history -f FILENAME' from the IPython Notebook
         interface will replace FILENAME even if it already exists *without*
         confirmation.
         """)
     @argument(
         '-g', dest='pattern', nargs='*', default=None,
         help="""
         treat the arg as a glob pattern to search for in (full) history.
         This includes the saved history (almost all commands ever written).
         The pattern may contain '?' to match one unknown character and '*'
         to match any number of unknown characters. Use '%%hist -g' to show
         full saved history (may be very long).
         """)
     @argument(
         '-l', dest='limit', type=int, nargs='?', default=_unspecified,
         help="""
         get the last n lines from all sessions. Specify n as a single
         arg, or the default is the last 10 lines.
         """)
     @argument(
         '-u', dest='unique', action='store_true',
         help="""
         when searching history using `-g`, show only unique history.
         """)
     @argument('range', nargs='*')
     @skip_doctest
     @line_magic
     def history(self, parameter_s = ''):
         """Print input history (_i<n> variables), with most recent last.
 
         By default, input history is printed without line numbers so it can be
         directly pasted into an editor. Use -n to show them.
 
         By default, all input history from the current session is displayed.
         Ranges of history can be indicated using the syntax:
         
         ``4``
             Line 4, current session
         ``4-6``
             Lines 4-6, current session
         ``243/1-5``
             Lines 1-5, session 243
         ``~2/7``
             Line 7, session 2 before current
         ``~8/1-~6/5``
             From the first line of 8 sessions ago, to the fifth line of 6
             sessions ago.
         
         Multiple ranges can be entered, separated by spaces
 
         The same syntax is used by %macro, %save, %edit, %rerun
 
         Examples
         --------
         ::
 
           In [6]: %history -n 4-6
           4:a = 12
           5:print a**2
           6:%history -n 4-6
 
         """
 
         args = parse_argstring(self.history, parameter_s)
 
         # For brevity
         history_manager = self.shell.history_manager
 
         def _format_lineno(session, line):
             """Helper function to format line numbers properly."""
             if session in (0, history_manager.session_number):
                 return str(line)
             return "%s/%s" % (session, line)
 
         # Check if output to specific file was requested.
         outfname = args.filename
         if not outfname:
             outfile = io.stdout  # default
             # We don't want to close stdout at the end!
             close_at_end = False
         else:
             if os.path.exists(outfname):
                 try:
                     ans = io.ask_yes_no("File %r exists. Overwrite?" % outfname)
                 except StdinNotImplementedError:
                     ans = True
                 if not ans:
                     print('Aborting.')
                     return
                 print("Overwriting file.")
             outfile = io_open(outfname, 'w', encoding='utf-8')
             close_at_end = True
 
         print_nums = args.print_nums
         get_output = args.get_output
         pyprompts = args.pyprompts
         raw = args.raw
 
         pattern = None
         limit = None if args.limit is _unspecified else args.limit
 
         if args.pattern is not None:
             if args.pattern:
                 pattern = "*" + " ".join(args.pattern) + "*"
             else:
                 pattern = "*"
             hist = history_manager.search(pattern, raw=raw, output=get_output,
                                           n=limit, unique=args.unique)
             print_nums = True
         elif args.limit is not _unspecified:
             n = 10 if limit is None else limit
             hist = history_manager.get_tail(n, raw=raw, output=get_output)
         else:
             if args.range:      # Get history by ranges
                 hist = history_manager.get_range_by_str(" ".join(args.range),
                                                         raw, get_output)
             else:               # Just get history for the current session
                 hist = history_manager.get_range(raw=raw, output=get_output)
 
         # We could be displaying the entire history, so let's not try to pull
         # it into a list in memory. Anything that needs more space will just
         # misalign.
         width = 4
 
         for session, lineno, inline in hist:
             # Print user history with tabs expanded to 4 spaces.  The GUI
             # clients use hard tabs for easier usability in auto-indented code,
             # but we want to produce PEP-8 compliant history for safe pasting
             # into an editor.
             if get_output:
                 inline, output = inline
             inline = inline.expandtabs(4).rstrip()
 
             multiline = "\n" in inline
             line_sep = '\n' if multiline else ' '
             if print_nums:
                 print(u'%s:%s' % (_format_lineno(session, lineno).rjust(width),
                         line_sep),  file=outfile, end=u'')
             if pyprompts:
                 print(u">>> ", end=u"", file=outfile)
                 if multiline:
                     inline = "\n... ".join(inline.splitlines()) + "\n..."
             print(inline, file=outfile)
             if get_output and output:
                 print(output, file=outfile)
 
         if close_at_end:
             outfile.close()
 
     @line_magic
     def recall(self, arg):
         r"""Repeat a command, or get command to input line for editing.
 
         %recall and %rep are equivalent.
 
         - %recall (no arguments):
 
         Place a string version of last computation result (stored in the
         special '_' variable) to the next input prompt. Allows you to create
         elaborate command lines without using copy-paste::
 
              In[1]: l = ["hei", "vaan"]
              In[2]: "".join(l)
             Out[2]: heivaan
              In[3]: %recall
              In[4]: heivaan_ <== cursor blinking
 
         %recall 45
 
         Place history line 45 on the next input prompt. Use %hist to find
         out the number.
 
         %recall 1-4
 
         Combine the specified lines into one cell, and place it on the next
         input prompt. See %history for the slice syntax.
 
         %recall foo+bar
 
         If foo+bar can be evaluated in the user namespace, the result is
         placed at the next input prompt. Otherwise, the history is searched
         for lines which contain that substring, and the most recent one is
         placed at the next input prompt.
         """
         if not arg:                 # Last output
             self.shell.set_next_input(str(self.shell.user_ns["_"]))
             return
                                     # Get history range
         histlines = self.shell.history_manager.get_range_by_str(arg)
         cmd = "\n".join(x[2] for x in histlines)
         if cmd:
             self.shell.set_next_input(cmd.rstrip())
             return
 
         try:                        # Variable in user namespace
             cmd = str(eval(arg, self.shell.user_ns))
         except Exception:           # Search for term in history
             histlines = self.shell.history_manager.search("*"+arg+"*")
             for h in reversed([x[2] for x in histlines]):
                 if 'recall' in h or 'rep' in h:
                     continue
                 self.shell.set_next_input(h.rstrip())
                 return
         else:
             self.shell.set_next_input(cmd.rstrip())
         print("Couldn't evaluate or find in history:", arg)
 
     @line_magic
     def rerun(self, parameter_s=''):
         """Re-run previous input
 
         By default, you can specify ranges of input history to be repeated
         (as with %history). With no arguments, it will repeat the last line.
 
         Options:
 
           -l <n> : Repeat the last n lines of input, not including the
           current command.
 
           -g foo : Repeat the most recent line which contains foo
         """
         opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
         if "l" in opts:         # Last n lines
             n = int(opts['l'])
             hist = self.shell.history_manager.get_tail(n)
         elif "g" in opts:       # Search
             p = "*"+opts['g']+"*"
             hist = list(self.shell.history_manager.search(p))
             for l in reversed(hist):
                 if "rerun" not in l[2]:
                     hist = [l]     # The last match which isn't a %rerun
                     break
             else:
                 hist = []          # No matches except %rerun
         elif args:              # Specify history ranges
             hist = self.shell.history_manager.get_range_by_str(args)
         else:                   # Last line
             hist = self.shell.history_manager.get_tail(1)
         hist = [x[2] for x in hist]
         if not hist:
             print("No lines in history match specification")
             return
         histlines = "\n".join(hist)
         print("=== Executing: ===")
         print(histlines)
         print("=== Output: ===")
         self.shell.run_cell("\n".join(hist), store_history=False)

IPython/core/magics/osm.py

@@ -1,725 +1,725 b''
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import io
 import os
 import re
 import sys
 from pprint import pformat
 
 # Our own packages
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
-from IPython.core.error import UsageError, StdinNotImplementedError
+from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils.io import file_read, nlprint
+from IPython.utils.io import nlprint
 from IPython.utils.openpy import source_to_unicode
-from IPython.utils.path import get_py_filename, unquote_filename
+from IPython.utils.path import unquote_filename
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print "Total number of aliases:", len(aliases)
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except:
             print oinspect.getdoc(self.alias)
         else:
             self.shell.alias_manager.soft_define_alias(alias, cmd)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         self.shell.alias_manager.undefine_alias(aname)
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
 
         syscmdlist = []
         # Now define isexec in a cross platform manner.
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwdu()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff):
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 self.shell.alias_manager.define_alias(
                                     ff.replace('.',''), ff)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(ff)
             else:
                 no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), ff)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(ff)
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         return os.getcwdu()
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
 
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names.
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
 
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         oldcwd = os.getcwdu()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print "No matching entry in directory history"
                 return
             else:
                 opts = {}
 
 
         else:
             #turn all non-space-escaping backslashes to slashes,
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps, target)
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # strip extra quotes on Windows, because os.chdir doesn't like them
         ps = unquote_filename(ps)
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwdu()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title('IPython: ' + '~')
             cwd = os.getcwdu()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     @line_magic
     def env(self, parameter_s=''):
         """List environment variables."""
 
         return dict(os.environ)
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(unquote_filename(parameter_s))
         cwd = os.getcwdu().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.magic('dirs')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print "popd ->",top
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print '%s ==\n%s' % (var, pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk, bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwdu()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         if not parameter_s:
             raise UsageError('Missing filename, URL, input history range, '
                              'or macro.')
 
         try :
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print "Error: no such file, variable, URL, history range or macro"
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--amend', action='store_true', default=False,
         help='Open file for amending if it exists'
     )
     @magic_arguments.argument(
         'filename', type=unicode,
         help='file to write'
     )
     @cell_magic
     def file(self, line, cell):
         """Write the contents of the cell to a file.
         
         For frontends that do not support stdin (Notebook), -f is implied.
         """
         args = magic_arguments.parse_argstring(self.file, line)
         filename = unquote_filename(args.filename)
         
         if os.path.exists(filename):
             if args.amend:
                 print "Amending to %s" % filename
             else:
                 print "Overwriting %s" % filename
         else:
             print "Writing %s" % filename
         
         mode = 'a' if args.amend else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)

IPython/core/magics/script.py

@@ -1,283 +1,280 b''
 """Magic functions for running cells in various scripts."""
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import errno
 import os
-import re
 import sys
 import signal
 import time
 from subprocess import Popen, PIPE
 import atexit
 
 # Our own packages
 from IPython.config.configurable import Configurable
 from IPython.core import magic_arguments
-from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, magics_class, line_magic, cell_magic
 )
 from IPython.lib.backgroundjobs import BackgroundJobManager
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import List, Dict
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 def script_args(f):
     """single decorator for adding script args"""
     args = [
         magic_arguments.argument(
             '--out', type=str,
             help="""The variable in which to store stdout from the script.
             If the script is backgrounded, this will be the stdout *pipe*,
             instead of the stderr text itself.
             """
         ),
         magic_arguments.argument(
             '--err', type=str,
             help="""The variable in which to store stderr from the script.
             If the script is backgrounded, this will be the stderr *pipe*,
             instead of the stderr text itself.
             """
         ),
         magic_arguments.argument(
             '--bg', action="store_true",
             help="""Whether to run the script in the background.
             If given, the only way to see the output of the command is
             with --out/err.
             """
         ),
         magic_arguments.argument(
             '--proc', type=str,
             help="""The variable in which to store Popen instance.
             This is used only when --bg option is given.
             """
         ),
     ]
     for arg in args:
         f = arg(f)
     return f
 
 @magics_class
 class ScriptMagics(Magics, Configurable):
     """Magics for talking to scripts
     
     This defines a base `%%script` cell magic for running a cell
     with a program in a subprocess, and registers a few top-level
     magics that call %%script with common interpreters.
     """
     script_magics = List(config=True,
         help="""Extra script cell magics to define
         
         This generates simple wrappers of `%%script foo` as `%%foo`.
         
         If you want to add script magics that aren't on your path,
         specify them in script_paths
         """,
     )
     def _script_magics_default(self):
         """default to a common list of programs"""
         
         defaults = [
             'sh',
             'bash',
             'perl',
             'ruby',
             'python',
             'python3',
             'pypy',
         ]
         if os.name == 'nt':
             defaults.extend([
                 'cmd',
                 'powershell',
             ])
         
         return defaults
     
     script_paths = Dict(config=True,
         help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
         
         Only necessary for items in script_magics where the default path will not
         find the right interpreter.
         """
     )
     
     def __init__(self, shell=None):
         Configurable.__init__(self, config=shell.config)
         self._generate_script_magics()
         Magics.__init__(self, shell=shell)
         self.job_manager = BackgroundJobManager()
         self.bg_processes = []
         atexit.register(self.kill_bg_processes)
 
     def __del__(self):
         self.kill_bg_processes()
     
     def _generate_script_magics(self):
         cell_magics = self.magics['cell']
         for name in self.script_magics:
             cell_magics[name] = self._make_script_magic(name)
     
     def _make_script_magic(self, name):
         """make a named magic, that calls %%script with a particular program"""
         # expand to explicit path if necessary:
         script = self.script_paths.get(name, name)
         
         @magic_arguments.magic_arguments()
         @script_args
         def named_script_magic(line, cell):
             # if line, add it as cl-flags
             if line:
                  line = "%s %s" % (script, line)
             else:
                 line = script
             return self.shebang(line, cell)
         
         # write a basic docstring:
         named_script_magic.__doc__ = \
         """%%{name} script magic
         
         Run cells with {script} in a subprocess.
         
         This is a shortcut for `%%script {script}`
         """.format(**locals())
         
         return named_script_magic
     
     @magic_arguments.magic_arguments()
     @script_args
     @cell_magic("script")
     def shebang(self, line, cell):
         """Run a cell via a shell command
         
         The `%%script` line is like the #! line of script,
         specifying a program (bash, perl, ruby, etc.) with which to run.
         
         The rest of the cell is run by that program.
         
         Examples
         --------
         ::
         
             In [1]: %%script bash
                ...: for i in 1 2 3; do
                ...:   echo $i
                ...: done
             1
             2
             3
         """
         argv = arg_split(line, posix = not sys.platform.startswith('win'))
         args, cmd = self.shebang.parser.parse_known_args(argv)
         
         try:
             p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
         except OSError as e:
             if e.errno == errno.ENOENT:
                 print "Couldn't find program: %r" % cmd[0]
                 return
             else:
                 raise
         
         cell = cell.encode('utf8', 'replace')
         if args.bg:
             self.bg_processes.append(p)
             self._gc_bg_processes()
             if args.out:
                 self.shell.user_ns[args.out] = p.stdout
             if args.err:
                 self.shell.user_ns[args.err] = p.stderr
             self.job_manager.new(self._run_script, p, cell, daemon=True)
             if args.proc:
                 self.shell.user_ns[args.proc] = p
             return
         
         try:
             out, err = p.communicate(cell)
         except KeyboardInterrupt:
             try:
                 p.send_signal(signal.SIGINT)
                 time.sleep(0.1)
                 if p.poll() is not None:
                     print "Process is interrupted."
                     return
                 p.terminate()
                 time.sleep(0.1)
                 if p.poll() is not None:
                     print "Process is terminated."
                     return
                 p.kill()
                 print "Process is killed."
             except OSError:
                 pass
             except Exception as e:
                 print "Error while terminating subprocess (pid=%i): %s" \
                     % (p.pid, e)
             return
         out = py3compat.bytes_to_str(out)
         err = py3compat.bytes_to_str(err)
         if args.out:
             self.shell.user_ns[args.out] = out
         else:
             sys.stdout.write(out)
             sys.stdout.flush()
         if args.err:
             self.shell.user_ns[args.err] = err
         else:
             sys.stderr.write(err)
             sys.stderr.flush()
     
     def _run_script(self, p, cell):
         """callback for running the script in the background"""
         p.stdin.write(cell)
         p.stdin.close()
         p.wait()
 
     @line_magic("killbgscripts")
     def killbgscripts(self, _nouse_=''):
         """Kill all BG processes started by %%script and its family."""
         self.kill_bg_processes()
         print "All background processes were killed."
 
     def kill_bg_processes(self):
         """Kill all BG processes which are still running."""
         for p in self.bg_processes:
             if p.poll() is None:
                 try:
                     p.send_signal(signal.SIGINT)
                 except:
                     pass
         time.sleep(0.1)
         for p in self.bg_processes:
             if p.poll() is None:
                 try:
                     p.terminate()
                 except:
                     pass
         time.sleep(0.1)
         for p in self.bg_processes:
             if p.poll() is None:
                 try:
                     p.kill()
                 except:
                     pass
         self._gc_bg_processes()
 
     def _gc_bg_processes(self):
         self.bg_processes = [p for p in self.bg_processes if p.poll() is None]

IPython/core/page.py

@@ -1,349 +1,348 b''
 # encoding: utf-8
 """
 Paging capabilities for IPython.core
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 
 For now this uses ipapi, so it can't be in IPython.utils.  If we can get
 rid of that dependency, we could move it there.
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import os
 import re
-import subprocess
 import sys
 import tempfile
 
 from io import UnsupportedOperation
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
 from IPython.utils.data import chop
 from IPython.utils import io
 from IPython.utils.process import system
 from IPython.utils.terminal import get_terminal_size
 from IPython.utils import py3compat
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print(os.linesep.join(screens[0]), file=io.stdout)
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print(last_escape + hunk, file=io.stdout)
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]), file=io.stdout)
 
 def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
     if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
         return screen_lines_def
     
     try:
         import termios
         import curses
     except ImportError:
         return screen_lines_def
     
     # There is a bug in curses, where *sometimes* it fails to properly
     # initialize, and then after the endwin() call is made, the
     # terminal is left in an unusable state.  Rather than trying to
     # check everytime for this (by requesting and comparing termios
     # flags each time), we just save the initial terminal state and
     # unconditionally reset it every time.  It's cheaper than making
     # the checks.
     term_flags = termios.tcgetattr(sys.stdout)
 
     # Curses modifies the stdout buffer size by default, which messes
     # up Python's normal stdout buffering.  This would manifest itself
     # to IPython users as delayed printing on stdout after having used
     # the pager.
     #
     # We can prevent this by manually setting the NCURSES_NO_SETBUF
     # environment variable.  For more details, see:
     # http://bugs.python.org/issue10144
     NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
     os.environ['NCURSES_NO_SETBUF'] = ''
 
     # Proceed with curses initialization
     try:
         scr = curses.initscr()
     except AttributeError:
         # Curses on Solaris may not be complete, so we can't use it there
         return screen_lines_def
     
     screen_lines_real,screen_cols = scr.getmaxyx()
     curses.endwin()
 
     # Restore environment
     if NCURSES_NO_SETBUF is None:
         del os.environ['NCURSES_NO_SETBUF']
     else:
         os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF
 
     # Restore terminal state in case endwin() didn't.
     termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
     # Now we have what we needed: the screen size in rows/columns
     return screen_lines_real
     #print '***Screen size:',screen_lines_real,'lines x',\
     #screen_cols,'columns.' # dbg
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
 
     # first, try the hook
     ip = ipapi.get()
     if ip:
         try:
             ip.hooks.show_in_pager(strng)
             return
         except TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print(strng)
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.splitlines()[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         try:
             screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint, file=io.stdout)
             return
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print(str_toprint, file=io.stdout)
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = open(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd, 'w')
                 try:
                     pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(py3compat.cast_bytes_py2(
                         strng, encoding=pager_encoding))
                 finally:
                     retval = pager.close()
             except IOError as msg:  # broken pipe when user quits
                 if msg.args == (32, 'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 
 def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         system(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print('Unable to show file',repr(fname))
 
 
 def get_pager_cmd(pager_cmd=None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one.
     """
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 
 def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 
 # (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         io.stdout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         io.stdout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print(header, end=' ')
     if len(str) < width:
         print(str)
         snip = 0
     else:
         whalf = int((width -5)/2)
         print(str[:whalf] + ' <...> ' + str[-whalf:])
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip

IPython/core/prefilter.py

@@ -1,754 +1,749 b''
 # encoding: utf-8
 """
 Prefiltering components.
 
 Prefilters transform user input before it is exec'd by Python.  These
 transforms are used to implement additional syntax such as !ls and %magic.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Dan Milstein
 * Ville Vainio
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-import __builtin__
-import codeop
 import re
 
-from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
 from IPython.config.configurable import Configurable
 from IPython.core.inputsplitter import (
     ESC_SHELL,
     ESC_SH_CAP,
     ESC_HELP,
     ESC_MAGIC,
     ESC_MAGIC2,
     ESC_QUOTE,
     ESC_QUOTE2,
     ESC_PAREN,
 )
 from IPython.core.macro import Macro
-from IPython.core.splitinput import split_user_input, LineInfo
-from IPython.core import page
+from IPython.core.splitinput import LineInfo
 
 from IPython.utils.traitlets import (
-    List, Integer, Any, Unicode, CBool, Bool, Instance, CRegExp
+    List, Integer, Unicode, CBool, Bool, Instance, CRegExp
 )
-from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
 # Global utilities, errors and constants
 #-----------------------------------------------------------------------------
 
 
 class PrefilterError(Exception):
     pass
 
 
 # RegExp to identify potential function names
 re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
 
 # RegExp to exclude strings with this start from autocalling.  In
 # particular, all binary operators should be excluded, so that if foo is
 # callable, foo OP bar doesn't become foo(OP bar), which is invalid.  The
 # characters '!=()' don't need to be checked for, as the checkPythonChars
 # routine explicitely does so, to catch direct calls and rebindings of
 # existing names.
 
 # Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
 # it affects the rest of the group in square brackets.
 re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
                              r'|^is |^not |^in |^and |^or ')
 
 # try to catch also methods for stuff in lists/tuples/dicts: off
 # (experimental). For this to work, the line_split regexp would need
 # to be modified so it wouldn't break things at '['. That line is
 # nasty enough that I shouldn't change it until I can test it _well_.
 #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
 
 
 # Handler Check Utilities
 def is_shadowed(identifier, ip):
     """Is the given identifier defined in one of the namespaces which shadow
     the alias and magic namespaces?  Note that an identifier is different
     than ifun, because it can not contain a '.' character."""
     # This is much safer than calling ofind, which can change state
     return (identifier in ip.user_ns \
             or identifier in ip.user_global_ns \
             or identifier in ip.ns_table['builtin'])
 
 
 #-----------------------------------------------------------------------------
 # Main Prefilter manager
 #-----------------------------------------------------------------------------
 
 
 class PrefilterManager(Configurable):
     """Main prefilter component.
 
     The IPython prefilter is run on all user input before it is run.  The
     prefilter consumes lines of input and produces transformed lines of
     input.
 
     The iplementation consists of two phases:
 
     1. Transformers
     2. Checkers and handlers
 
     Over time, we plan on deprecating the checkers and handlers and doing
     everything in the transformers.
 
     The transformers are instances of :class:`PrefilterTransformer` and have
     a single method :meth:`transform` that takes a line and returns a
     transformed line.  The transformation can be accomplished using any
     tool, but our current ones use regular expressions for speed.
 
     After all the transformers have been run, the line is fed to the checkers,
     which are instances of :class:`PrefilterChecker`.  The line is passed to
     the :meth:`check` method, which either returns `None` or a
     :class:`PrefilterHandler` instance.  If `None` is returned, the other
     checkers are tried.  If an :class:`PrefilterHandler` instance is returned,
     the line is passed to the :meth:`handle` method of the returned
     handler and no further checkers are tried.
 
     Both transformers and checkers have a `priority` attribute, that determines
     the order in which they are called.  Smaller priorities are tried first.
 
     Both transformers and checkers also have `enabled` attribute, which is
     a boolean that determines if the instance is used.
 
     Users or developers can change the priority or enabled attribute of
     transformers or checkers, but they must call the :meth:`sort_checkers`
     or :meth:`sort_transformers` method after changing the priority.
     """
 
     multi_line_specials = CBool(True, config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def __init__(self, shell=None, config=None):
         super(PrefilterManager, self).__init__(shell=shell, config=config)
         self.shell = shell
         self.init_transformers()
         self.init_handlers()
         self.init_checkers()
 
     #-------------------------------------------------------------------------
     # API for managing transformers
     #-------------------------------------------------------------------------
 
     def init_transformers(self):
         """Create the default transformers."""
         self._transformers = []
         for transformer_cls in _default_transformers:
             transformer_cls(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     def sort_transformers(self):
         """Sort the transformers by priority.
 
         This must be called after the priority of a transformer is changed.
         The :meth:`register_transformer` method calls this automatically.
         """
         self._transformers.sort(key=lambda x: x.priority)
 
     @property
     def transformers(self):
         """Return a list of checkers, sorted by priority."""
         return self._transformers
 
     def register_transformer(self, transformer):
         """Register a transformer instance."""
         if transformer not in self._transformers:
             self._transformers.append(transformer)
             self.sort_transformers()
 
     def unregister_transformer(self, transformer):
         """Unregister a transformer instance."""
         if transformer in self._transformers:
             self._transformers.remove(transformer)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_checkers(self):
         """Create the default checkers."""
         self._checkers = []
         for checker in _default_checkers:
             checker(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     def sort_checkers(self):
         """Sort the checkers by priority.
 
         This must be called after the priority of a checker is changed.
         The :meth:`register_checker` method calls this automatically.
         """
         self._checkers.sort(key=lambda x: x.priority)
 
     @property
     def checkers(self):
         """Return a list of checkers, sorted by priority."""
         return self._checkers
 
     def register_checker(self, checker):
         """Register a checker instance."""
         if checker not in self._checkers:
             self._checkers.append(checker)
             self.sort_checkers()
 
     def unregister_checker(self, checker):
         """Unregister a checker instance."""
         if checker in self._checkers:
             self._checkers.remove(checker)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_handlers(self):
         """Create the default handlers."""
         self._handlers = {}
         self._esc_handlers = {}
         for handler in _default_handlers:
             handler(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     @property
     def handlers(self):
         """Return a dict of all the handlers."""
         return self._handlers
 
     def register_handler(self, name, handler, esc_strings):
         """Register a handler instance by name with esc_strings."""
         self._handlers[name] = handler
         for esc_str in esc_strings:
             self._esc_handlers[esc_str] = handler
 
     def unregister_handler(self, name, handler, esc_strings):
         """Unregister a handler instance by name with esc_strings."""
         try:
             del self._handlers[name]
         except KeyError:
             pass
         for esc_str in esc_strings:
             h = self._esc_handlers.get(esc_str)
             if h is handler:
                 del self._esc_handlers[esc_str]
 
     def get_handler_by_name(self, name):
         """Get a handler by its name."""
         return self._handlers.get(name)
 
     def get_handler_by_esc(self, esc_str):
         """Get a handler by its escape string."""
         return self._esc_handlers.get(esc_str)
 
     #-------------------------------------------------------------------------
     # Main prefiltering API
     #-------------------------------------------------------------------------
 
     def prefilter_line_info(self, line_info):
         """Prefilter a line that has been converted to a LineInfo object.
 
         This implements the checker/handler part of the prefilter pipe.
         """
         # print "prefilter_line_info: ", line_info
         handler = self.find_handler(line_info)
         return handler.handle(line_info)
 
     def find_handler(self, line_info):
         """Find a handler for the line_info by trying checkers."""
         for checker in self.checkers:
             if checker.enabled:
                 handler = checker.check(line_info)
                 if handler:
                     return handler
         return self.get_handler_by_name('normal')
 
     def transform_line(self, line, continue_prompt):
         """Calls the enabled transformers in order of increasing priority."""
         for transformer in self.transformers:
             if transformer.enabled:
                 line = transformer.transform(line, continue_prompt)
         return line
 
     def prefilter_line(self, line, continue_prompt=False):
         """Prefilter a single input line as text.
 
         This method prefilters a single line of text by calling the
         transformers and then the checkers/handlers.
         """
 
         # print "prefilter_line: ", line, continue_prompt
         # All handlers *must* return a value, even if it's blank ('').
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self.shell._last_input_line = line
 
         if not line:
             # Return immediately on purely empty lines, so that if the user
             # previously typed some whitespace that started a continuation
             # prompt, he can break out of that loop with just an empty line.
             # This is how the default python prompt works.
             return ''
 
         # At this point, we invoke our transformers.
         if not continue_prompt or (continue_prompt and self.multi_line_specials):
             line = self.transform_line(line, continue_prompt)
 
         # Now we compute line_info for the checkers and handlers
         line_info = LineInfo(line, continue_prompt)
 
         # the input history needs to track even empty lines
         stripped = line.strip()
 
         normal_handler = self.get_handler_by_name('normal')
         if not stripped:
             return normal_handler.handle(line_info)
 
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.multi_line_specials:
             return normal_handler.handle(line_info)
 
         prefiltered = self.prefilter_line_info(line_info)
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
     def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of
         input.  This simply calls :meth:`prefilter_line` for each line of
         input.
 
         This covers cases where there are multiple lines in the user entry,
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
         llines = lines.rstrip('\n').split('\n')
         # We can get multiple lines in one shot, where multiline input 'blends'
         # into one line, in cases like recalling from the readline history
         # buffer.  We need to make sure that in such cases, we correctly
         # communicate downstream which line is first and which are continuation
         # ones.
         if len(llines) > 1:
             out = '\n'.join([self.prefilter_line(line, lnum>0)
                              for lnum, line in enumerate(llines) ])
         else:
             out = self.prefilter_line(llines[0], continue_prompt)
 
         return out
 
 #-----------------------------------------------------------------------------
 # Prefilter transformers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterTransformer(Configurable):
     """Transform a line of user input."""
 
     priority = Integer(100, config=True)
     # Transformers don't currently use shell or prefilter_manager, but as we
     # move away from checkers and handlers, they will need them.
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     enabled = Bool(True, config=True)
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterTransformer, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_transformer(self)
 
     def transform(self, line, continue_prompt):
         """Transform a line, returning the new one."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterChecker(Configurable):
     """Inspect an input line and return a handler for that line."""
 
     priority = Integer(100, config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     enabled = Bool(True, config=True)
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterChecker, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_checker(self)
 
     def check(self, line_info):
         """Inspect line_info and return a handler instance or None."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 class EmacsChecker(PrefilterChecker):
 
     priority = Integer(100, config=True)
     enabled = Bool(False, config=True)
 
     def check(self, line_info):
         "Emacs ipython-mode tags certain input lines."
         if line_info.line.endswith('# PYTHON-MODE'):
             return self.prefilter_manager.get_handler_by_name('emacs')
         else:
             return None
 
 
 class MacroChecker(PrefilterChecker):
 
     priority = Integer(250, config=True)
 
     def check(self, line_info):
         obj = self.shell.user_ns.get(line_info.ifun)
         if isinstance(obj, Macro):
             return self.prefilter_manager.get_handler_by_name('macro')
         else:
             return None
 
 
 class IPyAutocallChecker(PrefilterChecker):
 
     priority = Integer(300, config=True)
 
     def check(self, line_info):
         "Instances of IPyAutocall in user_ns get autocalled immediately"
         obj = self.shell.user_ns.get(line_info.ifun, None)
         if isinstance(obj, IPyAutocall):
             obj.set_ip(self.shell)
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 class AssignmentChecker(PrefilterChecker):
 
     priority = Integer(600, config=True)
 
     def check(self, line_info):
         """Check to see if user is assigning to a var for the first time, in
         which case we want to avoid any sort of automagic / autocall games.
 
         This allows users to assign to either alias or magic names true python
         variables (the magic/alias systems always take second seat to true
         python code).  E.g. ls='hi', or ls,that=1,2"""
         if line_info.the_rest:
             if line_info.the_rest[0] in '=,':
                 return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutoMagicChecker(PrefilterChecker):
 
     priority = Integer(700, config=True)
 
     def check(self, line_info):
         """If the ifun is magic, and automagic is on, run it.  Note: normal,
         non-auto magic would already have been triggered via '%' in
         check_esc_chars. This just checks for automagic.  Also, before
         triggering the magic handler, make sure that there is nothing in the
         user namespace which could shadow it."""
         if not self.shell.automagic or not self.shell.find_magic(line_info.ifun):
             return None
 
         # We have a likely magic method.  Make sure we should actually call it.
         if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:
             return None
 
         head = line_info.ifun.split('.',1)[0]
         if is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('magic')
 
 
 class AliasChecker(PrefilterChecker):
 
     priority = Integer(800, config=True)
 
     def check(self, line_info):
         "Check if the initital identifier on the line is an alias."
         # Note: aliases can not contain '.'
         head = line_info.ifun.split('.',1)[0]
         if line_info.ifun not in self.shell.alias_manager \
                or head not in self.shell.alias_manager \
                or is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('alias')
 
 
 class PythonOpsChecker(PrefilterChecker):
 
     priority = Integer(900, config=True)
 
     def check(self, line_info):
         """If the 'rest' of the line begins with a function call or pretty much
         any python operator, we should simply execute the line (regardless of
         whether or not there's a possible autocall expansion).  This avoids
         spurious (and very confusing) geattr() accesses."""
         if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
             return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutocallChecker(PrefilterChecker):
 
     priority = Integer(1000, config=True)
 
     function_name_regexp = CRegExp(re_fun_name, config=True,
         help="RegExp to identify potential function names.")
     exclude_regexp = CRegExp(re_exclude_auto, config=True,
         help="RegExp to exclude strings with this start from autocalling.")
 
     def check(self, line_info):
         "Check if the initial word/function is callable and autocall is on."
         if not self.shell.autocall:
             return None
 
         oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
         if not oinfo['found']:
             return None
 
         if callable(oinfo['obj']) \
                and (not self.exclude_regexp.match(line_info.the_rest)) \
                and self.function_name_regexp.match(line_info.ifun):
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 #-----------------------------------------------------------------------------
 # Prefilter handlers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterHandler(Configurable):
 
     handler_name = Unicode('normal')
     esc_strings = List([])
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterHandler, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_handler(
             self.handler_name,
             self,
             self.esc_strings
         )
 
     def handle(self, line_info):
         # print "normal: ", line_info
         """Handle normal input lines. Use as a template for handlers."""
 
         # With autoindent on, we need some way to exit the input loop, and I
         # don't want to force the user to have to backspace all the way to
         # clear the line.  The rule will be in this case, that either two
         # lines of pure whitespace in a row, or a line of pure whitespace but
         # of a size different to the indent level, will exit the input loop.
         line = line_info.line
         continue_prompt = line_info.continue_prompt
 
         if (continue_prompt and
             self.shell.autoindent and
             line.isspace() and
             0 < abs(len(line) - self.shell.indent_current_nsp) <= 2):
             line = ''
 
         return line
 
     def __str__(self):
         return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
 
 
 class AliasHandler(PrefilterHandler):
 
     handler_name = Unicode('alias')
 
     def handle(self, line_info):
         """Handle alias input lines. """
         transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)
         # pre is needed, because it carries the leading whitespace.  Otherwise
         # aliases won't work in indented sections.
         line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, transformed)
 
         return line_out
 
 
 class MacroHandler(PrefilterHandler):
     handler_name = Unicode("macro")
 
     def handle(self, line_info):
         obj = self.shell.user_ns.get(line_info.ifun)
         pre_space = line_info.pre_whitespace
         line_sep = "\n" + pre_space
         return pre_space + line_sep.join(obj.value.splitlines())
 
 
 class MagicHandler(PrefilterHandler):
 
     handler_name = Unicode('magic')
     esc_strings = List([ESC_MAGIC])
 
     def handle(self, line_info):
         """Execute magic functions."""
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         cmd = '%sget_ipython().magic(%r)' % (line_info.pre_whitespace,
                                                     (ifun + " " + the_rest))
         return cmd
 
 
 class AutoHandler(PrefilterHandler):
 
     handler_name = Unicode('auto')
     esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
 
     def handle(self, line_info):
         """Handle lines which can be auto-executed, quoting if requested."""
         line    = line_info.line
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         pre     = line_info.pre
         esc     = line_info.esc
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self.shell)['obj']
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
 
         # User objects sometimes raise exceptions on attribute access other
         # than AttributeError (we've seen it in the past), so it's safest to be
         # ultra-conservative here and catch all.
         try:
             auto_rewrite = obj.rewrite
         except Exception:
             auto_rewrite = True
 
         if esc == ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
         elif esc == ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (ifun,the_rest)
         elif esc == ESC_PAREN:
             newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
         else:
             # Auto-paren.       
             if force_auto:
                 # Don't rewrite if it is already a call.
                 do_rewrite = not the_rest.startswith('(')
             else:
                 if not the_rest:
                     # We only apply it to argument-less calls if the autocall
                     # parameter is set to 2.
                     do_rewrite = (self.shell.autocall >= 2)
                 elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):
                     # Don't autocall in this case: item access for an object
                     # which is BOTH callable and implements __getitem__.
                     do_rewrite = False
                 else:
                     do_rewrite = True
 
             # Figure out the rewritten command
             if do_rewrite:
                 if the_rest.endswith(';'):
                     newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
                 else:
                     newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)                
             else:
                 normal_handler = self.prefilter_manager.get_handler_by_name('normal')
                 return normal_handler.handle(line_info)
         
         # Display the rewritten call
         if auto_rewrite:
             self.shell.auto_rewrite_input(newcmd)
 
         return newcmd
 
 
 class EmacsHandler(PrefilterHandler):
 
     handler_name = Unicode('emacs')
     esc_strings = List([])
 
     def handle(self, line_info):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
         return line_info.line
 
 
 #-----------------------------------------------------------------------------
 # Defaults
 #-----------------------------------------------------------------------------
 
 
 _default_transformers = [
 ]
 
 _default_checkers = [
     EmacsChecker,
     MacroChecker,
     IPyAutocallChecker,
     AssignmentChecker,
     AutoMagicChecker,
     AliasChecker,
     PythonOpsChecker,
     AutocallChecker
 ]
 
 _default_handlers = [
     PrefilterHandler,
     AliasHandler,
     MacroHandler,
     MagicHandler,
     AutoHandler,
     EmacsHandler
 ]

IPython/core/profileapp.py

@@ -1,307 +1,306 b''
 # encoding: utf-8
 """
 An application for managing IPython profiles.
 
 To be invoked as the `ipython profile` subcommand.
 
 Authors:
 
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-import logging
 import os
 
-from IPython.config.application import Application, boolean_flag
+from IPython.config.application import Application
 from IPython.core.application import (
-    BaseIPythonApplication, base_flags, base_aliases
+    BaseIPythonApplication, base_flags
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.traitlets import Unicode, Bool, Dict
 
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 create_help = """Create an IPython profile by name
 
 Create an ipython profile directory by its name or
 profile directory path. Profile directories contain
 configuration, log and security related files and are named
 using the convention 'profile_<name>'. By default they are
 located in your ipython directory. Once created, you will
 can edit the configuration files in the profile
 directory to configure IPython. Most users will create a
 profile directory by name,
 `ipython profile create myprofile`, which will put the directory
 in `<ipython_dir>/profile_myprofile`.
 """
 list_help = """List available IPython profiles
 
 List all available profiles, by profile location, that can
 be found in the current working directly or in the ipython
 directory. Profile directories are named using the convention
 'profile_<profile>'.
 """
 profile_help = """Manage IPython profiles
 
 Profile directories contain
 configuration, log and security related files and are named
 using the convention 'profile_<name>'. By default they are
 located in your ipython directory.  You can create profiles
 with `ipython profile create <name>`, or see the profiles you
 already have with `ipython profile list`
 
 To get started configuring IPython, simply do:
 
 $> ipython profile create
 
 and IPython will create the default profile in <ipython_dir>/profile_default,
 where you can edit ipython_config.py to start configuring IPython.
 
 """
 
 _list_examples = "ipython profile list  # list all profiles"
 
 _create_examples = """
 ipython profile create foo         # create profile foo w/ default config files
 ipython profile create foo --reset # restage default config files over current
 ipython profile create foo --parallel # also stage parallel config files
 """
 
 _main_examples = """
 ipython profile create -h  # show the help string for the create subcommand
 ipython profile list -h    # show the help string for the list subcommand
 
 ipython locate profile foo # print the path to the directory for profile 'foo'
 """
 
 #-----------------------------------------------------------------------------
 # Profile Application Class (for `ipython profile` subcommand)
 #-----------------------------------------------------------------------------
 
 
 def list_profiles_in(path):
     """list profiles in a given root directory"""
     files = os.listdir(path)
     profiles = []
     for f in files:
         full_path = os.path.join(path, f)
         if os.path.isdir(full_path) and f.startswith('profile_'):
             profiles.append(f.split('_',1)[-1])
     return profiles
 
 
 def list_bundled_profiles():
     """list profiles that are bundled with IPython."""
     path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
     files = os.listdir(path)
     profiles = []
     for profile in files:
         full_path = os.path.join(path, profile)
         if os.path.isdir(full_path) and profile != "__pycache__":
             profiles.append(profile)
     return profiles
 
 
 class ProfileLocate(BaseIPythonApplication):
     description = """print the path an IPython profile dir"""
     
     def parse_command_line(self, argv=None):
         super(ProfileLocate, self).parse_command_line(argv)
         if self.extra_args:
             self.profile = self.extra_args[0]
     
     def start(self):
         print self.profile_dir.location
 
 
 class ProfileList(Application):
     name = u'ipython-profile'
     description = list_help
     examples = _list_examples
 
     aliases = Dict({
         'ipython-dir' : 'ProfileList.ipython_dir',
         'log-level' : 'Application.log_level',
     })
     flags = Dict(dict(
         debug = ({'Application' : {'log_level' : 0}},
             "Set Application.log_level to 0, maximizing log output."
         )
     ))
 
     ipython_dir = Unicode(get_ipython_dir(), config=True,
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This options can also be specified through
         the environment variable IPYTHONDIR.
         """
     )
 
 
     def _print_profiles(self, profiles):
         """print list of profiles, indented."""
         for profile in profiles:
             print '    %s' % profile
 
     def list_profile_dirs(self):
         profiles = list_bundled_profiles()
         if profiles:
             print
             print "Available profiles in IPython:"
             self._print_profiles(profiles)
             print
             print "    The first request for a bundled profile will copy it"
             print "    into your IPython directory (%s)," % self.ipython_dir
             print "    where you can customize it."
         
         profiles = list_profiles_in(self.ipython_dir)
         if profiles:
             print
             print "Available profiles in %s:" % self.ipython_dir
             self._print_profiles(profiles)
         
         profiles = list_profiles_in(os.getcwdu())
         if profiles:
             print
             print "Available profiles in current directory (%s):" % os.getcwdu()
             self._print_profiles(profiles)
         
         print
         print "To use any of the above profiles, start IPython with:"
         print "    ipython --profile=<name>"
         print
 
     def start(self):
         self.list_profile_dirs()
 
 
 create_flags = {}
 create_flags.update(base_flags)
 # don't include '--init' flag, which implies running profile create in other apps
 create_flags.pop('init')
 create_flags['reset'] = ({'ProfileCreate': {'overwrite' : True}},
                         "reset config files in this profile to the defaults.")
 create_flags['parallel'] = ({'ProfileCreate': {'parallel' : True}},
                         "Include the config files for parallel "
                         "computing apps (ipengine, ipcontroller, etc.)")
 
 
 class ProfileCreate(BaseIPythonApplication):
     name = u'ipython-profile'
     description = create_help
     examples = _create_examples
     auto_create = Bool(True, config=False)
 
     def _copy_config_files_default(self):
         return True
 
     parallel = Bool(False, config=True,
         help="whether to include parallel computing config files")
     def _parallel_changed(self, name, old, new):
         parallel_files = [   'ipcontroller_config.py',
                             'ipengine_config.py',
                             'ipcluster_config.py'
                         ]
         if new:
             for cf in parallel_files:
                 self.config_files.append(cf)
         else:
             for cf in parallel_files:
                 if cf in self.config_files:
                     self.config_files.remove(cf)
 
     def parse_command_line(self, argv):
         super(ProfileCreate, self).parse_command_line(argv)
         # accept positional arg as profile name
         if self.extra_args:
             self.profile = self.extra_args[0]
 
     flags = Dict(create_flags)
 
     classes = [ProfileDir]
 
     def init_config_files(self):
         super(ProfileCreate, self).init_config_files()
         # use local imports, since these classes may import from here
         from IPython.frontend.terminal.ipapp import TerminalIPythonApp
         apps = [TerminalIPythonApp]
         try:
             from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
         except Exception:
             # this should be ImportError, but under weird circumstances
             # this might be an AttributeError, or possibly others
             # in any case, nothing should cause the profile creation to crash.
             pass
         else:
             apps.append(IPythonQtConsoleApp)
         try:
             from IPython.frontend.html.notebook.notebookapp import NotebookApp
         except ImportError:
             pass
         except Exception:
             self.log.debug('Unexpected error when importing NotebookApp',
                            exc_info=True
             )
         else:
             apps.append(NotebookApp)
         if self.parallel:
             from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
             from IPython.parallel.apps.ipengineapp import IPEngineApp
             from IPython.parallel.apps.ipclusterapp import IPClusterStart
             from IPython.parallel.apps.iploggerapp import IPLoggerApp
             apps.extend([
                 IPControllerApp,
                 IPEngineApp,
                 IPClusterStart,
                 IPLoggerApp,
             ])
         for App in apps:
             app = App()
             app.config.update(self.config)
             app.log = self.log
             app.overwrite = self.overwrite
             app.copy_config_files=True
             app.profile = self.profile
             app.init_profile_dir()
             app.init_config_files()
 
     def stage_default_config_file(self):
         pass
 
 
 class ProfileApp(Application):
     name = u'ipython-profile'
     description = profile_help
     examples = _main_examples
 
     subcommands = Dict(dict(
         create = (ProfileCreate, ProfileCreate.description.splitlines()[0]),
         list = (ProfileList, ProfileList.description.splitlines()[0]),
     ))
 
     def start(self):
         if self.subapp is None:
             print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
             print
             self.print_description()
             self.print_subcommands()
             self.exit(1)
         else:
             return self.subapp.start()
 

IPython/core/profiledir.py

@@ -1,228 +1,226 b''
 # encoding: utf-8
 """
 An object for managing IPython profile directories.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import shutil
-import sys
 
 from IPython.config.configurable import LoggingConfigurable
-from IPython.config.loader import Config
 from IPython.utils.path import get_ipython_package_dir, expand_path
-from IPython.utils.traitlets import List, Unicode, Bool
+from IPython.utils.traitlets import Unicode, Bool
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class ProfileDirError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Class for managing profile directories
 #-----------------------------------------------------------------------------
 
 class ProfileDir(LoggingConfigurable):
     """An object to manage the profile directory and its resources.
 
     The profile directory is used by all IPython applications, to manage
     configuration, logging and security.
 
     This object knows how to find, create and manage these directories. This
     should be used by any code that wants to handle profiles.
     """
 
     security_dir_name = Unicode('security')
     log_dir_name = Unicode('log')
     startup_dir_name = Unicode('startup')
     pid_dir_name = Unicode('pid')
     security_dir = Unicode(u'')
     log_dir = Unicode(u'')
     startup_dir = Unicode(u'')
     pid_dir = Unicode(u'')
 
     location = Unicode(u'', config=True,
         help="""Set the profile location directly. This overrides the logic used by the
         `profile` option.""",
         )
 
     _location_isset = Bool(False) # flag for detecting multiply set location
 
     def _location_changed(self, name, old, new):
         if self._location_isset:
             raise RuntimeError("Cannot set profile location more than once.")
         self._location_isset = True
         if not os.path.isdir(new):
             os.makedirs(new)
         
         # ensure config files exist:
         self.security_dir = os.path.join(new, self.security_dir_name)
         self.log_dir = os.path.join(new, self.log_dir_name)
         self.startup_dir = os.path.join(new, self.startup_dir_name)
         self.pid_dir = os.path.join(new, self.pid_dir_name)
         self.check_dirs()
 
     def _log_dir_changed(self, name, old, new):
         self.check_log_dir()
 
     def check_log_dir(self):
         if not os.path.isdir(self.log_dir):
             os.mkdir(self.log_dir)
 
     def _startup_dir_changed(self, name, old, new):
         self.check_startup_dir()
 
     def check_startup_dir(self):
         if not os.path.isdir(self.startup_dir):
             os.mkdir(self.startup_dir)
         readme = os.path.join(self.startup_dir, 'README')
         src = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'README_STARTUP')
         if not os.path.exists(readme):
             shutil.copy(src, readme)
 
     def _security_dir_changed(self, name, old, new):
         self.check_security_dir()
 
     def check_security_dir(self):
         if not os.path.isdir(self.security_dir):
             os.mkdir(self.security_dir, 0o700)
         else:
             try:
                 os.chmod(self.security_dir, 0o700)
             except OSError:
                 self.log.warn("Could not set security dir permissions to private.")
 
     def _pid_dir_changed(self, name, old, new):
         self.check_pid_dir()
 
     def check_pid_dir(self):
         if not os.path.isdir(self.pid_dir):
             os.mkdir(self.pid_dir, 0o700)
         else:
             try:
                 os.chmod(self.pid_dir, 0o700)
             except OSError:
                 self.log.warn("Could not set pid dir permissions to private.")
 
     def check_dirs(self):
         self.check_security_dir()
         self.check_log_dir()
         self.check_pid_dir()
         self.check_startup_dir()
 
     def copy_config_file(self, config_file, path=None, overwrite=False):
         """Copy a default config file into the active profile directory.
 
         Default configuration files are kept in :mod:`IPython.config.default`.
         This function moves these from that location to the working profile
         directory.
         """
         dst = os.path.join(self.location, config_file)
         if os.path.isfile(dst) and not overwrite:
             return False
         if path is None:
             path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
         src = os.path.join(path, config_file)
         shutil.copy(src, dst)
         return True
 
     @classmethod
     def create_profile_dir(cls, profile_dir, config=None):
         """Create a new profile directory given a full path.
 
         Parameters
         ----------
         profile_dir : str
             The full path to the profile directory.  If it does exist, it will
             be used.  If not, it will be created.
         """
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def create_profile_dir_by_name(cls, path, name=u'default', config=None):
         """Create a profile dir by profile name and path.
 
         Parameters
         ----------
         path : unicode
             The path (directory) to put the profile directory in.
         name : unicode
             The name of the profile.  The name of the profile directory will
             be "profile_<profile>".
         """
         if not os.path.isdir(path):
             raise ProfileDirError('Directory not found: %s' % path)
         profile_dir = os.path.join(path, u'profile_' + name)
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
         """Find an existing profile dir by profile name, return its ProfileDir.
 
         This searches through a sequence of paths for a profile dir.  If it
         is not found, a :class:`ProfileDirError` exception will be raised.
 
         The search path algorithm is:
         1. ``os.getcwdu()``
         2. ``ipython_dir``
 
         Parameters
         ----------
         ipython_dir : unicode or str
             The IPython directory to use.
         name : unicode or str
             The name of the profile.  The name of the profile directory
             will be "profile_<profile>".
         """
         dirname = u'profile_' + name
         paths = [os.getcwdu(), ipython_dir]
         for p in paths:
             profile_dir = os.path.join(p, dirname)
             if os.path.isdir(profile_dir):
                 return cls(location=profile_dir, config=config)
         else:
             raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_profile_dir(cls, profile_dir, config=None):
         """Find/create a profile dir and return its ProfileDir.
 
         This will create the profile directory if it doesn't exist.
 
         Parameters
         ----------
         profile_dir : unicode or str
             The path of the profile directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
         profile_dir = expand_path(profile_dir)
         if not os.path.isdir(profile_dir):
             raise ProfileDirError('Profile directory not found: %s' % profile_dir)
         return cls(location=profile_dir, config=config)
 
 

IPython/core/ultratb.py

@@ -1,1247 +1,1246 b''
 # -*- coding: utf-8 -*-
 """
 ultratb.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB:
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 * VerboseTB
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 Note:
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 
 Installation instructions for ColorTB:
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 * Color schemes
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from __future__ import unicode_literals
 
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 try:                           # Python 2
     generate_tokens = tokenize.generate_tokens
 except AttributeError:         # Python 3
     generate_tokens = tokenize.tokenize
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython.core import debugger, ipapi
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils import pyfile
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
-from IPython.utils.openpy import read_py_file
 from IPython.utils.warn import info, error
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with Python >= 2.5
 inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = ulinecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         ipinst = ipapi.get()
         if ipinst is not None:
             scheme = ipinst.colors
         else:
             scheme = DEFAULT_SCHEME
 
     _line_format = _parser.format2
 
     for line in lines:
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str', scheme)
         if not err: line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num,
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num,
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(object):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing io.stdout until runtime.  The way
         # things are written now, the io.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
         to io.stdout.  This ensures compatibility with most tools, including
         Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return io.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor', call_pdb=False, ostream=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         value : object
           Data stored in the exception
 
         elist : list
           List of frames, see class docstring for details.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance value is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
         # Note: this code originally read:
 
         ## for line in lines[:-1]:
         ##     out_list.append(" "+line)
         ## out_list.append(lines[-1])
 
         # This means it was indenting everything but the last line by a little
         # bit.  I've disabled this for now, but if we see ugliness somewhre we
         # can restore it.
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal,
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         #from pprint import pformat; print 'LISTTB', pformat(list) # dbg
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = Colors.excName + etype.__name__ + Colors.Normal
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append( py3compat.cast_unicode(stype) + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 #print 'filename is',filename  # dbg
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = ulinecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                         (Colors.normalEm,
                          Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                          Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset-1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal) )
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
         # sync with user hooks
         if have_filedata:
             ipinst = ipapi.get()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value, [])
 
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              context=5):
         """Return a nice text document describing the traceback."""
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
                                            pyver, date.rjust(75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
                                      'Traceback (most recent call last)'.\
                                                   rjust(75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         ##self.check_cache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context, tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             if not file:
                 file = '?'
             elif not(file.startswith(str("<")) and file.endswith(str(">"))):
                 # Guess that filenames like <string> aren't real filenames, so
                 # don't call abspath on them.                    
                 try:
                     file = abspath(file)
                 except OSError:
                     # Not sure if this can still happen: abspath now works with
                     # file names like <string>
                     pass
             file = py3compat.cast_unicode(file, util_path.fs_encoding)
             link = tpl_link % file
             args, varargs, varkw, locals = inspect.getargvalues(frame)
 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # This happens in situations like errors inside generator
                     # expressions, where local variables are listed in the
                     # line, but can't be extracted from the frame.  I'm not
                     # 100% sure this isn't actually a bug in inspect itself,
                     # but since there's no info for us to compute with, the
                     # best we can do is report the failure and move on.  Here
                     # we must *not* call any traceback construction again,
                     # because that would mess up use of %debug later on.  So we
                     # simply report the failure and move on.  The only
                     # limitation will be that this frame won't have locals
                     # listed in the call signature.  Quite subtle problem...
                     # I can't think of a good way to validate this in a unit
                     # test, but running a script consisting of:
                     #  dict( (k,v.strip()) for (k,v) in range(10) )
                     # will illustrate the error, if this exception catch is
                     # disabled.
                     call = tpl_call_fail % func
             
             # Don't attempt to tokenize binary files.
             if file.endswith(('.so', '.pyd', '.dll')):
                 frames.append('%s %s\n' % (link,call))
                 continue
             elif file.endswith(('.pyc','.pyo')):
                 # Look up the corresponding source file.
                 file = pyfile.source_from_cache(file)
 
             def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 names = []
                 name_cont = False
                 
                 for token_type, token, start, end, line in generate_tokens(linereader):
                     # build composite names
                     if token_type == tokenize.NAME and token not in keyword.kwlist:
                         if name_cont:
                             # Continuation of a dotted name
                             try:
                                 names[-1].append(token)
                             except IndexError:
                                 names.append([token])
                             name_cont = False
                         else:
                             # Regular new names.  We append everything, the caller
                             # will be responsible for pruning the list later.  It's
                             # very tricky to try to prune as we go, b/c composite
                             # names can fool us.  The pruning at the end is easy
                             # to do (or the caller can print a list with repeated
                             # names if so desired.
                             names.append([token])
                     elif token == '.':
                         name_cont = True
                     elif token_type == tokenize.NEWLINE:
                         break
                         
             except (IndexError, UnicodeDecodeError):
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError as msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
 
             # Join composite names (e.g. "dict.fromkeys")
             names = ['.'.join(n) for n in names]
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if name_base in locals:
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if name_base in frame.f_globals:
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _format_traceback_lines(lnum,index,lines,Colors,lvals,
                                             col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, py3compat.cast_unicode(evalue_str))]
         if (not py3compat.PY3) and type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, py3compat.cast_unicode(evalue_str)))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         # vds: >>
         if records:
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
              ipinst = ipapi.get()
              if ipinst is not None:
                  ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
 
         # return all our info assembled as a single string
         # return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
         return [head] + frames + [''.join(exception[0])]
 
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self,'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
 
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, context
             )
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
                 self, etype, value, elist, tb_offset, context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <'+mode+'>\n'
                              'Valid modes: '+str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, context=5):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, context)
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self,color_scheme = 'NoColor'):
         ListTB.__init__(self,color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
         ListTB.__call__(self,etype,value,elist)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, d_e):
         (d, e) = d_e
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
 
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         handler(*sys.exc_info())
     print ''
 
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         handler(*sys.exc_info())
     print ''
 

IPython/lib/backgroundjobs.py

@@ -1,483 +1,483 b''
 # -*- coding: utf-8 -*-
 """Manage background (threaded) jobs conveniently from an interactive shell.
 
 This module provides a BackgroundJobManager class.  This is the main class
 meant for public usage, it implements an object which can create and manage
 new background jobs.
 
 It also provides the actual job classes managed by these BackgroundJobManager
 objects, see their docstrings below.
 
 
 This system was inspired by discussions with B. Granger and the
 BackgroundCommand class described in the book Python Scripting for
 Computational Science, by H. P. Langtangen:
 
 http://folk.uio.no/hpl/scripting
 
 (although ultimately no code from this text was used, as IPython's system is a
 separate implementation).
 
 An example notebook is provided in our documentation illustrating interactive
 use of the system.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2005-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Code begins
 import sys
 import threading
 
 from IPython.core.ultratb import AutoFormattedTB
-from IPython.utils.warn import warn, error
+from IPython.utils.warn import error
 
 
 class BackgroundJobManager(object):
     """Class to manage a pool of backgrounded threaded jobs.
 
     Below, we assume that 'jobs' is a BackgroundJobManager instance.
     
     Usage summary (see the method docstrings for details):
 
       jobs.new(...) -> start a new job
       
       jobs() or jobs.status() -> print status summary of all jobs
 
       jobs[N] -> returns job number N.
 
       foo = jobs[N].result -> assign to variable foo the result of job N
 
       jobs[N].traceback() -> print the traceback of dead job N
 
       jobs.remove(N) -> remove (finished) job N
 
       jobs.flush() -> remove all finished jobs
       
     As a convenience feature, BackgroundJobManager instances provide the
     utility result and traceback methods which retrieve the corresponding
     information from the jobs list:
 
       jobs.result(N) <--> jobs[N].result
       jobs.traceback(N) <--> jobs[N].traceback()
 
     While this appears minor, it allows you to use tab completion
     interactively on the job manager instance.
     """
 
     def __init__(self):
         # Lists for job management, accessed via a property to ensure they're
         # up to date.x
         self._running  = []
         self._completed = []
         self._dead = []
         # A dict of all jobs, so users can easily access any of them
         self.all = {}
         # For reporting
         self._comp_report = []
         self._dead_report = []
         # Store status codes locally for fast lookups
         self._s_created   = BackgroundJobBase.stat_created_c
         self._s_running   = BackgroundJobBase.stat_running_c
         self._s_completed = BackgroundJobBase.stat_completed_c
         self._s_dead      = BackgroundJobBase.stat_dead_c
 
     @property
     def running(self):
         self._update_status()
         return self._running
 
     @property
     def dead(self):
         self._update_status()
         return self._dead
 
     @property
     def completed(self):
         self._update_status()
         return self._completed
 
     def new(self, func_or_exp, *args, **kwargs):
         """Add a new background job and start it in a separate thread.
 
         There are two types of jobs which can be created:
 
         1. Jobs based on expressions which can be passed to an eval() call.
         The expression must be given as a string.  For example:
 
           job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
 
         The given expression is passed to eval(), along with the optional
         global/local dicts provided.  If no dicts are given, they are
         extracted automatically from the caller's frame.
         
         A Python statement is NOT a valid eval() expression.  Basically, you
         can only use as an eval() argument something which can go on the right
         of an '=' sign and be assigned to a variable.
 
         For example,"print 'hello'" is not valid, but '2+3' is.
 
         2. Jobs given a function object, optionally passing additional
         positional arguments:
 
           job_manager.new(myfunc, x, y)
 
         The function is called with the given arguments.
 
         If you need to pass keyword arguments to your function, you must
         supply them as a dict named kw:
 
           job_manager.new(myfunc, x, y, kw=dict(z=1))
 
         The reason for this assymmetry is that the new() method needs to
         maintain access to its own keywords, and this prevents name collisions
         between arguments to new() and arguments to your own functions.
 
         In both cases, the result is stored in the job.result field of the
         background job object.
 
         You can set `daemon` attribute of the thread by giving the keyword
         argument `daemon`.
 
         Notes and caveats:
 
         1. All threads running share the same standard output.  Thus, if your
         background jobs generate output, it will come out on top of whatever
         you are currently writing.  For this reason, background jobs are best
         used with silent functions which simply return their output.
 
         2. Threads also all work within the same global namespace, and this
         system does not lock interactive variables.  So if you send job to the
         background which operates on a mutable object for a long time, and
         start modifying that same mutable object interactively (or in another
         backgrounded job), all sorts of bizarre behaviour will occur.
 
         3. If a background job is spending a lot of time inside a C extension
         module which does not release the Python Global Interpreter Lock
         (GIL), this will block the IPython prompt.  This is simply because the
         Python interpreter can only switch between threads at Python
         bytecodes.  While the execution is inside C code, the interpreter must
         simply wait unless the extension module releases the GIL.
 
         4. There is no way, due to limitations in the Python threads library,
         to kill a thread once it has started."""
         
         if callable(func_or_exp):
             kw  = kwargs.get('kw',{})
             job = BackgroundJobFunc(func_or_exp,*args,**kw)
         elif isinstance(func_or_exp, basestring):
             if not args:
                 frame = sys._getframe(1)
                 glob, loc = frame.f_globals, frame.f_locals
             elif len(args)==1:
                 glob = loc = args[0]
             elif len(args)==2:
                 glob,loc = args
             else:
                 raise ValueError(
                       'Expression jobs take at most 2 args (globals,locals)')
             job = BackgroundJobExpr(func_or_exp, glob, loc)
         else:
             raise TypeError('invalid args for new job')
 
         if kwargs.get('daemon', False):
             job.daemon = True
         job.num = len(self.all)+1 if self.all else 0
         self.running.append(job)
         self.all[job.num] = job
         print 'Starting job # %s in a separate thread.' % job.num
         job.start()
         return job
 
     def __getitem__(self, job_key):
         num = job_key if isinstance(job_key, int) else job_key.num
         return self.all[num]
 
     def __call__(self):
         """An alias to self.status(),
 
         This allows you to simply call a job manager instance much like the
         Unix `jobs` shell command."""
 
         return self.status()
 
     def _update_status(self):
         """Update the status of the job lists.
 
         This method moves finished jobs to one of two lists:
           - self.completed: jobs which completed successfully
           - self.dead: jobs which finished but died.
 
         It also copies those jobs to corresponding _report lists.  These lists
         are used to report jobs completed/dead since the last update, and are
         then cleared by the reporting function after each call."""
 
         # Status codes
         srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
         # State lists, use the actual lists b/c the public names are properties
         # that call this very function on access
         running, completed, dead = self._running, self._completed, self._dead
 
         # Now, update all state lists
         for num, job in enumerate(running):
             stat = job.stat_code
             if stat == srun:
                 continue
             elif stat == scomp:
                 completed.append(job)
                 self._comp_report.append(job)
                 running[num] = False
             elif stat == sdead:
                 dead.append(job)
                 self._dead_report.append(job)
                 running[num] = False
         # Remove dead/completed jobs from running list
         running[:] = filter(None, running)
 
     def _group_report(self,group,name):
         """Report summary for a given job group.
 
         Return True if the group had any elements."""
 
         if group:
             print '%s jobs:' % name
             for job in group:
                 print '%s : %s' % (job.num,job)
             print
             return True
 
     def _group_flush(self,group,name):
         """Flush a given job group
 
         Return True if the group had any elements."""
 
         njobs = len(group)
         if njobs:
             plural = {1:''}.setdefault(njobs,'s')
             print 'Flushing %s %s job%s.' % (njobs,name,plural)
             group[:] = []
             return True
         
     def _status_new(self):
         """Print the status of newly finished jobs.
 
         Return True if any new jobs are reported.
 
         This call resets its own state every time, so it only reports jobs
         which have finished since the last time it was called."""
 
         self._update_status()
         new_comp = self._group_report(self._comp_report, 'Completed')
         new_dead = self._group_report(self._dead_report,
                                       'Dead, call jobs.traceback() for details')
         self._comp_report[:] = []
         self._dead_report[:] = []
         return new_comp or new_dead
                 
     def status(self,verbose=0):
         """Print a status of all jobs currently being managed."""
 
         self._update_status()
         self._group_report(self.running,'Running')
         self._group_report(self.completed,'Completed')
         self._group_report(self.dead,'Dead')
         # Also flush the report queues
         self._comp_report[:] = []
         self._dead_report[:] = []
 
     def remove(self,num):
         """Remove a finished (completed or dead) job."""
 
         try:
             job = self.all[num]
         except KeyError:
             error('Job #%s not found' % num)
         else:
             stat_code = job.stat_code
             if stat_code == self._s_running:
                 error('Job #%s is still running, it can not be removed.' % num)
                 return
             elif stat_code == self._s_completed:
                 self.completed.remove(job)
             elif stat_code == self._s_dead:
                 self.dead.remove(job)
 
     def flush(self):
         """Flush all finished jobs (completed and dead) from lists.
 
         Running jobs are never flushed.
 
         It first calls _status_new(), to update info. If any jobs have
         completed since the last _status_new() call, the flush operation
         aborts."""
 
         # Remove the finished jobs from the master dict
         alljobs = self.all
         for job in self.completed+self.dead:
             del(alljobs[job.num])
 
         # Now flush these lists completely
         fl_comp = self._group_flush(self.completed, 'Completed')
         fl_dead = self._group_flush(self.dead, 'Dead')
         if not (fl_comp or fl_dead):
             print 'No jobs to flush.'
 
     def result(self,num):
         """result(N) -> return the result of job N."""
         try:
             return self.all[num].result
         except KeyError:
             error('Job #%s not found' % num)
 
     def _traceback(self, job):
         num = job if isinstance(job, int) else job.num
         try:
             self.all[num].traceback()
         except KeyError:
             error('Job #%s not found' % num)
 
     def traceback(self, job=None):
         if job is None:
             self._update_status()
             for deadjob in self.dead:
                 print "Traceback for: %r" % deadjob
                 self._traceback(deadjob)
                 print
         else:
             self._traceback(job)
 
 
 class BackgroundJobBase(threading.Thread):
     """Base class to build BackgroundJob classes.
 
     The derived classes must implement:
 
     - Their own __init__, since the one here raises NotImplementedError.  The
     derived constructor must call self._init() at the end, to provide common
     initialization.
 
     - A strform attribute used in calls to __str__.
 
     - A call() method, which will make the actual execution call and must
     return a value to be held in the 'result' field of the job object."""
 
     # Class constants for status, in string and as numerical codes (when
     # updating jobs lists, we don't want to do string comparisons).  This will
     # be done at every user prompt, so it has to be as fast as possible
     stat_created   = 'Created'; stat_created_c = 0
     stat_running   = 'Running'; stat_running_c = 1
     stat_completed = 'Completed'; stat_completed_c = 2
     stat_dead      = 'Dead (Exception), call jobs.traceback() for details'
     stat_dead_c = -1
 
     def __init__(self):
         raise NotImplementedError("This class can not be instantiated directly.")
 
     def _init(self):
         """Common initialization for all BackgroundJob objects"""
         
         for attr in ['call','strform']:
             assert hasattr(self,attr), "Missing attribute <%s>" % attr
         
         # The num tag can be set by an external job manager
         self.num = None
       
         self.status    = BackgroundJobBase.stat_created
         self.stat_code = BackgroundJobBase.stat_created_c
         self.finished  = False
         self.result    = '<BackgroundJob has not completed>'
         
         # reuse the ipython traceback handler if we can get to it, otherwise
         # make a new one
         try:
             make_tb = get_ipython().InteractiveTB.text
         except:
             make_tb = AutoFormattedTB(mode = 'Context',
                                       color_scheme='NoColor',
                                       tb_offset = 1).text
         # Note that the actual API for text() requires the three args to be
         # passed in, so we wrap it in a simple lambda.
         self._make_tb = lambda : make_tb(None, None, None)
 
         # Hold a formatted traceback if one is generated.
         self._tb = None
         
         threading.Thread.__init__(self)
 
     def __str__(self):
         return self.strform
 
     def __repr__(self):
         return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
 
     def traceback(self):
         print self._tb
         
     def run(self):
         try:
             self.status    = BackgroundJobBase.stat_running
             self.stat_code = BackgroundJobBase.stat_running_c
             self.result    = self.call()
         except:
             self.status    = BackgroundJobBase.stat_dead
             self.stat_code = BackgroundJobBase.stat_dead_c
             self.finished  = None
             self.result    = ('<BackgroundJob died, call jobs.traceback() for details>')
             self._tb       = self._make_tb()
         else:
             self.status    = BackgroundJobBase.stat_completed
             self.stat_code = BackgroundJobBase.stat_completed_c
             self.finished  = True
 
 
 class BackgroundJobExpr(BackgroundJobBase):
     """Evaluate an expression as a background job (uses a separate thread)."""
 
     def __init__(self, expression, glob=None, loc=None):
         """Create a new job from a string which can be fed to eval().
 
         global/locals dicts can be provided, which will be passed to the eval
         call."""
 
         # fail immediately if the given expression can't be compiled
         self.code = compile(expression,'<BackgroundJob compilation>','eval')
                 
         glob = {} if glob is None else glob
         loc = {} if loc is None else loc
         self.expression = self.strform = expression
         self.glob = glob
         self.loc = loc
         self._init()
         
     def call(self):
         return eval(self.code,self.glob,self.loc)
 
 
 class BackgroundJobFunc(BackgroundJobBase):
     """Run a function call as a background job (uses a separate thread)."""
 
     def __init__(self, func, *args, **kwargs):
         """Create a new job from a callable object.
 
         Any positional arguments and keyword args given to this constructor
         after the initial callable are passed directly to it."""
 
         if not callable(func):
             raise TypeError(
                 'first argument to BackgroundJobFunc must be callable')
         
         self.func = func
         self.args = args
         self.kwargs = kwargs
         # The string form will only include the function passed, because
         # generating string representations of the arguments is a potentially
         # _very_ expensive operation (e.g. with large arrays).
         self.strform = str(func)
         self._init()
 
     def call(self):
         return self.func(*self.args, **self.kwargs)

IPython/lib/clipboard.py

@@ -1,56 +1,55 b''
 """ Utilities for accessing the platform's clipboard.
 """
 
 import subprocess
-import sys
 
 from IPython.core.error import TryNext
 import IPython.utils.py3compat as py3compat
 
 def win32_clipboard_get():
     """ Get the current clipboard's text on Windows.
 
     Requires Mark Hammond's pywin32 extensions.
     """
     try:
         import win32clipboard
     except ImportError:
         raise TryNext("Getting text from the clipboard requires the pywin32 "
                       "extensions: http://sourceforge.net/projects/pywin32/")
     win32clipboard.OpenClipboard()
     text = win32clipboard.GetClipboardData(win32clipboard.CF_TEXT)
     # FIXME: convert \r\n to \n?
     win32clipboard.CloseClipboard()
     return text
 
 def osx_clipboard_get():
     """ Get the clipboard's text on OS X.
     """
     p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],
         stdout=subprocess.PIPE)
     text, stderr = p.communicate()
     # Text comes in with old Mac \r line endings. Change them to \n.
     text = text.replace(b'\r', b'\n')
     text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
     return text
 
 def tkinter_clipboard_get():
     """ Get the clipboard's text using Tkinter.
 
     This is the default on systems that are not Windows or OS X. It may
     interfere with other UI toolkits and should be replaced with an
     implementation that uses that toolkit.
     """
     try:
         import Tkinter
     except ImportError:
         raise TryNext("Getting text from the clipboard on this platform "
                       "requires Tkinter.")
     root = Tkinter.Tk()
     root.withdraw()
     text = root.clipboard_get()
     root.destroy()
     text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
     return text
 
 

IPython/lib/demo.py

@@ -1,583 +1,582 b''
 """Module for interactive demos using IPython.
 
 This module implements a few classes for running Python scripts interactively
 in IPython for demonstrations.  With very simple markup (a few tags in
 comments), you can control points where the script stops executing and returns
 control to IPython.
 
 
 Provided classes
 ----------------
 
 The classes are (see their docstrings for further details):
 
  - Demo: pure python demos
 
  - IPythonDemo: demos with input to be processed by IPython as if it had been
  typed interactively (so magics work, as well as any other special syntax you
  may have added via input prefilters).
 
  - LineDemo: single-line version of the Demo class.  These demos are executed
  one line at a time, and require no markup.
 
  - IPythonLineDemo: IPython version of the LineDemo class (the demo is
  executed a line at a time, but processed via IPython).
 
  - ClearMixin: mixin to make Demo classes with less visual clutter.  It
    declares an empty marquee and a pre_cmd that clears the screen before each
    block (see Subclassing below).
 
  - ClearDemo, ClearIPDemo: mixin-enabled versions of the Demo and IPythonDemo
    classes.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.demo
    :parts: 3
 
 Subclassing
 -----------
 
 The classes here all include a few methods meant to make customization by
 subclassing more convenient.  Their docstrings below have some more details:
 
   - marquee(): generates a marquee to provide visible on-screen markers at each
     block start and end.
 
   - pre_cmd(): run right before the execution of each block.
 
   - post_cmd(): run right after the execution of each block.  If the block
     raises an exception, this is NOT called.
 
 
 Operation
 ---------
 
 The file is run in its own empty namespace (though you can pass it a string of
 arguments as if in a command line environment, and it will see those as
 sys.argv).  But at each stop, the global IPython namespace is updated with the
 current internal demo namespace, so you can work interactively with the data
 accumulated so far.
 
 By default, each block of code is printed (with syntax highlighting) before
 executing it and you have to confirm execution.  This is intended to show the
 code to an audience first so you can discuss it, and only proceed with
 execution once you agree.  There are a few tags which allow you to modify this
 behavior.
 
 The supported tags are:
 
 # <demo> stop
 
   Defines block boundaries, the points where IPython stops execution of the
   file and returns to the interactive prompt.
 
   You can optionally mark the stop tag with extra dashes before and after the
   word 'stop', to help visually distinguish the blocks in a text editor:
 
   # <demo> --- stop ---
 
 
 # <demo> silent
 
   Make a block execute silently (and hence automatically).  Typically used in
   cases where you have some boilerplate or initialization code which you need
   executed but do not want to be seen in the demo.
 
 # <demo> auto
 
   Make a block execute automatically, but still being printed.  Useful for
   simple code which does not warrant discussion, since it avoids the extra
   manual confirmation.
 
 # <demo> auto_all
 
   This tag can _only_ be in the first block, and if given it overrides the
   individual auto tags to make the whole demo fully automatic (no block asks
   for confirmation).  It can also be given at creation time (or the attribute
   set later) to override what's in the file.
 
 While _any_ python file can be run as a Demo instance, if there are no stop
 tags the whole file will run in a single block (no different that calling
 first %pycat and then %run).  The minimal markup to make this useful is to
 place a set of stop tags; the other tags are only there to let you fine-tune
 the execution.
 
 This is probably best explained with the simple example file below.  You can
 copy this into a file named ex_demo.py, and try running it via::
 
     from IPython.demo import Demo
     d = Demo('ex_demo.py')
     d()
 
 Each time you call the demo object, it runs the next block.  The demo object
 has a few useful methods for navigation, like again(), edit(), jump(), seek()
 and back().  It can be reset for a new run via reset() or reloaded from disk
 (in case you've edited the source) via reload().  See their docstrings below.
 
 Note: To make this simpler to explore, a file called "demo-exercizer.py" has
 been added to the "docs/examples/core" directory.  Just cd to this directory in
 an IPython session, and type::
 
   %run demo-exercizer.py
 
 and then follow the directions.
 
 Example
 -------
 
 The following is a very simple example of a valid demo file.
 
 ::
 
     #################### EXAMPLE DEMO <ex_demo.py> ###############################
     '''A simple interactive demo to illustrate the use of IPython's Demo class.'''
 
     print 'Hello, welcome to an interactive IPython demo.'
 
     # The mark below defines a block boundary, which is a point where IPython will
     # stop execution and return to the interactive prompt. The dashes are actually
     # optional and used only as a visual aid to clearly separate blocks while
     # editing the demo code.
     # <demo> stop
 
     x = 1
     y = 2
 
     # <demo> stop
 
     # the mark below makes this block as silent
     # <demo> silent
 
     print 'This is a silent block, which gets executed but not printed.'
 
     # <demo> stop
     # <demo> auto
     print 'This is an automatic block.'
     print 'It is executed without asking for confirmation, but printed.'
     z = x+y
 
     print 'z=',x
 
     # <demo> stop
     # This is just another normal block.
     print 'z is now:', z
 
     print 'bye!'
     ################### END EXAMPLE DEMO <ex_demo.py> ############################
 """
 
 from __future__ import unicode_literals
 
 #*****************************************************************************
 #     Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import os
 import re
 import shlex
 import sys
 
-from IPython.utils.PyColorize import Parser
 from IPython.utils import io
-from IPython.utils.io import file_read, file_readlines
+from IPython.utils.io import file_read
 from IPython.utils.text import marquee
 from IPython.utils import openpy
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
 class DemoError(Exception): pass
 
 def re_mark(mark):
     return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)
 
 class Demo(object):
 
     re_stop     = re_mark('-*\s?stop\s?-*')
     re_silent   = re_mark('silent')
     re_auto     = re_mark('auto')
     re_auto_all = re_mark('auto_all')
 
     def __init__(self,src,title='',arg_str='',auto_all=None):
         """Make a new demo object.  To run the demo, simply call the object.
 
         See the module docstring for full details and an example (you can use
         IPython.Demo? in IPython to see it).
 
         Inputs:
 
           - src is either a file, or file-like object, or a
               string that can be resolved to a filename.
 
         Optional inputs:
 
           - title: a string to use as the demo name.  Of most use when the demo
           you are making comes from an object that has no filename, or if you
           want an alternate denotation distinct from the filename.
 
           - arg_str(''): a string of arguments, internally converted to a list
           just like sys.argv, so the demo script can see a similar
           environment.
 
           - auto_all(None): global flag to run all blocks automatically without
           confirmation.  This attribute overrides the block-level tags and
           applies to the whole demo.  It is an attribute of the object, and
           can be changed at runtime simply by reassigning it to a boolean
           value.
           """
         if hasattr(src, "read"):
              # It seems to be a file or a file-like object
             self.fname = "from a file-like object"
             if title == '':
                 self.title = "from a file-like object"
             else:
                 self.title = title
         else:
              # Assume it's a string or something that can be converted to one
             self.fname = src
             if title == '':
                 (filepath, filename) = os.path.split(src)
                 self.title = filename
             else:
                 self.title = title
         self.sys_argv = [src] + shlex.split(arg_str)
         self.auto_all = auto_all
         self.src = src
 
         # get a few things from ipython.  While it's a bit ugly design-wise,
         # it ensures that things like color scheme and the like are always in
         # sync with the ipython mode being used.  This class is only meant to
         # be used inside ipython anyways,  so it's OK.
         ip = get_ipython()  # this is in builtins whenever IPython is running
         self.ip_ns       = ip.user_ns
         self.ip_colorize = ip.pycolorize
         self.ip_showtb   = ip.showtraceback
         self.ip_run_cell = ip.run_cell
         self.shell       = ip
 
         # load user data and initialize data structures
         self.reload()
 
     def fload(self):
         """Load file object."""
         # read data and parse into blocks
         if hasattr(self, 'fobj') and self.fobj is not None:
            self.fobj.close()
         if hasattr(self.src, "read"):
              # It seems to be a file or a file-like object
             self.fobj = self.src
         else:
              # Assume it's a string or something that can be converted to one
             self.fobj = openpy.open(self.fname)
 
     def reload(self):
         """Reload source from disk and initialize state."""
         self.fload()
 
         self.src     = "".join(openpy.strip_encoding_cookie(self.fobj))
         src_b        = [b.strip() for b in self.re_stop.split(self.src) if b]
         self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
         self._auto   = [bool(self.re_auto.findall(b)) for b in src_b]
 
         # if auto_all is not given (def. None), we read it from the file
         if self.auto_all is None:
             self.auto_all = bool(self.re_auto_all.findall(src_b[0]))
         else:
             self.auto_all = bool(self.auto_all)
 
         # Clean the sources from all markup so it doesn't get displayed when
         # running the demo
         src_blocks = []
         auto_strip = lambda s: self.re_auto.sub('',s)
         for i,b in enumerate(src_b):
             if self._auto[i]:
                 src_blocks.append(auto_strip(b))
             else:
                 src_blocks.append(b)
         # remove the auto_all marker
         src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])
 
         self.nblocks = len(src_blocks)
         self.src_blocks = src_blocks
 
         # also build syntax-highlighted source
         self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
 
         # ensure clean namespace and seek offset
         self.reset()
 
     def reset(self):
         """Reset the namespace and seek pointer to restart the demo"""
         self.user_ns     = {}
         self.finished    = False
         self.block_index = 0
 
     def _validate_index(self,index):
         if index<0 or index>=self.nblocks:
             raise ValueError('invalid block index %s' % index)
 
     def _get_index(self,index):
         """Get the current block index, validating and checking status.
 
         Returns None if the demo is finished"""
 
         if index is None:
             if self.finished:
                 print('Demo finished.  Use <demo_name>.reset() if you want to rerun it.', file=io.stdout)
                 return None
             index = self.block_index
         else:
             self._validate_index(index)
         return index
 
     def seek(self,index):
         """Move the current seek pointer to the given block.
 
         You can use negative indices to seek from the end, with identical
         semantics to those of Python lists."""
         if index<0:
             index = self.nblocks + index
         self._validate_index(index)
         self.block_index = index
         self.finished = False
 
     def back(self,num=1):
         """Move the seek pointer back num blocks (default is 1)."""
         self.seek(self.block_index-num)
 
     def jump(self,num=1):
         """Jump a given number of blocks relative to the current one.
 
         The offset can be positive or negative, defaults to 1."""
         self.seek(self.block_index+num)
 
     def again(self):
         """Move the seek pointer back one block and re-execute."""
         self.back(1)
         self()
 
     def edit(self,index=None):
         """Edit a block.
 
         If no number is given, use the last block executed.
 
         This edits the in-memory copy of the demo, it does NOT modify the
         original source file.  If you want to do that, simply open the file in
         an editor and use reload() when you make changes to the file.  This
         method is meant to let you change a block during a demonstration for
         explanatory purposes, without damaging your original script."""
 
         index = self._get_index(index)
         if index is None:
             return
         # decrease the index by one (unless we're at the very beginning), so
         # that the default demo.edit() call opens up the sblock we've last run
         if index>0:
             index -= 1
 
         filename = self.shell.mktempfile(self.src_blocks[index])
         self.shell.hooks.editor(filename,1)
         new_block = file_read(filename)
         # update the source and colored block
         self.src_blocks[index] = new_block
         self.src_blocks_colored[index] = self.ip_colorize(new_block)
         self.block_index = index
         # call to run with the newly edited index
         self()
 
     def show(self,index=None):
         """Show a single block on screen"""
 
         index = self._get_index(index)
         if index is None:
             return
 
         print(self.marquee('<%s> block # %s (%s remaining)' %
                            (self.title,index,self.nblocks-index-1)), file=io.stdout)
         print((self.src_blocks_colored[index]), file=io.stdout)
         sys.stdout.flush()
 
     def show_all(self):
         """Show entire demo on screen, block by block"""
 
         fname = self.title
         title = self.title
         nblocks = self.nblocks
         silent = self._silent
         marquee = self.marquee
         for index,block in enumerate(self.src_blocks_colored):
             if silent[index]:
                 print(marquee('<%s> SILENT block # %s (%s remaining)' %
                               (title,index,nblocks-index-1)), file=io.stdout)
             else:
                 print(marquee('<%s> block # %s (%s remaining)' %
                               (title,index,nblocks-index-1)), file=io.stdout)
             print(block, end=' ', file=io.stdout)
         sys.stdout.flush()
 
     def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
         exec source in self.user_ns
 
     def __call__(self,index=None):
         """run a block of the demo.
 
         If index is given, it should be an integer >=1 and <= nblocks.  This
         means that the calling convention is one off from typical Python
         lists.  The reason for the inconsistency is that the demo always
         prints 'Block n/N, and N is the total, so it would be very odd to use
         zero-indexing here."""
 
         index = self._get_index(index)
         if index is None:
             return
         try:
             marquee = self.marquee
             next_block = self.src_blocks[index]
             self.block_index += 1
             if self._silent[index]:
                 print(marquee('Executing silent block # %s (%s remaining)' %
                               (index,self.nblocks-index-1)), file=io.stdout)
             else:
                 self.pre_cmd()
                 self.show(index)
                 if self.auto_all or self._auto[index]:
                     print(marquee('output:'), file=io.stdout)
                 else:
                     print(marquee('Press <q> to quit, <Enter> to execute...'), end=' ', file=io.stdout)
                     ans = raw_input().strip()
                     if ans:
                         print(marquee('Block NOT executed'), file=io.stdout)
                         return
             try:
                 save_argv = sys.argv
                 sys.argv = self.sys_argv
                 self.run_cell(next_block)
                 self.post_cmd()
             finally:
                 sys.argv = save_argv
 
         except:
             self.ip_showtb(filename=self.fname)
         else:
             self.ip_ns.update(self.user_ns)
 
         if self.block_index == self.nblocks:
             mq1 = self.marquee('END OF DEMO')
             if mq1:
                 # avoid spurious print >>io.stdout,s if empty marquees are used
                 print(file=io.stdout)
                 print(mq1, file=io.stdout)
                 print(self.marquee('Use <demo_name>.reset() if you want to rerun it.'), file=io.stdout)
             self.finished = True
 
     # These methods are meant to be overridden by subclasses who may wish to
     # customize the behavior of of their demos.
     def marquee(self,txt='',width=78,mark='*'):
         """Return the input string centered in a 'marquee'."""
         return marquee(txt,width,mark)
 
     def pre_cmd(self):
         """Method called before executing each block."""
         pass
 
     def post_cmd(self):
         """Method called after executing each block."""
         pass
 
 
 class IPythonDemo(Demo):
     """Class for interactive demos with IPython's input processing applied.
 
     This subclasses Demo, but instead of executing each block by the Python
     interpreter (via exec), it actually calls IPython on it, so that any input
     filters which may be in place are applied to the input block.
 
     If you have an interactive environment which exposes special input
     processing, you can use this class instead to write demo scripts which
     operate exactly as if you had typed them interactively.  The default Demo
     class requires the input to be valid, pure Python code.
     """
 
     def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
         self.shell.run_cell(source)
 
 class LineDemo(Demo):
     """Demo where each line is executed as a separate block.
 
     The input script should be valid Python code.
 
     This class doesn't require any markup at all, and it's meant for simple
     scripts (with no nesting or any kind of indentation) which consist of
     multiple lines of input to be executed, one at a time, as if they had been
     typed in the interactive prompt.
 
     Note: the input can not have *any* indentation, which means that only
     single-lines of input are accepted, not even function definitions are
     valid."""
 
     def reload(self):
         """Reload source from disk and initialize state."""
         # read data and parse into blocks
         self.fload()
         lines           = self.fobj.readlines()
         src_b           = [l for l in lines if l.strip()]
         nblocks         = len(src_b)
         self.src        = ''.join(lines)
         self._silent    = [False]*nblocks
         self._auto      = [True]*nblocks
         self.auto_all   = True
         self.nblocks    = nblocks
         self.src_blocks = src_b
 
         # also build syntax-highlighted source
         self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
 
         # ensure clean namespace and seek offset
         self.reset()
 
 
 class IPythonLineDemo(IPythonDemo,LineDemo):
     """Variant of the LineDemo class whose input is processed by IPython."""
     pass
 
 
 class ClearMixin(object):
     """Use this mixin to make Demo classes with less visual clutter.
 
     Demos using this mixin will clear the screen before every block and use
     blank marquees.
 
     Note that in order for the methods defined here to actually override those
     of the classes it's mixed with, it must go /first/ in the inheritance
     tree.  For example:
 
         class ClearIPDemo(ClearMixin,IPythonDemo): pass
 
     will provide an IPythonDemo class with the mixin's features.
     """
 
     def marquee(self,txt='',width=78,mark='*'):
         """Blank marquee that returns '' no matter what the input."""
         return ''
 
     def pre_cmd(self):
         """Method called before executing each block.
 
         This one simply clears the screen."""
         from IPython.utils.terminal import term_clear
         term_clear()
 
 class ClearDemo(ClearMixin,Demo):
     pass
 
 
 class ClearIPDemo(ClearMixin,IPythonDemo):
     pass

IPython/lib/inputhookwx.py

@@ -1,165 +1,163 b''
 # encoding: utf-8
 
 """
 Enable wxPython to be used interacive by setting PyOS_InputHook.
 
 Authors:  Robin Dunn, Brian Granger, Ondrej Certik
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
 import signal
-import sys
 import time
 from timeit import default_timer as clock
 import wx
 
 from IPython.lib.inputhook import stdin_ready
 
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def inputhook_wx1():
     """Run the wx event loop by processing pending events only.
 
     This approach seems to work, but its performance is not great as it
     relies on having PyOS_InputHook called regularly.
     """
     try:
         app = wx.GetApp()
         if app is not None:
             assert wx.Thread_IsMain()
 
             # Make a temporary event loop and process system events until
             # there are no more waiting, then allow idle events (which
             # will also deal with pending or posted wx events.)
             evtloop = wx.EventLoop()
             ea = wx.EventLoopActivator(evtloop)
             while evtloop.Pending():
                 evtloop.Dispatch()
             app.ProcessIdle()
             del ea
     except KeyboardInterrupt:
         pass
     return 0
 
 class EventLoopTimer(wx.Timer):
 
     def __init__(self, func):
         self.func = func
         wx.Timer.__init__(self)
 
     def Notify(self):
         self.func()
 
 class EventLoopRunner(object):
 
     def Run(self, time):
         self.evtloop = wx.EventLoop()
         self.timer = EventLoopTimer(self.check_stdin)
         self.timer.Start(time)
         self.evtloop.Run()
 
     def check_stdin(self):
         if stdin_ready():
             self.timer.Stop()
             self.evtloop.Exit()
 
 def inputhook_wx2():
     """Run the wx event loop, polling for stdin.
 
     This version runs the wx eventloop for an undetermined amount of time,
     during which it periodically checks to see if anything is ready on
     stdin.  If anything is ready on stdin, the event loop exits.
 
     The argument to elr.Run controls how often the event loop looks at stdin.
     This determines the responsiveness at the keyboard.  A setting of 1000
     enables a user to type at most 1 char per second.  I have found that a
     setting of 10 gives good keyboard response.  We can shorten it further,
     but eventually performance would suffer from calling select/kbhit too
     often.
     """
     try:
         app = wx.GetApp()
         if app is not None:
             assert wx.Thread_IsMain()
             elr = EventLoopRunner()
             # As this time is made shorter, keyboard response improves, but idle
             # CPU load goes up.  10 ms seems like a good compromise.
             elr.Run(time=10)  # CHANGE time here to control polling interval
     except KeyboardInterrupt:
         pass
     return 0
 
 def inputhook_wx3():
     """Run the wx event loop by processing pending events only.
 
     This is like inputhook_wx1, but it keeps processing pending events
     until stdin is ready.  After processing all pending events, a call to
     time.sleep is inserted.  This is needed, otherwise, CPU usage is at 100%.
     This sleep time should be tuned though for best performance.
     """
     # We need to protect against a user pressing Control-C when IPython is
     # idle and this is running. We trap KeyboardInterrupt and pass.
     try:
         app = wx.GetApp()
         if app is not None:
             assert wx.Thread_IsMain()
 
             # The import of wx on Linux sets the handler for signal.SIGINT
             # to 0.  This is a bug in wx or gtk.  We fix by just setting it
             # back to the Python default.
             if not callable(signal.getsignal(signal.SIGINT)):
                 signal.signal(signal.SIGINT, signal.default_int_handler)
 
             evtloop = wx.EventLoop()
             ea = wx.EventLoopActivator(evtloop)
             t = clock()
             while not stdin_ready():
                 while evtloop.Pending():
                     t = clock()
                     evtloop.Dispatch()
                 app.ProcessIdle()
                 # We need to sleep at this point to keep the idle CPU load
                 # low.  However, if sleep to long, GUI response is poor.  As
                 # a compromise, we watch how often GUI events are being processed
                 # and switch between a short and long sleep time.  Here are some
                 # stats useful in helping to tune this.
                 # time    CPU load
                 # 0.001   13%
                 # 0.005   3%
                 # 0.01    1.5%
                 # 0.05    0.5%
                 used_time = clock() - t
                 if used_time > 5*60.0:
                     # print 'Sleep for 5 s'  # dbg
                     time.sleep(5.0)
                 elif used_time > 10.0:
                     # print 'Sleep for 1 s'  # dbg
                     time.sleep(1.0)
                 elif used_time > 0.1:
                     # Few GUI events coming in, so we can sleep longer
                     # print 'Sleep for 0.05 s'  # dbg
                     time.sleep(0.05)
                 else:
                     # Many GUI events coming in, so sleep only very little
                     time.sleep(0.001)
             del ea
     except KeyboardInterrupt:
         pass
     return 0
 
 # This is our default implementation
 inputhook_wx = inputhook_wx3

IPython/lib/latextools.py

@@ -1,252 +1,252 b''
 # -*- coding: utf-8 -*-
 """Tools for handling LaTeX.
 
 Authors:
 
 * Brian Granger
 """
 #-----------------------------------------------------------------------------
 # Copyright (C) 2010 IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from io import BytesIO
 from base64 import encodestring
 import os
 import tempfile
 import shutil
 import subprocess
 
 from IPython.utils.process import find_cmd, FindCmdError
 from IPython.config.configurable import SingletonConfigurable
-from IPython.utils.traitlets import Instance, List, CBool, CUnicode
+from IPython.utils.traitlets import List, CBool, CUnicode
 from IPython.utils.py3compat import bytes_to_str
 
 #-----------------------------------------------------------------------------
 # Tools
 #-----------------------------------------------------------------------------
 
 
 class LaTeXTool(SingletonConfigurable):
     """An object to store configuration of the LaTeX tool."""
 
     backends = List(
         CUnicode, ["matplotlib", "dvipng"],
         help="Preferred backend to draw LaTeX math equations. "
         "Backends in the list are checked one by one and the first "
         "usable one is used.  Note that `matplotlib` backend "
         "is usable only for inline style equations.  To draw  "
         "display style equations, `dvipng` backend must be specified. ",
         # It is a List instead of Enum, to make configuration more
         # flexible.  For example, to use matplotlib mainly but dvipng
         # for display style, the default ["matplotlib", "dvipng"] can
         # be used.  To NOT use dvipng so that other repr such as
         # unicode pretty printing is used, you can use ["matplotlib"].
         config=True)
 
     use_breqn = CBool(
         True,
         help="Use breqn.sty to automatically break long equations. "
         "This configuration takes effect only for dvipng backend.",
         config=True)
 
     packages = List(
         ['amsmath', 'amsthm', 'amssymb', 'bm'],
         help="A list of packages to use for dvipng backend. "
         "'breqn' will be automatically appended when use_breqn=True.",
         config=True)
 
     preamble = CUnicode(
         help="Additional preamble to use when generating LaTeX source "
         "for dvipng backend.",
         config=True)
 
 
 def latex_to_png(s, encode=False, backend=None, wrap=False):
     """Render a LaTeX string to PNG.
 
     Parameters
     ----------
     s : str
         The raw string containing valid inline LaTeX.
     encode : bool, optional
         Should the PNG data bebase64 encoded to make it JSON'able.
     backend : {matplotlib, dvipng}
         Backend for producing PNG data.
     wrap : bool
         If true, Automatically wrap `s` as a LaTeX equation.
 
     None is returned when the backend cannot be used.
 
     """
     allowed_backends = LaTeXTool.instance().backends
     if backend is None:
         backend = allowed_backends[0]
     if backend not in allowed_backends:
         return None
     if backend == 'matplotlib':
         f = latex_to_png_mpl
     elif backend == 'dvipng':
         f = latex_to_png_dvipng
     else:
         raise ValueError('No such backend {0}'.format(backend))
     bin_data = f(s, wrap)
     if encode and bin_data:
         bin_data = encodestring(bin_data)
     return bin_data
 
 
 def latex_to_png_mpl(s, wrap):
     try:
         from matplotlib import mathtext
     except ImportError:
         return None
 
     if wrap:
         s = '${0}$'.format(s)
     mt = mathtext.MathTextParser('bitmap')
     f = BytesIO()
     mt.to_png(f, s, fontsize=12)
     return f.getvalue()
 
 
 def latex_to_png_dvipng(s, wrap):
     try:
         find_cmd('latex')
         find_cmd('dvipng')
     except FindCmdError:
         return None
     try:
         workdir = tempfile.mkdtemp()
         tmpfile = os.path.join(workdir, "tmp.tex")
         dvifile = os.path.join(workdir, "tmp.dvi")
         outfile = os.path.join(workdir, "tmp.png")
 
         with open(tmpfile, "w") as f:
             f.writelines(genelatex(s, wrap))
 
         with open(os.devnull, 'w') as devnull:
             subprocess.check_call(
                 ["latex", "-halt-on-error", tmpfile], cwd=workdir,
                 stdout=devnull, stderr=devnull)
 
             subprocess.check_call(
                 ["dvipng", "-T", "tight", "-x", "1500", "-z", "9",
                  "-bg", "transparent", "-o", outfile, dvifile], cwd=workdir,
                 stdout=devnull, stderr=devnull)
 
         with open(outfile, "rb") as f:
             bin_data = f.read()
     finally:
         shutil.rmtree(workdir)
     return bin_data
 
 
 def kpsewhich(filename):
     """Invoke kpsewhich command with an argument `filename`."""
     try:
         find_cmd("kpsewhich")
         proc = subprocess.Popen(
             ["kpsewhich", filename],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (stdout, stderr) = proc.communicate()
         return stdout.strip()
     except FindCmdError:
         pass
 
 
 def genelatex(body, wrap):
     """Generate LaTeX document for dvipng backend."""
     lt = LaTeXTool.instance()
     breqn = wrap and lt.use_breqn and kpsewhich("breqn.sty")
     yield r'\documentclass{article}'
     packages = lt.packages
     if breqn:
         packages = packages + ['breqn']
     for pack in packages:
         yield r'\usepackage{{{0}}}'.format(pack)
     yield r'\pagestyle{empty}'
     if lt.preamble:
         yield lt.preamble
     yield r'\begin{document}'
     if breqn:
         yield r'\begin{dmath*}'
         yield body
         yield r'\end{dmath*}'
     elif wrap:
         yield '$${0}$$'.format(body)
     else:
         yield body
     yield r'\end{document}'
 
 
 _data_uri_template_png = """<img src="data:image/png;base64,%s" alt=%s />"""
 
 def latex_to_html(s, alt='image'):
     """Render LaTeX to HTML with embedded PNG data using data URIs.
 
     Parameters
     ----------
     s : str
         The raw string containing valid inline LateX.
     alt : str
         The alt text to use for the HTML.
     """
     base64_data = bytes_to_str(latex_to_png(s, encode=True), 'ascii')
     if base64_data:
         return _data_uri_template_png  % (base64_data, alt)
 
 
 # From matplotlib, thanks to mdboom. Once this is in matplotlib releases, we
 # will remove.
 def math_to_image(s, filename_or_obj, prop=None, dpi=None, format=None):
     """
     Given a math expression, renders it in a closely-clipped bounding
     box to an image file.
 
     *s*
        A math expression.  The math portion should be enclosed in
        dollar signs.
 
     *filename_or_obj*
        A filepath or writable file-like object to write the image data
        to.
 
     *prop*
        If provided, a FontProperties() object describing the size and
        style of the text.
 
     *dpi*
        Override the output dpi, otherwise use the default associated
        with the output format.
 
     *format*
        The output format, eg. 'svg', 'pdf', 'ps' or 'png'.  If not
        provided, will be deduced from the filename.
     """
     from matplotlib import figure
     # backend_agg supports all of the core output formats
     from matplotlib.backends import backend_agg
     from matplotlib.font_manager import FontProperties
     from matplotlib.mathtext import MathTextParser
 
     if prop is None:
         prop = FontProperties()
 
     parser = MathTextParser('path')
     width, height, depth, _, _ = parser.parse(s, dpi=72, prop=prop)
 
     fig = figure.Figure(figsize=(width / 72.0, height / 72.0))
     fig.text(0, depth/height, s, fontproperties=prop)
     backend_agg.FigureCanvasAgg(fig)
     fig.savefig(filename_or_obj, dpi=dpi, format=format)
 
     return depth
 

IPython/utils/_process_posix.py

@@ -1,198 +1,197 b''
 """Posix-specific implementation of process utilities.
 
 This file is only meant to be imported by process.py, not by end-users.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import subprocess as sp
 import sys
 
 from IPython.external import pexpect
 
 # Our own
 from .autoattr import auto_attr
 from ._process_common import getoutput, arg_split
-from IPython.utils import text
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
 
 def _find_cmd(cmd):
     """Find the full path to a command using which."""
 
     path = sp.Popen(['/usr/bin/env', 'which', cmd],
                     stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
     return py3compat.bytes_to_str(path)
 
 
 class ProcessHandler(object):
     """Execute subprocesses under the control of pexpect.
     """
     # Timeout in seconds to wait on each reading of the subprocess' output.
     # This should not be set too low to avoid cpu overusage from our side,
     # since we read in a loop whose period is controlled by this timeout.
     read_timeout = 0.05
 
     # Timeout to give a process if we receive SIGINT, between sending the
     # SIGINT to the process and forcefully terminating it.
     terminate_timeout = 0.2
 
     # File object where stdout and stderr of the subprocess will be written
     logfile = None
 
     # Shell to call for subprocesses to execute
     sh = None
 
     @auto_attr
     def sh(self):
         sh = pexpect.which('sh')
         if sh is None:
             raise OSError('"sh" shell not found')
         return sh
 
     def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
         """Arguments are used for pexpect calls."""
         self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
                              None else read_timeout)
         self.terminate_timeout = (ProcessHandler.terminate_timeout if
                                   terminate_timeout is None else
                                   terminate_timeout)
         self.logfile = sys.stdout if logfile is None else logfile
 
     def getoutput(self, cmd):
         """Run a command and return its stdout/stderr as a string.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         output : str
           A string containing the combination of stdout and stderr from the
         subprocess, in whatever order the subprocess originally wrote to its
         file descriptors (so the order of the information in this string is the
         correct order as would be seen if running the command in a terminal).
         """
         try:
             return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
         except KeyboardInterrupt:
             print('^C', file=sys.stderr, end='')
 
     def getoutput_pexpect(self, cmd):
         """Run a command and return its stdout/stderr as a string.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         output : str
           A string containing the combination of stdout and stderr from the
         subprocess, in whatever order the subprocess originally wrote to its
         file descriptors (so the order of the information in this string is the
         correct order as would be seen if running the command in a terminal).
         """
         try:
             return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
         except KeyboardInterrupt:
             print('^C', file=sys.stderr, end='')
 
     def system(self, cmd):
         """Execute a command in a subshell.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         int : child's exitstatus
         """
         # Get likely encoding for the output.
         enc = DEFAULT_ENCODING
         
         # Patterns to match on the output, for pexpect.  We read input and
         # allow either a short timeout or EOF
         patterns = [pexpect.TIMEOUT, pexpect.EOF]
         # the index of the EOF pattern in the list.
         # even though we know it's 1, this call means we don't have to worry if
         # we change the above list, and forget to change this value:
         EOF_index = patterns.index(pexpect.EOF)
         # The size of the output stored so far in the process output buffer.
         # Since pexpect only appends to this buffer, each time we print we
         # record how far we've printed, so that next time we only print *new*
         # content from the buffer.
         out_size = 0
         try:
             # Since we're not really searching the buffer for text patterns, we
             # can set pexpect's search window to be tiny and it won't matter.
             # We only search for the 'patterns' timeout or EOF, which aren't in
             # the text itself.
             #child = pexpect.spawn(pcmd, searchwindowsize=1)
             if hasattr(pexpect, 'spawnb'):
                 child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
             else:
                 child = pexpect.spawn(self.sh, args=['-c', cmd])  # Vanilla Pexpect
             flush = sys.stdout.flush
             while True:
                 # res is the index of the pattern that caused the match, so we
                 # know whether we've finished (if we matched EOF) or not
                 res_idx = child.expect_list(patterns, self.read_timeout)
                 print(child.before[out_size:].decode(enc, 'replace'), end='')
                 flush()
                 if res_idx==EOF_index:
                     break
                 # Update the pointer to what we've already printed
                 out_size = len(child.before)
         except KeyboardInterrupt:
             # We need to send ^C to the process.  The ascii code for '^C' is 3
             # (the character is known as ETX for 'End of Text', see
             # curses.ascii.ETX).
             child.sendline(chr(3))
             # Read and print any more output the program might produce on its
             # way out.
             try:
                 out_size = len(child.before)
                 child.expect_list(patterns, self.terminate_timeout)
                 print(child.before[out_size:].decode(enc, 'replace'), end='')
                 sys.stdout.flush()
             except KeyboardInterrupt:
                 # Impatient users tend to type it multiple times
                 pass
             finally:
                 # Ensure the subprocess really is terminated
                 child.terminate(force=True)
         # add isalive check, to ensure exitstatus is set:
         child.isalive()
         return child.exitstatus
 
 
 # Make system() with a functional interface for outside use.  Note that we use
 # getoutput() from the _common utils, which is built on top of popen(). Using
 # pexpect to get subprocess output produces difficult to parse output, since
 # programs think they are talking to a tty and produce highly formatted output
 # (ls is a good example) that makes them hard.
 system = ProcessHandler().system
 
 
 

IPython/utils/_process_win32.py

@@ -1,189 +1,188 b''
 """Windows-specific implementation of process utilities.
 
 This file is only meant to be imported by process.py, not by end-users.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # stdlib
 import os
 import sys
 import ctypes
 import msvcrt
 
 from ctypes import c_int, POINTER
 from ctypes.wintypes import LPCWSTR, HLOCAL
 from subprocess import STDOUT
 
 # our own imports
 from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
 from . import py3compat
-from . import text
 from .encoding import DEFAULT_ENCODING
 
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
 
 class AvoidUNCPath(object):
     """A context manager to protect command execution from UNC paths.
 
     In the Win32 API, commands can't be invoked with the cwd being a UNC path.
     This context manager temporarily changes directory to the 'C:' drive on
     entering, and restores the original working directory on exit.
 
     The context manager returns the starting working directory *if* it made a
     change and None otherwise, so that users can apply the necessary adjustment
     to their system calls in the event of a change.
 
     Example
     -------
     ::
         cmd = 'dir'
         with AvoidUNCPath() as path:
             if path is not None:
                 cmd = '"pushd %s &&"%s' % (path, cmd)
             os.system(cmd)
     """
     def __enter__(self):
         self.path = os.getcwdu()
         self.is_unc_path = self.path.startswith(r"\\")
         if self.is_unc_path:
             # change to c drive (as cmd.exe cannot handle UNC addresses)
             os.chdir("C:")
             return self.path
         else:
             # We return None to signal that there was no change in the working
             # directory
             return None
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.is_unc_path:
             os.chdir(self.path)
 
 
 def _find_cmd(cmd):
     """Find the full path to a .bat or .exe using the win32api module."""
     try:
         from win32api import SearchPath
     except ImportError:
         raise ImportError('you need to have pywin32 installed for this to work')
     else:
         PATH = os.environ['PATH']
         extensions = ['.exe', '.com', '.bat', '.py']
         path = None
         for ext in extensions:
             try:
                 path = SearchPath(PATH, cmd + ext)[0]
             except:
                 pass
         if path is None:
             raise OSError("command %r not found" % cmd)
         else:
             return path
 
 
 def _system_body(p):
     """Callback for _system."""
     enc = DEFAULT_ENCODING
     for line in read_no_interrupt(p.stdout).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stdout)
     for line in read_no_interrupt(p.stderr).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stderr)
 
     # Wait to finish for returncode
     return p.wait()
 
 
 def system(cmd):
     """Win32 version of os.system() that works with network shares.
 
     Note that this implementation returns None, as meant for use in IPython.
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     None : we explicitly do NOT return the subprocess status code, as this
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
     # The controller provides interactivity with both
     # stdin and stdout
     #import _process_win32_controller
     #_process_win32_controller.system(cmd)
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         return process_handler(cmd, _system_body)
 
 def getoutput(cmd):
     """Return standard output of executing cmd in a shell.
 
     Accepts the same arguments as os.system().
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     stdout : str
     """
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
 
     if out is None:
         out = b''
     return py3compat.bytes_to_str(out)
 
 try:
     CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
     CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
     CommandLineToArgvW.restype = POINTER(LPCWSTR)
     LocalFree = ctypes.windll.kernel32.LocalFree
     LocalFree.res_type = HLOCAL
     LocalFree.arg_types = [HLOCAL]
     
     def arg_split(commandline, posix=False, strict=True):
         """Split a command line's arguments in a shell-like manner.
 
         This is a special version for windows that use a ctypes call to CommandLineToArgvW
         to do the argv splitting. The posix paramter is ignored.
         
         If strict=False, process_common.arg_split(...strict=False) is used instead.
         """
         #CommandLineToArgvW returns path to executable if called with empty string.
         if commandline.strip() == "":
             return []
         if not strict:
             # not really a cl-arg, fallback on _process_common
             return py_arg_split(commandline, posix=posix, strict=strict)
         argvn = c_int()
         result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
         result_array_type = LPCWSTR * argvn.value
         result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
         retval = LocalFree(result_pointer)
         return result
 except AttributeError:
     arg_split = py_arg_split

IPython/utils/jsonutil.py

@@ -1,213 +1,211 b''
 """Utilities to manipulate JSON objects.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import math
 import re
-import sys
 import types
 from datetime import datetime
 
 try:
     # base64.encodestring is deprecated in Python 3.x
     from base64 import encodebytes
 except ImportError:
     # Python 2.x
     from base64 import encodestring as encodebytes
 
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
-from IPython.utils import text
 next_attr_name = '__next__' if py3compat.PY3 else 'next'
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # timestamp formats
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json."""
     for k in dikt.iterkeys():
         if isinstance(k, basestring):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 
 def extract_dates(obj):
     """extract ISO8601 dates from unpacked JSON"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in obj.iteritems():
             obj[k] = extract_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ extract_dates(o) for o in obj ]
     elif isinstance(obj, basestring):
         if ISO8601_PAT.match(obj):
             obj = datetime.strptime(obj, ISO8601)
     return obj
 
 def squash_dates(obj):
     """squash datetime objects into ISO8601 strings"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in obj.iteritems():
             obj[k] = squash_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ squash_dates(o) for o in obj ]
     elif isinstance(obj, datetime):
         obj = obj.strftime(ISO8601)
     return obj
 
 def date_default(obj):
     """default function for packing datetime objects in JSON."""
     if isinstance(obj, datetime):
         return obj.strftime(ISO8601)
     else:
         raise TypeError("%r is not JSON serializable"%obj)
 
 
 # constants for identifying png/jpeg data
 PNG = b'\x89PNG\r\n\x1a\n'
 JPEG = b'\xff\xd8'
 
 def encode_images(format_dict):
     """b64-encodes images in a displaypub format dict
 
     Perhaps this should be handled in json_clean itself?
 
     Parameters
     ----------
 
     format_dict : dict
         A dictionary of display data keyed by mime-type
 
     Returns
     -------
 
     format_dict : dict
         A copy of the same dictionary,
         but binary image data ('image/png' or 'image/jpeg')
         is base64-encoded.
 
     """
     encoded = format_dict.copy()
     pngdata = format_dict.get('image/png')
     if isinstance(pngdata, bytes) and pngdata[:8] == PNG:
         encoded['image/png'] = encodebytes(pngdata).decode('ascii')
     jpegdata = format_dict.get('image/jpeg')
     if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:
         encoded['image/jpeg'] = encodebytes(jpegdata).decode('ascii')
     return encoded
 
 
 def json_clean(obj):
     """Clean an object to ensure it's safe to encode in JSON.
 
     Atomic, immutable objects are returned unmodified.  Sets and tuples are
     converted to lists, lists are copied and dicts are also copied.
 
     Note: dicts whose keys could cause collisions upon encoding (such as a dict
     with both the number 1 and the string '1' as keys) will cause a ValueError
     to be raised.
 
     Parameters
     ----------
     obj : any python object
 
     Returns
     -------
     out : object
 
       A version of the input which will not cause an encoding error when
       encoded as JSON.  Note that this function does not *encode* its inputs,
       it simply sanitizes it so that there will be no encoding errors later.
 
     Examples
     --------
     >>> json_clean(4)
     4
     >>> json_clean(range(10))
     [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     >>> sorted(json_clean(dict(x=1, y=2)).items())
     [('x', 1), ('y', 2)]
     >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
     [('x', 1), ('y', 2), ('z', [1, 2, 3])]
     >>> json_clean(True)
     True
     """
     # types that are 'atomic' and ok in json as-is.  bool doesn't need to be
     # listed explicitly because bools pass as int instances
     atomic_ok = (unicode, int, types.NoneType)
 
     # containers that we need to convert into lists
     container_to_list = (tuple, set, types.GeneratorType)
 
     if isinstance(obj, float):
         # cast out-of-range floats to their reprs
         if math.isnan(obj) or math.isinf(obj):
             return repr(obj)
         return obj
 
     if isinstance(obj, atomic_ok):
         return obj
 
     if isinstance(obj, bytes):
         return obj.decode(DEFAULT_ENCODING, 'replace')
 
     if isinstance(obj, container_to_list) or (
         hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
         obj = list(obj)
 
     if isinstance(obj, list):
         return [json_clean(x) for x in obj]
 
     if isinstance(obj, dict):
         # First, validate that the dict won't lose data in conversion due to
         # key collisions after stringification.  This can happen with keys like
         # True and 'true' or 1 and '1', which collide in JSON.
         nkeys = len(obj)
         nkeys_collapsed = len(set(map(str, obj)))
         if nkeys != nkeys_collapsed:
             raise ValueError('dict can not be safely converted to JSON: '
                              'key collision would lead to dropped values')
         # If all OK, proceed by making the new dict that will be json-safe
         out = {}
         for k,v in obj.iteritems():
             out[str(k)] = json_clean(v)
         return out
 
     # If we get here, we don't know how to handle the object, so we just get
     # its repr and return that.  This will catch lambdas, open sockets, class
     # objects, and any other complicated contraption that json can't encode
     return repr(obj)

IPython/utils/pickleutil.py

@@ -1,325 +1,324 b''
 # encoding: utf-8
 
 """Pickle related utilities. Perhaps this should be called 'can'."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import copy
 import logging
 import sys
 from types import FunctionType
 
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
 
 try:
     import numpy
 except:
     numpy = None
 
-import codeutil
 import py3compat
 from importstring import import_item
 
 from IPython.config import Application
 
 if py3compat.PY3:
     buffer = memoryview
     class_type = type
 else:
     from types import ClassType
     class_type = (type, ClassType)
 
 #-------------------------------------------------------------------------------
 # Classes
 #-------------------------------------------------------------------------------
 
 
 class CannedObject(object):
     def __init__(self, obj, keys=[]):
         self.keys = keys
         self.obj = copy.copy(obj)
         for key in keys:
             setattr(self.obj, key, can(getattr(obj, key)))
         
         self.buffers = []
 
     def get_object(self, g=None):
         if g is None:
             g = {}
         for key in self.keys:
             setattr(self.obj, key, uncan(getattr(self.obj, key), g))
         return self.obj
     
 
 class Reference(CannedObject):
     """object for wrapping a remote reference by name."""
     def __init__(self, name):
         if not isinstance(name, basestring):
             raise TypeError("illegal name: %r"%name)
         self.name = name
         self.buffers = []
 
     def __repr__(self):
         return "<Reference: %r>"%self.name
 
     def get_object(self, g=None):
         if g is None:
             g = {}
         
         return eval(self.name, g)
 
 
 class CannedFunction(CannedObject):
 
     def __init__(self, f):
         self._check_type(f)
         self.code = f.func_code
         if f.func_defaults:
             self.defaults = [ can(fd) for fd in f.func_defaults ]
         else:
             self.defaults = None
         self.module = f.__module__ or '__main__'
         self.__name__ = f.__name__
         self.buffers = []
 
     def _check_type(self, obj):
         assert isinstance(obj, FunctionType), "Not a function type"
 
     def get_object(self, g=None):
         # try to load function back into its module:
         if not self.module.startswith('__'):
             __import__(self.module)
             g = sys.modules[self.module].__dict__
 
         if g is None:
             g = {}
         if self.defaults:
             defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
         else:
             defaults = None
         newFunc = FunctionType(self.code, g, self.__name__, defaults)
         return newFunc
 
 class CannedClass(CannedObject):
 
     def __init__(self, cls):
         self._check_type(cls)
         self.name = cls.__name__
         self.old_style = not isinstance(cls, type)
         self._canned_dict = {}
         for k,v in cls.__dict__.items():
             if k not in ('__weakref__', '__dict__'):
                 self._canned_dict[k] = can(v)
         if self.old_style:
             mro = []
         else:
             mro = cls.mro()
         
         self.parents = [ can(c) for c in mro[1:] ]
         self.buffers = []
 
     def _check_type(self, obj):
         assert isinstance(obj, class_type), "Not a class type"
 
     def get_object(self, g=None):
         parents = tuple(uncan(p, g) for p in self.parents)
         return type(self.name, parents, uncan_dict(self._canned_dict, g=g))
 
 class CannedArray(CannedObject):
     def __init__(self, obj):
         self.shape = obj.shape
         self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
         if sum(obj.shape) == 0:
             # just pickle it
             self.buffers = [pickle.dumps(obj, -1)]
         else:
             # ensure contiguous
             obj = numpy.ascontiguousarray(obj, dtype=None)
             self.buffers = [buffer(obj)]
     
     def get_object(self, g=None):
         data = self.buffers[0]
         if sum(self.shape) == 0:
             # no shape, we just pickled it
             return pickle.loads(data)
         else:
             return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
 
 
 class CannedBytes(CannedObject):
     wrap = bytes
     def __init__(self, obj):
         self.buffers = [obj]
     
     def get_object(self, g=None):
         data = self.buffers[0]
         return self.wrap(data)
 
 def CannedBuffer(CannedBytes):
     wrap = buffer
 
 #-------------------------------------------------------------------------------
 # Functions
 #-------------------------------------------------------------------------------
 
 def _logger():
     """get the logger for the current Application
     
     the root logger will be used if no Application is running
     """
     if Application.initialized():
         logger = Application.instance().log
     else:
         logger = logging.getLogger()
         if not logger.handlers:
             logging.basicConfig()
     
     return logger
 
 def _import_mapping(mapping, original=None):
     """import any string-keys in a type mapping
     
     """
     log = _logger()
     log.debug("Importing canning map")
     for key,value in mapping.items():
         if isinstance(key, basestring):
             try:
                 cls = import_item(key)
             except Exception:
                 if original and key not in original:
                     # only message on user-added classes
                     log.error("cannning class not importable: %r", key, exc_info=True)
                 mapping.pop(key)
             else:
                 mapping[cls] = mapping.pop(key)
 
 def istype(obj, check):
     """like isinstance(obj, check), but strict
     
     This won't catch subclasses.
     """
     if isinstance(check, tuple):
         for cls in check:
             if type(obj) is cls:
                 return True
         return False
     else:
         return type(obj) is check
 
 def can(obj):
     """prepare an object for pickling"""
     
     import_needed = False
     
     for cls,canner in can_map.iteritems():
         if isinstance(cls, basestring):
             import_needed = True
             break
         elif istype(obj, cls):
             return canner(obj)
     
     if import_needed:
         # perform can_map imports, then try again
         # this will usually only happen once
         _import_mapping(can_map, _original_can_map)
         return can(obj)
     
     return obj
 
 def can_class(obj):
     if isinstance(obj, class_type) and obj.__module__ == '__main__':
         return CannedClass(obj)
     else:
         return obj
 
 def can_dict(obj):
     """can the *values* of a dict"""
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = can(v)
         return newobj
     else:
         return obj
 
 def can_sequence(obj):
     """can the elements of a sequence"""
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([can(i) for i in obj])
     else:
         return obj
 
 def uncan(obj, g=None):
     """invert canning"""
     
     import_needed = False
     for cls,uncanner in uncan_map.iteritems():
         if isinstance(cls, basestring):
             import_needed = True
             break
         elif isinstance(obj, cls):
             return uncanner(obj, g)
     
     if import_needed:
         # perform uncan_map imports, then try again
         # this will usually only happen once
         _import_mapping(uncan_map, _original_uncan_map)
         return uncan(obj, g)
     
     return obj
 
 def uncan_dict(obj, g=None):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = uncan(v,g)
         return newobj
     else:
         return obj
 
 def uncan_sequence(obj, g=None):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([uncan(i,g) for i in obj])
     else:
         return obj
 
 
 #-------------------------------------------------------------------------------
 # API dictionaries
 #-------------------------------------------------------------------------------
 
 # These dicts can be extended for custom serialization of new objects
 
 can_map = {
     'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
     'numpy.ndarray' : CannedArray,
     FunctionType : CannedFunction,
     bytes : CannedBytes,
     buffer : CannedBuffer,
     class_type : can_class,
 }
 
 uncan_map = {
     CannedObject : lambda obj, g: obj.get_object(g),
 }
 
 # for use in _import_mapping:
 _original_can_map = can_map.copy()
 _original_uncan_map = uncan_map.copy()

IPython/utils/process.py

@@ -1,126 +1,125 b''
 # encoding: utf-8
 """
 Utilities for working with external processes.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import os
 import sys
 import shlex
 
 # Our own
 if sys.platform == 'win32':
     from ._process_win32 import _find_cmd, system, getoutput, AvoidUNCPath, arg_split
 else:
     from ._process_posix import _find_cmd, system, getoutput, arg_split
 
 
 from ._process_common import getoutputerror
-from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class FindCmdError(Exception):
     pass
 
 
 def find_cmd(cmd):
     """Find absolute path to executable cmd in a cross platform manner.
 
     This function tries to determine the full path to a command line program
     using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
     time it will use the version that is first on the users `PATH`.  If
     cmd is `python` return `sys.executable`.
 
     Warning, don't use this to find IPython command line programs as there
     is a risk you will find the wrong one.  Instead find those using the
     following code and looking for the application itself::
 
         from IPython.utils.path import get_ipython_module_path
         from IPython.utils.process import pycmd2argv
         argv = pycmd2argv(get_ipython_module_path('IPython.frontend.terminal.ipapp'))
 
     Parameters
     ----------
     cmd : str
         The command line program to look for.
     """
     if cmd == 'python':
         return os.path.abspath(sys.executable)
     try:
         path = _find_cmd(cmd).rstrip()
     except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':
         raise FindCmdError('command could not be found: %s' % cmd)
     return os.path.abspath(path)
 
 
 def is_cmd_found(cmd):
     """Check whether executable `cmd` exists or not and return a bool."""
     try:
         find_cmd(cmd)
         return True
     except FindCmdError:
         return False
 
 
 def pycmd2argv(cmd):
     r"""Take the path of a python command and return a list (argv-style).
 
     This only works on Python based command line programs and will find the
     location of the ``python`` executable using ``sys.executable`` to make
     sure the right version is used.
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and [, cmd] otherwise.
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         return [sys.executable, cmd]
 
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwdu().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))

IPython/utils/rlineimpl.py

@@ -1,124 +1,119 b''
 # -*- coding: utf-8 -*-
 """ Imports and provides the 'correct' version of readline for the platform.
 
 Readline is used throughout IPython as::
 
     import IPython.utils.rlineimpl as readline
 
 In addition to normal readline stuff, this module provides have_readline
 boolean and _outputfile variable used in IPython.utils.
 """
 
-import os
-import re
 import sys
-import time
 import warnings
 
-from subprocess import Popen, PIPE
-
 if sys.platform == 'darwin':
     # dirty trick, to skip the system readline, because pip-installed readline
     # will never be found on OSX, since lib-dynload always comes ahead of site-packages
     from distutils import sysconfig
     lib_dynload = sysconfig.get_config_var('DESTSHARED')
     del sysconfig
     try:
         dynload_idx = sys.path.index(lib_dynload)
     except ValueError:
         dynload_idx = None
     else:
         sys.path.pop(dynload_idx)
 try:
     from readline import *
     import readline as _rl
     have_readline = True
 except ImportError:
     try:
         from pyreadline import *
         import pyreadline as _rl
         have_readline = True
     except ImportError:
         have_readline = False
 
 if sys.platform == 'darwin':
     # dirty trick, part II:
     if dynload_idx is not None:
         # restore path
         sys.path.insert(dynload_idx, lib_dynload)
         if not have_readline:
             # *only* have system readline, try import again
             try:
                 from readline import *
                 import readline as _rl
                 have_readline = True
             except ImportError:
                 have_readline = False
             else:
                 # if we want to warn about EPD / Fink having bad readline
                 # we would do it here
                 pass
     # cleanup dirty trick vars
     del dynload_idx, lib_dynload
 
 if have_readline and hasattr(_rl, 'rlmain'):
     # patch add_history to allow for strings in pyreadline <= 1.5:
     # fix copied from pyreadline 1.6
     import pyreadline
     if pyreadline.release.version <= '1.5':
         def add_history(line):
             """add a line to the history buffer."""
             from pyreadline import lineobj
             if not isinstance(line, lineobj.TextLine):
                 line = lineobj.TextLine(line)
             return _rl.add_history(line)
 
 if (sys.platform == 'win32' or sys.platform == 'cli') and have_readline:
     try:
         _outputfile=_rl.GetOutputFile()
     except AttributeError:
         warnings.warn("Failed GetOutputFile")
         have_readline = False
 
 # Test to see if libedit is being used instead of GNU readline.
 # Thanks to Boyd Waters for the original patch.
 uses_libedit = False
 
 if have_readline:
     # Official Python docs state that 'libedit' is in the docstring for libedit readline:
     uses_libedit = _rl.__doc__ and 'libedit' in _rl.__doc__
     # Note that many non-System Pythons also do not use proper readline,
     # but do not report libedit at all, nor are they linked dynamically against libedit.
     # known culprits of this include: EPD, Fink
     # There is not much we can do to detect this, until we find a specific failure
     # case, rather than relying on the readline module to self-identify as broken.
 
 if uses_libedit and sys.platform == 'darwin':
     _rl.parse_and_bind("bind ^I rl_complete")
     warnings.warn('\n'.join(['', "*"*78,
         "libedit detected - readline will not be well behaved, including but not limited to:",
         "   * crashes on tab completion",
         "   * incorrect history navigation",
         "   * corrupting long-lines",
         "   * failure to wrap or indent lines properly",
         "It is highly recommended that you install readline, which is easy_installable:",
         "     easy_install readline",
         "Note that `pip install readline` generally DOES NOT WORK, because",
         "it installs to site-packages, which come *after* lib-dynload in sys.path,",
         "where readline is located.  It must be `easy_install readline`, or to a custom",
         "location on your PYTHONPATH (even --user comes after lib-dyload).",
         "*"*78]),
         RuntimeWarning)
 
 # the clear_history() function was only introduced in Python 2.4 and is
 # actually optional in the readline API, so we must explicitly check for its
 # existence.  Some known platforms actually don't have it.  This thread:
 # http://mail.python.org/pipermail/python-dev/2003-August/037845.html
 # has the original discussion.
 
 if have_readline:
     try:
         _rl.clear_history
     except AttributeError:
         def clear_history(): pass
         _rl.clear_history = clear_history

IPython/utils/sysinfo.py

@@ -1,168 +1,166 b''
 # encoding: utf-8
 """
 Utilities for getting information about IPython and the system it's running in.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import platform
 import pprint
 import sys
 import subprocess
 
-from ConfigParser import ConfigParser
-
 from IPython.core import release
 from IPython.utils import py3compat, _sysinfo, encoding
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def pkg_commit_hash(pkg_path):
     """Get short form of commit hash given directory `pkg_path`
 
     We get the commit hash from (in order of preference):
 
     * IPython.utils._sysinfo.commit
     * git output, if we are in a git repository
 
     If these fail, we return a not-found placeholder tuple
 
     Parameters
     ----------
     pkg_path : str
        directory containing package
        only used for getting commit from active repo
 
     Returns
     -------
     hash_from : str
        Where we got the hash from - description
     hash_str : str
        short form of hash
     """
     # Try and get commit from written commit text file
     if _sysinfo.commit:
         return "installation", _sysinfo.commit
 
     # maybe we are in a repository
     proc = subprocess.Popen('git rev-parse --short HEAD',
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             cwd=pkg_path, shell=True)
     repo_commit, _ = proc.communicate()
     if repo_commit:
         return 'repository', repo_commit.strip()
     return '(none found)', '<not found>'
 
 
 def pkg_info(pkg_path):
     """Return dict describing the context of this package
 
     Parameters
     ----------
     pkg_path : str
        path containing __init__.py for package
 
     Returns
     -------
     context : dict
        with named parameters of interest
     """
     src, hsh = pkg_commit_hash(pkg_path)
     return dict(
         ipython_version=release.version,
         ipython_path=pkg_path,
         commit_source=src,
         commit_hash=hsh,
         sys_version=sys.version,
         sys_executable=sys.executable,
         sys_platform=sys.platform,
         platform=platform.platform(),
         os_name=os.name,
         default_encoding=encoding.DEFAULT_ENCODING,
         )
 
 
 @py3compat.doctest_refactor_print
 def sys_info():
     """Return useful information about IPython and the system, as a string.
 
     Example
     -------
     In [2]: print sys_info()
     {'commit_hash': '144fdae',      # random
      'commit_source': 'repository',
      'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
      'ipython_version': '0.11.dev',
      'os_name': 'posix',
      'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
      'sys_executable': '/usr/bin/python',
      'sys_platform': 'linux2',
      'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
    """
     p = os.path
     path = p.dirname(p.abspath(p.join(__file__, '..')))
     return pprint.pformat(pkg_info(path))
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
 
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus